<commit_before>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package helpers

import (
	"bytes"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/cilium/cilium/api/v1/models"

	log "github.com/sirupsen/logrus"
)

const (
	MaxRetries = 30
)

//Cilium struct helper. Struct that has a Node and logCxt with all relevant
//functions across cilium actions
type Cilium struct {
	Node *Node

	logCxt *log.Entry
}

//CreateCilium returns a Cilium struct
func CreateCilium(target string, log *log.Entry) *Cilium {
	log.Infof("Cilium: set target to '%s'", target)
	node := CreateNodeFromTarget(target)
	if node == nil {
		return nil
	}

	return &Cilium{
		Node:   node,
		logCxt: log,
	}
}

//Exec runs a Cilium command and returns the resultant cmdRes.
func (c *Cilium) Exec(cmd string) *CmdRes {
	stdout := new(bytes.Buffer)
	stderr := new(bytes.Buffer)
	command := fmt.Sprintf("cilium %s", cmd)
	exit := c.Node.ExecWithSudo(command, stdout, stderr)
	return &CmdRes{
		cmd:    command,
		stdout: stdout,
		stderr: stderr,
		exit:   exit,
	}
}

//EndpointGet returns the output of cilium endpoint get for the provided endpoint ID
func (c *Cilium) EndpointGet(id string) *models.Endpoint {

	var data []models.Endpoint
	err := c.Exec(fmt.Sprintf("endpoint get %s", id)).UnMarshal(&data)
	if err != nil {
		c.logCxt.Errorf("EndpointGet fail %d: %s", id, err)
		return nil
	}
	if len(data) > 0 {
		return &data[0]
	}
	return nil
}

//EndpointSetConfig sets the provided configuration option to the provided
//value for the endpoint with the provided id.
func (c *Cilium) EndpointSetConfig(id, option, value string) bool {
	// on grep we use an space, so we are sure that only match the key that we want.

	logger := c.logCxt.WithFields(log.Fields{"EndpointId": id})
	res := c.Exec(fmt.Sprintf(
		"endpoint config %s | grep '%s ' | awk '{print $2}'", id, option))

	if res.SingleOut() == value {
		return res.WasSuccessful()
	}

	before := c.EndpointGet(id)
	if before == nil {
		return false
	}
	data := c.Exec(fmt.Sprintf("endpoint config %s %s=%s", id, option, value))
	if !data.WasSuccessful() {
		logger.Errorf("Can't set endoint config %s=%s", option, value)
		return false
	}
	err := WithTimeout(func() bool {
		status := c.EndpointGet(id)
		if len(status.Status) > len(before.Status) {
			return true
		}
		logger.Info("Endpoint is not regenerated")
		return false
	}, "Endpoint is not regenerated", &TimeoutConfig{Timeout: 100})
	if err != nil {
		logger.Errorf("Endpoint set failed:%s", err)
		return false
	}
	return true
}

var EndpointWaitUntilReadyRetry int = 0 //List how many retries EndpointWaitUntilReady should have

//EndpointWaitUntilReady This function wait until all the endpoints are in the ready status
func (c *Cilium) EndpointWaitUntilReady(validation ...bool) bool {

	logger := c.logCxt.WithFields(log.Fields{"EndpointWaitReady": ""})

	getEpsStatus := func(data []models.Endpoint) map[int64]int {
		result := make(map[int64]int)
		for _, v := range data {
			result[v.ID] = len(v.Status)
		}
		return result
	}

	var data []models.Endpoint

	if err := c.GetEndpoints().UnMarshal(&data); err != nil {
		logger.Info("Can't get endpoints: %s", err)
		if EndpointWaitUntilReadyRetry > MaxRetries {
			logger.Errorf("Can not get endpoints, die :%s", err)
			return false
		}
		EndpointWaitUntilReadyRetry++
		Sleep(5)
		return c.EndpointWaitUntilReady(validation...)
	}
	EndpointWaitUntilReadyRetry = 0 //Reset to 0
	epsStatus := getEpsStatus(data)

	body := func() bool {
		var data []models.Endpoint

		if err := c.GetEndpoints().UnMarshal(&data); err != nil {
			logger.Info("Can't get endpoints: %s", err)
			return false
		}
		var valid, invalid int
		for _, eps := range data {
			if eps.State != "ready" {
				invalid++
			} else {
				valid++
			}
			if len(validation) > 0 && validation[0] {
				if originalVal, _ := epsStatus[eps.ID]; len(eps.Status) <= originalVal {
					logger.Infof("Endpoint '%d' is not regenerated", eps.ID)
					return false
				}
			}
		}

		if invalid == 0 {
			return true
		}
		logger.Infof("Endpoints are not ready valid='%d' invalid='%d'", valid, invalid)
		return false
	}
	err := WithTimeout(body, "Endpoints are not ready", &TimeoutConfig{Timeout: 300})
	if err != nil {
		return false
	}
	return true
}

//GetEndpoints Return the output of cilium endpoint list -o json
func (c *Cilium) GetEndpoints() *CmdRes {
	return c.Exec("endpoint list -o json")
}

//GetEndpointsIds returns a mapping of a Docker container name to to its
//corresponding endpoint ID, and an error if the list of endpoints cannot be
//retrieved via the Cilium CLI.
func (c *Cilium) GetEndpointsIds() (map[string]string, error) {
	// cilium endpoint list -o jsonpath='{range [*]}{@.container-name}{"="}{@.id}{"\n"}{end}'
	filter := `{range [*]}{@.container-name}{"="}{@.id}{"\n"}{end}`
	endpoints := c.Exec(fmt.Sprintf("endpoint list -o jsonpath='%s'", filter))
	if !endpoints.WasSuccessful() {
		return nil, fmt.Errorf("Can't get endpoint list: %s", endpoints.CombineOutput())
	}
	return endpoints.KVOutput(), nil
}

//GetEndpointsNames returns the container-name field of each Cilium endpoint.
func (c *Cilium) GetEndpointsNames() ([]string, error) {
	data := c.GetEndpoints()
	if data.WasSuccessful() == false {
		return nil, fmt.Errorf("Could't get endpoints")
	}
	result, err := data.Filter("{ [*].container-name }")
	if err != nil {
		return nil, err
	}

	return strings.Split(result.String(), " "), nil
}

//ManifestsPath returns the manifest path
func (c *Cilium) ManifestsPath() string {
	return fmt.Sprintf("%s/runtime/manifests/", basePath)
}

//GetFullPath returns the valid path for a file
func (c *Cilium) GetFullPath(name string) string {
	return fmt.Sprintf("%s%s", c.ManifestsPath(), name)
}

//PolicyEndpointsSummary returns the count of whether policy enforcement is
//enabled, disabled, and the total number of endpoints, and an error if the
//Cilium endpoint metadata cannot be retrieved via the API.
func (c *Cilium) PolicyEndpointsSummary() (map[string]int, error) {
	result := map[string]int{
		"enabled":  0,
		"disabled": 0,
		"total":    0,
	}

	endpoints, err := c.GetEndpoints().Filter("{ [*].policy-enabled }")
	if err != nil {
		return result, fmt.Errorf("Can't get the endpoints")
	}
	status := strings.Split(endpoints.String(), " ")
	result["enabled"], result["total"] = CountValues("true", status)
	result["disabled"], result["total"] = CountValues("false", status)
	return result, nil
}

//PolicyEnforcementSet sets the PolicyEnforcement configuration value for the
//Cilium agent to the provided status.
func (c *Cilium) PolicyEnforcementSet(status string, waitReady ...bool) *CmdRes {
	// We check before setting PolicyEnforcement; if we do not, EndpointWait
	// will fail due to the status of the endpoints not changing.
	res := c.Exec("config | grep PolicyEnforcement | awk '{print $2}'")
	if res.SingleOut() == status {
		return res
	}
	res = c.Exec(fmt.Sprintf("config PolicyEnforcement=%s", status))
	if len(waitReady) > 0 && waitReady[0] {
		c.EndpointWaitUntilReady(true)
	}
	return res
}

//PolicyDel delete a policy with the given ID
func (c *Cilium) PolicyDel(id string) *CmdRes {
	return c.Exec(fmt.Sprintf("policy delete %s", id))
}

//PolicyGet return the policy value
func (c *Cilium) PolicyGet(id string) *CmdRes {
	return c.Exec(fmt.Sprintf("policy get %s", id))
}

//PolicyGetRevision Get the current Policy revision
func (c *Cilium) PolicyGetRevision() (int, error) {
	//FIXME GH-1725
	rev := c.Exec("policy get | grep Revision| awk '{print $2}'")
	return rev.IntOutput()
}

//PolicyImport imports a new policy into Cilium and waits until the policy
//revision number increments.
func (c *Cilium) PolicyImport(path string, timeout time.Duration) (int, error) {
	revision, err := c.PolicyGetRevision()
	if err != nil {
		return -1, fmt.Errorf("Can't get policy revision: %s", err)
	}
	c.logCxt.Infof("PolicyImport: %s and current policy revision is '%d'", path, revision)
	res := c.Exec(fmt.Sprintf("policy import %s", path))
	if res.WasSuccessful() == false {
		fmt.Println(res.Output())
		c.logCxt.Errorf("Could not import policy: %s", res.CombineOutput())
		return -1, fmt.Errorf("Could not import policy %s", path)
	}
	body := func() bool {
		currentRev, _ := c.PolicyGetRevision()
		if currentRev > revision {
			c.PolicyWait(currentRev)
			return true
		}
		c.logCxt.Infof("PolicyImport: current revision %d same as %d", currentRev, revision)
		return false
	}
	err = WithTimeout(body, "Could not import policy revision", &TimeoutConfig{Timeout: timeout})
	if err != nil {
		return -1, err
	}
	revision, err = c.PolicyGetRevision()
	c.logCxt.Infof("PolicyImport: finished '%s' with revision '%d'", path, revision)
	return revision, err
}

//PolicyWait executes cilium policy wait, which waits until all endpoints are
//updated to the given policy revision.
func (c *Cilium) PolicyWait(revisionNum int) *CmdRes {
	return c.Exec(fmt.Sprintf("policy wait %d", revisionNum))
}

//ReportFailed gathers relevant Cilium runtime data and logs for debugging purposes
func (c *Cilium) ReportFailed(commands ...string) {
	wr := c.logCxt.Logger.Out
	fmt.Fprint(wr, "StackTrace Begin\n")

	//FIXME: Ginkgo PR383 add here --since option
	res := c.Node.Exec("sudo journalctl --no-pager -u cilium")
	fmt.Fprint(wr, res.Output())

	res = c.Node.Exec("sudo cilium endpoint list")
	fmt.Fprint(wr, res.Output())

	for _, cmd := range commands {
		fmt.Fprintf(wr, "Command '%s': \n", cmd)
		res = c.Node.Exec(fmt.Sprintf("%s", cmd))
		fmt.Fprint(wr, res.Output())
	}
	fmt.Fprint(wr, "StackTrace Ends\n")
}

//ServiceAdd Creates a new cilium service
func (c *Cilium) ServiceAdd(id int, frontend string, backends []string, rev int) *CmdRes {
	cmd := fmt.Sprintf(
		"service update --frontend '%s' --backends '%s' --id '%d' --rev '%d'",
		frontend, strings.Join(backends, ","), id, rev)
	return c.Exec(cmd)
}

//ServiceGet Get a service from cilium
func (c *Cilium) ServiceGet(id int) *CmdRes {
	return c.Exec(fmt.Sprintf("service get '%d'", id))
}

//ServiceDel delete the service ID
func (c *Cilium) ServiceDel(id int) *CmdRes {
	return c.Exec(fmt.Sprintf("service delete '%d'", id))
}

//SetUp setup cilium config
func (c *Cilium) SetUp() error {
	template := `
PATH=/usr/lib/llvm-3.8/bin:/usr/local/sbin:/usr/local/bin:/usr/bin:/usr/sbin:/sbin:/bin
CILIUM_OPTS=--kvstore consul --kvstore-opt consul.address=127.0.0.1:8500 --debug
INITSYSTEM=SYSTEMD`

	err := RenderTemplateToFile("cilium", template, 0777)
	if err != nil {
		return err
	}
	defer os.Remove("cilium")

	res := c.Node.Exec("sudo cp /vagrant/cilium /etc/sysconfig/cilium")
	if !res.WasSuccessful() {
		return fmt.Errorf("%s", res.CombineOutput())
	}
	res = c.Node.Exec("sudo systemctl restart cilium")
	if !res.WasSuccessful() {
		return fmt.Errorf("%s", res.CombineOutput())
	}
	return nil
}

//WaitUntilReady waits until the output of cilium status returns with code
//zero. Returns an error if the output of cilium status returns a nonzero
//return code after the specified timeout duration has elapsed.
func (c *Cilium) WaitUntilReady(timeout time.Duration) error {

	body := func() bool {
		res := c.Node.Exec("sudo cilium status")
		c.logCxt.Infof("Cilium status is %t", res.WasSuccessful())
		return res.WasSuccessful()
	}
	err := WithTimeout(body, "Cilium is not ready", &TimeoutConfig{Timeout: timeout})
	return err
}
<commit_msg>helpers/cilium: Remove extraneous terminal output<commit_after>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package helpers

import (
	"bytes"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/cilium/cilium/api/v1/models"

	log "github.com/sirupsen/logrus"
)

const (
	MaxRetries = 30
)

//Cilium struct helper. Struct that has a Node and logCxt with all relevant
//functions across cilium actions
type Cilium struct {
	Node *Node

	logCxt *log.Entry
}

//CreateCilium returns a Cilium struct
func CreateCilium(target string, log *log.Entry) *Cilium {
	log.Infof("Cilium: set target to '%s'", target)
	node := CreateNodeFromTarget(target)
	if node == nil {
		return nil
	}

	return &Cilium{
		Node:   node,
		logCxt: log,
	}
}

//Exec runs a Cilium command and returns the resultant cmdRes.
func (c *Cilium) Exec(cmd string) *CmdRes {
	stdout := new(bytes.Buffer)
	stderr := new(bytes.Buffer)
	command := fmt.Sprintf("cilium %s", cmd)
	exit := c.Node.ExecWithSudo(command, stdout, stderr)
	return &CmdRes{
		cmd:    command,
		stdout: stdout,
		stderr: stderr,
		exit:   exit,
	}
}

//EndpointGet returns the output of cilium endpoint get for the provided endpoint ID
func (c *Cilium) EndpointGet(id string) *models.Endpoint {

	var data []models.Endpoint
	err := c.Exec(fmt.Sprintf("endpoint get %s", id)).UnMarshal(&data)
	if err != nil {
		c.logCxt.Errorf("EndpointGet fail %d: %s", id, err)
		return nil
	}
	if len(data) > 0 {
		return &data[0]
	}
	return nil
}

//EndpointSetConfig sets the provided configuration option to the provided
//value for the endpoint with the provided id.
func (c *Cilium) EndpointSetConfig(id, option, value string) bool {
	// on grep we use an space, so we are sure that only match the key that we want.

	logger := c.logCxt.WithFields(log.Fields{"EndpointId": id})
	res := c.Exec(fmt.Sprintf(
		"endpoint config %s | grep '%s ' | awk '{print $2}'", id, option))

	if res.SingleOut() == value {
		return res.WasSuccessful()
	}

	before := c.EndpointGet(id)
	if before == nil {
		return false
	}
	data := c.Exec(fmt.Sprintf("endpoint config %s %s=%s", id, option, value))
	if !data.WasSuccessful() {
		logger.Errorf("Can't set endoint config %s=%s", option, value)
		return false
	}
	err := WithTimeout(func() bool {
		status := c.EndpointGet(id)
		if len(status.Status) > len(before.Status) {
			return true
		}
		logger.Info("Endpoint is not regenerated")
		return false
	}, "Endpoint is not regenerated", &TimeoutConfig{Timeout: 100})
	if err != nil {
		logger.Errorf("Endpoint set failed:%s", err)
		return false
	}
	return true
}

var EndpointWaitUntilReadyRetry int = 0 //List how many retries EndpointWaitUntilReady should have

//EndpointWaitUntilReady This function wait until all the endpoints are in the ready status
func (c *Cilium) EndpointWaitUntilReady(validation ...bool) bool {

	logger := c.logCxt.WithFields(log.Fields{"EndpointWaitReady": ""})

	getEpsStatus := func(data []models.Endpoint) map[int64]int {
		result := make(map[int64]int)
		for _, v := range data {
			result[v.ID] = len(v.Status)
		}
		return result
	}

	var data []models.Endpoint

	if err := c.GetEndpoints().UnMarshal(&data); err != nil {
		logger.Info("Can't get endpoints: %s", err)
		if EndpointWaitUntilReadyRetry > MaxRetries {
			logger.Errorf("Can not get endpoints, die :%s", err)
			return false
		}
		EndpointWaitUntilReadyRetry++
		Sleep(5)
		return c.EndpointWaitUntilReady(validation...)
	}
	EndpointWaitUntilReadyRetry = 0 //Reset to 0
	epsStatus := getEpsStatus(data)

	body := func() bool {
		var data []models.Endpoint

		if err := c.GetEndpoints().UnMarshal(&data); err != nil {
			logger.Info("Can't get endpoints: %s", err)
			return false
		}
		var valid, invalid int
		for _, eps := range data {
			if eps.State != "ready" {
				invalid++
			} else {
				valid++
			}
			if len(validation) > 0 && validation[0] {
				if originalVal, _ := epsStatus[eps.ID]; len(eps.Status) <= originalVal {
					logger.Infof("Endpoint '%d' is not regenerated", eps.ID)
					return false
				}
			}
		}

		if invalid == 0 {
			return true
		}
		logger.Infof("Endpoints are not ready valid='%d' invalid='%d'", valid, invalid)
		return false
	}
	err := WithTimeout(body, "Endpoints are not ready", &TimeoutConfig{Timeout: 300})
	if err != nil {
		return false
	}
	return true
}

//GetEndpoints Return the output of cilium endpoint list -o json
func (c *Cilium) GetEndpoints() *CmdRes {
	return c.Exec("endpoint list -o json")
}

//GetEndpointsIds returns a mapping of a Docker container name to to its
//corresponding endpoint ID, and an error if the list of endpoints cannot be
//retrieved via the Cilium CLI.
func (c *Cilium) GetEndpointsIds() (map[string]string, error) {
	// cilium endpoint list -o jsonpath='{range [*]}{@.container-name}{"="}{@.id}{"\n"}{end}'
	filter := `{range [*]}{@.container-name}{"="}{@.id}{"\n"}{end}`
	endpoints := c.Exec(fmt.Sprintf("endpoint list -o jsonpath='%s'", filter))
	if !endpoints.WasSuccessful() {
		return nil, fmt.Errorf("Can't get endpoint list: %s", endpoints.CombineOutput())
	}
	return endpoints.KVOutput(), nil
}

//GetEndpointsNames returns the container-name field of each Cilium endpoint.
func (c *Cilium) GetEndpointsNames() ([]string, error) {
	data := c.GetEndpoints()
	if data.WasSuccessful() == false {
		return nil, fmt.Errorf("Could't get endpoints")
	}
	result, err := data.Filter("{ [*].container-name }")
	if err != nil {
		return nil, err
	}

	return strings.Split(result.String(), " "), nil
}

//ManifestsPath returns the manifest path
func (c *Cilium) ManifestsPath() string {
	return fmt.Sprintf("%s/runtime/manifests/", basePath)
}

//GetFullPath returns the valid path for a file
func (c *Cilium) GetFullPath(name string) string {
	return fmt.Sprintf("%s%s", c.ManifestsPath(), name)
}

//PolicyEndpointsSummary returns the count of whether policy enforcement is
//enabled, disabled, and the total number of endpoints, and an error if the
//Cilium endpoint metadata cannot be retrieved via the API.
func (c *Cilium) PolicyEndpointsSummary() (map[string]int, error) {
	result := map[string]int{
		"enabled":  0,
		"disabled": 0,
		"total":    0,
	}

	endpoints, err := c.GetEndpoints().Filter("{ [*].policy-enabled }")
	if err != nil {
		return result, fmt.Errorf("Can't get the endpoints")
	}
	status := strings.Split(endpoints.String(), " ")
	result["enabled"], result["total"] = CountValues("true", status)
	result["disabled"], result["total"] = CountValues("false", status)
	return result, nil
}

//PolicyEnforcementSet sets the PolicyEnforcement configuration value for the
//Cilium agent to the provided status.
func (c *Cilium) PolicyEnforcementSet(status string, waitReady ...bool) *CmdRes {
	// We check before setting PolicyEnforcement; if we do not, EndpointWait
	// will fail due to the status of the endpoints not changing.
	res := c.Exec("config | grep PolicyEnforcement | awk '{print $2}'")
	if res.SingleOut() == status {
		return res
	}
	res = c.Exec(fmt.Sprintf("config PolicyEnforcement=%s", status))
	if len(waitReady) > 0 && waitReady[0] {
		c.EndpointWaitUntilReady(true)
	}
	return res
}

//PolicyDel delete a policy with the given ID
func (c *Cilium) PolicyDel(id string) *CmdRes {
	return c.Exec(fmt.Sprintf("policy delete %s", id))
}

//PolicyGet return the policy value
func (c *Cilium) PolicyGet(id string) *CmdRes {
	return c.Exec(fmt.Sprintf("policy get %s", id))
}

//PolicyGetRevision Get the current Policy revision
func (c *Cilium) PolicyGetRevision() (int, error) {
	//FIXME GH-1725
	rev := c.Exec("policy get | grep Revision| awk '{print $2}'")
	return rev.IntOutput()
}

//PolicyImport imports a new policy into Cilium and waits until the policy
//revision number increments.
func (c *Cilium) PolicyImport(path string, timeout time.Duration) (int, error) {
	revision, err := c.PolicyGetRevision()
	if err != nil {
		return -1, fmt.Errorf("Can't get policy revision: %s", err)
	}
	c.logCxt.Infof("PolicyImport: %s and current policy revision is '%d'", path, revision)
	res := c.Exec(fmt.Sprintf("policy import %s", path))
	if res.WasSuccessful() == false {
		c.logCxt.Errorf("Could not import policy: %s", res.CombineOutput())
		return -1, fmt.Errorf("Could not import policy %s", path)
	}
	body := func() bool {
		currentRev, _ := c.PolicyGetRevision()
		if currentRev > revision {
			c.PolicyWait(currentRev)
			return true
		}
		c.logCxt.Infof("PolicyImport: current revision %d same as %d", currentRev, revision)
		return false
	}
	err = WithTimeout(body, "Could not import policy revision", &TimeoutConfig{Timeout: timeout})
	if err != nil {
		return -1, err
	}
	revision, err = c.PolicyGetRevision()
	c.logCxt.Infof("PolicyImport: finished '%s' with revision '%d'", path, revision)
	return revision, err
}

//PolicyWait executes cilium policy wait, which waits until all endpoints are
//updated to the given policy revision.
func (c *Cilium) PolicyWait(revisionNum int) *CmdRes {
	return c.Exec(fmt.Sprintf("policy wait %d", revisionNum))
}

//ReportFailed gathers relevant Cilium runtime data and logs for debugging purposes
func (c *Cilium) ReportFailed(commands ...string) {
	wr := c.logCxt.Logger.Out
	fmt.Fprint(wr, "StackTrace Begin\n")

	//FIXME: Ginkgo PR383 add here --since option
	res := c.Node.Exec("sudo journalctl --no-pager -u cilium")
	fmt.Fprint(wr, res.Output())

	res = c.Node.Exec("sudo cilium endpoint list")
	fmt.Fprint(wr, res.Output())

	for _, cmd := range commands {
		fmt.Fprintf(wr, "Command '%s': \n", cmd)
		res = c.Node.Exec(fmt.Sprintf("%s", cmd))
		fmt.Fprint(wr, res.Output())
	}
	fmt.Fprint(wr, "StackTrace Ends\n")
}

//ServiceAdd Creates a new cilium service
func (c *Cilium) ServiceAdd(id int, frontend string, backends []string, rev int) *CmdRes {
	cmd := fmt.Sprintf(
		"service update --frontend '%s' --backends '%s' --id '%d' --rev '%d'",
		frontend, strings.Join(backends, ","), id, rev)
	return c.Exec(cmd)
}

//ServiceGet Get a service from cilium
func (c *Cilium) ServiceGet(id int) *CmdRes {
	return c.Exec(fmt.Sprintf("service get '%d'", id))
}

//ServiceDel delete the service ID
func (c *Cilium) ServiceDel(id int) *CmdRes {
	return c.Exec(fmt.Sprintf("service delete '%d'", id))
}

//SetUp setup cilium config
func (c *Cilium) SetUp() error {
	template := `
PATH=/usr/lib/llvm-3.8/bin:/usr/local/sbin:/usr/local/bin:/usr/bin:/usr/sbin:/sbin:/bin
CILIUM_OPTS=--kvstore consul --kvstore-opt consul.address=127.0.0.1:8500 --debug
INITSYSTEM=SYSTEMD`

	err := RenderTemplateToFile("cilium", template, 0777)
	if err != nil {
		return err
	}
	defer os.Remove("cilium")

	res := c.Node.Exec("sudo cp /vagrant/cilium /etc/sysconfig/cilium")
	if !res.WasSuccessful() {
		return fmt.Errorf("%s", res.CombineOutput())
	}
	res = c.Node.Exec("sudo systemctl restart cilium")
	if !res.WasSuccessful() {
		return fmt.Errorf("%s", res.CombineOutput())
	}
	return nil
}

//WaitUntilReady waits until the output of cilium status returns with code
//zero. Returns an error if the output of cilium status returns a nonzero
//return code after the specified timeout duration has elapsed.
func (c *Cilium) WaitUntilReady(timeout time.Duration) error {

	body := func() bool {
		res := c.Node.Exec("sudo cilium status")
		c.logCxt.Infof("Cilium status is %t", res.WasSuccessful())
		return res.WasSuccessful()
	}
	err := WithTimeout(body, "Cilium is not ready", &TimeoutConfig{Timeout: timeout})
	return err
}
<|endoftext|>


<commit_before>// Package mime helps retrieving mimetypes given extensions.
// This is an alternative to the "mime" package, and has fallbacks for the most common types.
package mime

import (
	"bytes"
	"io/ioutil"
	"net/http"
	"sync"
)

var fallback = map[string]string{
	"tar.gz":  "application/x-gzip-compressed-tar",
	"tar.bz":  "application/x-bzip-compressed-tar",
	"tar.bz2": "application/x-bzip-compressed-tar",
	"tar.xz":  "application/x-xz-compressed-tar",
	"tgz":     "application/x-gzip-compressed-tar",
	"tbz":     "application/x-bzip-compressed-tar",
	"tbz2":    "application/x-bzip-compressed-tar",
	"txz":     "application/x-xz-compressed-tar",
	"gz":      "application/x-gzip",
	"bz2":     "application/x-bzip2",
	"xz":      "application/x-xz",
	"html":    "text/html",
	"css":     "text/css",
	"js":      "application/javascript",
	"json":    "application/javascript",
	"txt":     "text/plain",
	"png":     "image/png",
	"jpg":     "image/jpg",
	"svg":     "image/svg+xml",
	"xml":     "text/xml",
	"rss":     "application/rss+xml",
	"zip":     "application/zip",
	"tar":     "application/x-tar",
	"exe":     "application/vnd.microsoft.portable-executable",
	"com":     "application/octet-stream",
	"elf":     "application/octet-stream",
	"7z":      "application/octet-stream",
	"rar":     "application/octet-stream",
	"arj":     "application/octet-stream",
	"lz":      "application/octet-stream",
	"torrent": "application/x-bittorrent",
}

// Reader caches the contents of a mime info text file
type Reader struct {
	filename  string
	utf8      bool
	mimetypes map[string]string
	mu        sync.Mutex
}

// New creates a new Reader. The filename is a list of mimetypes and extensions.
// If utf8 is true, "; charset=utf-8" will be added when setting http headers.
func New(filename string, utf8 bool) *Reader {
	return &Reader{filename, utf8, nil, sync.Mutex{}}
}

// Read a mimetype text file. Return a hash map from ext to mimetype.
func readMimetypes(filename string) (map[string]string, error) {
	mimetypes := make(map[string]string)
	// Read the mimetype file
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	// For each line, store extensions and mimetypes in the hash map
	for _, line := range bytes.Split(data, []byte("\n")) {
		fields := bytes.Fields(line)
		if len(fields) > 1 {
			for _, ext := range fields[1:] {
				mimetypes[string(ext)] = string(fields[0])
			}
		}
	}
	return mimetypes, nil
}

// Get returns the mimetype, or an empty string if no mimetype or mimetype source is found
func (mr *Reader) Get(ext string) string {
	var err error
	// No extension
	if len(ext) == 0 {
		return ""
	}
	// Strip the leading dot
	if ext[0] == '.' {
		ext = ext[1:]
	}
	mr.mu.Lock()
	defer mr.mu.Unlock()
	if mr.mimetypes == nil {
		mr.mimetypes, err = readMimetypes(mr.filename)
		if err != nil {
			// Using the fallback hash map
			if mime, ok := fallback[ext]; ok {
				return mime
			}
			// Unable to find the mime type for the given extension
			return ""
		}
	}
	// Use the value from the hash map
	if mime, ok := mr.mimetypes[ext]; ok {
		return mime
	}
	// Using the fallback hash map
	if mime, ok := fallback[ext]; ok {
		return mime
	}
	// Unable to find the mime type for the given extension
	return ""
}

// SetHeader sets the Content-Type for a given ResponseWriter and filename extension
func (mr *Reader) SetHeader(w http.ResponseWriter, ext string) {
	mimestring := mr.Get(ext)
	if mimestring == "" {
		// Default mime type
		mimestring = "application/octet-stream"
	}
	if mr.utf8 {
		mimestring += "; charset=utf-8"
	}
	w.Header().Add("Content-Type", mimestring)
}
<commit_msg>Add a few basic mime types to the fallback list<commit_after>// Package mime helps retrieving mimetypes given extensions.
// This is an alternative to the "mime" package, and has fallbacks for the most common types.
package mime

import (
	"bytes"
	"io/ioutil"
	"net/http"
	"sync"
)

var fallback = map[string]string{
	"7z":      "application/octet-stream",
	"arj":     "application/octet-stream",
	"bmp":     "image/bmp",
	"bz2":     "application/x-bzip2",
	"com":     "application/octet-stream",
	"css":     "text/css",
	"elf":     "application/octet-stream",
	"exe":     "application/vnd.microsoft.portable-executable",
	"gif":     "image/gif",
	"gz":      "application/x-gzip",
	"html":    "text/html",
	"ico":     "image/x-icon",
	"jpeg":    "image/jpg",
	"jpg":     "image/jpg",
	"js":      "application/javascript",
	"json":    "application/javascript",
	"lz":      "application/octet-stream",
	"mkv":     "video/x-matroska",
	"ogg":     "video/ogg",
	"pdf":     "application/pdf",
	"png":     "image/png",
	"rar":     "application/octet-stream",
	"rss":     "application/rss+xml",
	"svg":     "image/svg+xml",
	"tar":     "application/x-tar",
	"tar.bz":  "application/x-bzip-compressed-tar",
	"tar.bz2": "application/x-bzip-compressed-tar",
	"tar.gz":  "application/x-gzip-compressed-tar",
	"tar.xz":  "application/x-xz-compressed-tar",
	"tbz":     "application/x-bzip-compressed-tar",
	"tbz2":    "application/x-bzip-compressed-tar",
	"tgz":     "application/x-gzip-compressed-tar",
	"torrent": "application/x-bittorrent",
	"txt":     "text/plain",
	"txz":     "application/x-xz-compressed-tar",
	"wasm":    "application/wasm",
	"webm":    "video/webm",
	"webp":    "image/webp",
	"xml":     "text/xml",
	"xz":      "application/x-xz",
	"zip":     "application/zip",
}

// Reader caches the contents of a mime info text file
type Reader struct {
	filename  string
	utf8      bool
	mimetypes map[string]string
	mu        sync.Mutex
}

// New creates a new Reader. The filename is a list of mimetypes and extensions.
// If utf8 is true, "; charset=utf-8" will be added when setting http headers.
func New(filename string, utf8 bool) *Reader {
	return &Reader{filename, utf8, nil, sync.Mutex{}}
}

// Read a mimetype text file. Return a hash map from ext to mimetype.
func readMimetypes(filename string) (map[string]string, error) {
	mimetypes := make(map[string]string)
	// Read the mimetype file
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	// For each line, store extensions and mimetypes in the hash map
	for _, line := range bytes.Split(data, []byte("\n")) {
		fields := bytes.Fields(line)
		if len(fields) > 1 {
			for _, ext := range fields[1:] {
				mimetypes[string(ext)] = string(fields[0])
			}
		}
	}
	return mimetypes, nil
}

// Get returns the mimetype, or an empty string if no mimetype or mimetype source is found
func (mr *Reader) Get(ext string) string {
	var err error
	// No extension
	if len(ext) == 0 {
		return ""
	}
	// Strip the leading dot
	if ext[0] == '.' {
		ext = ext[1:]
	}
	mr.mu.Lock()
	defer mr.mu.Unlock()
	if mr.mimetypes == nil {
		mr.mimetypes, err = readMimetypes(mr.filename)
		if err != nil {
			// Using the fallback hash map
			if mime, ok := fallback[ext]; ok {
				return mime
			}
			// Unable to find the mime type for the given extension
			return ""
		}
	}
	// Use the value from the hash map
	if mime, ok := mr.mimetypes[ext]; ok {
		return mime
	}
	// Using the fallback hash map
	if mime, ok := fallback[ext]; ok {
		return mime
	}
	// Unable to find the mime type for the given extension
	return ""
}

// SetHeader sets the Content-Type for a given ResponseWriter and filename extension
func (mr *Reader) SetHeader(w http.ResponseWriter, ext string) {
	mimestring := mr.Get(ext)
	if mimestring == "" {
		// Default mime type
		mimestring = "application/octet-stream"
	}
	if mr.utf8 {
		mimestring += "; charset=utf-8"
	}
	w.Header().Add("Content-Type", mimestring)
}
<|endoftext|>


<commit_before>package acceptance_test

import (
	"fmt"
	"time"

	"github.com/lib/pq"
	"github.com/pivotal-golang/lager/lagertest"
	"github.com/sclevine/agouti"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/sclevine/agouti/matchers"

	"github.com/cloudfoundry/gunk/urljoiner"
	"github.com/concourse/atc"
	"github.com/concourse/atc/db"
	"github.com/concourse/atc/event"
)

var _ = Describe("Job Builds", func() {
	var atcProcess ifrit.Process
	var dbListener *pq.Listener
	var atcPort uint16
	var pipelineDBFactory db.PipelineDBFactory
	var pipelineDB db.PipelineDB

	BeforeEach(func() {
		dbLogger := lagertest.NewTestLogger("test")
		postgresRunner.Truncate()
		dbConn = db.Wrap(postgresRunner.Open())
		dbListener = pq.NewListener(postgresRunner.DataSourceName(), time.Second, time.Minute, nil)
		bus := db.NewNotificationsBus(dbListener, dbConn)
		sqlDB = db.NewSQL(dbLogger, dbConn, bus)
		pipelineDBFactory = db.NewPipelineDBFactory(dbLogger, dbConn, bus, sqlDB)
		atcProcess, atcPort = startATC(atcBin, 1, true, BASIC_AUTH)
		_, err := dbConn.Query(`DELETE FROM teams WHERE name = 'main'`)
		Expect(err).NotTo(HaveOccurred())
	})

	AfterEach(func() {
		ginkgomon.Interrupt(atcProcess)

		Expect(dbConn.Close()).To(Succeed())
		Expect(dbListener.Close()).To(Succeed())
	})

	Describe("viewing a jobs builds", func() {
		var page *agouti.Page

		BeforeEach(func() {
			var err error
			page, err = agoutiDriver.NewPage()
			Expect(err).NotTo(HaveOccurred())
		})

		AfterEach(func() {
			Expect(page.Destroy()).To(Succeed())
		})

		homepage := func() string {
			return fmt.Sprintf("http://127.0.0.1:%d/pipelines/%s", atcPort, atc.DefaultPipelineName)
		}

		withPath := func(path string) string {
			return urljoiner.Join(homepage(), path)
		}

		Context("with a job in the configuration", func() {
			var build db.Build
			var buildInput = db.BuildInput{
				Name: "build-input-1",
				VersionedResource: db.VersionedResource{
					Resource:     "my-resource",
					PipelineName: atc.DefaultPipelineName,
					Version: db.Version{
						"ref": "thing",
					},
				},
			}
			var buildOutput = db.VersionedResource{
				Resource:     "some-output",
				PipelineName: atc.DefaultPipelineName,
				Version: db.Version{
					"thing": "output-version",
				},
			}
			var teamName = atc.DefaultTeamName

			BeforeEach(func() {
				team, err := sqlDB.SaveTeam(db.Team{Name: teamName})
				Expect(err).NotTo(HaveOccurred())

				// job build data
				_, err = sqlDB.SaveConfig(team.Name, atc.DefaultPipelineName, atc.Config{
					Jobs: atc.JobConfigs{
						{Name: "job-name"},
					},
					Resources: atc.ResourceConfigs{
						{Name: "my-resource"},
						{Name: "some-output"},
					},
				}, db.ConfigVersion(1), db.PipelineUnpaused)
				Expect(err).NotTo(HaveOccurred())

				pipelineDB, err = pipelineDBFactory.BuildWithTeamNameAndName(team.Name, atc.DefaultPipelineName)
				Expect(err).NotTo(HaveOccurred())

				build, err = pipelineDB.CreateJobBuild("job-name")
				Expect(err).NotTo(HaveOccurred())

				_, err = sqlDB.StartBuild(build.ID, "", "")
				Expect(err).NotTo(HaveOccurred())

				sqlDB.SaveBuildEvent(build.ID, event.Log{
					Origin: event.Origin{
						Source: event.OriginSourceStdout,
						ID:     "some-id",
					},
					Payload: "hello this is a payload",
				})

				Expect(sqlDB.FinishBuild(build.ID, db.StatusSucceeded)).To(Succeed())

				_, err = sqlDB.SaveBuildInput(teamName, build.ID, buildInput)
				Expect(err).NotTo(HaveOccurred())

				_, err = sqlDB.SaveBuildOutput(teamName, build.ID, buildOutput, true)
				Expect(err).NotTo(HaveOccurred())
			})

			Context("with more then 100 job builds", func() {
				var testBuilds []db.Build

				BeforeEach(func() {
					for i := 1; i < 103; i++ {
						build, err := pipelineDB.CreateJobBuild("job-name")
						Expect(err).NotTo(HaveOccurred())
						testBuilds = append(testBuilds, build)
					}
				})

				It("can have paginated results", func() {
					// homepage -> job detail w/build info
					Expect(page.Navigate(homepage())).To(Succeed())
					// we will need to authenticate later to prove it is working for our page
					Authenticate(page, "admin", "password")
					Eventually(page.FindByLink("job-name")).Should(BeFound())
					Expect(page.FindByLink("job-name").Click()).To(Succeed())

					Eventually(page.All("#builds li").Count).Should(Equal(103))

					// job detail w/build info -> job detail
					Eventually(page.Find("h1 a")).Should(BeFound())
					Expect(page.Find("h1 a").Click()).To(Succeed())
					Eventually(page).Should(HaveURL(withPath("jobs/job-name")))
					Eventually(page.All(".js-build").Count).Should(Equal(100))

					Expect(page.First(".pagination .disabled .fa-arrow-left")).Should(BeFound())
					Expect(page.First(".pagination .fa-arrow-right").Click()).To(Succeed())
					Eventually(page.All(".js-build").Count).Should(Equal(3))

					Expect(page.First(".pagination .disabled .fa-arrow-right")).Should(BeFound())
					Expect(page.First(".pagination .fa-arrow-left").Click()).To(Succeed())
					Eventually(page.All(".js-build").Count).Should(Equal(100))
				})
			})

			It("can view resource information of a job build", func() {
				// homepage -> job detail w/build info
				Expect(page.Navigate(homepage())).To(Succeed())
				// we will need to authenticate later to prove it is working for our page
				Authenticate(page, "admin", "password")
				Eventually(page.FindByLink("job-name")).Should(BeFound())
				Expect(page.FindByLink("job-name").Click()).To(Succeed())

				// job detail w/build info -> job detail
				Eventually(page).Should(HaveURL(withPath(fmt.Sprintf("jobs/job-name/builds/%d", build.ID))))
				Eventually(page.Find("h1")).Should(HaveText(fmt.Sprintf("job-name #%d", build.ID)))
				Expect(page.Find("h1 a").Click()).To(Succeed())
				Eventually(page).Should(HaveURL(withPath("jobs/job-name")))

				Eventually(page.Find("#page-header.succeeded")).Should(BeFound())

				Eventually(page.All(".builds-list li")).Should(HaveCount(1))

				Expect(page.Find(".builds-list li:first-child a")).To(HaveText(fmt.Sprintf("#%d", build.ID)))
				Eventually(page.Find(".builds-list li:first-child a.succeeded")).Should(BeFound())

				buildTimes, err := page.Find(".builds-list li:first-child .build-duration").Text()
				Expect(err).NotTo(HaveOccurred())
				Expect(buildTimes).To(ContainSubstring("started"))
				Expect(buildTimes).To(MatchRegexp("started \\ds ago"))
				Expect(buildTimes).To(MatchRegexp("finished \\ds ago"))
				Expect(buildTimes).To(MatchRegexp("duration \\ds"))

				Eventually(page.Find(".builds-list li:first-child .inputs .resource-name")).Should(BeFound())
				Expect(page.Find(".builds-list li:first-child .inputs .resource-name")).To(HaveText("my-resource"))
				Expect(page.Find(".builds-list li:first-child .inputs .resource-version .dict-key")).To(HaveText("ref"))
				Expect(page.Find(".builds-list li:first-child .inputs .resource-version .dict-value")).To(HaveText("thing"))

				Expect(page.Find(".builds-list li:first-child .outputs .resource-name")).To(HaveText("some-output"))
				Expect(page.Find(".builds-list li:first-child .outputs .resource-version .dict-key")).To(HaveText("thing"))
				Expect(page.Find(".builds-list li:first-child .outputs .resource-version .dict-value")).To(HaveText("output-version"))
			})
		})
	})
})
<commit_msg>be flexible with slower running tests<commit_after>package acceptance_test

import (
	"fmt"
	"time"

	"github.com/lib/pq"
	"github.com/pivotal-golang/lager/lagertest"
	"github.com/sclevine/agouti"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/sclevine/agouti/matchers"

	"github.com/cloudfoundry/gunk/urljoiner"
	"github.com/concourse/atc"
	"github.com/concourse/atc/db"
	"github.com/concourse/atc/event"
)

var _ = Describe("Job Builds", func() {
	var atcProcess ifrit.Process
	var dbListener *pq.Listener
	var atcPort uint16
	var pipelineDBFactory db.PipelineDBFactory
	var pipelineDB db.PipelineDB

	BeforeEach(func() {
		dbLogger := lagertest.NewTestLogger("test")
		postgresRunner.Truncate()
		dbConn = db.Wrap(postgresRunner.Open())
		dbListener = pq.NewListener(postgresRunner.DataSourceName(), time.Second, time.Minute, nil)
		bus := db.NewNotificationsBus(dbListener, dbConn)
		sqlDB = db.NewSQL(dbLogger, dbConn, bus)
		pipelineDBFactory = db.NewPipelineDBFactory(dbLogger, dbConn, bus, sqlDB)
		atcProcess, atcPort = startATC(atcBin, 1, true, BASIC_AUTH)
		_, err := dbConn.Query(`DELETE FROM teams WHERE name = 'main'`)
		Expect(err).NotTo(HaveOccurred())
	})

	AfterEach(func() {
		ginkgomon.Interrupt(atcProcess)

		Expect(dbConn.Close()).To(Succeed())
		Expect(dbListener.Close()).To(Succeed())
	})

	Describe("viewing a jobs builds", func() {
		var page *agouti.Page

		BeforeEach(func() {
			var err error
			page, err = agoutiDriver.NewPage()
			Expect(err).NotTo(HaveOccurred())
		})

		AfterEach(func() {
			Expect(page.Destroy()).To(Succeed())
		})

		homepage := func() string {
			return fmt.Sprintf("http://127.0.0.1:%d/pipelines/%s", atcPort, atc.DefaultPipelineName)
		}

		withPath := func(path string) string {
			return urljoiner.Join(homepage(), path)
		}

		Context("with a job in the configuration", func() {
			var build db.Build
			var buildInput = db.BuildInput{
				Name: "build-input-1",
				VersionedResource: db.VersionedResource{
					Resource:     "my-resource",
					PipelineName: atc.DefaultPipelineName,
					Version: db.Version{
						"ref": "thing",
					},
				},
			}
			var buildOutput = db.VersionedResource{
				Resource:     "some-output",
				PipelineName: atc.DefaultPipelineName,
				Version: db.Version{
					"thing": "output-version",
				},
			}
			var teamName = atc.DefaultTeamName

			BeforeEach(func() {
				team, err := sqlDB.SaveTeam(db.Team{Name: teamName})
				Expect(err).NotTo(HaveOccurred())

				// job build data
				_, err = sqlDB.SaveConfig(team.Name, atc.DefaultPipelineName, atc.Config{
					Jobs: atc.JobConfigs{
						{Name: "job-name"},
					},
					Resources: atc.ResourceConfigs{
						{Name: "my-resource"},
						{Name: "some-output"},
					},
				}, db.ConfigVersion(1), db.PipelineUnpaused)
				Expect(err).NotTo(HaveOccurred())

				pipelineDB, err = pipelineDBFactory.BuildWithTeamNameAndName(team.Name, atc.DefaultPipelineName)
				Expect(err).NotTo(HaveOccurred())

				build, err = pipelineDB.CreateJobBuild("job-name")
				Expect(err).NotTo(HaveOccurred())

				_, err = sqlDB.StartBuild(build.ID, "", "")
				Expect(err).NotTo(HaveOccurred())

				sqlDB.SaveBuildEvent(build.ID, event.Log{
					Origin: event.Origin{
						Source: event.OriginSourceStdout,
						ID:     "some-id",
					},
					Payload: "hello this is a payload",
				})

				Expect(sqlDB.FinishBuild(build.ID, db.StatusSucceeded)).To(Succeed())

				_, err = sqlDB.SaveBuildInput(teamName, build.ID, buildInput)
				Expect(err).NotTo(HaveOccurred())

				_, err = sqlDB.SaveBuildOutput(teamName, build.ID, buildOutput, true)
				Expect(err).NotTo(HaveOccurred())
			})

			Context("with more then 100 job builds", func() {
				var testBuilds []db.Build

				BeforeEach(func() {
					for i := 1; i < 103; i++ {
						build, err := pipelineDB.CreateJobBuild("job-name")
						Expect(err).NotTo(HaveOccurred())
						testBuilds = append(testBuilds, build)
					}
				})

				It("can have paginated results", func() {
					// homepage -> job detail w/build info
					Expect(page.Navigate(homepage())).To(Succeed())
					// we will need to authenticate later to prove it is working for our page
					Authenticate(page, "admin", "password")
					Eventually(page.FindByLink("job-name")).Should(BeFound())
					Expect(page.FindByLink("job-name").Click()).To(Succeed())

					Eventually(page.All("#builds li").Count).Should(Equal(103))

					// job detail w/build info -> job detail
					Eventually(page.Find("h1 a")).Should(BeFound())
					Expect(page.Find("h1 a").Click()).To(Succeed())
					Eventually(page).Should(HaveURL(withPath("jobs/job-name")))
					Eventually(page.All(".js-build").Count).Should(Equal(100))

					Expect(page.First(".pagination .disabled .fa-arrow-left")).Should(BeFound())
					Expect(page.First(".pagination .fa-arrow-right").Click()).To(Succeed())
					Eventually(page.All(".js-build").Count).Should(Equal(3))

					Expect(page.First(".pagination .disabled .fa-arrow-right")).Should(BeFound())
					Expect(page.First(".pagination .fa-arrow-left").Click()).To(Succeed())
					Eventually(page.All(".js-build").Count).Should(Equal(100))
				})
			})

			It("can view resource information of a job build", func() {
				// homepage -> job detail w/build info
				Expect(page.Navigate(homepage())).To(Succeed())
				// we will need to authenticate later to prove it is working for our page
				Authenticate(page, "admin", "password")
				Eventually(page.FindByLink("job-name")).Should(BeFound())
				Expect(page.FindByLink("job-name").Click()).To(Succeed())

				// job detail w/build info -> job detail
				Eventually(page).Should(HaveURL(withPath(fmt.Sprintf("jobs/job-name/builds/%d", build.ID))))
				Eventually(page.Find("h1")).Should(HaveText(fmt.Sprintf("job-name #%d", build.ID)))
				Expect(page.Find("h1 a").Click()).To(Succeed())
				Eventually(page).Should(HaveURL(withPath("jobs/job-name")))

				Eventually(page.Find("#page-header.succeeded")).Should(BeFound())

				Eventually(page.All(".builds-list li")).Should(HaveCount(1))

				Expect(page.Find(".builds-list li:first-child a")).To(HaveText(fmt.Sprintf("#%d", build.ID)))
				Eventually(page.Find(".builds-list li:first-child a.succeeded")).Should(BeFound())

				buildTimes, err := page.Find(".builds-list li:first-child .build-duration").Text()
				Expect(err).NotTo(HaveOccurred())
				Expect(buildTimes).To(ContainSubstring("started"))
				Expect(buildTimes).To(MatchRegexp("started \\d+s ago"))
				Expect(buildTimes).To(MatchRegexp("finished \\d+s ago"))
				Expect(buildTimes).To(MatchRegexp("duration \\d+s"))

				Eventually(page.Find(".builds-list li:first-child .inputs .resource-name")).Should(BeFound())
				Expect(page.Find(".builds-list li:first-child .inputs .resource-name")).To(HaveText("my-resource"))
				Expect(page.Find(".builds-list li:first-child .inputs .resource-version .dict-key")).To(HaveText("ref"))
				Expect(page.Find(".builds-list li:first-child .inputs .resource-version .dict-value")).To(HaveText("thing"))

				Expect(page.Find(".builds-list li:first-child .outputs .resource-name")).To(HaveText("some-output"))
				Expect(page.Find(".builds-list li:first-child .outputs .resource-version .dict-key")).To(HaveText("thing"))
				Expect(page.Find(".builds-list li:first-child .outputs .resource-version .dict-value")).To(HaveText("output-version"))
			})
		})
	})
})
<|endoftext|>


<commit_before>package dht

import (
	"fmt"
	"hash/fnv"
	"net"

	"github.com/anacrolix/missinggo/v2"
	"github.com/anacrolix/stm/stmutil"

	"github.com/anacrolix/dht/v2/krpc"
)

func mustListen(addr string) net.PacketConn {
	ret, err := net.ListenPacket("udp", addr)
	if err != nil {
		panic(err)
	}
	return ret
}

func addrResolver(addr string) func() ([]Addr, error) {
	return func() ([]Addr, error) {
		ua, err := net.ResolveUDPAddr("udp", addr)
		return []Addr{NewAddr(ua)}, err
	}
}

type addrMaybeId struct {
	Addr krpc.NodeAddr
	Id   *int160
}

func (me addrMaybeId) String() string {
	if me.Id == nil {
		return fmt.Sprintf("unknown id at %s", me.Addr)
	} else {
		return fmt.Sprintf("%x at %v", *me.Id, me.Addr)
	}
}

func (l addrMaybeId) closerThan(r addrMaybeId, target int160) bool {
	var ml missinggo.MultiLess
	ml.NextBool(r.Id == nil, l.Id == nil)
	if l.Id != nil && r.Id != nil {
		d := distance(*l.Id, target).Cmp(distance(*r.Id, target))
		ml.StrictNext(d == 0, d < 0)
	}
	// TODO: Use bytes/hash when it's available (go1.14?), and have a unique seed for each
	// instance.
	hashString := func(s string) uint64 {
		h := fnv.New64a()
		h.Write([]byte(s))
		return h.Sum64()
	}
	lh := hashString(l.Addr.String())
	rh := hashString(r.Addr.String())
	ml.StrictNext(lh == rh, lh < rh)
	//ml.StrictNext(l.Addr.String() == r.Addr.String(), l.Addr.String() < r.Addr.String())
	return ml.Less()

}

func nodesByDistance(target int160) stmutil.Settish {
	return stmutil.NewSortedSet(func(l, r interface{}) bool {
		return l.(addrMaybeId).closerThan(r.(addrMaybeId), target)
	})
}
<commit_msg>Fix Id formatting in addrMaybeId.String<commit_after>package dht

import (
	"fmt"
	"hash/fnv"
	"net"

	"github.com/anacrolix/missinggo/v2"
	"github.com/anacrolix/stm/stmutil"

	"github.com/anacrolix/dht/v2/krpc"
)

func mustListen(addr string) net.PacketConn {
	ret, err := net.ListenPacket("udp", addr)
	if err != nil {
		panic(err)
	}
	return ret
}

func addrResolver(addr string) func() ([]Addr, error) {
	return func() ([]Addr, error) {
		ua, err := net.ResolveUDPAddr("udp", addr)
		return []Addr{NewAddr(ua)}, err
	}
}

type addrMaybeId struct {
	Addr krpc.NodeAddr
	Id   *int160
}

func (me addrMaybeId) String() string {
	if me.Id == nil {
		return fmt.Sprintf("unknown id at %s", me.Addr)
	} else {
		return fmt.Sprintf("%v at %v", *me.Id, me.Addr)
	}
}

func (l addrMaybeId) closerThan(r addrMaybeId, target int160) bool {
	var ml missinggo.MultiLess
	ml.NextBool(r.Id == nil, l.Id == nil)
	if l.Id != nil && r.Id != nil {
		d := distance(*l.Id, target).Cmp(distance(*r.Id, target))
		ml.StrictNext(d == 0, d < 0)
	}
	// TODO: Use bytes/hash when it's available (go1.14?), and have a unique seed for each
	// instance.
	hashString := func(s string) uint64 {
		h := fnv.New64a()
		h.Write([]byte(s))
		return h.Sum64()
	}
	lh := hashString(l.Addr.String())
	rh := hashString(r.Addr.String())
	ml.StrictNext(lh == rh, lh < rh)
	//ml.StrictNext(l.Addr.String() == r.Addr.String(), l.Addr.String() < r.Addr.String())
	return ml.Less()

}

func nodesByDistance(target int160) stmutil.Settish {
	return stmutil.NewSortedSet(func(l, r interface{}) bool {
		return l.(addrMaybeId).closerThan(r.(addrMaybeId), target)
	})
}
<|endoftext|>


<commit_before>package mitm

import (
	"crypto/tls"
	"errors"
	"io"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"sync"
	"time"
)

type ServerParam struct {
	CA        *tls.Certificate // the Root CA for generatng on the fly MITM certificates
	TLSConfig *tls.Config      // a template TLS config for the server.
}

func (p ServerParam) cert(names ...string) (*tls.Certificate, error) {
	return GenerateCert(p.CA, names...)
}

// A ServerConn is a net.Conn that holds its clients SNI header in ServerName
// after the handshake.
type ServerConn struct {
	*tls.Conn

	// ServerName is set during Conn's handshake to the client's requested
	// server name set in the SNI header. It is not safe to access across
	// multiple goroutines while Conn is performing the handshake.
	ServerName string
}

// Server wraps cn with a ServerConn configured with p so that during its
// Handshake, it will generate a new certificate using p.CA. After a successful
// Handshake, its ServerName field will be set to the clients requested
// ServerName in the SNI header.
func Server(cn net.Conn, p ServerParam) *ServerConn {
	conf := new(tls.Config)
	if p.TLSConfig != nil {
		*conf = *p.TLSConfig
	}
	sc := new(ServerConn)
	conf.GetCertificate = func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
		sc.ServerName = hello.ServerName
		return GenerateCert(p.CA, hello.ServerName)
	}
	sc.Conn = tls.Server(cn, conf)
	return sc
}

type listener struct {
	net.Listener
	ca   *tls.Certificate
	conf *tls.Config
}

// NewListener returns a net.Listener that generates a new cert from ca for
// each new Accept. It uses SNI to generate the cert, and herefore only
// works with clients that send SNI headers.
//
// This is useful for building transparent MITM proxies.
func NewListener(inner net.Listener, ca *tls.Certificate, conf *tls.Config) net.Listener {
	return &listener{inner, ca, conf}
}

func (l *listener) Accept() (net.Conn, error) {
	cn, err := l.Listener.Accept()
	if err != nil {
		return nil, err
	}
	sc := Server(cn, ServerParam{
		CA:        l.ca,
		TLSConfig: l.conf,
	})
	return sc, nil
}

// Proxy is a forward proxy that substitutes its own certificate
// for incoming TLS connections in place of the upstream server's
// certificate.
type Proxy struct {
	// Wrap specifies a function for optionally wrapping upstream for
	// inspecting the decrypted HTTP request and response.
	Wrap func(upstream http.Handler) http.Handler

	// CA specifies the root CA for generating leaf certs for each incoming
	// TLS request.
	CA *tls.Certificate

	// TLSServerConfig specifies the tls.Config to use when generating leaf
	// cert using CA.
	TLSServerConfig *tls.Config

	// TLSClientConfig specifies the tls.Config to use when establishing
	// an upstream connection for proxying.
	TLSClientConfig *tls.Config

	// FlushInterval specifies the flush interval
	// to flush to the client while copying the
	// response body.
	// If zero, no periodic flushing is done.
	FlushInterval time.Duration
}

var (
	okHeader           = "HTTP/1.1 200 OK\r\n\r\n"
	noUpstreamHeader   = "HTTP/1.1 503 No Upstream\r\n\r\n"
	noDownstreamHeader = "HTTP/1.1 503 No Downstream\r\n\r\n"
	errHeader          = "HTTP/1.1 500 Internal Server Error\r\n\r\n"
)

func (p *Proxy) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	if req.Method != "CONNECT" {
		rp := &httputil.ReverseProxy{
			Director:      httpDirector,
			FlushInterval: p.FlushInterval,
		}
		p.Wrap(rp).ServeHTTP(w, req)
		return
	}

	cn, _, err := w.(http.Hijacker).Hijack()
	if err != nil {
		log.Println("Hijack:", err)
		http.Error(w, "No Upstream", 503)
		return
	}
	defer cn.Close()

	_, err = io.WriteString(cn, okHeader)
	if err != nil {
		log.Println("Write:", err)
		return
	}

	sc, ok := cn.(*ServerConn)
	if !ok {
		name := dnsName(req.Host)
		if name == "" {
			log.Println("cannot determine cert name for " + req.Host)
			io.WriteString(cn, noDownstreamHeader)
			return
		}
		sc = Server(cn, ServerParam{
			CA:        p.CA,
			TLSConfig: p.TLSServerConfig,
		})
		if err := sc.Handshake(); err != nil {
			log.Println("Server Handshake:", err)
			return
		}
	}

	cc, err := p.tlsDial(req.Host, sc.ServerName)
	if err != nil {
		log.Println("tlsDial:", err)
		io.WriteString(cn, noUpstreamHeader)
		return
	}
	p.proxyMITM(sc, cc)
}

func (p *Proxy) tlsDial(addr, serverName string) (net.Conn, error) {
	conf := new(tls.Config)
	if p.TLSClientConfig != nil {
		*conf = *p.TLSClientConfig
	}
	conf.ServerName = serverName
	return tls.Dial("tcp", addr, conf)
}

func (p *Proxy) proxyMITM(upstream, downstream net.Conn) {
	var mu sync.Mutex
	dial := func(network, addr string) (net.Conn, error) {
		mu.Lock()
		defer mu.Unlock()
		if downstream == nil {
			return nil, io.EOF
		}
		cn := downstream
		downstream = nil
		return cn, nil
	}
	rp := &httputil.ReverseProxy{
		Director:      httpsDirector,
		Transport:     &http.Transport{DialTLS: dial},
		FlushInterval: p.FlushInterval,
	}
	ch := make(chan struct{})
	wc := &onCloseConn{upstream, func() { ch <- struct{}{} }}
	http.Serve(&oneShotListener{wc}, p.Wrap(rp))
	<-ch
}

func httpDirector(r *http.Request) {
	r.URL.Host = r.Host
	r.URL.Scheme = "http"
}

func httpsDirector(req *http.Request) {
	req.URL.Host = req.Host
	req.URL.Scheme = "https"
}

// A oneShotListener implements net.Listener whos Accept only returns a
// net.Conn as specified by c followed by an error for each subsequent Accept.
type oneShotListener struct {
	c net.Conn
}

func (l *oneShotListener) Accept() (net.Conn, error) {
	if l.c == nil {
		return nil, errors.New("closed")
	}
	c := l.c
	l.c = nil
	return c, nil
}

func (l *oneShotListener) Close() error {
	return nil
}

func (l *oneShotListener) Addr() net.Addr {
	return l.c.LocalAddr()
}

// A onCloseConn implements net.Conn and calls its f on Close.
type onCloseConn struct {
	net.Conn
	f func()
}

func (c *onCloseConn) Close() error {
	if c.f != nil {
		c.f()
		c.f = nil
	}
	return c.Conn.Close()
}

// dnsName returns the DNS name in addr, if any.
func dnsName(addr string) string {
	host, _, err := net.SplitHostPort(addr)
	if err != nil {
		return ""
	}
	return host
}
<commit_msg>Remove dead code<commit_after>package mitm

import (
	"crypto/tls"
	"errors"
	"io"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"sync"
	"time"
)

type ServerParam struct {
	CA        *tls.Certificate // the Root CA for generatng on the fly MITM certificates
	TLSConfig *tls.Config      // a template TLS config for the server.
}

// A ServerConn is a net.Conn that holds its clients SNI header in ServerName
// after the handshake.
type ServerConn struct {
	*tls.Conn

	// ServerName is set during Conn's handshake to the client's requested
	// server name set in the SNI header. It is not safe to access across
	// multiple goroutines while Conn is performing the handshake.
	ServerName string
}

// Server wraps cn with a ServerConn configured with p so that during its
// Handshake, it will generate a new certificate using p.CA. After a successful
// Handshake, its ServerName field will be set to the clients requested
// ServerName in the SNI header.
func Server(cn net.Conn, p ServerParam) *ServerConn {
	conf := new(tls.Config)
	if p.TLSConfig != nil {
		*conf = *p.TLSConfig
	}
	sc := new(ServerConn)
	conf.GetCertificate = func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
		sc.ServerName = hello.ServerName
		return GenerateCert(p.CA, hello.ServerName)
	}
	sc.Conn = tls.Server(cn, conf)
	return sc
}

type listener struct {
	net.Listener
	ca   *tls.Certificate
	conf *tls.Config
}

// NewListener returns a net.Listener that generates a new cert from ca for
// each new Accept. It uses SNI to generate the cert, and herefore only
// works with clients that send SNI headers.
//
// This is useful for building transparent MITM proxies.
func NewListener(inner net.Listener, ca *tls.Certificate, conf *tls.Config) net.Listener {
	return &listener{inner, ca, conf}
}

func (l *listener) Accept() (net.Conn, error) {
	cn, err := l.Listener.Accept()
	if err != nil {
		return nil, err
	}
	sc := Server(cn, ServerParam{
		CA:        l.ca,
		TLSConfig: l.conf,
	})
	return sc, nil
}

// Proxy is a forward proxy that substitutes its own certificate
// for incoming TLS connections in place of the upstream server's
// certificate.
type Proxy struct {
	// Wrap specifies a function for optionally wrapping upstream for
	// inspecting the decrypted HTTP request and response.
	Wrap func(upstream http.Handler) http.Handler

	// CA specifies the root CA for generating leaf certs for each incoming
	// TLS request.
	CA *tls.Certificate

	// TLSServerConfig specifies the tls.Config to use when generating leaf
	// cert using CA.
	TLSServerConfig *tls.Config

	// TLSClientConfig specifies the tls.Config to use when establishing
	// an upstream connection for proxying.
	TLSClientConfig *tls.Config

	// FlushInterval specifies the flush interval
	// to flush to the client while copying the
	// response body.
	// If zero, no periodic flushing is done.
	FlushInterval time.Duration
}

var (
	okHeader           = "HTTP/1.1 200 OK\r\n\r\n"
	noUpstreamHeader   = "HTTP/1.1 503 No Upstream\r\n\r\n"
	noDownstreamHeader = "HTTP/1.1 503 No Downstream\r\n\r\n"
	errHeader          = "HTTP/1.1 500 Internal Server Error\r\n\r\n"
)

func (p *Proxy) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	if req.Method != "CONNECT" {
		rp := &httputil.ReverseProxy{
			Director:      httpDirector,
			FlushInterval: p.FlushInterval,
		}
		p.Wrap(rp).ServeHTTP(w, req)
		return
	}

	cn, _, err := w.(http.Hijacker).Hijack()
	if err != nil {
		log.Println("Hijack:", err)
		http.Error(w, "No Upstream", 503)
		return
	}
	defer cn.Close()

	_, err = io.WriteString(cn, okHeader)
	if err != nil {
		log.Println("Write:", err)
		return
	}

	sc, ok := cn.(*ServerConn)
	if !ok {
		name := dnsName(req.Host)
		if name == "" {
			log.Println("cannot determine cert name for " + req.Host)
			io.WriteString(cn, noDownstreamHeader)
			return
		}
		sc = Server(cn, ServerParam{
			CA:        p.CA,
			TLSConfig: p.TLSServerConfig,
		})
		if err := sc.Handshake(); err != nil {
			log.Println("Server Handshake:", err)
			return
		}
	}

	cc, err := p.tlsDial(req.Host, sc.ServerName)
	if err != nil {
		log.Println("tlsDial:", err)
		io.WriteString(cn, noUpstreamHeader)
		return
	}
	p.proxyMITM(sc, cc)
}

func (p *Proxy) tlsDial(addr, serverName string) (net.Conn, error) {
	conf := new(tls.Config)
	if p.TLSClientConfig != nil {
		*conf = *p.TLSClientConfig
	}
	conf.ServerName = serverName
	return tls.Dial("tcp", addr, conf)
}

func (p *Proxy) proxyMITM(upstream, downstream net.Conn) {
	var mu sync.Mutex
	dial := func(network, addr string) (net.Conn, error) {
		mu.Lock()
		defer mu.Unlock()
		if downstream == nil {
			return nil, io.EOF
		}
		cn := downstream
		downstream = nil
		return cn, nil
	}
	rp := &httputil.ReverseProxy{
		Director:      httpsDirector,
		Transport:     &http.Transport{DialTLS: dial},
		FlushInterval: p.FlushInterval,
	}
	ch := make(chan struct{})
	wc := &onCloseConn{upstream, func() { ch <- struct{}{} }}
	http.Serve(&oneShotListener{wc}, p.Wrap(rp))
	<-ch
}

func httpDirector(r *http.Request) {
	r.URL.Host = r.Host
	r.URL.Scheme = "http"
}

func httpsDirector(req *http.Request) {
	req.URL.Host = req.Host
	req.URL.Scheme = "https"
}

// A oneShotListener implements net.Listener whos Accept only returns a
// net.Conn as specified by c followed by an error for each subsequent Accept.
type oneShotListener struct {
	c net.Conn
}

func (l *oneShotListener) Accept() (net.Conn, error) {
	if l.c == nil {
		return nil, errors.New("closed")
	}
	c := l.c
	l.c = nil
	return c, nil
}

func (l *oneShotListener) Close() error {
	return nil
}

func (l *oneShotListener) Addr() net.Addr {
	return l.c.LocalAddr()
}

// A onCloseConn implements net.Conn and calls its f on Close.
type onCloseConn struct {
	net.Conn
	f func()
}

func (c *onCloseConn) Close() error {
	if c.f != nil {
		c.f()
		c.f = nil
	}
	return c.Conn.Close()
}

// dnsName returns the DNS name in addr, if any.
func dnsName(addr string) string {
	host, _, err := net.SplitHostPort(addr)
	if err != nil {
		return ""
	}
	return host
}
<|endoftext|>


<commit_before>/*
Debootstrap Action

Construct the target rootfs with debootstrap tool.

Yaml syntax:
 - action: debootstrap
   mirror: URL
   suite: "name"
   components: <list of components>
   variant: "name"
   keyring-package:
   keyring-file:

Mandatory properties:

- suite -- release code name or symbolic name (e.g. "stable")

Optional properties:

- check-gpg -- verify GPG signatures on Release files, true by default

- mirror -- URL with Debian-compatible repository

- variant -- name of the bootstrap script variant to use

- components -- list of components to use for packages selection.
Example:
 components: [ main, contrib ]

- keyring-package -- keyring for package validation.

- keyring-file -- keyring file for repository validation.

- merged-usr -- use merged '/usr' filesystem, true by default.
*/
package actions

import (
	"fmt"
	"io"
	"os"
	"path"
	"strings"

	"github.com/go-debos/debos"
)

type DebootstrapAction struct {
	debos.BaseAction `yaml:",inline"`
	Suite            string
	Mirror           string
	Variant          string
	KeyringPackage   string `yaml:"keyring-package"`
	KeyringFile      string `yaml:"keyring-file"`
	Components       []string
	MergedUsr        bool `yaml:"merged-usr"`
	CheckGpg         bool `yaml:"check-gpg"`
}

func NewDebootstrapAction() *DebootstrapAction {
	d := DebootstrapAction{}
	// Use filesystem with merged '/usr' by default
	d.MergedUsr = true
	// Be secure by default
	d.CheckGpg = true
	return &d

}

func (d *DebootstrapAction) RunSecondStage(context debos.DebosContext) error {
	cmdline := []string{
		"/debootstrap/debootstrap",
		"--no-check-gpg",
		"--second-stage"}

	if d.Components != nil {
		s := strings.Join(d.Components, ",")
		cmdline = append(cmdline, fmt.Sprintf("--components=%s", s))
	}

	c := debos.NewChrootCommandForContext(context)
	// Can't use nspawn for debootstrap as it wants to create device nodes
	c.ChrootMethod = debos.CHROOT_METHOD_CHROOT

	err := c.Run("Debootstrap (stage 2)", cmdline...)

	if (err != nil) {
		log := path.Join(context.Rootdir, "debootstrap/debootstrap.log")
		_ = debos.Command{}.Run("debootstrap.log", "cat", log)
	}

	return err
}

func (d *DebootstrapAction) Run(context *debos.DebosContext) error {
	d.LogStart()
	cmdline := []string{"debootstrap"}

	if d.MergedUsr {
		cmdline = append(cmdline, "--merged-usr")
	}

	if !d.CheckGpg {
		cmdline = append(cmdline, fmt.Sprintf("--no-check-gpg"))
	} else if d.KeyringFile != "" {
		path := debos.CleanPathAt(d.KeyringFile, context.RecipeDir)
		cmdline = append(cmdline, fmt.Sprintf("--keyring=%s", path))
	}

	if d.KeyringPackage != "" {
		cmdline = append(cmdline, fmt.Sprintf("--include=%s", d.KeyringPackage))
	}

	if d.Components != nil {
		s := strings.Join(d.Components, ",")
		cmdline = append(cmdline, fmt.Sprintf("--components=%s", s))
	}

	/* FIXME drop the hardcoded amd64 assumption" */
	foreign := context.Architecture != "amd64"

	if foreign {
		cmdline = append(cmdline, "--foreign")
		cmdline = append(cmdline, fmt.Sprintf("--arch=%s", context.Architecture))

	}

	if d.Variant != "" {
		cmdline = append(cmdline, fmt.Sprintf("--variant=%s", d.Variant))
	}

	cmdline = append(cmdline, d.Suite)
	cmdline = append(cmdline, context.Rootdir)
	cmdline = append(cmdline, d.Mirror)
	cmdline = append(cmdline, "/usr/share/debootstrap/scripts/unstable")

	err := debos.Command{}.Run("Debootstrap", cmdline...)

	if err != nil {
		log := path.Join(context.Rootdir, "debootstrap/debootstrap.log")
		_ = debos.Command{}.Run("debootstrap.log", "cat", log)
		return err
	}

	if foreign {
		err = d.RunSecondStage(*context)
		if err != nil {
			return err
		}
	}

	/* HACK */
	srclist, err := os.OpenFile(path.Join(context.Rootdir, "etc/apt/sources.list"),
		os.O_RDWR|os.O_CREATE, 0755)
	if err != nil {
		return err
	}
	_, err = io.WriteString(srclist, fmt.Sprintf("deb %s %s %s\n",
		d.Mirror,
		d.Suite,
		strings.Join(d.Components, " ")))
	if err != nil {
		return err
	}
	srclist.Close()

	c := debos.NewChrootCommandForContext(*context)

	return c.Run("apt clean", "/usr/bin/apt-get", "clean")
}
<commit_msg>debootstrap: Explicitly disable merged /usr if not requested<commit_after>/*
Debootstrap Action

Construct the target rootfs with debootstrap tool.

Yaml syntax:
 - action: debootstrap
   mirror: URL
   suite: "name"
   components: <list of components>
   variant: "name"
   keyring-package:
   keyring-file:

Mandatory properties:

- suite -- release code name or symbolic name (e.g. "stable")

Optional properties:

- check-gpg -- verify GPG signatures on Release files, true by default

- mirror -- URL with Debian-compatible repository

- variant -- name of the bootstrap script variant to use

- components -- list of components to use for packages selection.
Example:
 components: [ main, contrib ]

- keyring-package -- keyring for package validation.

- keyring-file -- keyring file for repository validation.

- merged-usr -- use merged '/usr' filesystem, true by default.
*/
package actions

import (
	"fmt"
	"io"
	"os"
	"path"
	"strings"

	"github.com/go-debos/debos"
)

type DebootstrapAction struct {
	debos.BaseAction `yaml:",inline"`
	Suite            string
	Mirror           string
	Variant          string
	KeyringPackage   string `yaml:"keyring-package"`
	KeyringFile      string `yaml:"keyring-file"`
	Components       []string
	MergedUsr        bool `yaml:"merged-usr"`
	CheckGpg         bool `yaml:"check-gpg"`
}

func NewDebootstrapAction() *DebootstrapAction {
	d := DebootstrapAction{}
	// Use filesystem with merged '/usr' by default
	d.MergedUsr = true
	// Be secure by default
	d.CheckGpg = true
	return &d

}

func (d *DebootstrapAction) RunSecondStage(context debos.DebosContext) error {
	cmdline := []string{
		"/debootstrap/debootstrap",
		"--no-check-gpg",
		"--second-stage"}

	if d.Components != nil {
		s := strings.Join(d.Components, ",")
		cmdline = append(cmdline, fmt.Sprintf("--components=%s", s))
	}

	c := debos.NewChrootCommandForContext(context)
	// Can't use nspawn for debootstrap as it wants to create device nodes
	c.ChrootMethod = debos.CHROOT_METHOD_CHROOT

	err := c.Run("Debootstrap (stage 2)", cmdline...)

	if (err != nil) {
		log := path.Join(context.Rootdir, "debootstrap/debootstrap.log")
		_ = debos.Command{}.Run("debootstrap.log", "cat", log)
	}

	return err
}

func (d *DebootstrapAction) Run(context *debos.DebosContext) error {
	d.LogStart()
	cmdline := []string{"debootstrap"}

	if d.MergedUsr {
		cmdline = append(cmdline, "--merged-usr")
	} else {
		cmdline = append(cmdline, "--no-merged-usr")
	}

	if !d.CheckGpg {
		cmdline = append(cmdline, fmt.Sprintf("--no-check-gpg"))
	} else if d.KeyringFile != "" {
		path := debos.CleanPathAt(d.KeyringFile, context.RecipeDir)
		cmdline = append(cmdline, fmt.Sprintf("--keyring=%s", path))
	}

	if d.KeyringPackage != "" {
		cmdline = append(cmdline, fmt.Sprintf("--include=%s", d.KeyringPackage))
	}

	if d.Components != nil {
		s := strings.Join(d.Components, ",")
		cmdline = append(cmdline, fmt.Sprintf("--components=%s", s))
	}

	/* FIXME drop the hardcoded amd64 assumption" */
	foreign := context.Architecture != "amd64"

	if foreign {
		cmdline = append(cmdline, "--foreign")
		cmdline = append(cmdline, fmt.Sprintf("--arch=%s", context.Architecture))

	}

	if d.Variant != "" {
		cmdline = append(cmdline, fmt.Sprintf("--variant=%s", d.Variant))
	}

	cmdline = append(cmdline, d.Suite)
	cmdline = append(cmdline, context.Rootdir)
	cmdline = append(cmdline, d.Mirror)
	cmdline = append(cmdline, "/usr/share/debootstrap/scripts/unstable")

	err := debos.Command{}.Run("Debootstrap", cmdline...)

	if err != nil {
		log := path.Join(context.Rootdir, "debootstrap/debootstrap.log")
		_ = debos.Command{}.Run("debootstrap.log", "cat", log)
		return err
	}

	if foreign {
		err = d.RunSecondStage(*context)
		if err != nil {
			return err
		}
	}

	/* HACK */
	srclist, err := os.OpenFile(path.Join(context.Rootdir, "etc/apt/sources.list"),
		os.O_RDWR|os.O_CREATE, 0755)
	if err != nil {
		return err
	}
	_, err = io.WriteString(srclist, fmt.Sprintf("deb %s %s %s\n",
		d.Mirror,
		d.Suite,
		strings.Join(d.Components, " ")))
	if err != nil {
		return err
	}
	srclist.Close()

	c := debos.NewChrootCommandForContext(*context)

	return c.Run("apt clean", "/usr/bin/apt-get", "clean")
}
<|endoftext|>


<commit_before>/*
Debootstrap Action

Construct the target rootfs with debootstrap tool.

Yaml syntax:
 - action: debootstrap
   mirror: URL
   suite: "name"
   components: <list of components>
   variant: "name"
   keyring-package:

Mandatory properties:

- suite -- release code name or symbolic name (e.g. "stable")

Optional properties:

- mirror -- URL with Debian-compatible repository

- variant -- name of the bootstrap script variant to use

- components -- list of components to use for packages selection.
Example:
 components: [ main, contrib ]

- keyring-package -- keyring for packages validation. Currently ignored.
*/
package actions

import (
	"fmt"
	"io"
	"os"
	"path"
	"strings"

	"github.com/go-debos/debos"
)

type DebootstrapAction struct {
	debos.BaseAction `yaml:",inline"`
	Suite            string
	Mirror           string
	Variant          string
	KeyringPackage   string `yaml:"keyring-package"`
	Components       []string
	MergedUsr        bool `yaml:"merged-usr"`
}

func NewDebootstrapAction() *DebootstrapAction {
	d := DebootstrapAction{}
	// Use filesystem with merged '/usr' by default
	d.MergedUsr = true
	return &d

}

func (d *DebootstrapAction) RunSecondStage(context debos.DebosContext) error {
	cmdline := []string{
		"/debootstrap/debootstrap",
		"--no-check-gpg",
		"--second-stage"}

	if d.Components != nil {
		s := strings.Join(d.Components, ",")
		cmdline = append(cmdline, fmt.Sprintf("--components=%s", s))
	}

	c := debos.NewChrootCommand(context.Rootdir, context.Architecture)
	// Can't use nspawn for debootstrap as it wants to create device nodes
	c.ChrootMethod = debos.CHROOT_METHOD_CHROOT

	return c.Run("Debootstrap (stage 2)", cmdline...)
}

func (d *DebootstrapAction) Run(context *debos.DebosContext) error {
	d.LogStart()
	cmdline := []string{"debootstrap", "--no-check-gpg"}

	if d.MergedUsr {
		cmdline = append(cmdline, "--merged-usr")
	}

	if d.KeyringPackage != "" {
		cmdline = append(cmdline, fmt.Sprintf("--keyring=%s", d.KeyringPackage))
	}

	if d.Components != nil {
		s := strings.Join(d.Components, ",")
		cmdline = append(cmdline, fmt.Sprintf("--components=%s", s))
	}

	/* FIXME drop the hardcoded amd64 assumption" */
	foreign := context.Architecture != "amd64"

	if foreign {
		cmdline = append(cmdline, "--foreign")
		cmdline = append(cmdline, fmt.Sprintf("--arch=%s", context.Architecture))

	}

	if d.Variant != "" {
		cmdline = append(cmdline, fmt.Sprintf("--variant=%s", d.Variant))
	}

	cmdline = append(cmdline, d.Suite)
	cmdline = append(cmdline, context.Rootdir)
	cmdline = append(cmdline, d.Mirror)
	cmdline = append(cmdline, "/usr/share/debootstrap/scripts/unstable")

	err := debos.Command{}.Run("Debootstrap", cmdline...)

	if err != nil {
		return err
	}

	if foreign {
		err = d.RunSecondStage(*context)
		if err != nil {
			return err
		}
	}

	/* HACK */
	srclist, err := os.OpenFile(path.Join(context.Rootdir, "etc/apt/sources.list"),
		os.O_RDWR|os.O_CREATE, 0755)
	if err != nil {
		return err
	}
	_, err = io.WriteString(srclist, fmt.Sprintf("deb %s %s %s\n",
		d.Mirror,
		d.Suite,
		strings.Join(d.Components, " ")))
	if err != nil {
		return err
	}
	srclist.Close()

	c := debos.NewChrootCommand(context.Rootdir, context.Architecture)

	return c.Run("apt clean", "/usr/bin/apt-get", "clean")
}
<commit_msg>actions: debootstrap: description of 'merged-usr'<commit_after>/*
Debootstrap Action

Construct the target rootfs with debootstrap tool.

Yaml syntax:
 - action: debootstrap
   mirror: URL
   suite: "name"
   components: <list of components>
   variant: "name"
   keyring-package:

Mandatory properties:

- suite -- release code name or symbolic name (e.g. "stable")

Optional properties:

- mirror -- URL with Debian-compatible repository

- variant -- name of the bootstrap script variant to use

- components -- list of components to use for packages selection.
Example:
 components: [ main, contrib ]

- keyring-package -- keyring for packages validation. Currently ignored.

- merged-usr -- use merged '/usr' filesystem, true by default.
*/
package actions

import (
	"fmt"
	"io"
	"os"
	"path"
	"strings"

	"github.com/go-debos/debos"
)

type DebootstrapAction struct {
	debos.BaseAction `yaml:",inline"`
	Suite            string
	Mirror           string
	Variant          string
	KeyringPackage   string `yaml:"keyring-package"`
	Components       []string
	MergedUsr        bool `yaml:"merged-usr"`
}

func NewDebootstrapAction() *DebootstrapAction {
	d := DebootstrapAction{}
	// Use filesystem with merged '/usr' by default
	d.MergedUsr = true
	return &d

}

func (d *DebootstrapAction) RunSecondStage(context debos.DebosContext) error {
	cmdline := []string{
		"/debootstrap/debootstrap",
		"--no-check-gpg",
		"--second-stage"}

	if d.Components != nil {
		s := strings.Join(d.Components, ",")
		cmdline = append(cmdline, fmt.Sprintf("--components=%s", s))
	}

	c := debos.NewChrootCommand(context.Rootdir, context.Architecture)
	// Can't use nspawn for debootstrap as it wants to create device nodes
	c.ChrootMethod = debos.CHROOT_METHOD_CHROOT

	return c.Run("Debootstrap (stage 2)", cmdline...)
}

func (d *DebootstrapAction) Run(context *debos.DebosContext) error {
	d.LogStart()
	cmdline := []string{"debootstrap", "--no-check-gpg"}

	if d.MergedUsr {
		cmdline = append(cmdline, "--merged-usr")
	}

	if d.KeyringPackage != "" {
		cmdline = append(cmdline, fmt.Sprintf("--keyring=%s", d.KeyringPackage))
	}

	if d.Components != nil {
		s := strings.Join(d.Components, ",")
		cmdline = append(cmdline, fmt.Sprintf("--components=%s", s))
	}

	/* FIXME drop the hardcoded amd64 assumption" */
	foreign := context.Architecture != "amd64"

	if foreign {
		cmdline = append(cmdline, "--foreign")
		cmdline = append(cmdline, fmt.Sprintf("--arch=%s", context.Architecture))

	}

	if d.Variant != "" {
		cmdline = append(cmdline, fmt.Sprintf("--variant=%s", d.Variant))
	}

	cmdline = append(cmdline, d.Suite)
	cmdline = append(cmdline, context.Rootdir)
	cmdline = append(cmdline, d.Mirror)
	cmdline = append(cmdline, "/usr/share/debootstrap/scripts/unstable")

	err := debos.Command{}.Run("Debootstrap", cmdline...)

	if err != nil {
		return err
	}

	if foreign {
		err = d.RunSecondStage(*context)
		if err != nil {
			return err
		}
	}

	/* HACK */
	srclist, err := os.OpenFile(path.Join(context.Rootdir, "etc/apt/sources.list"),
		os.O_RDWR|os.O_CREATE, 0755)
	if err != nil {
		return err
	}
	_, err = io.WriteString(srclist, fmt.Sprintf("deb %s %s %s\n",
		d.Mirror,
		d.Suite,
		strings.Join(d.Components, " ")))
	if err != nil {
		return err
	}
	srclist.Close()

	c := debos.NewChrootCommand(context.Rootdir, context.Architecture)

	return c.Run("apt clean", "/usr/bin/apt-get", "clean")
}
<|endoftext|>


<commit_before>package modd

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/signal"
	"path/filepath"
	"time"

	"github.com/cortesi/modd/conf"
	"github.com/cortesi/modd/notify"
	"github.com/cortesi/modd/shell"
	"github.com/cortesi/moddwatch"
	"github.com/cortesi/termlog"
)

// Version is the modd release version
const Version = "0.3"

const lullTime = time.Millisecond * 100

const shellVarName = "@shell"

// CommonExcludes is a list of commonly excluded files suitable for passing in
// the excludes parameter to Watch - includes repo directories, temporary
// files, and so forth.
var CommonExcludes = []string{
	// VCS
	"**/.git/**",
	"**/.hg/**",
	"**/.svn/**",
	"**/.bzr/**",

	// OSX
	"**/.DS_Store/**",

	// Temporary files
	"**.tmp",
	"**~",
	"**#",
	"**.bak",
	"**.swp",
	"**.___jb_old___",
	"**.___jb_bak___",

	// Python
	"**.py[cod]",

	// Node
	"**/node_modules/**",
}

// ModRunner coordinates running the modd command
type ModRunner struct {
	Log        termlog.TermLog
	Config     *conf.Config
	ConfPath   string
	ConfReload bool
	Notifiers  []notify.Notifier
}

// NewModRunner constructs a new ModRunner
func NewModRunner(confPath string, log termlog.TermLog, notifiers []notify.Notifier, confreload bool) (*ModRunner, error) {
	mr := &ModRunner{
		Log:        log,
		ConfPath:   confPath,
		ConfReload: confreload,
		Notifiers:  notifiers,
	}
	err := mr.ReadConfig()
	if err != nil {
		return nil, err
	}
	return mr, nil
}

// ReadConfig parses the configuration file in ConfPath
func (mr *ModRunner) ReadConfig() error {
	ret, err := ioutil.ReadFile(mr.ConfPath)
	if err != nil {
		return fmt.Errorf("Error reading config file %s: %s", mr.ConfPath, err)
	}
	newcnf, err := conf.Parse(mr.ConfPath, string(ret))
	if err != nil {
		return fmt.Errorf("Error reading config file %s: %s", mr.ConfPath, err)
	}

	shellMethod := newcnf.GetVariables()[shellVarName]
	if !shell.Has(shellMethod) {
		return fmt.Errorf("No shell interface %q", shellMethod)
	}

	newcnf.CommonExcludes(CommonExcludes)
	mr.Config = newcnf
	return nil
}

// PrepOnly runs all prep functions and exits
func (mr *ModRunner) PrepOnly(initial bool) error {
	for _, b := range mr.Config.Blocks {
		err := RunPreps(b, mr.Config.GetVariables(), nil, mr.Log, mr.Notifiers, initial)
		if err != nil {
			return err
		}
	}
	return nil
}

func (mr *ModRunner) runBlock(b conf.Block, mod *moddwatch.Mod, dpen *DaemonPen) {
	if b.InDir != "" {
		currentDir, err := os.Getwd()
		if err != nil {
			mr.Log.Shout("Error getting current working directory: %s", err)
			return
		}
		err = os.Chdir(b.InDir)
		if err != nil {
			mr.Log.Shout(
				"Error changing to indir directory \"%s\": %s",
				b.InDir,
				err,
			)
			return
		}
		defer func() {
			err := os.Chdir(currentDir)
			if err != nil {
				mr.Log.Shout("Error returning to original directory: %s", err)
			}
		}()
	}
	err := RunPreps(
		b,
		mr.Config.GetVariables(),
		mod, mr.Log,
		mr.Notifiers,
		mod == nil,
	)
	if err != nil {
		if _, ok := err.(ProcError); !ok {
			mr.Log.Shout("Error running prep: %s", err)
		}
		return
	}
	dpen.Restart()
}

func (mr *ModRunner) trigger(mod *moddwatch.Mod, dworld *DaemonWorld) {
	for i, b := range mr.Config.Blocks {
		lmod := mod
		if lmod != nil {
			var err error
			lmod, err = mod.Filter(b.Include, b.Exclude)
			if err != nil {
				mr.Log.Shout("Error filtering events: %s", err)
				continue
			}
			if lmod.Empty() {
				continue
			}
		}
		mr.runBlock(b, lmod, dworld.DaemonPens[i])
	}
}

// Gives control of chan to caller
func (mr *ModRunner) runOnChan(modchan chan *moddwatch.Mod, readyCallback func()) error {
	dworld, err := NewDaemonWorld(mr.Config, mr.Log)
	if err != nil {
		return err
	}
	defer dworld.Shutdown(os.Kill)

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill)
	defer signal.Reset(os.Interrupt, os.Kill)
	go func() {
		dworld.Shutdown(<-c)
		os.Exit(0)
	}()

	watchpaths := mr.Config.WatchPatterns()
	if mr.ConfReload {
		watchpaths = append(watchpaths, filepath.Dir(mr.ConfPath))
	}

	// FIXME: This takes a long time. We could start it in parallel with the
	// first process run in a goroutine
	watcher, err := moddwatch.Watch(watchpaths, lullTime, modchan)
	if err != nil {
		return fmt.Errorf("Error watching: %s", err)
	}
	defer watcher.Stop()

	mr.trigger(nil, dworld)
	go readyCallback()
	for mod := range modchan {
		if mod == nil {
			break
		}

		if mr.ConfReload && mod.Has(mr.ConfPath) {
			mr.Log.Notice("Reloading config %s", mr.ConfPath)
			err := mr.ReadConfig()
			if err != nil {
				mr.Log.Warn("%s", err)
				continue
			} else {
				return nil
			}
		}

		mr.Log.SayAs("debug", "Delta: \n%s", mod.String())
		mr.trigger(mod, dworld)
	}
	return nil
}

// Run is the top-level runner for modd
func (mr *ModRunner) Run() error {
	for {
		modchan := make(chan *moddwatch.Mod, 1024)
		err := mr.runOnChan(modchan, func() {})
		if err != nil {
			return err
		}
	}
}
<commit_msg>Bump version<commit_after>package modd

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/signal"
	"path/filepath"
	"time"

	"github.com/cortesi/modd/conf"
	"github.com/cortesi/modd/notify"
	"github.com/cortesi/modd/shell"
	"github.com/cortesi/moddwatch"
	"github.com/cortesi/termlog"
)

// Version is the modd release version
const Version = "0.4"

const lullTime = time.Millisecond * 100

const shellVarName = "@shell"

// CommonExcludes is a list of commonly excluded files suitable for passing in
// the excludes parameter to Watch - includes repo directories, temporary
// files, and so forth.
var CommonExcludes = []string{
	// VCS
	"**/.git/**",
	"**/.hg/**",
	"**/.svn/**",
	"**/.bzr/**",

	// OSX
	"**/.DS_Store/**",

	// Temporary files
	"**.tmp",
	"**~",
	"**#",
	"**.bak",
	"**.swp",
	"**.___jb_old___",
	"**.___jb_bak___",

	// Python
	"**.py[cod]",

	// Node
	"**/node_modules/**",
}

// ModRunner coordinates running the modd command
type ModRunner struct {
	Log        termlog.TermLog
	Config     *conf.Config
	ConfPath   string
	ConfReload bool
	Notifiers  []notify.Notifier
}

// NewModRunner constructs a new ModRunner
func NewModRunner(confPath string, log termlog.TermLog, notifiers []notify.Notifier, confreload bool) (*ModRunner, error) {
	mr := &ModRunner{
		Log:        log,
		ConfPath:   confPath,
		ConfReload: confreload,
		Notifiers:  notifiers,
	}
	err := mr.ReadConfig()
	if err != nil {
		return nil, err
	}
	return mr, nil
}

// ReadConfig parses the configuration file in ConfPath
func (mr *ModRunner) ReadConfig() error {
	ret, err := ioutil.ReadFile(mr.ConfPath)
	if err != nil {
		return fmt.Errorf("Error reading config file %s: %s", mr.ConfPath, err)
	}
	newcnf, err := conf.Parse(mr.ConfPath, string(ret))
	if err != nil {
		return fmt.Errorf("Error reading config file %s: %s", mr.ConfPath, err)
	}

	shellMethod := newcnf.GetVariables()[shellVarName]
	if !shell.Has(shellMethod) {
		return fmt.Errorf("No shell interface %q", shellMethod)
	}

	newcnf.CommonExcludes(CommonExcludes)
	mr.Config = newcnf
	return nil
}

// PrepOnly runs all prep functions and exits
func (mr *ModRunner) PrepOnly(initial bool) error {
	for _, b := range mr.Config.Blocks {
		err := RunPreps(b, mr.Config.GetVariables(), nil, mr.Log, mr.Notifiers, initial)
		if err != nil {
			return err
		}
	}
	return nil
}

func (mr *ModRunner) runBlock(b conf.Block, mod *moddwatch.Mod, dpen *DaemonPen) {
	if b.InDir != "" {
		currentDir, err := os.Getwd()
		if err != nil {
			mr.Log.Shout("Error getting current working directory: %s", err)
			return
		}
		err = os.Chdir(b.InDir)
		if err != nil {
			mr.Log.Shout(
				"Error changing to indir directory \"%s\": %s",
				b.InDir,
				err,
			)
			return
		}
		defer func() {
			err := os.Chdir(currentDir)
			if err != nil {
				mr.Log.Shout("Error returning to original directory: %s", err)
			}
		}()
	}
	err := RunPreps(
		b,
		mr.Config.GetVariables(),
		mod, mr.Log,
		mr.Notifiers,
		mod == nil,
	)
	if err != nil {
		if _, ok := err.(ProcError); !ok {
			mr.Log.Shout("Error running prep: %s", err)
		}
		return
	}
	dpen.Restart()
}

func (mr *ModRunner) trigger(mod *moddwatch.Mod, dworld *DaemonWorld) {
	for i, b := range mr.Config.Blocks {
		lmod := mod
		if lmod != nil {
			var err error
			lmod, err = mod.Filter(b.Include, b.Exclude)
			if err != nil {
				mr.Log.Shout("Error filtering events: %s", err)
				continue
			}
			if lmod.Empty() {
				continue
			}
		}
		mr.runBlock(b, lmod, dworld.DaemonPens[i])
	}
}

// Gives control of chan to caller
func (mr *ModRunner) runOnChan(modchan chan *moddwatch.Mod, readyCallback func()) error {
	dworld, err := NewDaemonWorld(mr.Config, mr.Log)
	if err != nil {
		return err
	}
	defer dworld.Shutdown(os.Kill)

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill)
	defer signal.Reset(os.Interrupt, os.Kill)
	go func() {
		dworld.Shutdown(<-c)
		os.Exit(0)
	}()

	watchpaths := mr.Config.WatchPatterns()
	if mr.ConfReload {
		watchpaths = append(watchpaths, filepath.Dir(mr.ConfPath))
	}

	// FIXME: This takes a long time. We could start it in parallel with the
	// first process run in a goroutine
	watcher, err := moddwatch.Watch(watchpaths, lullTime, modchan)
	if err != nil {
		return fmt.Errorf("Error watching: %s", err)
	}
	defer watcher.Stop()

	mr.trigger(nil, dworld)
	go readyCallback()
	for mod := range modchan {
		if mod == nil {
			break
		}

		if mr.ConfReload && mod.Has(mr.ConfPath) {
			mr.Log.Notice("Reloading config %s", mr.ConfPath)
			err := mr.ReadConfig()
			if err != nil {
				mr.Log.Warn("%s", err)
				continue
			} else {
				return nil
			}
		}

		mr.Log.SayAs("debug", "Delta: \n%s", mod.String())
		mr.trigger(mod, dworld)
	}
	return nil
}

// Run is the top-level runner for modd
func (mr *ModRunner) Run() error {
	for {
		modchan := make(chan *moddwatch.Mod, 1024)
		err := mr.runOnChan(modchan, func() {})
		if err != nil {
			return err
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"spool-mock/config"
	"net"
	"spool-mock/client"
	"strings"
	"io"
	"net/textproto"
	"bufio"
	"bytes"
	"spool-mock/dotreader"
)

func Quit(conn *client.Conn, tok []string) {
	conn.Send("205 Bye.")
}

func Unsupported(conn *client.Conn, tok []string) {
	fmt.Println(fmt.Sprintf("WARN: C(%s): Unsupported cmd %s", conn.RemoteAddr(), tok[0]))
	conn.Send("500 Unsupported.")
}

func read(conn *client.Conn, msgid string, msgtype string) {
	var code string
	if msgtype == "ARTICLE" {
		code = "220"
	} else if msgtype == "HEAD" {
		code = "221"
	} else if msgtype == "BODY" {
		code = "222"
	} else {
		panic("Should not get here")
	}

	if msgid == "<aaa@bb.cc>" {
		conn.Send("500 msgid means fivehundred err")
		return
	}

	raw := `Path: asg009!abp002.ams.xsnews.nl!abuse.newsxs.nl!not-for-mail
From: Zinitzio <x8F4zpNLByt8Vhh1hyFBTcarWqKeqTszySrxYJUNrGyj64VA761YahKczcyROsOv.N5UyksLragucHTY7hXbIf3OraQSwtjjJX6PcYubvlsh6oPDUGuY1j0b4Z7i6xnio@47a00b01.16110764.10.1443172883.1.NL.v8r0DMvyrMxvrV9wjB9RklWe-p-p1ZChfS4lxGsMNtRWMbyLXZonEJ6Lp3usHDsLnG>
Subject: Mkv Tool Nix 8.4.0 NL | Zgp
Newsgroups: free.pt
Message-ID: <pTgQyybcKwYEhIFVg2wH7@spot.net>
X-Newsreader: Spotnet 2.0.0.114
X-XML: <Spotnet><Posting><Key>7</Key><Created>1443172883</Created><Poster>Zinitzio</Poster><Tag>Zgp</Tag><Title>Mkv Tool Nix 8.4.0 NL</Title><Description>Iedere Mkv (x264) film heeft meerdere sporen. Met dit programma kun je sporen verwijderen of toevoegen. Heb je een film zonder ondertitel dan kun je die makkelijk toevoegen.[br][br]In deze spot zitten de volgende onderdelen:[br][br]Mkv Tool Nix 8.4.0</Description><Image Width='350' Height='350'><Segment>Ldqj0ABsZDMEhIFVgyrLc@spot.net</Segment></Image><Size>16110764</Size><Category>0404a0004b01</Category><NZB><Segment>sm0Ls136Ir4EhIFVgj4Dg@spot.net</Segment></NZB></Posting></Spotnet>
X-XML-Signature: mMXtDVvEzuAz5soJzKcpsd042VQY2M306o418-pOYtLIxv7DN5lDzAO3rB3EakfZT
X-User-Key: <RSAKeyValue><Modulus>x8F4zpNLByt8Vhh1hyFBTcarWqKeqTszySrxYJUNrGyj64VA761YahKczcyROsOv</Modulus><Exponent>AQAB</Exponent></RSAKeyValue>
X-User-Signature: N5UyksLragucHTY7hXbIf3OraQSwtjjJX6PcYubvlsh6oPDUGuY1j0b4Z7i6xnio
Content-Type: text/plain; charset=ISO-8859-1
Content-Transfer-Encoding: 8bit
X-Complaints-To: abuse@newsxs.nl
Organization: Newsxs
Date: Fri, 25 Sep 2015 11:21:23 +0200
Lines: 5
NNTP-Posting-Date: Fri, 25 Sep 2015 11:21:23 +0200

Iedere Mkv (x264) film heeft meerdere sporen. Met dit programma kun je sporen verwijderen of toevoegen. Heb je een film zonder ondertitel dan kun je die makkelijk toevoegen.

In deze spot zitten de volgende onderdelen:

Mkv Tool Nix 8.4.0`
	raw = strings.Replace(raw, "\n", "\r\n", -1)

	conn.Send(code + " " + msgid)

	if msgid == "<aab@bb.cc>" {
		// fake a broken
		conn.Send(raw[0:50])
		conn.Close()
	} else {
		conn.Send(raw)
	}
	conn.Send("\r\n.") // additional \r\n auto-added
	if msgid == "<close@bb.cc>" {
		conn.Close()
	}
}

func PostArticle(conn *client.Conn) {
	conn.Send("340 Start posting.")

	b := new(bytes.Buffer)
	br := bufio.NewReader(conn.GetReader())
	r := textproto.NewReader(br)

	fmt.Println("PostArticle head.")
	m, e := r.ReadMIMEHeader()
	if e != nil {
		conn.Send("440 Failed reading header")
		return
	}

	fmt.Println("PostArticle body.")
	if _, e := io.Copy(b, dotreader.New(br)); e != nil {
		conn.Send("440 Failed reading body")
		return
	}

	if val := m.Get("X-Accept"); val == "DENY" {
		conn.Send("440 Deny test.")
		return
	}

	if b.String() != "\r\nBody.\r\nBody1\r\nBody2 ohyeay?\r\n.\r\n" {
		conn.Send("500 Body does not match hardcoded compare value.")
		return
	}
	conn.Send("240 Posted.")
}

func Article(conn *client.Conn, tok []string) {
	if len(tok) != 2 {
		conn.Send("501 Invalid syntax.")
		return
	}
	read(conn, tok[1], "ARTICLE")
}

func Head(conn *client.Conn, tok []string) {
	if len(tok) != 2 {
		conn.Send("501 Invalid syntax.")
		return
	}
	read(conn, tok[1], "HEAD")
}

func Body(conn *client.Conn, tok []string) {
	if len(tok) != 2 {
		conn.Send("501 Invalid syntax.")
		return
	}
	read(conn, tok[1], "BODY")
}

func Group(conn *client.Conn, tok []string) {
	if len(tok) != 2 {
		conn.Send("501 Invalid syntax.")
		return
	}
	if tok[1] == "nosuch.group" {
		conn.Send("411 No such group.")
		return
	} else if tok[1] == "standard.group" {
		conn.Send("211 300007627 8974530000 9274537627 standard.group")
		return
	}

	conn.Send("501 No test for given groupname")
}

func req(conn *client.Conn) {
	conn.Send("200 StoreD")
	for {
		tok, e := conn.ReadLine()
		if e != nil {
			fmt.Println(fmt.Sprintf("WARN: C(%s): %s", conn.RemoteAddr(), e.Error()))
			break
		}

		cmd := strings.ToUpper(tok[0])
		if cmd == "QUIT" {
			Quit(conn, tok)
			break
		} else if cmd == "ARTICLE" {
			Article(conn, tok)
		} else if cmd == "HEAD" {
			Head(conn, tok)
		} else if cmd == "BODY" {
			Body(conn, tok)
		} else if cmd == "AUTHINFO" {
			sub := strings.ToUpper(tok[1])
			if sub == "USER" {
				conn.Send("381 Need more.")
			} else if sub == "PASS" {
				if tok[2] == "test" {
					conn.Send("281 Authentication accepted.")
				}
			}
		} else if cmd == "GROUP" {
			// GROUP x
			Group(conn, tok)
		} else if cmd == "NOOP" {
			conn.Send("500 Unsupported.")
		} else if cmd == "POST" {
			PostArticle(conn)
		} else {
			Unsupported(conn, tok)
			break
		}
	}

	conn.Close()
	if config.Verbose {
		fmt.Println(fmt.Sprintf("C(%s) Closed", conn.RemoteAddr()))
	}
}

func nntpListen(listen string) error {
	sock, err := net.Listen("tcp", listen)
	if err != nil {
		return err
	}
	if config.Verbose {
		fmt.Println("nntpd listening on " + listen)
	}

	for {
		conn, err := sock.Accept()
		if err != nil {
			panic(err)
		}
		if config.Verbose {
			fmt.Println(fmt.Sprintf("C(%s) New", conn.RemoteAddr()))
		}

		go req(client.New(conn))
	}
}<commit_msg>Test. Add suppor for HEAD+BODY<commit_after>package main

import (
	"fmt"
	"spool-mock/config"
	"net"
	"spool-mock/client"
	"strings"
	"io"
	"net/textproto"
	"bufio"
	"bytes"
	"spool-mock/dotreader"
)

func Quit(conn *client.Conn, tok []string) {
	conn.Send("205 Bye.")
}

func Unsupported(conn *client.Conn, tok []string) {
	fmt.Println(fmt.Sprintf("WARN: C(%s): Unsupported cmd %s", conn.RemoteAddr(), tok[0]))
	conn.Send("500 Unsupported.")
}

func read(conn *client.Conn, msgid string, msgtype string) {
	var code string
	head := true
	body := true

	if msgtype == "ARTICLE" {
		code = "220"
	} else if msgtype == "HEAD" {
		code = "221"
		body = false
	} else if msgtype == "BODY" {
		code = "222"
		head = false
	} else {
		panic("Should not get here")
	}

	if msgid == "<aaa@bb.cc>" {
		conn.Send("500 msgid means fivehundred err")
		return
	}

	var raw string
	if head {
	raw += `Path: asg009!abp002.ams.xsnews.nl!abuse.newsxs.nl!not-for-mail
From: Zinitzio <x8F4zpNLByt8Vhh1hyFBTcarWqKeqTszySrxYJUNrGyj64VA761YahKczcyROsOv.N5UyksLragucHTY7hXbIf3OraQSwtjjJX6PcYubvlsh6oPDUGuY1j0b4Z7i6xnio@47a00b01.16110764.10.1443172883.1.NL.v8r0DMvyrMxvrV9wjB9RklWe-p-p1ZChfS4lxGsMNtRWMbyLXZonEJ6Lp3usHDsLnG>
Subject: Mkv Tool Nix 8.4.0 NL | Zgp
Newsgroups: free.pt
Message-ID: <pTgQyybcKwYEhIFVg2wH7@spot.net>
X-Newsreader: Spotnet 2.0.0.114
X-XML: <Spotnet><Posting><Key>7</Key><Created>1443172883</Created><Poster>Zinitzio</Poster><Tag>Zgp</Tag><Title>Mkv Tool Nix 8.4.0 NL</Title><Description>Iedere Mkv (x264) film heeft meerdere sporen. Met dit programma kun je sporen verwijderen of toevoegen. Heb je een film zonder ondertitel dan kun je die makkelijk toevoegen.[br][br]In deze spot zitten de volgende onderdelen:[br][br]Mkv Tool Nix 8.4.0</Description><Image Width='350' Height='350'><Segment>Ldqj0ABsZDMEhIFVgyrLc@spot.net</Segment></Image><Size>16110764</Size><Category>0404a0004b01</Category><NZB><Segment>sm0Ls136Ir4EhIFVgj4Dg@spot.net</Segment></NZB></Posting></Spotnet>
X-XML-Signature: mMXtDVvEzuAz5soJzKcpsd042VQY2M306o418-pOYtLIxv7DN5lDzAO3rB3EakfZT
X-User-Key: <RSAKeyValue><Modulus>x8F4zpNLByt8Vhh1hyFBTcarWqKeqTszySrxYJUNrGyj64VA761YahKczcyROsOv</Modulus><Exponent>AQAB</Exponent></RSAKeyValue>
X-User-Signature: N5UyksLragucHTY7hXbIf3OraQSwtjjJX6PcYubvlsh6oPDUGuY1j0b4Z7i6xnio
Content-Type: text/plain; charset=ISO-8859-1
Content-Transfer-Encoding: 8bit
X-Complaints-To: abuse@newsxs.nl
Organization: Newsxs
Date: Fri, 25 Sep 2015 11:21:23 +0200
Lines: 5
NNTP-Posting-Date: Fri, 25 Sep 2015 11:21:23 +0200`
}
if head && body {
	raw += "\n\n"
}
if body {
raw += `Iedere Mkv (x264) film heeft meerdere sporen. Met dit programma kun je sporen verwijderen of toevoegen. Heb je een film zonder ondertitel dan kun je die makkelijk toevoegen.

In deze spot zitten de volgende onderdelen:

Mkv Tool Nix 8.4.0`
}

	raw = strings.Replace(raw, "\n", "\r\n", -1)

	conn.Send(code + " " + msgid)

	if msgid == "<aab@bb.cc>" {
		// fake a broken
		conn.Send(raw[0:50])
		conn.Close()
	} else {
		conn.Send(raw)
	}
	conn.Send("\r\n.") // additional \r\n auto-added
	if msgid == "<close@bb.cc>" {
		conn.Close()
	}
}

func PostArticle(conn *client.Conn) {
	conn.Send("340 Start posting.")

	b := new(bytes.Buffer)
	br := bufio.NewReader(conn.GetReader())
	r := textproto.NewReader(br)

	fmt.Println("PostArticle head.")
	m, e := r.ReadMIMEHeader()
	if e != nil {
		conn.Send("440 Failed reading header")
		return
	}

	fmt.Println("PostArticle body.")
	if _, e := io.Copy(b, dotreader.New(br)); e != nil {
		conn.Send("440 Failed reading body")
		return
	}

	if val := m.Get("X-Accept"); val == "DENY" {
		conn.Send("440 Deny test.")
		return
	}

	if b.String() != "\r\nBody.\r\nBody1\r\nBody2 ohyeay?\r\n.\r\n" {
		conn.Send("500 Body does not match hardcoded compare value.")
		return
	}
	conn.Send("240 Posted.")
}

func Article(conn *client.Conn, tok []string) {
	if len(tok) != 2 {
		conn.Send("501 Invalid syntax.")
		return
	}
	read(conn, tok[1], "ARTICLE")
}

func Head(conn *client.Conn, tok []string) {
	if len(tok) != 2 {
		conn.Send("501 Invalid syntax.")
		return
	}
	read(conn, tok[1], "HEAD")
}

func Body(conn *client.Conn, tok []string) {
	if len(tok) != 2 {
		conn.Send("501 Invalid syntax.")
		return
	}
	read(conn, tok[1], "BODY")
}

func Group(conn *client.Conn, tok []string) {
	if len(tok) != 2 {
		conn.Send("501 Invalid syntax.")
		return
	}
	if tok[1] == "nosuch.group" {
		conn.Send("411 No such group.")
		return
	} else if tok[1] == "standard.group" {
		conn.Send("211 300007627 8974530000 9274537627 standard.group")
		return
	}

	conn.Send("501 No test for given groupname")
}

func req(conn *client.Conn) {
	conn.Send("200 StoreD")
	for {
		tok, e := conn.ReadLine()
		if e != nil {
			fmt.Println(fmt.Sprintf("WARN: C(%s): %s", conn.RemoteAddr(), e.Error()))
			break
		}

		cmd := strings.ToUpper(tok[0])
		if cmd == "QUIT" {
			Quit(conn, tok)
			break
		} else if cmd == "ARTICLE" {
			Article(conn, tok)
		} else if cmd == "HEAD" {
			Head(conn, tok)
		} else if cmd == "BODY" {
			Body(conn, tok)
		} else if cmd == "AUTHINFO" {
			sub := strings.ToUpper(tok[1])
			if sub == "USER" {
				conn.Send("381 Need more.")
			} else if sub == "PASS" {
				if tok[2] == "test" {
					conn.Send("281 Authentication accepted.")
				}
			}
		} else if cmd == "GROUP" {
			// GROUP x
			Group(conn, tok)
		} else if cmd == "NOOP" {
			conn.Send("500 Unsupported.")
		} else if cmd == "POST" {
			PostArticle(conn)
		} else {
			Unsupported(conn, tok)
			break
		}
	}

	conn.Close()
	if config.Verbose {
		fmt.Println(fmt.Sprintf("C(%s) Closed", conn.RemoteAddr()))
	}
}

func nntpListen(listen string) error {
	sock, err := net.Listen("tcp", listen)
	if err != nil {
		return err
	}
	if config.Verbose {
		fmt.Println("nntpd listening on " + listen)
	}

	for {
		conn, err := sock.Accept()
		if err != nil {
			panic(err)
		}
		if config.Verbose {
			fmt.Println(fmt.Sprintf("C(%s) New", conn.RemoteAddr()))
		}

		go req(client.New(conn))
	}
}<|endoftext|>