<commit_before>package hostdb

import (
	"math/big"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

var (
	// Because most weights would otherwise be fractional, we set the base
	// weight to be very large.
	baseWeight = types.NewCurrency(new(big.Int).Exp(big.NewInt(10), big.NewInt(50), nil))

	// collateralExponentiation is the number of times that the collateral is
	// multiplied into the price.
	//
	// NOTE: Changing this value downwards needs that the baseWeight will need
	// to be increased.
	collateralExponentiation = 2

	// priceDiveNormalization reduces the raw value of the price so that not so
	// many digits are needed when operating on the weight. This also allows the
	// base weight to be a lot lower.
	priceDivNormalization = types.SiacoinPrecision.Div64(100)

	// Set a mimimum price, below which setting lower prices will no longer put
	// this host at an advatnage. This price is considered the bar for
	// 'essentially free', and is kept to a minimum to prevent certain Sybil
	// attack related attack vectors.
	//
	// NOTE: This needs to be intelligently adjusted down as the practical price
	// of storage changes, and as the price of the siacoin changes.
	minDivPrice = types.SiacoinPrecision.Mul64(250)

	// priceExponentiation is the number of times that the weight is divided by
	// the price.
	//
	// NOTE: Changing this value upwards means that the baseWeight will need to
	// be increased.
	priceExponentiation = 4

	// requiredStorage indicates the amount of storage that the host must be
	// offering in order to be considered a valuable/worthwhile host.
	requiredStorage = func() uint64 {
		switch build.Release {
		case "dev":
			return 1e6
		case "standard":
			return 5e9
		case "testing":
			return 1e3
		default:
			panic("incorrect/missing value for requiredStorage constant")
		}
	}()

	// uptimeExponentiation is the number of times the uptime percentage is
	// multiplied by itself when determining host uptime penalty.
	uptimeExponentiation = 18
)

// collateralAdjustments improves the host's weight according to the amount of
// collateral that they have provided.
//
// NOTE: For any reasonable value of collateral, there will be a huge blowup,
// allowing for the base weight to be a lot lower, as the collateral is
// accounted for before anything else.
func collateralAdjustments(entry modules.HostDBEntry, weight types.Currency) types.Currency {
	if entry.Collateral.IsZero() {
		// Instead of zeroing out the weight, just return the weight as though
		// the collateral is 1 hasting. Competitively speaking, this is
		// effectively zero.
		return weight
	}
	for i := 0; i < collateralExponentiation; i++ {
		weight = weight.Mul(entry.Collateral)
	}
	return weight
}

// priceAdjustments will adjust the weight of the entry according to the prices
// that it has set.
func priceAdjustments(entry modules.HostDBEntry, weight types.Currency) types.Currency {
	// Sanity checks - the constants values need to have certain relationships
	// to eachother
	if build.DEBUG {
		// If the minDivPrice is not much larger than the divNormalization,
		// there will be problems with granularity after the divNormalization is
		// applied.
		if minDivPrice.Div64(100).Cmp(priceDivNormalization) < 0 {
			build.Critical("Maladjusted minDivePrice and divNormalization constants in hostdb package")
		}
	}

	// Prices tiered as follows:
	//    - the storage price is presented as 'per block per byte'
	//    - the contract price is presented as a flat rate
	//    - the upload bandwidth price is per byte
	//    - the download bandwidth price is per byte
	//
	// The hostdb will naively assume the following for now:
	//    - each contract covers 6 weeks of storage (default is 12 weeks, but
	//      renewals occur at midpoint) - 6048 blocks - and 10GB of storage.
	//    - uploads happen once per 12 weeks (average lifetime of a file is 12 weeks)
	//    - downloads happen once per 6 weeks (files are on average downloaded twice throughout lifetime)
	//
	// In the future, the renter should be able to track average user behavior
	// and adjust accordingly. This flexibility will be added later.
	adjustedContractPrice := entry.ContractPrice.Div64(6048).Div64(10e9) // Adjust contract price to match 10GB for 6 weeks.
	adjustedUploadPrice := entry.UploadBandwidthPrice.Div64(24192)       // Adjust upload price to match a single upload over 24 weeks.
	adjustedDownloadPrice := entry.DownloadBandwidthPrice.Div64(12096)   // Adjust download price to match one download over 12 weeks.
	siafundFee := adjustedContractPrice.Add(adjustedUploadPrice).Add(adjustedDownloadPrice).Add(entry.Collateral).MulTax()
	totalPrice := entry.StoragePrice.Add(adjustedContractPrice).Add(adjustedUploadPrice).Add(adjustedDownloadPrice).Add(siafundFee)

	// Set the divPrice, which is closely related to the totalPrice, but
	// adjusted both to make the math more computationally friendly and also
	// given a hard minimum to prevent certain classes of Sybil attacks -
	// attacks where the attacker tries to esacpe the need to burn coins by
	// setting an extremely low price.
	divPrice := totalPrice
	if divPrice.Cmp(minDivPrice) < 0 {
		divPrice = minDivPrice
	}
	// Shrink the div price so that the math can be a lot less intense. Without
	// this step, the base price would need to be closer to 10e150 as opposed to
	// 10e50.
	divPrice = divPrice.Div(priceDivNormalization)
	for i := 0; i < priceExponentiation; i++ {
		weight = weight.Div(divPrice)
	}
	return weight
}

// storageRemainingAdjustments adjusts the weight of the entry according to how
// much storage it has remaining.
func storageRemainingAdjustments(entry modules.HostDBEntry) float64 {
	base := float64(1)
	if entry.RemainingStorage < 200*requiredStorage {
		base = base / 2 // 2x total penalty
	}
	if entry.RemainingStorage < 100*requiredStorage {
		base = base / 3 // 6x total penalty
	}
	if entry.RemainingStorage < 50*requiredStorage {
		base = base / 4 // 24x total penalty
	}
	if entry.RemainingStorage < 25*requiredStorage {
		base = base / 5 // 95x total penalty
	}
	if entry.RemainingStorage < 10*requiredStorage {
		base = base / 6 // 570x total penalty
	}
	if entry.RemainingStorage < 5*requiredStorage {
		base = base / 10 // 5,700x total penalty
	}
	if entry.RemainingStorage < requiredStorage {
		base = base / 100 // 570,000x total penalty
	}
	return base
}

// versionAdjustments will adjust the weight of the entry according to the siad
// version reported by the host.
func versionAdjustments(entry modules.HostDBEntry) float64 {
	base := float64(1)
	if build.VersionCmp(entry.Version, "1.0.3") < 0 {
		base = base / 5 // 5x total penalty.
	}
	if build.VersionCmp(entry.Version, "1.0.0") < 0 {
		base = base / 20 // 100x total penalty.
	}
	return base
}

// lifetimeAdjustments will adjust the weight of the host according to the total
// amount of time that has passed since the host's original announcement.
func (hdb *HostDB) lifetimeAdjustments(entry modules.HostDBEntry) float64 {
	base := float64(1)
	if hdb.blockHeight >= entry.FirstSeen {
		age := hdb.blockHeight - entry.FirstSeen
		if age < 6000 {
			base = base / 2 // 2x total
		}
		if age < 4000 {
			base = base / 2 // 4x total
		}
		if age < 2000 {
			base = base / 4 // 16x total
		}
		if age < 1000 {
			base = base / 4 // 64x total
		}
		if age < 288 {
			base = base / 2 // 128x total
		}
	} else {
		// Shouldn't happen, but the usecase is covered anyway.
		base = base / 1000 // Because something weird is happening, don't trust this host very much.
		hdb.log.Critical("Hostdb has witnessed a host where the FirstSeen height is higher than the current block height.")
	}
	return base
}

// uptimeAdjustments penalizes the host for having poor uptime, and for being
// offline.
func (hdb *HostDB) uptimeAdjustments(entry modules.HostDBEntry) float64 {
	// Special case: if we have scanned the host twice or fewer, don't perform
	// uptime math.
	if len(entry.ScanHistory) == 0 {
		return 0.001 // Shouldn't happen.
	}
	if len(entry.ScanHistory) == 1 {
		if entry.ScanHistory[0].Success {
			return 0.75
		}
		return 0.25
	}
	if len(entry.ScanHistory) == 2 {
		if entry.ScanHistory[0].Success && entry.ScanHistory[1].Success {
			return 0.85
		}
		if entry.ScanHistory[0].Success || entry.ScanHistory[1].Success {
			return 0.50
		}
		return 0.05
	}

	// Compute the total measured uptime and total measured downtime for this
	// host.
	var uptime time.Duration
	var downtime time.Duration
	recentTime := entry.ScanHistory[0].Timestamp
	for _, scan := range entry.ScanHistory[1:] {
		if recentTime.After(scan.Timestamp) {
			hdb.log.Critical("Host entry scan history not sorted.")
		}
		if scan.Success {
			uptime += recentTime.Sub(scan.Timestamp)
		} else {
			downtime += recentTime.Sub(scan.Timestamp)
		}
		recentTime = scan.Timestamp
	}

	// Calculate the penalty for low uptime.
	uptimePenalty := float64(1)
	uptimeRatio := float64(uptime) / float64(uptime+downtime)
	if uptimeRatio > 0.97 {
		uptimeRatio = 0.97
	}
	uptimeRatio += 0.03
	for i := 0; i < uptimeExponentiation; i++ {
		uptimePenalty *= uptimeRatio
	}

	// Calculate the penalty for consecutive downtime.
	var consecutiveDowntime time.Duration
	scanLen := len(entry.ScanHistory)
	startTime := time.Now()
	for i := scanLen - 1; i >= 0; i-- {
		if entry.ScanHistory[i].Success {
			break
		}
		consecutiveDowntime = startTime.Sub(entry.ScanHistory[i].Timestamp)
	}
	// Penalize by a factor of 2 for each consecutive day of downtime,
	// including penalizing by a factor of 2 for any leftover downtime.
	for i := 0; consecutiveDowntime > 0 && i < 15; i++ {
		uptimePenalty = uptimePenalty / 2
		consecutiveDowntime -= time.Hour * 24
	}
	return uptimePenalty
}

// calculateHostWeight returns the weight of a host according to the settings of
// the host database entry. Currently, only the price is considered.
func (hdb *HostDB) calculateHostWeight(entry modules.HostDBEntry) types.Currency {
	// Perform the high resolution adjustments.
	weight := baseWeight
	weight = collateralAdjustments(entry, weight)
	weight = priceAdjustments(entry, weight)

	// Perform the lower resolution adjustments.
	storageRemainingPenalty := storageRemainingAdjustments(entry)
	versionPenalty := versionAdjustments(entry)
	lifetimePenalty := hdb.lifetimeAdjustments(entry)
	uptimePenalty := hdb.uptimeAdjustments(entry)

	// Combine the adjustments.
	fullPenalty := storageRemainingPenalty * versionPenalty * lifetimePenalty * uptimePenalty
	weight = weight.MulFloat(fullPenalty)

	if weight.IsZero() {
		// A weight of zero is problematic for for the host tree.
		return types.NewCurrency64(1)
	}
	return weight
}
<commit_msg>better host downtime management<commit_after>package hostdb

import (
	"math/big"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

var (
	// Because most weights would otherwise be fractional, we set the base
	// weight to be very large.
	baseWeight = types.NewCurrency(new(big.Int).Exp(big.NewInt(10), big.NewInt(75), nil))

	// collateralExponentiation is the number of times that the collateral is
	// multiplied into the price.
	//
	// NOTE: Changing this value downwards needs that the baseWeight will need
	// to be increased.
	collateralExponentiation = 1

	// priceDiveNormalization reduces the raw value of the price so that not so
	// many digits are needed when operating on the weight. This also allows the
	// base weight to be a lot lower.
	priceDivNormalization = types.SiacoinPrecision.Div64(100)

	// minCollateral is the amount of collateral we weight all hosts as having,
	// even if they do not have any collateral. This is to temporarily prop up
	// weak / cheap hosts on the network while the network is bootstrapping.
	minCollateral = types.SiacoinPrecision.Mul64(25)

	// Set a mimimum price, below which setting lower prices will no longer put
	// this host at an advatnage. This price is considered the bar for
	// 'essentially free', and is kept to a minimum to prevent certain Sybil
	// attack related attack vectors.
	//
	// NOTE: This needs to be intelligently adjusted down as the practical price
	// of storage changes, and as the price of the siacoin changes.
	minDivPrice = types.SiacoinPrecision.Mul64(250)

	// priceExponentiation is the number of times that the weight is divided by
	// the price.
	//
	// NOTE: Changing this value upwards means that the baseWeight will need to
	// be increased.
	priceExponentiation = 4

	// requiredStorage indicates the amount of storage that the host must be
	// offering in order to be considered a valuable/worthwhile host.
	requiredStorage = func() uint64 {
		switch build.Release {
		case "dev":
			return 1e6
		case "standard":
			return 5e9
		case "testing":
			return 1e3
		default:
			panic("incorrect/missing value for requiredStorage constant")
		}
	}()

	// uptimeExponentiation is the number of times the uptime percentage is
	// multiplied by itself when determining host uptime penalty.
	uptimeExponentiation = 18
)

// collateralAdjustments improves the host's weight according to the amount of
// collateral that they have provided.
//
// NOTE: For any reasonable value of collateral, there will be a huge blowup,
// allowing for the base weight to be a lot lower, as the collateral is
// accounted for before anything else.
func collateralAdjustments(entry modules.HostDBEntry, weight types.Currency) types.Currency {
	usedCollateral := entry.Collateral
	if entry.Collateral.Cmp(minCollateral) < 0 {
		usedCollateral = minCollateral
	}
	for i := 0; i < collateralExponentiation; i++ {
		weight = weight.Mul(usedCollateral)
	}
	return weight
}

// priceAdjustments will adjust the weight of the entry according to the prices
// that it has set.
func priceAdjustments(entry modules.HostDBEntry, weight types.Currency) types.Currency {
	// Sanity checks - the constants values need to have certain relationships
	// to eachother
	if build.DEBUG {
		// If the minDivPrice is not much larger than the divNormalization,
		// there will be problems with granularity after the divNormalization is
		// applied.
		if minDivPrice.Div64(100).Cmp(priceDivNormalization) < 0 {
			build.Critical("Maladjusted minDivePrice and divNormalization constants in hostdb package")
		}
	}

	// Prices tiered as follows:
	//    - the storage price is presented as 'per block per byte'
	//    - the contract price is presented as a flat rate
	//    - the upload bandwidth price is per byte
	//    - the download bandwidth price is per byte
	//
	// The hostdb will naively assume the following for now:
	//    - each contract covers 6 weeks of storage (default is 12 weeks, but
	//      renewals occur at midpoint) - 6048 blocks - and 10GB of storage.
	//    - uploads happen once per 12 weeks (average lifetime of a file is 12 weeks)
	//    - downloads happen once per 6 weeks (files are on average downloaded twice throughout lifetime)
	//
	// In the future, the renter should be able to track average user behavior
	// and adjust accordingly. This flexibility will be added later.
	adjustedContractPrice := entry.ContractPrice.Div64(6048).Div64(10e9) // Adjust contract price to match 10GB for 6 weeks.
	adjustedUploadPrice := entry.UploadBandwidthPrice.Div64(24192)       // Adjust upload price to match a single upload over 24 weeks.
	adjustedDownloadPrice := entry.DownloadBandwidthPrice.Div64(12096)   // Adjust download price to match one download over 12 weeks.
	siafundFee := adjustedContractPrice.Add(adjustedUploadPrice).Add(adjustedDownloadPrice).Add(entry.Collateral).MulTax()
	totalPrice := entry.StoragePrice.Add(adjustedContractPrice).Add(adjustedUploadPrice).Add(adjustedDownloadPrice).Add(siafundFee)

	// Set the divPrice, which is closely related to the totalPrice, but
	// adjusted both to make the math more computationally friendly and also
	// given a hard minimum to prevent certain classes of Sybil attacks -
	// attacks where the attacker tries to esacpe the need to burn coins by
	// setting an extremely low price.
	divPrice := totalPrice
	if divPrice.Cmp(minDivPrice) < 0 {
		divPrice = minDivPrice
	}
	// Shrink the div price so that the math can be a lot less intense. Without
	// this step, the base price would need to be closer to 10e150 as opposed to
	// 10e50.
	divPrice = divPrice.Div(priceDivNormalization)
	for i := 0; i < priceExponentiation; i++ {
		weight = weight.Div(divPrice)
	}
	return weight
}

// storageRemainingAdjustments adjusts the weight of the entry according to how
// much storage it has remaining.
func storageRemainingAdjustments(entry modules.HostDBEntry) float64 {
	base := float64(1)
	if entry.RemainingStorage < 200*requiredStorage {
		base = base / 2 // 2x total penalty
	}
	if entry.RemainingStorage < 100*requiredStorage {
		base = base / 3 // 6x total penalty
	}
	if entry.RemainingStorage < 50*requiredStorage {
		base = base / 4 // 24x total penalty
	}
	if entry.RemainingStorage < 25*requiredStorage {
		base = base / 5 // 95x total penalty
	}
	if entry.RemainingStorage < 10*requiredStorage {
		base = base / 6 // 570x total penalty
	}
	if entry.RemainingStorage < 5*requiredStorage {
		base = base / 10 // 5,700x total penalty
	}
	if entry.RemainingStorage < requiredStorage {
		base = base / 100 // 570,000x total penalty
	}
	return base
}

// versionAdjustments will adjust the weight of the entry according to the siad
// version reported by the host.
func versionAdjustments(entry modules.HostDBEntry) float64 {
	base := float64(1)
	if build.VersionCmp(entry.Version, "1.0.3") < 0 {
		base = base / 5 // 5x total penalty.
	}
	if build.VersionCmp(entry.Version, "1.0.0") < 0 {
		base = base / 20 // 100x total penalty.
	}
	return base
}

// lifetimeAdjustments will adjust the weight of the host according to the total
// amount of time that has passed since the host's original announcement.
func (hdb *HostDB) lifetimeAdjustments(entry modules.HostDBEntry) float64 {
	base := float64(1)
	if hdb.blockHeight >= entry.FirstSeen {
		age := hdb.blockHeight - entry.FirstSeen
		if age < 6000 {
			base = base / 2 // 2x total
		}
		if age < 4000 {
			base = base / 2 // 4x total
		}
		if age < 2000 {
			base = base / 4 // 16x total
		}
		if age < 1000 {
			base = base / 4 // 64x total
		}
		if age < 288 {
			base = base / 2 // 128x total
		}
	} else {
		// Shouldn't happen, but the usecase is covered anyway.
		base = base / 1000 // Because something weird is happening, don't trust this host very much.
		hdb.log.Critical("Hostdb has witnessed a host where the FirstSeen height is higher than the current block height.")
	}
	return base
}

// uptimeAdjustments penalizes the host for having poor uptime, and for being
// offline.
func (hdb *HostDB) uptimeAdjustments(entry modules.HostDBEntry) float64 {
	// Special case: if we have scanned the host twice or fewer, don't perform
	// uptime math.
	if len(entry.ScanHistory) == 0 {
		return 0.001 // Shouldn't happen.
	}
	if len(entry.ScanHistory) == 1 {
		if entry.ScanHistory[0].Success {
			return 0.75
		}
		return 0.25
	}
	if len(entry.ScanHistory) == 2 {
		if entry.ScanHistory[0].Success && entry.ScanHistory[1].Success {
			return 0.85
		}
		if entry.ScanHistory[0].Success || entry.ScanHistory[1].Success {
			return 0.50
		}
		return 0.05
	}

	// Compute the total measured uptime and total measured downtime for this
	// host.
	var uptime time.Duration
	var downtime time.Duration
	recentTime := entry.ScanHistory[0].Timestamp
	recentSuccess := entry.ScanHistory[0].Success
	for _, scan := range entry.ScanHistory[1:] {
		if recentTime.After(scan.Timestamp) {
			hdb.log.Critical("Host entry scan history not sorted.")
		}
		if recentSuccess {
			uptime += scan.Timestamp.Sub(recentTime)
		} else {
			downtime += scan.Timestamp.Sub(recentTime)
		}
		recentTime = scan.Timestamp
		recentSuccess = scan.Success
	}
	// Sanity check against 0 total time.
	if uptime == 0 && downtime == 0 {
		return 0.001 // Shouldn't happen.
	}

	// Calculate the penalty for low uptime.
	uptimePenalty := float64(1)
	uptimeRatio := float64(uptime) / float64(uptime+downtime)
	if uptimeRatio > 0.97 {
		uptimeRatio = 0.97
	}
	uptimeRatio += 0.03
	for i := 0; i < uptimeExponentiation; i++ {
		uptimePenalty *= uptimeRatio
	}

	// Calculate the penalty for downtime across consecutive scans.
	scanLen := len(entry.ScanHistory)
	for i := scanLen - 1; i >= 0; i-- {
		if entry.ScanHistory[i].Success {
			break
		}
		uptimePenalty = uptimePenalty / 2
	}
	return uptimePenalty
}

// calculateHostWeight returns the weight of a host according to the settings of
// the host database entry. Currently, only the price is considered.
func (hdb *HostDB) calculateHostWeight(entry modules.HostDBEntry) types.Currency {
	// Perform the high resolution adjustments.
	weight := baseWeight
	weight = collateralAdjustments(entry, weight)
	weight = priceAdjustments(entry, weight)

	// Perform the lower resolution adjustments.
	storageRemainingPenalty := storageRemainingAdjustments(entry)
	versionPenalty := versionAdjustments(entry)
	lifetimePenalty := hdb.lifetimeAdjustments(entry)
	uptimePenalty := hdb.uptimeAdjustments(entry)

	// Combine the adjustments.
	fullPenalty := storageRemainingPenalty * versionPenalty * lifetimePenalty * uptimePenalty
	weight = weight.MulFloat(fullPenalty)

	if weight.IsZero() {
		// A weight of zero is problematic for for the host tree.
		return types.NewCurrency64(1)
	}
	return weight
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"time"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/labels"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = framework.KubeDescribe("Kibana Logging Instances Is Alive [Feature:Elasticsearch]", func() {
	f := framework.NewDefaultFramework("kibana-logging")

	BeforeEach(func() {
		// TODO: For now assume we are only testing cluster logging with Elasticsearch
		// and Kibana on GCE. Once we are sure that Elasticsearch and Kibana cluster level logging
		// works for other providers we should widen this scope of this test.
		framework.SkipUnlessProviderIs("gce")
	})

	It("should check that the Kibana logging instance is alive", func() {
		ClusterLevelLoggingWithKibana(f)
	})
})

const (
	kibanaKey   = "k8s-app"
	kibanaValue = "kibana-logging"
)

// ClusterLevelLoggingWithKibana is an end to end test that checks to see if Kibana is alive.
func ClusterLevelLoggingWithKibana(f *framework.Framework) {
	// graceTime is how long to keep retrying requests for status information.
	const graceTime = 2 * time.Minute

	// Check for the existence of the Kibana service.
	By("Checking the Kibana service exists.")
	s := f.ClientSet.Core().Services(api.NamespaceSystem)
	// Make a few attempts to connect. This makes the test robust against
	// being run as the first e2e test just after the e2e cluster has been created.
	var err error
	for start := time.Now(); time.Since(start) < graceTime; time.Sleep(5 * time.Second) {
		if _, err = s.Get("kibana-logging"); err == nil {
			break
		}
		framework.Logf("Attempt to check for the existence of the Kibana service failed after %v", time.Since(start))
	}
	Expect(err).NotTo(HaveOccurred())

	// Wait for the Kibana pod(s) to enter the running state.
	By("Checking to make sure the Kibana pods are running")
	label := labels.SelectorFromSet(labels.Set(map[string]string{kibanaKey: kibanaValue}))
	options := api.ListOptions{LabelSelector: label}
	pods, err := f.ClientSet.Core().Pods(api.NamespaceSystem).List(options)
	Expect(err).NotTo(HaveOccurred())
	for _, pod := range pods.Items {
		err = framework.WaitForPodRunningInNamespace(f.ClientSet, &pod)
		Expect(err).NotTo(HaveOccurred())
	}

	By("Checking to make sure we get a response from the Kibana UI.")
	err = nil
	for start := time.Now(); time.Since(start) < graceTime; time.Sleep(5 * time.Second) {
		proxyRequest, errProxy := framework.GetServicesProxyRequest(f.ClientSet, f.ClientSet.Core().RESTClient().Get())
		if errProxy != nil {
			framework.Logf("After %v failed to get services proxy request: %v", time.Since(start), errProxy)
			continue
		}
		// Query against the root URL for Kibana.
		_, err = proxyRequest.Namespace(api.NamespaceSystem).
			Name("kibana-logging").
			DoRaw()
		if err != nil {
			framework.Logf("After %v proxy call to kibana-logging failed: %v", time.Since(start), err)
			continue
		}
		break
	}
	Expect(err).NotTo(HaveOccurred())
}
<commit_msg>Fixed kibana test problem<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"time"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/labels"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = framework.KubeDescribe("Kibana Logging Instances Is Alive [Feature:Elasticsearch]", func() {
	f := framework.NewDefaultFramework("kibana-logging")

	BeforeEach(func() {
		// TODO: For now assume we are only testing cluster logging with Elasticsearch
		// and Kibana on GCE. Once we are sure that Elasticsearch and Kibana cluster level logging
		// works for other providers we should widen this scope of this test.
		framework.SkipUnlessProviderIs("gce")
	})

	It("should check that the Kibana logging instance is alive", func() {
		ClusterLevelLoggingWithKibana(f)
	})
})

const (
	kibanaKey   = "k8s-app"
	kibanaValue = "kibana-logging"
)

// ClusterLevelLoggingWithKibana is an end to end test that checks to see if Kibana is alive.
func ClusterLevelLoggingWithKibana(f *framework.Framework) {
	// graceTime is how long to keep retrying requests for status information.
	const graceTime = 10 * time.Minute

	// Check for the existence of the Kibana service.
	By("Checking the Kibana service exists.")
	s := f.ClientSet.Core().Services(api.NamespaceSystem)
	// Make a few attempts to connect. This makes the test robust against
	// being run as the first e2e test just after the e2e cluster has been created.
	var err error
	for start := time.Now(); time.Since(start) < graceTime; time.Sleep(5 * time.Second) {
		if _, err = s.Get("kibana-logging"); err == nil {
			break
		}
		framework.Logf("Attempt to check for the existence of the Kibana service failed after %v", time.Since(start))
	}
	Expect(err).NotTo(HaveOccurred())

	// Wait for the Kibana pod(s) to enter the running state.
	By("Checking to make sure the Kibana pods are running")
	label := labels.SelectorFromSet(labels.Set(map[string]string{kibanaKey: kibanaValue}))
	options := api.ListOptions{LabelSelector: label}
	pods, err := f.ClientSet.Core().Pods(api.NamespaceSystem).List(options)
	Expect(err).NotTo(HaveOccurred())
	for _, pod := range pods.Items {
		err = framework.WaitForPodRunningInNamespace(f.ClientSet, &pod)
		Expect(err).NotTo(HaveOccurred())
	}

	By("Checking to make sure we get a response from the Kibana UI.")
	err = nil
	for start := time.Now(); time.Since(start) < graceTime; time.Sleep(5 * time.Second) {
		proxyRequest, errProxy := framework.GetServicesProxyRequest(f.ClientSet, f.ClientSet.Core().RESTClient().Get())
		if errProxy != nil {
			framework.Logf("After %v failed to get services proxy request: %v", time.Since(start), errProxy)
			continue
		}
		// Query against the root URL for Kibana.
		_, err = proxyRequest.Namespace(api.NamespaceSystem).
			Name("kibana-logging").
			DoRaw()
		if err != nil {
			framework.Logf("After %v proxy call to kibana-logging failed: %v", time.Since(start), err)
			continue
		}
		break
	}
	Expect(err).NotTo(HaveOccurred())
}
<|endoftext|>


<commit_before>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

// This program hung when run under the C/C++ ThreadSanitizer. TSAN installs a
// libc interceptor that writes signal handlers to a global variable within the
// TSAN runtime instead of making a sigaction system call. A bug in
// syscall.runtime_AfterForkInChild corrupted TSAN's signal forwarding table
// during calls to (*os/exec.Cmd).Run, causing the parent process to fail to
// invoke signal handlers.

import (
	"fmt"
	"os"
	"os/exec"
	"os/signal"
	"syscall"
)

import "C"

func main() {
	ch := make(chan os.Signal)
	signal.Notify(ch, syscall.SIGUSR1)

	if err := exec.Command("true").Run(); err != nil {
		fmt.Fprintf(os.Stderr, "Unexpected error from `true`: %v", err)
		os.Exit(1)
	}

	syscall.Kill(syscall.Getpid(), syscall.SIGUSR1)
	<-ch
}
<commit_msg>[release-branch.go1.18] misc/cgo/testsanitizers: use buffered channel in tsan12.go<commit_after>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

// This program hung when run under the C/C++ ThreadSanitizer. TSAN installs a
// libc interceptor that writes signal handlers to a global variable within the
// TSAN runtime instead of making a sigaction system call. A bug in
// syscall.runtime_AfterForkInChild corrupted TSAN's signal forwarding table
// during calls to (*os/exec.Cmd).Run, causing the parent process to fail to
// invoke signal handlers.

import (
	"fmt"
	"os"
	"os/exec"
	"os/signal"
	"syscall"
)

import "C"

func main() {
	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGUSR1)

	if err := exec.Command("true").Run(); err != nil {
		fmt.Fprintf(os.Stderr, "Unexpected error from `true`: %v", err)
		os.Exit(1)
	}

	syscall.Kill(syscall.Getpid(), syscall.SIGUSR1)
	<-ch
}
<|endoftext|>


<commit_before>// Copyright 2017 Cristian Greco <cristian@regolo.cc>
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package collapser_test

import (
	"fmt"
	"time"

	"github.com/TypeTwo/go-collapser"
)

// Consider a server application accessing a database.
// Running the query execution function within a Collapser will alleviate
// database load by reducing many simultaneous requests to a single access.
func Example_databaseQuery() {

	query := "select count(*) from books"

	dbExec := func(q string) int {
		fmt.Println("Query hit database!")
		time.Sleep(1 * time.Second)
		return 42 // After much thought.
	}

	out := make(chan string)

	c := collapser.NewCollapser()

	for i := 0; i < 3; i++ {
		// Launch a goroutine to execute the query.
		go func(i int) {
			res := c.Do(query, func() interface{} {
				return dbExec(query) // Only one query will hit the database.
			})
			out <- fmt.Sprintf("Query #%d: %d", i, res.Get().(int))
		}(i)
	}

	// Wait for all goroutines to complete.
	fmt.Println(<-out)
	fmt.Println(<-out)
	fmt.Println(<-out)

	// Unordered output:
	// Query hit database!
	// Query #0: 42
	// Query #1: 42
	// Query #2: 42
}

<commit_msg>reduce sleep<commit_after>// Copyright 2017 Cristian Greco <cristian@regolo.cc>
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package collapser_test

import (
	"fmt"
	"time"

	"github.com/TypeTwo/go-collapser"
)

// Consider a server application accessing a database.
// Running the query execution function within a Collapser will alleviate
// database load by reducing many simultaneous requests to a single access.
func Example_databaseQuery() {

	query := "select count(*) from books"

	dbExec := func(q string) int {
		fmt.Println("Query hit database!")
		time.Sleep(100 * time.Millisecond)
		return 42 // After much thought.
	}

	out := make(chan string)

	c := collapser.NewCollapser()

	for i := 0; i < 3; i++ {
		// Launch a goroutine to execute the query.
		go func(i int) {
			res := c.Do(query, func() interface{} {
				return dbExec(query) // Only one query will hit the database.
			})
			out <- fmt.Sprintf("Query #%d: %d", i, res.Get().(int))
		}(i)
	}

	// Wait for all goroutines to complete.
	fmt.Println(<-out)
	fmt.Println(<-out)
	fmt.Println(<-out)

	// Unordered output:
	// Query hit database!
	// Query #0: 42
	// Query #1: 42
	// Query #2: 42
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package approvers

import (
	"encoding/json"
	"math/rand"
	"sort"
	"strings"

	"bytes"
	"fmt"
	"path/filepath"

	"k8s.io/kubernetes/pkg/util/sets"
	"k8s.io/test-infra/mungegithub/features"
	c "k8s.io/test-infra/mungegithub/mungers/matchers/comment"
)

const (
	ownersFileName           = "OWNERS"
	ApprovalNotificationName = "ApprovalNotifier"
)

type RepoInterface interface {
	Approvers(path string) sets.String
	LeafApprovers(path string) sets.String
	FindApproverOwnersForPath(path string) string
}

type RepoAlias struct {
	repo  RepoInterface
	alias features.Aliases
}

func NewRepoAlias(repo RepoInterface, alias features.Aliases) *RepoAlias {
	return &RepoAlias{
		repo:  repo,
		alias: alias,
	}
}

func (r *RepoAlias) Approvers(path string) sets.String {
	return r.alias.Expand(r.repo.Approvers(path))
}

func (r *RepoAlias) LeafApprovers(path string) sets.String {
	return r.alias.Expand(r.repo.LeafApprovers(path))
}
func (r *RepoAlias) FindApproverOwnersForPath(path string) string {
	return r.repo.FindApproverOwnersForPath(path)
}

type Owners struct {
	filenames []string
	repo      RepoInterface
	seed      int64
}

func NewOwners(filenames []string, r RepoInterface, s int64) Owners {
	return Owners{filenames: filenames, repo: r, seed: s}
}

// GetApprovers returns a map from ownersFiles -> people that are approvers in them
func (o Owners) GetApprovers() map[string]sets.String {
	ownersToApprovers := map[string]sets.String{}

	for fn := range o.GetOwnersSet() {
		ownersToApprovers[fn] = o.repo.Approvers(fn)
	}

	return ownersToApprovers
}

// GetLeafApprovers returns a map from ownersFiles -> people that are approvers in them (only the leaf)
func (o Owners) GetLeafApprovers() map[string]sets.String {
	ownersToApprovers := map[string]sets.String{}

	for fn := range o.GetOwnersSet() {
		ownersToApprovers[fn] = o.repo.LeafApprovers(fn)
	}

	return ownersToApprovers
}

// GetAllPotentialApprovers returns the people from relevant owners files needed to get the PR approved
func (o Owners) GetAllPotentialApprovers() []string {
	approversOnly := []string{}
	for _, approverList := range o.GetLeafApprovers() {
		for approver := range approverList {
			approversOnly = append(approversOnly, approver)
		}
	}
	sort.Strings(approversOnly)
	return approversOnly
}

// GetReverseMap returns a map from people -> OWNERS files for which they are an approver
func (o Owners) GetReverseMap() map[string]sets.String {
	approverOwnersfiles := map[string]sets.String{}
	for ownersFile, approvers := range o.GetLeafApprovers() {
		for approver := range approvers {
			if _, ok := approverOwnersfiles[approver]; ok {
				approverOwnersfiles[approver].Insert(ownersFile)
			} else {
				approverOwnersfiles[approver] = sets.NewString(ownersFile)
			}
		}
	}
	return approverOwnersfiles
}

func findMostCoveringApprover(allApprovers []string, reverseMap map[string]sets.String, unapproved sets.String) string {
	maxCovered := 0
	var bestPerson string
	for _, approver := range allApprovers {
		filesCanApprove := reverseMap[approver]
		if filesCanApprove.Intersection(unapproved).Len() > maxCovered {
			maxCovered = len(filesCanApprove)
			bestPerson = approver
		}
	}
	return bestPerson
}

// GetSuggestedApprovers solves the exact cover problem, finding an approver capable of
// approving every OWNERS file in the PR
func (o Owners) GetSuggestedApprovers() sets.String {
	randomizedApprovers := o.GetShuffledApprovers()
	reverseMap := o.GetReverseMap()

	ap := NewApprovers(o)
	for !ap.IsApproved() {
		ap.AddApprover(findMostCoveringApprover(randomizedApprovers, reverseMap, ap.UnapprovedFiles()), "", "")
	}

	return ap.GetCurrentApproversSet()
}

// GetOwnersSet returns a set containing all the Owners files necessary to get the PR approved
func (o Owners) GetOwnersSet() sets.String {
	owners := sets.NewString()
	for _, fn := range o.filenames {
		owners.Insert(o.repo.FindApproverOwnersForPath(fn))
	}
	return removeSubdirs(owners.List())
}

// Shuffles the potential approvers so that we don't always suggest the same people
func (o Owners) GetShuffledApprovers() []string {
	approversList := o.GetAllPotentialApprovers()
	order := rand.New(rand.NewSource(o.seed)).Perm(len(approversList))
	people := make([]string, 0, len(approversList))
	for _, i := range order {
		people = append(people, approversList[i])
	}
	return people
}

// removeSubdirs takes a list of directories as an input and returns a set of directories with all
// subdirectories removed.  E.g. [/a,/a/b/c,/d/e,/d/e/f] -> [/a, /d/e]
func removeSubdirs(dirList []string) sets.String {
	toDel := sets.String{}
	for i := 0; i < len(dirList)-1; i++ {
		for j := i + 1; j < len(dirList); j++ {
			// ex /a/b has prefix /a so if remove /a/b since its already covered
			if strings.HasPrefix(dirList[i], dirList[j]) {
				toDel.Insert(dirList[i])
			} else if strings.HasPrefix(dirList[j], dirList[i]) {
				toDel.Insert(dirList[j])
			}
		}
	}
	finalSet := sets.NewString(dirList...)
	finalSet.Delete(toDel.List()...)
	return finalSet
}

// Approval has the information about each approval on a PR
type Approval struct {
	Login     string // Login of the approver
	How       string // How did the approver approved
	Reference string // Where did the approver approved
}

// String creates a link for the approval. Use `Login` if you just want the name.
func (a Approval) String() string {
	return fmt.Sprintf(
		`*<a href="%s" title="%s">%s</a>*`,
		a.Reference,
		a.How,
		a.Login,
	)
}

type Approvers struct {
	owners    Owners
	approvers map[string]Approval
}

// IntersectSetsCase runs the intersection between to sets.String in a
// case-insensitive way. It returns the name with the case of "one".
func IntersectSetsCase(one, other sets.String) sets.String {
	lower := sets.NewString()
	for item := range other {
		lower.Insert(strings.ToLower(item))
	}

	intersection := sets.NewString()
	for item := range one {
		if lower.Has(strings.ToLower(item)) {
			intersection.Insert(item)
		}
	}
	return intersection
}

// NewApprovers create a new "Approvers" with no approval.
func NewApprovers(owners Owners) Approvers {
	return Approvers{
		owners:    owners,
		approvers: map[string]Approval{},
	}
}

// AddApprover adds a new approval to "Approvers".
func (ap *Approvers) AddApprover(login, how, reference string) {
	ap.approvers[login] = Approval{
		Login:     login,
		How:       how,
		Reference: reference,
	}
}

// RemoveApprover removes an approver from the list.
func (ap *Approvers) RemoveApprover(login string) {
	delete(ap.approvers, login)
}

// GetCurrentApproversSet returns the set of approvers (login only)
func (ap Approvers) GetCurrentApproversSet() sets.String {
	currentApprovers := sets.NewString()

	for approver := range ap.approvers {
		currentApprovers.Insert(approver)
	}

	return currentApprovers
}

// GetFilesApprovers returns a map from files -> list of current approvers.
func (ap Approvers) GetFilesApprovers() map[string]sets.String {
	filesApprovers := map[string]sets.String{}
	currentApprovers := ap.GetCurrentApproversSet()

	for fn, potentialApprovers := range ap.owners.GetApprovers() {
		// The order of parameter matters here:
		// - currentApprovers is the list of github handle that have approved
		// - potentialApprovers is the list of handle in OWNERSa
		// files that can approve each file.
		//
		// We want to keep the syntax of the github handle
		// rather than the potential mis-cased username found in
		// the OWNERS file, that's why it's the first parameter.
		filesApprovers[fn] = IntersectSetsCase(currentApprovers, potentialApprovers)
	}

	return filesApprovers
}

// UnapprovedFiles returns owners files that still need approval
func (ap Approvers) UnapprovedFiles() sets.String {
	unapproved := sets.NewString()
	for fn, approvers := range ap.GetFilesApprovers() {
		if len(approvers) == 0 {
			unapproved.Insert(fn)
		}
	}
	return unapproved
}

// UnapprovedFiles returns owners files that still need approval
func (ap Approvers) GetFiles() []File {
	allOwnersFiles := []File{}
	filesApprovers := ap.GetFilesApprovers()
	for _, fn := range ap.owners.GetOwnersSet().List() {
		if len(filesApprovers[fn]) == 0 {
			allOwnersFiles = append(allOwnersFiles, UnapprovedFile{fn})
		} else {
			allOwnersFiles = append(allOwnersFiles, ApprovedFile{fn, filesApprovers[fn]})
		}
	}

	return allOwnersFiles
}

func (ap Approvers) GetCCs() []string {
	approvers := []string{}

	reverseMap := ap.owners.GetReverseMap()
	unapproved := ap.UnapprovedFiles()

	for _, suggestedApprover := range ap.owners.GetSuggestedApprovers().List() {
		if reverseMap[suggestedApprover].Intersection(unapproved).Len() != 0 {
			approvers = append(approvers, suggestedApprover)
		}
	}

	return approvers
}

// IsApproved returns a bool indicating whether or not the PR is approved
func (ap Approvers) IsApproved() bool {
	return ap.UnapprovedFiles().Len() == 0
}

// ListString returns the list of all approvers along with links and tooltips.
func (ap Approvers) ListString() string {
	approvals := []string{}

	for _, approver := range ap.approvers {
		approvals = append(approvals, approver.String())
	}

	return strings.Join(approvals, ", ")
}

type File interface {
	toString(string, string) string
}

type ApprovedFile struct {
	filepath  string
	approvers sets.String
}

type UnapprovedFile struct {
	filepath string
}

func (a ApprovedFile) toString(org, project string) string {
	fullOwnersPath := filepath.Join(a.filepath, ownersFileName)
	link := fmt.Sprintf("https://github.com/%s/%s/blob/master/%v", org, project, fullOwnersPath)
	return fmt.Sprintf("- ~~[%s](%s)~~ [%v]\n", fullOwnersPath, link, strings.Join(a.approvers.List(), ","))
}

func (ua UnapprovedFile) toString(org, project string) string {
	fullOwnersPath := filepath.Join(ua.filepath, ownersFileName)
	link := fmt.Sprintf("https://github.com/%s/%s/blob/master/%v", org, project, fullOwnersPath)
	return fmt.Sprintf("- **[%s](%s)** \n", fullOwnersPath, link)
}

// getMessage returns the comment body that we want the approval-handler to display on PRs
// The comment shows:
// 	- a list of approvers files (and links) needed to get the PR approved
// 	- a list of approvers files with strikethroughs that already have an approver's approval
// 	- a suggested list of people from each OWNERS files that can fully approve the PR
// 	- how an approver can indicate their approval
// 	- how an approver can cancel their approval
func GetMessage(ap Approvers, org, project string) string {
	formatStr := "The following people have approved this PR: %v\n\nNeeds approval from an approver in each of these OWNERS Files:\n"
	context := bytes.NewBufferString(fmt.Sprintf(formatStr, ap.ListString()))
	for _, ownersFile := range ap.GetFiles() {
		context.WriteString(ownersFile.toString(org, project))
	}

	CCs := ap.GetCCs()
	if len(CCs) != 0 {
		context.WriteString("\nWe suggest the following people:\ncc ")
		for _, person := range CCs {
			context.WriteString("" + person + " ")
		}
	}
	context.WriteString("\n You can indicate your approval by writing `/approve` in a comment\n You can cancel your approval by writing `/approve cancel` in a comment")
	title := "This PR is **NOT APPROVED**"
	if ap.IsApproved() {
		title = "This PR is **APPROVED**"
	}
	context.WriteString(getGubernatorMeta(CCs))
	return (&c.Notification{ApprovalNotificationName, title, context.String()}).String()
}

// gets the meta data gubernator uses for
func getGubernatorMeta(toBeAssigned []string) string {
	forMachine := map[string][]string{"approvers": toBeAssigned}
	bytes, err := json.Marshal(forMachine)
	if err == nil {
		return fmt.Sprintf("\n<!-- META=%s -->", bytes)
	}
	return ""
}
<commit_msg>approval: Fix random test TestStringList<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package approvers

import (
	"encoding/json"
	"math/rand"
	"sort"
	"strings"

	"bytes"
	"fmt"
	"path/filepath"

	"k8s.io/kubernetes/pkg/util/sets"
	"k8s.io/test-infra/mungegithub/features"
	c "k8s.io/test-infra/mungegithub/mungers/matchers/comment"
)

const (
	ownersFileName           = "OWNERS"
	ApprovalNotificationName = "ApprovalNotifier"
)

type RepoInterface interface {
	Approvers(path string) sets.String
	LeafApprovers(path string) sets.String
	FindApproverOwnersForPath(path string) string
}

type RepoAlias struct {
	repo  RepoInterface
	alias features.Aliases
}

func NewRepoAlias(repo RepoInterface, alias features.Aliases) *RepoAlias {
	return &RepoAlias{
		repo:  repo,
		alias: alias,
	}
}

func (r *RepoAlias) Approvers(path string) sets.String {
	return r.alias.Expand(r.repo.Approvers(path))
}

func (r *RepoAlias) LeafApprovers(path string) sets.String {
	return r.alias.Expand(r.repo.LeafApprovers(path))
}
func (r *RepoAlias) FindApproverOwnersForPath(path string) string {
	return r.repo.FindApproverOwnersForPath(path)
}

type Owners struct {
	filenames []string
	repo      RepoInterface
	seed      int64
}

func NewOwners(filenames []string, r RepoInterface, s int64) Owners {
	return Owners{filenames: filenames, repo: r, seed: s}
}

// GetApprovers returns a map from ownersFiles -> people that are approvers in them
func (o Owners) GetApprovers() map[string]sets.String {
	ownersToApprovers := map[string]sets.String{}

	for fn := range o.GetOwnersSet() {
		ownersToApprovers[fn] = o.repo.Approvers(fn)
	}

	return ownersToApprovers
}

// GetLeafApprovers returns a map from ownersFiles -> people that are approvers in them (only the leaf)
func (o Owners) GetLeafApprovers() map[string]sets.String {
	ownersToApprovers := map[string]sets.String{}

	for fn := range o.GetOwnersSet() {
		ownersToApprovers[fn] = o.repo.LeafApprovers(fn)
	}

	return ownersToApprovers
}

// GetAllPotentialApprovers returns the people from relevant owners files needed to get the PR approved
func (o Owners) GetAllPotentialApprovers() []string {
	approversOnly := []string{}
	for _, approverList := range o.GetLeafApprovers() {
		for approver := range approverList {
			approversOnly = append(approversOnly, approver)
		}
	}
	sort.Strings(approversOnly)
	return approversOnly
}

// GetReverseMap returns a map from people -> OWNERS files for which they are an approver
func (o Owners) GetReverseMap() map[string]sets.String {
	approverOwnersfiles := map[string]sets.String{}
	for ownersFile, approvers := range o.GetLeafApprovers() {
		for approver := range approvers {
			if _, ok := approverOwnersfiles[approver]; ok {
				approverOwnersfiles[approver].Insert(ownersFile)
			} else {
				approverOwnersfiles[approver] = sets.NewString(ownersFile)
			}
		}
	}
	return approverOwnersfiles
}

func findMostCoveringApprover(allApprovers []string, reverseMap map[string]sets.String, unapproved sets.String) string {
	maxCovered := 0
	var bestPerson string
	for _, approver := range allApprovers {
		filesCanApprove := reverseMap[approver]
		if filesCanApprove.Intersection(unapproved).Len() > maxCovered {
			maxCovered = len(filesCanApprove)
			bestPerson = approver
		}
	}
	return bestPerson
}

// GetSuggestedApprovers solves the exact cover problem, finding an approver capable of
// approving every OWNERS file in the PR
func (o Owners) GetSuggestedApprovers() sets.String {
	randomizedApprovers := o.GetShuffledApprovers()
	reverseMap := o.GetReverseMap()

	ap := NewApprovers(o)
	for !ap.IsApproved() {
		ap.AddApprover(findMostCoveringApprover(randomizedApprovers, reverseMap, ap.UnapprovedFiles()), "", "")
	}

	return ap.GetCurrentApproversSet()
}

// GetOwnersSet returns a set containing all the Owners files necessary to get the PR approved
func (o Owners) GetOwnersSet() sets.String {
	owners := sets.NewString()
	for _, fn := range o.filenames {
		owners.Insert(o.repo.FindApproverOwnersForPath(fn))
	}
	return removeSubdirs(owners.List())
}

// Shuffles the potential approvers so that we don't always suggest the same people
func (o Owners) GetShuffledApprovers() []string {
	approversList := o.GetAllPotentialApprovers()
	order := rand.New(rand.NewSource(o.seed)).Perm(len(approversList))
	people := make([]string, 0, len(approversList))
	for _, i := range order {
		people = append(people, approversList[i])
	}
	return people
}

// removeSubdirs takes a list of directories as an input and returns a set of directories with all
// subdirectories removed.  E.g. [/a,/a/b/c,/d/e,/d/e/f] -> [/a, /d/e]
func removeSubdirs(dirList []string) sets.String {
	toDel := sets.String{}
	for i := 0; i < len(dirList)-1; i++ {
		for j := i + 1; j < len(dirList); j++ {
			// ex /a/b has prefix /a so if remove /a/b since its already covered
			if strings.HasPrefix(dirList[i], dirList[j]) {
				toDel.Insert(dirList[i])
			} else if strings.HasPrefix(dirList[j], dirList[i]) {
				toDel.Insert(dirList[j])
			}
		}
	}
	finalSet := sets.NewString(dirList...)
	finalSet.Delete(toDel.List()...)
	return finalSet
}

// Approval has the information about each approval on a PR
type Approval struct {
	Login     string // Login of the approver
	How       string // How did the approver approved
	Reference string // Where did the approver approved
}

// String creates a link for the approval. Use `Login` if you just want the name.
func (a Approval) String() string {
	return fmt.Sprintf(
		`*<a href="%s" title="%s">%s</a>*`,
		a.Reference,
		a.How,
		a.Login,
	)
}

type Approvers struct {
	owners    Owners
	approvers map[string]Approval
}

// IntersectSetsCase runs the intersection between to sets.String in a
// case-insensitive way. It returns the name with the case of "one".
func IntersectSetsCase(one, other sets.String) sets.String {
	lower := sets.NewString()
	for item := range other {
		lower.Insert(strings.ToLower(item))
	}

	intersection := sets.NewString()
	for item := range one {
		if lower.Has(strings.ToLower(item)) {
			intersection.Insert(item)
		}
	}
	return intersection
}

// NewApprovers create a new "Approvers" with no approval.
func NewApprovers(owners Owners) Approvers {
	return Approvers{
		owners:    owners,
		approvers: map[string]Approval{},
	}
}

// AddApprover adds a new approval to "Approvers".
func (ap *Approvers) AddApprover(login, how, reference string) {
	ap.approvers[login] = Approval{
		Login:     login,
		How:       how,
		Reference: reference,
	}
}

// RemoveApprover removes an approver from the list.
func (ap *Approvers) RemoveApprover(login string) {
	delete(ap.approvers, login)
}

// GetCurrentApproversSet returns the set of approvers (login only)
func (ap Approvers) GetCurrentApproversSet() sets.String {
	currentApprovers := sets.NewString()

	for approver := range ap.approvers {
		currentApprovers.Insert(approver)
	}

	return currentApprovers
}

// GetFilesApprovers returns a map from files -> list of current approvers.
func (ap Approvers) GetFilesApprovers() map[string]sets.String {
	filesApprovers := map[string]sets.String{}
	currentApprovers := ap.GetCurrentApproversSet()

	for fn, potentialApprovers := range ap.owners.GetApprovers() {
		// The order of parameter matters here:
		// - currentApprovers is the list of github handle that have approved
		// - potentialApprovers is the list of handle in OWNERSa
		// files that can approve each file.
		//
		// We want to keep the syntax of the github handle
		// rather than the potential mis-cased username found in
		// the OWNERS file, that's why it's the first parameter.
		filesApprovers[fn] = IntersectSetsCase(currentApprovers, potentialApprovers)
	}

	return filesApprovers
}

// UnapprovedFiles returns owners files that still need approval
func (ap Approvers) UnapprovedFiles() sets.String {
	unapproved := sets.NewString()
	for fn, approvers := range ap.GetFilesApprovers() {
		if len(approvers) == 0 {
			unapproved.Insert(fn)
		}
	}
	return unapproved
}

// UnapprovedFiles returns owners files that still need approval
func (ap Approvers) GetFiles() []File {
	allOwnersFiles := []File{}
	filesApprovers := ap.GetFilesApprovers()
	for _, fn := range ap.owners.GetOwnersSet().List() {
		if len(filesApprovers[fn]) == 0 {
			allOwnersFiles = append(allOwnersFiles, UnapprovedFile{fn})
		} else {
			allOwnersFiles = append(allOwnersFiles, ApprovedFile{fn, filesApprovers[fn]})
		}
	}

	return allOwnersFiles
}

func (ap Approvers) GetCCs() []string {
	approvers := []string{}

	reverseMap := ap.owners.GetReverseMap()
	unapproved := ap.UnapprovedFiles()

	for _, suggestedApprover := range ap.owners.GetSuggestedApprovers().List() {
		if reverseMap[suggestedApprover].Intersection(unapproved).Len() != 0 {
			approvers = append(approvers, suggestedApprover)
		}
	}

	return approvers
}

// IsApproved returns a bool indicating whether or not the PR is approved
func (ap Approvers) IsApproved() bool {
	return ap.UnapprovedFiles().Len() == 0
}

// ListString returns the list of all approvers along with links and tooltips.
func (ap Approvers) ListString() string {
	approvals := []string{}

	for _, approver := range ap.GetCurrentApproversSet().List() {
		approvals = append(approvals, ap.approvers[approver].String())
	}

	return strings.Join(approvals, ", ")
}

type File interface {
	toString(string, string) string
}

type ApprovedFile struct {
	filepath  string
	approvers sets.String
}

type UnapprovedFile struct {
	filepath string
}

func (a ApprovedFile) toString(org, project string) string {
	fullOwnersPath := filepath.Join(a.filepath, ownersFileName)
	link := fmt.Sprintf("https://github.com/%s/%s/blob/master/%v", org, project, fullOwnersPath)
	return fmt.Sprintf("- ~~[%s](%s)~~ [%v]\n", fullOwnersPath, link, strings.Join(a.approvers.List(), ","))
}

func (ua UnapprovedFile) toString(org, project string) string {
	fullOwnersPath := filepath.Join(ua.filepath, ownersFileName)
	link := fmt.Sprintf("https://github.com/%s/%s/blob/master/%v", org, project, fullOwnersPath)
	return fmt.Sprintf("- **[%s](%s)** \n", fullOwnersPath, link)
}

// getMessage returns the comment body that we want the approval-handler to display on PRs
// The comment shows:
// 	- a list of approvers files (and links) needed to get the PR approved
// 	- a list of approvers files with strikethroughs that already have an approver's approval
// 	- a suggested list of people from each OWNERS files that can fully approve the PR
// 	- how an approver can indicate their approval
// 	- how an approver can cancel their approval
func GetMessage(ap Approvers, org, project string) string {
	formatStr := "The following people have approved this PR: %v\n\nNeeds approval from an approver in each of these OWNERS Files:\n"
	context := bytes.NewBufferString(fmt.Sprintf(formatStr, ap.ListString()))
	for _, ownersFile := range ap.GetFiles() {
		context.WriteString(ownersFile.toString(org, project))
	}

	CCs := ap.GetCCs()
	if len(CCs) != 0 {
		context.WriteString("\nWe suggest the following people:\ncc ")
		for _, person := range CCs {
			context.WriteString("" + person + " ")
		}
	}
	context.WriteString("\n You can indicate your approval by writing `/approve` in a comment\n You can cancel your approval by writing `/approve cancel` in a comment")
	title := "This PR is **NOT APPROVED**"
	if ap.IsApproved() {
		title = "This PR is **APPROVED**"
	}
	context.WriteString(getGubernatorMeta(CCs))
	return (&c.Notification{ApprovalNotificationName, title, context.String()}).String()
}

// gets the meta data gubernator uses for
func getGubernatorMeta(toBeAssigned []string) string {
	forMachine := map[string][]string{"approvers": toBeAssigned}
	bytes, err := json.Marshal(forMachine)
	if err == nil {
		return fmt.Sprintf("\n<!-- META=%s -->", bytes)
	}
	return ""
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"html"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"runtime"
	"strings"

	dapperish "github.com/opentracing/api-golang/examples/dapperish"
	"github.com/opentracing/api-golang/opentracing"
)

func client() {
	reader := bufio.NewReader(os.Stdin)
	for {
		span := opentracing.StartTrace("getInput")
		ctx := opentracing.BackgroundGoContextWithSpan(span)
		// Make sure that global trace tag propagation works.
		span.TraceContext().SetTraceAttribute("User", os.Getenv("USER"))
		span.Info("ctx: ", ctx)
		fmt.Print("\n\nEnter text (empty string to exit): ")
		text, _ := reader.ReadString('\n')
		text = strings.TrimSpace(text)
		if len(text) == 0 {
			fmt.Println("Exiting.")
			os.Exit(0)
		}

		span.Info(text)

		httpClient := &http.Client{}
		httpReq, _ := http.NewRequest("POST", "http://localhost:8080/", bytes.NewReader([]byte(text)))
		opentracing.AddTraceContextToHeader(
			span.TraceContext(), httpReq.Header, opentracing.DefaultTracer())
		resp, err := httpClient.Do(httpReq)
		if err != nil {
			span.Error("error: ", err)
		} else {
			span.Info("got response: ", resp)
		}

		span.Finish()
	}
}

func server() {
	http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		reqCtx, err := opentracing.TraceContextFromHeader(
			req.Header, opentracing.DefaultTracer())
		if err != nil {
			panic(err)
		}

		serverSpan := opentracing.JoinTrace(
			"serverSpan", reqCtx,
		).SetTag("component", "server")
		defer serverSpan.Finish()
		fullBody, err := ioutil.ReadAll(req.Body)
		if err != nil {
			serverSpan.Error("body read error", err)
		}
		serverSpan.Info("got request with body: " + string(fullBody))
		contextIDMap, tagsMap := opentracing.MarshalTraceContextStringMap(reqCtx)
		fmt.Fprintf(
			w,
			"Hello: %v // %v //  %q",
			contextIDMap,
			tagsMap,
			html.EscapeString(req.URL.Path))
	})

	log.Fatal(http.ListenAndServe(":8080", nil))
}

func main() {
	opentracing.InitDefaultTracer(dapperish.NewTracer("dapperish_tester"))

	go server()
	go client()

	runtime.Goexit()
}
<commit_msg>dehack<commit_after>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"html"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"runtime"
	"strings"

	"github.com/opentracing/api-golang/examples/dapperish"
	"github.com/opentracing/api-golang/opentracing"
)

func client() {
	reader := bufio.NewReader(os.Stdin)
	for {
		span := opentracing.StartTrace("getInput")
		ctx := opentracing.BackgroundGoContextWithSpan(span)
		// Make sure that global trace tag propagation works.
		span.TraceContext().SetTraceAttribute("User", os.Getenv("USER"))
		span.Info("ctx: ", ctx)
		fmt.Print("\n\nEnter text (empty string to exit): ")
		text, _ := reader.ReadString('\n')
		text = strings.TrimSpace(text)
		if len(text) == 0 {
			fmt.Println("Exiting.")
			os.Exit(0)
		}

		span.Info(text)

		httpClient := &http.Client{}
		httpReq, _ := http.NewRequest("POST", "http://localhost:8080/", bytes.NewReader([]byte(text)))
		opentracing.AddTraceContextToHeader(
			span.TraceContext(), httpReq.Header, opentracing.DefaultTracer())
		resp, err := httpClient.Do(httpReq)
		if err != nil {
			span.Error("error: ", err)
		} else {
			span.Info("got response: ", resp)
		}

		span.Finish()
	}
}

func server() {
	http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		reqCtx, err := opentracing.TraceContextFromHeader(
			req.Header, opentracing.DefaultTracer())
		if err != nil {
			panic(err)
		}

		serverSpan := opentracing.JoinTrace(
			"serverSpan", reqCtx,
		).SetTag("component", "server")
		defer serverSpan.Finish()
		fullBody, err := ioutil.ReadAll(req.Body)
		if err != nil {
			serverSpan.Error("body read error", err)
		}
		serverSpan.Info("got request with body: " + string(fullBody))
		contextIDMap, tagsMap := opentracing.MarshalTraceContextStringMap(reqCtx)
		fmt.Fprintf(
			w,
			"Hello: %v // %v //  %q",
			contextIDMap,
			tagsMap,
			html.EscapeString(req.URL.Path))
	})

	log.Fatal(http.ListenAndServe(":8080", nil))
}

func main() {
	opentracing.InitDefaultTracer(dapperish.NewTracer("dapperish_tester"))

	go server()
	go client()

	runtime.Goexit()
}
<|endoftext|>


<commit_before>package main

import (
	"reflect"
	"strings"
	"testing"
)

var simpleModule Module = Module{
	Name: "Simple",
	Codograms: []Codogram{
		{
			Name: "First",
			Fields: []Field{
				{Name: "i", Length: 2},
				{Name: "j", Length: 7},
				{Name: "k", Length: 16},
				{Name: "l", Length: 32},
			},
		},
	},
}

func TestParseJsonModule(t *testing.T) {
	m, err := ParseJsonModule("examples/simple_proto.json")
	if err != nil {
		t.Error(err)
	}
	orig := Module{}
	orig.Name = "Simple"
	orig.Codograms = make([]Codogram, 1)
	orig.Codograms[0].Name = "First"
	orig.Codograms[0].Fields = make([]Field, 4)
	orig.Codograms[0].Fields[0].Name = "i"
	orig.Codograms[0].Fields[0].Length = 2
	orig.Codograms[0].Fields[1].Name = "j"
	orig.Codograms[0].Fields[1].Length = 7
	orig.Codograms[0].Fields[2].Name = "k"
	orig.Codograms[0].Fields[2].Length = 16
	orig.Codograms[0].Fields[3].Name = "l"
	orig.Codograms[0].Fields[3].Length = 32

	if !reflect.DeepEqual(m, orig) {
		t.Error("m, orig not equal")
	}
}

func TestGenerateDotH(t *testing.T) {
	m := simpleModule
	err := m.AddCTypes()
	if err != nil {
		t.Error(err)
	}
	g, err := GenerateDotH(&m)
	if err != nil {
		t.Error(err)
	}
	orig := `
typedef struct First {
  uint8_t i; // 2 bits
  uint8_t j; // 7 bits
  uint16_t k; // 16 bits
  uint32_t l; // 32 bits
} First;
`
	replacer := strings.NewReplacer(" ", "",
		"\t", "",
		"\n", "")
	g = replacer.Replace(g)
	orig = replacer.Replace(orig)
	if g != orig {
		t.Errorf("g != orig:\n%s\n%s", g, orig)
	}
}
<commit_msg>Simplify TestParseJsonModule<commit_after>package main

import (
	"reflect"
	"strings"
	"testing"
)

var simpleModule Module = Module{
	Name: "Simple",
	Codograms: []Codogram{
		{
			Name: "First",
			Fields: []Field{
				{Name: "i", Length: 2},
				{Name: "j", Length: 7},
				{Name: "k", Length: 16},
				{Name: "l", Length: 32},
			},
		},
	},
}

func TestParseJsonModule(t *testing.T) {
	m, err := ParseJsonModule("examples/simple_proto.json")
	if err != nil {
		t.Error(err)
	}

	if !reflect.DeepEqual(m, simpleModule) {
		t.Error("m, orig not equal")
	}
}

func TestGenerateDotH(t *testing.T) {
	m := simpleModule
	err := m.AddCTypes()
	if err != nil {
		t.Error(err)
	}
	g, err := GenerateDotH(&m)
	if err != nil {
		t.Error(err)
	}
	orig := `
typedef struct First {
  uint8_t i; // 2 bits
  uint8_t j; // 7 bits
  uint16_t k; // 16 bits
  uint32_t l; // 32 bits
} First;
`
	replacer := strings.NewReplacer(" ", "",
		"\t", "",
		"\n", "")
	g = replacer.Replace(g)
	orig = replacer.Replace(orig)
	if g != orig {
		t.Errorf("g != orig:\n%s\n%s", g, orig)
	}
}
<|endoftext|>


<commit_before>package main

/**
 * a simple client that establishes a connection to the server, and confirms
 * that packets can be injected back to it on a listening UDP port.
 */

import (
	"flag"
	"github.com/google/gopacket"
	"github.com/google/gopacket/layers"
	"github.com/willscott/goturn/client"
	"github.com/willscott/sp3"
	"log"
	"net"
	"net/url"
	"os"
	"os/signal"
)

var server = flag.String("server", "localhost:80", "SP3 Server")

func main() {
	flag.Parse()

	interrupt := make(chan os.Signal, 1)
	signal.Notify(interrupt, os.Interrupt)

	// Learn external IP address.
	stun, err := net.Dial("udp", "stun.l.google.com:19302")
	if err != nil {
		panic(err)
	}
	defer stun.Close()
	stunclient := client.StunClient{Conn: stun}
	myPublicAddress, err := stunclient.Bind()
	if err != nil {
		panic(err)
	}
	udpAddr, err := net.ResolveUDPAddr(myPublicAddress.Network(), myPublicAddress.String())
	if err != nil {
		panic(err)
	}

	base := url.URL{}
	u, _ := base.Parse(*server)
	log.Printf("Connecting to SP3 at: %v", u)

	// Create a connection to the server
	conn, err := sp3.Dial(*u, udpAddr.IP, sp3.DirectAuth{}, nil)
	if err != nil {
		panic(err)
	}
	defer conn.Close()

	// Make a packet.
	buf := gopacket.NewSerializeBuffer()
	opts := gopacket.SerializeOptions{
		ComputeChecksums: true,
	}
	ip := &layers.IPv4{
		Version:  4,
		IHL:      5,
		TTL:      64,
		Protocol: 17,
		SrcIP:    net.IP{8, 8, 8, 8},
		DstIP:    udpAddr.IP,
	}
	udp := &layers.UDP{
		SrcPort: layers.UDPPort(53),
		DstPort: layers.UDPPort(udpAddr.Port),
	}
	udp.SetNetworkLayerForChecksum(ip)
	request := "Hello World!"
	ip.Length = 20 + 8 + uint16(len(request))
	udp.Length = 8 + uint16(len(request))
	payload := gopacket.Payload([]byte(request))
	if err = gopacket.SerializeLayers(buf, opts, ip, udp, payload); err != nil {
		panic(err)
	}

	// Send it.
	_, err = conn.WriteTo(buf.Bytes(), myPublicAddress)
	if err != nil {
		panic(err)
	}

	// Listen for it.
	pkt := make([]byte, 2048)
	n, err := stun.Read(pkt)
	if err != nil {
		panic(err)
	}
	panic("Got spoofed packet: " + string(pkt[0:n]))
}
<commit_msg>bug fixes, senderhello sent<commit_after>package main

/**
 * a simple client that establishes a connection to the server, and confirms
 * that packets can be injected back to it on a listening UDP port.
 */

import (
	"flag"
	"github.com/google/gopacket"
	"github.com/google/gopacket/layers"
	"github.com/willscott/goturn/client"
	"github.com/willscott/sp3"
	"log"
	"net"
	"net/url"
	"os"
	"os/signal"
)

var server = flag.String("server", "localhost:80", "SP3 Server")

func main() {
	flag.Parse()

	interrupt := make(chan os.Signal, 1)
	signal.Notify(interrupt, os.Interrupt)

	// Learn external IP address.
	stun, err := net.Dial("udp", "stun.l.google.com:19302")
	if err != nil {
		panic(err)
	}
	defer stun.Close()
	stunclient := client.StunClient{Conn: stun}
	myPublicAddress, err := stunclient.Bind()
	if err != nil {
		panic(err)
	}
	udpAddr, err := net.ResolveUDPAddr(myPublicAddress.Network(), myPublicAddress.String())
	if err != nil {
		panic(err)
	}

	base := url.URL{}
	u, _ := base.Parse(*server)
	log.Printf("Connecting to SP3 at: %v", u)

	// Create a connection to the server
	conn, err := sp3.Dial(*u, udpAddr.IP, sp3.DirectAuth{}, nil)
	if err != nil {
		panic(err)
	}
	defer conn.Close()

	log.Printf("Connection Established. Sending packet.")

	// Make a packet.
	buf := gopacket.NewSerializeBuffer()
	opts := gopacket.SerializeOptions{
		ComputeChecksums: true,
	}
	ip := &layers.IPv4{
		Version:  4,
		IHL:      5,
		TTL:      64,
		Protocol: 17,
		SrcIP:    net.IP{8, 8, 8, 8},
		DstIP:    udpAddr.IP,
	}
	udp := &layers.UDP{
		SrcPort: layers.UDPPort(53),
		DstPort: layers.UDPPort(udpAddr.Port),
	}
	udp.SetNetworkLayerForChecksum(ip)
	request := "Hello World!"
	ip.Length = 20 + 8 + uint16(len(request))
	udp.Length = 8 + uint16(len(request))
	payload := gopacket.Payload([]byte(request))
	if err = gopacket.SerializeLayers(buf, opts, ip, udp, payload); err != nil {
		panic(err)
	}

	// Send it.
	_, err = conn.WriteTo(buf.Bytes(), myPublicAddress)
	if err != nil {
		panic(err)
	}

	// Listen for it.
	pkt := make([]byte, 2048)
	n, err := stun.Read(pkt)
	if err != nil {
		panic(err)
	}
	panic("Got spoofed packet: " + string(pkt[0:n]))
}
<|endoftext|>


<commit_before>package handler

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"regexp"
	"runtime"
	"strconv"
	"time"

	"github.com/HouzuoGuo/laitos/lalog"
	"github.com/HouzuoGuo/laitos/misc"
	"github.com/HouzuoGuo/laitos/remotevm"
	"github.com/HouzuoGuo/laitos/toolbox"
)

const (
	/*
		DefaultLinuxDistributionURL is the download URL of PuppyLinux, recommended for use with remote virtual machine controls.
		PuppyLinux is lightweight yet functional, it has been thoroughly tested with the remote virtual machine control feature.
	*/
	DefaultLinuxDistributionURL = "http://distro.ibiblio.org/puppylinux/puppy-bionic/bionicpup64/bionicpup64-8.0-uefi.iso"

	// HandleVirtualMachinePage is the web template of the virtual machine remote control.
	HandleVirtualMachinePage = `

    <title>laitos remote virtual machine</title>

<form action="%s" method="post">

%s

		Info:
		<textarea rows="3" cols="80">%s</textarea>

		Virtual machine:
		<input type="submit" name="action" value="Refresh Screen"/>
		<input type="submit" name="action" value="Download OS"/>
		ISO URL:<input type="text" name="iso_url" value="%s"/>
		<input type="submit" name="action" value="Start"/>
		<input type="submit" name="action" value="Kill"/>

		Mouse:
		<input type="submit" name="action" value="LHold"/>
		<input type="submit" name="action" value="LRelease"/>
		<input type="submit" name="action" value="LDouble"/>
		<input type="submit" name="action" value="LClick"/>
		<input type="submit" name="action" value="RHold"/>
		<input type="submit" name="action" value="RRelease"/>
		<input type="submit" name="action" value="RDouble"/>
		<input type="submit" name="action" value="RClick"/>
		<input type="submit" name="action" value="Move To"/> X<input type="text" id="pointer_x" name="pointer_x" value="%d" size="2"/> Y<input type="text" id="pointer_y" name="pointer_y" value="%d" size="2"/>

		Keyboard:
		<input type="submit" name="action" value="Press Simultaneously"/>
		Codes: <input type="text" name="press_keys" value="%s" size="50"/> (e.g. ctrl shift s)

		Useful key codes:
		f1-f12, 1-9, a-z, minus, equal, bracket_left, bracket_right, backslash

		semicolon, apostrophe, comma, dot, slash, esc, backspace, tab, ret, spc

		ctrl, shift, alt, up, down, left, right, home, end, pgup, pgdn, insert, delete

</form>

`
)

// HandleVirtualMachine is an HTTP handler that offers remote virtual machine controls, excluding the screenshot itself.
type HandleVirtualMachine struct {
	LocalUtilityPortNumber    int                             `json:"LocalUtilityPortNumber"`
	ScreenshotEndpoint        string                          `json:"-"`
	ScreenshotHandlerInstance *HandleVirtualMachineScreenshot `json:"-"`
	VM                        *remotevm.VM                    `json:"-"`
	logger                    lalog.Logger
}

// Initialise internal state of the HTTP handler.
func (handler *HandleVirtualMachine) Initialise(logger lalog.Logger, _ *toolbox.CommandProcessor) error {
	handler.logger = logger
	// Calculate the number of CPUs and amount of memory to be granted to virtual machine
	// Give the virtual machine half of the system CPUs
	numCPUs := (runtime.NumCPU() + 1) / 2
	// Give each CPU 384MB of memory, or in total up to 25% of system main memory to work with.
	memSizeMB := numCPUs * 384
	if _, totalKB := misc.GetSystemMemoryUsageKB(); totalKB > 0 {
		if quarterOfMainMB := totalKB / 1024 / 4; quarterOfMainMB > memSizeMB {
			memSizeMB = quarterOfMainMB
		}
	}

	// Create virtual machine instance with adequate amount of RAM and CPU
	handler.VM = &remotevm.VM{
		NumCPU:    numCPUs,
		MemSizeMB: memSizeMB,
		// The TCP port for interacting with emulator comes from user configuration input
		QMPPort: handler.LocalUtilityPortNumber,
	}
	if err := handler.VM.Initialise(); err != nil {
		return err
	}
	// Screenshots are taken from the same VM
	handler.ScreenshotHandlerInstance.VM = handler.VM
	return nil
}

/*
renderRemoteVMPage renders the HTML page that offers virtual machine control.
Virtual machine screenshot sits in a  tag, though the image data is served by a differe, dedicated handler.
*/
func (handler *HandleVirtualMachine) renderRemoteVMPage(requestURL string, err error, isoURL string, pointerX, pointerY int, pressKeys string) []byte {
	var errStr string
	if err != nil {
		errStr = err.Error()
	}
	return []byte(fmt.Sprintf(HandleVirtualMachinePage,
		requestURL, errStr, handler.VM.GetDebugOutput(),
		isoURL,
		pointerX, pointerY,
		pressKeys,
		handler.ScreenshotEndpoint, time.Now().UnixNano()))
}

// parseSubmission reads form action (button) and form text fields input.
func (handler *HandleVirtualMachine) parseSubmission(r *http.Request) (button, isoURL string, pointerX, pointerY int, pressKeys string) {
	button = r.FormValue("action")
	isoURL = r.FormValue("iso_url")
	pointerX, _ = strconv.Atoi(r.FormValue("pointer_x"))
	pointerY, _ = strconv.Atoi(r.FormValue("pointer_y"))
	pressKeys = r.FormValue("press_keys")
	return
}

// getISODownloadLocation returns the file system location where downloaded OS ISO file is kept,.
func (handler *HandleVirtualMachine) getISODownloadLocation() string {
	// Prefer to use user's home directory over temp directory so that it won't be deleted when laitos restarts.
	parentDir, _ := os.UserHomeDir()
	if parentDir == "" {
		parentDir = os.TempDir()
	}
	return path.Join(parentDir, ".laitos-remote-vm-iso-download.iso")
}

// Handle renders HTML page, reads user input from HTML form submission, and carries out corresponding VM control operations.
func (handler *HandleVirtualMachine) Handle(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/html")
	NoCache(w)
	if r.Method == http.MethodGet {
		// Display the web page. Suggest user to download the default Linux distribution.
		_, _ = w.Write(handler.renderRemoteVMPage(r.RequestURI, nil, DefaultLinuxDistributionURL, 0, 0, ""))
	} else if r.Method == http.MethodPost {
		// Handle buttons
		button, isoURL, pointerX, pointerY, pressKeys := handler.parseSubmission(r)
		var actionErr error
		switch button {
		case "Refresh Screen":
			// Simply re-render the page, including the screenshot. No extra action is required.
		case "Download OS":
			go func() {
				_ = handler.VM.DownloadISO(isoURL, handler.getISODownloadLocation())
			}()
			actionErr = errors.New(`Download is in progress, use "Refresh Screen" button to monitor the progress from Info output.`)
		case "Start":
			// Kill the older VM (if it exists) and then start a new VM
			handler.VM.Kill()
			if _, isoErr := os.Stat(handler.getISODownloadLocation()); os.IsNotExist(isoErr) {
				// If an ISO file does not yet exist, download the default Linux distribution.
				actionErr = errors.New(`Downloading Linux distribution, use "Refresh Screen" to monitor the progress from Info output, and then press "Start" again.`)
				go func() {
					_ = handler.VM.DownloadISO(isoURL, handler.getISODownloadLocation())
				}()
			} else {
				actionErr = handler.VM.Start(handler.getISODownloadLocation())
			}
		case "Kill":
			handler.VM.Kill()
		case "LHold":
			actionErr = handler.VM.MoveMouse(pointerX, pointerY)
			if actionErr == nil {
				actionErr = handler.VM.HoldMouse(true, true)
			}
		case "LRelease":
			actionErr = handler.VM.MoveMouse(pointerX, pointerY)
			if actionErr == nil {
				actionErr = handler.VM.HoldMouse(true, false)
			}
		case "LDouble":
			actionErr = handler.VM.MoveMouse(pointerX, pointerY)
			if actionErr == nil {
				actionErr = handler.VM.DoubleClickMouse(true)
			}
		case "LClick":
			actionErr = handler.VM.MoveMouse(pointerX, pointerY)
			if actionErr == nil {
				actionErr = handler.VM.ClickMouse(true)
			}
		case "RHold":
			actionErr = handler.VM.MoveMouse(pointerX, pointerY)
			if actionErr == nil {
				actionErr = handler.VM.HoldMouse(false, true)
			}
		case "RRelease":
			actionErr = handler.VM.MoveMouse(pointerX, pointerY)
			if actionErr == nil {
				actionErr = handler.VM.HoldMouse(false, false)
			}
		case "RDouble":
			actionErr = handler.VM.MoveMouse(pointerX, pointerY)
			if actionErr == nil {
				actionErr = handler.VM.DoubleClickMouse(false)
			}
		case "RClick":
			actionErr = handler.VM.MoveMouse(pointerX, pointerY)
			if actionErr == nil {
				actionErr = handler.VM.ClickMouse(false)
			}
		case "Move To":
			actionErr = handler.VM.MoveMouse(pointerX, pointerY)
		case "Press Simultaneously":
			keys := regexp.MustCompile(`[a-zA-Z0-9_]+`).FindAllString(pressKeys, -1)
			if len(keys) > 0 {
				actionErr = handler.VM.ClickKeyboard(keys...)
			}
		default:
			actionErr = fmt.Errorf("Unknown button action: %s", button)
		}
		_, _ = w.Write(handler.renderRemoteVMPage(r.RequestURI, actionErr, isoURL, pointerX, pointerY, pressKeys))
	}
}

// GetRateLimitFactor returns 3, which is at least 3 actions/second, more than sufficient for a virtual machine operator.
func (_ *HandleVirtualMachine) GetRateLimitFactor() int {
	return 3
}

// SelfTest is not applicable to this HTTP handler.
func (_ *HandleVirtualMachine) SelfTest() error {
	return nil
}

// HandleVirtualMachineScreenshot is an HTTP handler that takes a screenshot of remote virtual machine and serves it in JPEG.
type HandleVirtualMachineScreenshot struct {
	VM *remotevm.VM `json:"-"`
}

// Initialise is not applicable to this HTTP handler, as its internal
func (_ *HandleVirtualMachineScreenshot) Initialise(lalog.Logger, *toolbox.CommandProcessor) error {
	// Initialised by HandleVirtualMachine.Initialise
	return nil
}

// Handle takes a virtual machine screenshot and responds with JPEG image data completed with appropriate HTTP headers.
func (handler *HandleVirtualMachineScreenshot) Handle(w http.ResponseWriter, r *http.Request) {
	NoCache(w)
	// Store screenshot picture in a temporary file
	screenshot, err := ioutil.TempFile("", "laitos-handle-vm-screenshot")
	if err != nil {
		http.Error(w, "Failed to create temporary file: "+err.Error(), http.StatusInternalServerError)
		return
	}
	_ = screenshot.Close()
	defer os.Remove(screenshot.Name())
	if err := handler.VM.TakeScreenshot(screenshot.Name()); err != nil {
		http.Error(w, "Failed to create temporary file: "+err.Error(), http.StatusInternalServerError)
		return
	}
	jpegContent, err := ioutil.ReadFile(screenshot.Name())
	if err != nil {
		http.Error(w, "Failed to read screenshot file: "+err.Error(), http.StatusInternalServerError)
		return
	}
	w.Header().Set("Content-Type", "image/jpeg")
	w.Header().Set("Content-Length", strconv.Itoa(len(jpegContent)))
	_, _ = w.Write(jpegContent)
}

// GetRateLimitFactor returns 3, which is at least 3 screenshots/second, more than sufficient for a virtual machine operator.
func (_ *HandleVirtualMachineScreenshot) GetRateLimitFactor() int {
	return 3
}

// SelfTest is not applicable to this HTTP handler.
func (_ *HandleVirtualMachineScreenshot) SelfTest() error {
	return nil
}
<commit_msg>use PuppyLinux fossa (based on ubuntu 20.04) for the default download URL of remote VM ISO<commit_after>package handler

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"regexp"
	"runtime"
	"strconv"
	"time"

	"github.com/HouzuoGuo/laitos/lalog"
	"github.com/HouzuoGuo/laitos/misc"
	"github.com/HouzuoGuo/laitos/remotevm"
	"github.com/HouzuoGuo/laitos/toolbox"
)

const (
	/*
		DefaultLinuxDistributionURL is the download URL of PuppyLinux, recommended for use with remote virtual machine controls.
		PuppyLinux is lightweight yet functional, it has been thoroughly tested with the remote virtual machine control feature.
	*/
	DefaultLinuxDistributionURL = "http://distro.ibiblio.org/puppylinux/puppy-fossa/fossapup64-9.5.iso"

	// HandleVirtualMachinePage is the web template of the virtual machine remote control.
	HandleVirtualMachinePage = `

    <title>laitos remote virtual machine</title>

<form action="%s" method="post">

%s

		Info:
		<textarea rows="3" cols="80">%s</textarea>

		Virtual machine:
		<input type="submit" name="action" value="Refresh Screen"/>
		<input type="submit" name="action" value="Download OS"/>
		ISO URL:<input type="text" name="iso_url" value="%s"/>
		<input type="submit" name="action" value="Start"/>
		<input type="submit" name="action" value="Kill"/>

		Mouse:
		<input type="submit" name="action" value="LHold"/>
		<input type="submit" name="action" value="LRelease"/>
		<input type="submit" name="action" value="LDouble"/>
		<input type="submit" name="action" value="LClick"/>
		<input type="submit" name="action" value="RHold"/>
		<input type="submit" name="action" value="RRelease"/>
		<input type="submit" name="action" value="RDouble"/>
		<input type="submit" name="action" value="RClick"/>
		<input type="submit" name="action" value="Move To"/> X<input type="text" id="pointer_x" name="pointer_x" value="%d" size="2"/> Y<input type="text" id="pointer_y" name="pointer_y" value="%d" size="2"/>

		Keyboard:
		<input type="submit" name="action" value="Press Simultaneously"/>
		Codes: <input type="text" name="press_keys" value="%s" size="50"/> (e.g. ctrl shift s)

		Useful key codes:
		f1-f12, 1-9, a-z, minus, equal, bracket_left, bracket_right, backslash

		semicolon, apostrophe, comma, dot, slash, esc, backspace, tab, ret, spc

		ctrl, shift, alt, up, down, left, right, home, end, pgup, pgdn, insert, delete

</form>

`
)

// HandleVirtualMachine is an HTTP handler that offers remote virtual machine controls, excluding the screenshot itself.
type HandleVirtualMachine struct {
	LocalUtilityPortNumber    int                             `json:"LocalUtilityPortNumber"`
	ScreenshotEndpoint        string                          `json:"-"`
	ScreenshotHandlerInstance *HandleVirtualMachineScreenshot `json:"-"`
	VM                        *remotevm.VM                    `json:"-"`
	logger                    lalog.Logger
}

// Initialise internal state of the HTTP handler.
func (handler *HandleVirtualMachine) Initialise(logger lalog.Logger, _ *toolbox.CommandProcessor) error {
	handler.logger = logger
	// Calculate the number of CPUs and amount of memory to be granted to virtual machine
	// Give the virtual machine half of the system CPUs
	numCPUs := (runtime.NumCPU() + 1) / 2
	// Give each CPU 384MB of memory, or in total up to 25% of system main memory to work with.
	memSizeMB := numCPUs * 384
	if _, totalKB := misc.GetSystemMemoryUsageKB(); totalKB > 0 {
		if quarterOfMainMB := totalKB / 1024 / 4; quarterOfMainMB > memSizeMB {
			memSizeMB = quarterOfMainMB
		}
	}

	// Create virtual machine instance with adequate amount of RAM and CPU
	handler.VM = &remotevm.VM{
		NumCPU:    numCPUs,
		MemSizeMB: memSizeMB,
		// The TCP port for interacting with emulator comes from user configuration input
		QMPPort: handler.LocalUtilityPortNumber,
	}
	if err := handler.VM.Initialise(); err != nil {
		return err
	}
	// Screenshots are taken from the same VM
	handler.ScreenshotHandlerInstance.VM = handler.VM
	return nil
}

/*
renderRemoteVMPage renders the HTML page that offers virtual machine control.
Virtual machine screenshot sits in a  tag, though the image data is served by a differe, dedicated handler.
*/
func (handler *HandleVirtualMachine) renderRemoteVMPage(requestURL string, err error, isoURL string, pointerX, pointerY int, pressKeys string) []byte {
	var errStr string
	if err != nil {
		errStr = err.Error()
	}
	return []byte(fmt.Sprintf(HandleVirtualMachinePage,
		requestURL, errStr, handler.VM.GetDebugOutput(),
		isoURL,
		pointerX, pointerY,
		pressKeys,
		handler.ScreenshotEndpoint, time.Now().UnixNano()))
}

// parseSubmission reads form action (button) and form text fields input.
func (handler *HandleVirtualMachine) parseSubmission(r *http.Request) (button, isoURL string, pointerX, pointerY int, pressKeys string) {
	button = r.FormValue("action")
	isoURL = r.FormValue("iso_url")
	pointerX, _ = strconv.Atoi(r.FormValue("pointer_x"))
	pointerY, _ = strconv.Atoi(r.FormValue("pointer_y"))
	pressKeys = r.FormValue("press_keys")
	return
}

// getISODownloadLocation returns the file system location where downloaded OS ISO file is kept,.
func (handler *HandleVirtualMachine) getISODownloadLocation() string {
	// Prefer to use user's home directory over temp directory so that it won't be deleted when laitos restarts.
	parentDir, _ := os.UserHomeDir()
	if parentDir == "" {
		parentDir = os.TempDir()
	}
	return path.Join(parentDir, ".laitos-remote-vm-iso-download.iso")
}

// Handle renders HTML page, reads user input from HTML form submission, and carries out corresponding VM control operations.
func (handler *HandleVirtualMachine) Handle(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/html")
	NoCache(w)
	if r.Method == http.MethodGet {
		// Display the web page. Suggest user to download the default Linux distribution.
		_, _ = w.Write(handler.renderRemoteVMPage(r.RequestURI, nil, DefaultLinuxDistributionURL, 0, 0, ""))
	} else if r.Method == http.MethodPost {
		// Handle buttons
		button, isoURL, pointerX, pointerY, pressKeys := handler.parseSubmission(r)
		var actionErr error
		switch button {
		case "Refresh Screen":
			// Simply re-render the page, including the screenshot. No extra action is required.
		case "Download OS":
			go func() {
				_ = handler.VM.DownloadISO(isoURL, handler.getISODownloadLocation())
			}()
			actionErr = errors.New(`Download is in progress, use "Refresh Screen" button to monitor the progress from Info output.`)
		case "Start":
			// Kill the older VM (if it exists) and then start a new VM
			handler.VM.Kill()
			if _, isoErr := os.Stat(handler.getISODownloadLocation()); os.IsNotExist(isoErr) {
				// If an ISO file does not yet exist, download the default Linux distribution.
				actionErr = errors.New(`Downloading Linux distribution, use "Refresh Screen" to monitor the progress from Info output, and then press "Start" again.`)
				go func() {
					_ = handler.VM.DownloadISO(isoURL, handler.getISODownloadLocation())
				}()
			} else {
				actionErr = handler.VM.Start(handler.getISODownloadLocation())
			}
		case "Kill":
			handler.VM.Kill()
		case "LHold":
			actionErr = handler.VM.MoveMouse(pointerX, pointerY)
			if actionErr == nil {
				actionErr = handler.VM.HoldMouse(true, true)
			}
		case "LRelease":
			actionErr = handler.VM.MoveMouse(pointerX, pointerY)
			if actionErr == nil {
				actionErr = handler.VM.HoldMouse(true, false)
			}
		case "LDouble":
			actionErr = handler.VM.MoveMouse(pointerX, pointerY)
			if actionErr == nil {
				actionErr = handler.VM.DoubleClickMouse(true)
			}
		case "LClick":
			actionErr = handler.VM.MoveMouse(pointerX, pointerY)
			if actionErr == nil {
				actionErr = handler.VM.ClickMouse(true)
			}
		case "RHold":
			actionErr = handler.VM.MoveMouse(pointerX, pointerY)
			if actionErr == nil {
				actionErr = handler.VM.HoldMouse(false, true)
			}
		case "RRelease":
			actionErr = handler.VM.MoveMouse(pointerX, pointerY)
			if actionErr == nil {
				actionErr = handler.VM.HoldMouse(false, false)
			}
		case "RDouble":
			actionErr = handler.VM.MoveMouse(pointerX, pointerY)
			if actionErr == nil {
				actionErr = handler.VM.DoubleClickMouse(false)
			}
		case "RClick":
			actionErr = handler.VM.MoveMouse(pointerX, pointerY)
			if actionErr == nil {
				actionErr = handler.VM.ClickMouse(false)
			}
		case "Move To":
			actionErr = handler.VM.MoveMouse(pointerX, pointerY)
		case "Press Simultaneously":
			keys := regexp.MustCompile(`[a-zA-Z0-9_]+`).FindAllString(pressKeys, -1)
			if len(keys) > 0 {
				actionErr = handler.VM.ClickKeyboard(keys...)
			}
		default:
			actionErr = fmt.Errorf("Unknown button action: %s", button)
		}
		_, _ = w.Write(handler.renderRemoteVMPage(r.RequestURI, actionErr, isoURL, pointerX, pointerY, pressKeys))
	}
}

// GetRateLimitFactor returns 3, which is at least 3 actions/second, more than sufficient for a virtual machine operator.
func (_ *HandleVirtualMachine) GetRateLimitFactor() int {
	return 3
}

// SelfTest is not applicable to this HTTP handler.
func (_ *HandleVirtualMachine) SelfTest() error {
	return nil
}

// HandleVirtualMachineScreenshot is an HTTP handler that takes a screenshot of remote virtual machine and serves it in JPEG.
type HandleVirtualMachineScreenshot struct {
	VM *remotevm.VM `json:"-"`
}

// Initialise is not applicable to this HTTP handler, as its internal
func (_ *HandleVirtualMachineScreenshot) Initialise(lalog.Logger, *toolbox.CommandProcessor) error {
	// Initialised by HandleVirtualMachine.Initialise
	return nil
}

// Handle takes a virtual machine screenshot and responds with JPEG image data completed with appropriate HTTP headers.
func (handler *HandleVirtualMachineScreenshot) Handle(w http.ResponseWriter, r *http.Request) {
	NoCache(w)
	// Store screenshot picture in a temporary file
	screenshot, err := ioutil.TempFile("", "laitos-handle-vm-screenshot")
	if err != nil {
		http.Error(w, "Failed to create temporary file: "+err.Error(), http.StatusInternalServerError)
		return
	}
	_ = screenshot.Close()
	defer os.Remove(screenshot.Name())
	if err := handler.VM.TakeScreenshot(screenshot.Name()); err != nil {
		http.Error(w, "Failed to create temporary file: "+err.Error(), http.StatusInternalServerError)
		return
	}
	jpegContent, err := ioutil.ReadFile(screenshot.Name())
	if err != nil {
		http.Error(w, "Failed to read screenshot file: "+err.Error(), http.StatusInternalServerError)
		return
	}
	w.Header().Set("Content-Type", "image/jpeg")
	w.Header().Set("Content-Length", strconv.Itoa(len(jpegContent)))
	_, _ = w.Write(jpegContent)
}

// GetRateLimitFactor returns 3, which is at least 3 screenshots/second, more than sufficient for a virtual machine operator.
func (_ *HandleVirtualMachineScreenshot) GetRateLimitFactor() int {
	return 3
}

// SelfTest is not applicable to this HTTP handler.
func (_ *HandleVirtualMachineScreenshot) SelfTest() error {
	return nil
}
<|endoftext|>