<commit_before>package main

import (
	"fmt"
	"time"

	"github.com/gbl08ma/disturbancesmlx/dataobjects"
	"github.com/heetch/sqalx"
)

// ComputeTypicalSeconds calculates and updates the TypicalSeconds
// for all the Connections and Transfers where that can be done using the registered Trips
// Current TypicalSeconds are ignored and discarded.
func ComputeTypicalSeconds(node sqalx.Node) error {
	tx, err := node.Beginx()
	if err != nil {
		return err
	}
	defer tx.Rollback()

	tripIDs, err := dataobjects.GetTripIDs(tx)
	if err != nil {
		return err
	}

	fmt.Printf("%d trip IDs\n", len(tripIDs))

	type TransferKey struct {
		Station string
		From    string
		To      string
	}

	transferCache := make(map[TransferKey]*dataobjects.Transfer)
	getTransfer := func(station, from, to string) (*dataobjects.Transfer, error) {
		transfer, cached := transferCache[TransferKey{station, from, to}]
		if !cached {
			var err error
			transfer, err = dataobjects.GetTransfer(tx, station, from, to)
			if err != nil {
				return nil, err
			}
			transferCache[TransferKey{station, from, to}] = transfer
		}
		return transfer, nil
	}
	// we can use pointers as keys in the following maps because getTransfer and transferCache
	// make sure only one instance of each transfer is created throughout the execution of this function
	transferAvgNumerator := make(map[*dataobjects.Transfer]float64)
	transferAvgDenominator := make(map[*dataobjects.Transfer]float64)

	type ConnectionKey struct {
		From string
		To   string
	}

	connectionCache := make(map[ConnectionKey]*dataobjects.Connection)
	getConnection := func(from, to string) (*dataobjects.Connection, error) {
		connection, cached := connectionCache[ConnectionKey{from, to}]
		if !cached {
			var err error
			connection, err = dataobjects.GetConnection(tx, from, to)
			if err != nil {
				return nil, err
			}
			connectionCache[ConnectionKey{from, to}] = connection
		}
		return connection, nil
	}
	// we can use pointers as keys in the following maps because getConnection and connectionCache
	// make sure only one instance of each connection is created throughout the execution of this function
	connectionAvgNumerator := make(map[*dataobjects.Connection]float64)
	connectionAvgDenominator := make(map[*dataobjects.Connection]float64)
	connectionStopAvgNumerator := make(map[*dataobjects.Connection]float64)
	connectionStopAvgDenominator := make(map[*dataobjects.Connection]float64)
	connectionWaitAvgNumerator := make(map[*dataobjects.Connection]float64)
	connectionWaitAvgDenominator := make(map[*dataobjects.Connection]float64)

	processTransfer := func(transfer *dataobjects.Transfer, use *dataobjects.StationUse) error {
		seconds := use.LeaveTime.Sub(use.EntryTime).Seconds()
		transferAvgNumerator[transfer] += seconds - 20
		transferAvgDenominator[transfer]++
		return nil
	}

	processConnection := func(connection *dataobjects.Connection, sourceUse *dataobjects.StationUse, targetUse *dataobjects.StationUse) error {
		seconds := targetUse.EntryTime.Sub(sourceUse.LeaveTime).Seconds()
		connectionAvgNumerator[connection] += seconds + 20
		connectionAvgDenominator[connection]++

		waitSeconds := sourceUse.LeaveTime.Sub(sourceUse.EntryTime).Seconds()
		if sourceUse.Type == dataobjects.NetworkEntry {
			connectionWaitAvgNumerator[connection] += waitSeconds - 20
			connectionWaitAvgDenominator[connection]++
		} else if sourceUse.Type == dataobjects.GoneThrough && waitSeconds < 60*3 {
			connectionStopAvgNumerator[connection] += waitSeconds - 20
			connectionStopAvgDenominator[connection]++
		}
		return nil
	}

	processTrip := func(trip *dataobjects.Trip) error {
		if len(trip.StationUses) <= 1 {
			// station visit or invalid trip
			// can't extract any data about connections
			return nil
		}

		for useIdx := 0; useIdx < len(trip.StationUses)-1; useIdx++ {
			sourceUse := trip.StationUses[useIdx]

			if sourceUse.Manual {
				// manual path extensions don't contain valid time data
				// skip
				continue
			}

			// if this is a transfer, process it
			if sourceUse.Type == dataobjects.Interchange {
				transfer, err := getTransfer(sourceUse.Station.ID, sourceUse.SourceLine.ID, sourceUse.TargetLine.ID)
				if err != nil {
					// transfer might no longer exist (closed stations, etc.)
					// move on
					fmt.Printf("Transfer on %s from %s to %s skipped\n", sourceUse.Station.ID, sourceUse.SourceLine.ID, sourceUse.TargetLine.ID)
					return nil
				}

				if err = processTransfer(transfer, sourceUse); err != nil {
					return err
				}
			}

			targetUse := trip.StationUses[useIdx+1]

			if targetUse.Manual {
				// manual path extensions don't contain valid time data
				// skip
				continue
			}

			connection, err := getConnection(sourceUse.Station.ID, targetUse.Station.ID)
			if err != nil {
				// connection might no longer exist (closed stations, etc.)
				// move on
				fmt.Printf("Connection from %s to %s skipped\n", sourceUse.Station.ID, targetUse.Station.ID)
				continue
			}
			if useIdx+2 < len(trip.StationUses) && trip.StationUses[useIdx+2].EntryTime.Sub(targetUse.EntryTime) < 1*time.Second {
				// this station use is certainly a forced extension to make up for a station the client did not capture correct times for
				// skip
				continue
			}

			if err = processConnection(connection, sourceUse, targetUse); err != nil {
				return err
			}
		}
		return nil
	}

	// instantiate each trip from DB individually
	// (instead of using dataobjects.GetTrips)
	// to reduce memory usage
	for _, tripID := range tripIDs {
		trip, err := dataobjects.GetTrip(tx, tripID)
		if err != nil {
			return err
		}

		if err = processTrip(trip); err != nil {
			return err
		}
	}

	for transfer, denominator := range transferAvgDenominator {
		if denominator < 2 {
			// data is not significant enough
			continue
		}
		average := transferAvgNumerator[transfer] / denominator
		// TODO: add math.Round to int cast once Go 1.10 is released
		transfer.TypicalSeconds = int(average)
		fmt.Printf("Updating transfer from %s to %s with %d (%f)\n", transfer.From.ID, transfer.To.ID, transfer.TypicalSeconds, denominator)
		err := transfer.Update(tx)
		if err != nil {
			return err
		}
	}

	for connection, denominator := range connectionAvgDenominator {
		if denominator < 2 {
			// data is not significant enough
			continue
		}
		average := connectionAvgNumerator[connection] / denominator
		// TODO: add math.Round to int cast once Go 1.10 is released
		connection.TypicalSeconds = int(average)
		fmt.Printf("Updating connection from %s to %s with %d (%f)\n", connection.From.ID, connection.To.ID, connection.TypicalSeconds, denominator)
		err := connection.Update(tx)
		if err != nil {
			return err
		}
	}

	for connection, denominator := range connectionStopAvgDenominator {
		if denominator < 2 {
			// data is not significant enough
			continue
		}
		average := connectionStopAvgNumerator[connection] / denominator
		// TODO: add math.Round to int cast once Go 1.10 is released
		connection.TypicalStopSeconds = int(average)
		fmt.Printf("Updating connection from %s to %s with stop %d (%f)\n", connection.From.ID, connection.To.ID, connection.TypicalStopSeconds, denominator)
		err := connection.Update(tx)
		if err != nil {
			return err
		}
	}

	for connection, denominator := range connectionWaitAvgDenominator {
		if denominator < 2 {
			// data is not significant enough
			continue
		}
		average := connectionWaitAvgNumerator[connection] / denominator
		// TODO: add math.Round to int cast once Go 1.10 is released
		connection.TypicalWaitingSeconds = int(average)
		fmt.Printf("Updating connection from %s to %s with wait %d (%f)\n", connection.From.ID, connection.To.ID, connection.TypicalWaitingSeconds, denominator)
		err := connection.Update(tx)
		if err != nil {
			return err
		}
	}

	return tx.Commit()
}

// ComputeAverageSpeed returns the average service speed in km/h
// based on the trips in the specified time range
func ComputeAverageSpeed(node sqalx.Node, fromTime time.Time, toTime time.Time) (float64, error) {
	tx, err := node.Beginx()
	if err != nil {
		return 0, err
	}
	defer tx.Commit() // read-only tx

	tripIDs, err := dataobjects.GetTripIDsBetween(tx, fromTime, toTime)
	if err != nil {
		return 0, err
	}

	if len(tripIDs) == 0 {
		return 0, nil
	}

	type TransferKey struct {
		Station string
		From    string
		To      string
	}

	type ConnectionKey struct {
		From string
		To   string
	}

	connectionCache := make(map[ConnectionKey]*dataobjects.Connection)
	getConnection := func(from, to string) (*dataobjects.Connection, error) {
		connection, cached := connectionCache[ConnectionKey{from, to}]
		if !cached {
			var err error
			connection, err = dataobjects.GetConnection(tx, from, to)
			if err != nil {
				return nil, err
			}
			connectionCache[ConnectionKey{from, to}] = connection
		}
		return connection, nil
	}

	var totalTime time.Duration
	var totalDistance int64

	processTrip := func(trip *dataobjects.Trip) error {
		if len(trip.StationUses) <= 1 {
			// station visit or invalid trip
			// can't extract any data about connections
			return nil
		}

		var startTime, endTime time.Time
		for useIdx := 0; useIdx < len(trip.StationUses)-1; useIdx++ {
			sourceUse := trip.StationUses[useIdx]

			if sourceUse.Manual {
				// manual path extensions don't contain valid time data
				// skip
				continue
			}

			if sourceUse.Type == dataobjects.Interchange ||
				sourceUse.Type == dataobjects.Visit {
				continue
			}

			targetUse := trip.StationUses[useIdx+1]

			if targetUse.Manual {
				// manual path extensions don't contain valid time data
				// skip
				continue
			}

			connection, err := getConnection(sourceUse.Station.ID, targetUse.Station.ID)
			if err != nil {
				// connection might no longer exist (closed stations, etc.)
				// move on
				fmt.Printf("Connection from %s to %s skipped\n", sourceUse.Station.ID, targetUse.Station.ID)
				return nil
			}

			totalDistance += int64(connection.WorldLength)
			if startTime.IsZero() {
				startTime = sourceUse.LeaveTime
			}
			endTime = targetUse.EntryTime
		}
		totalTime += endTime.Sub(startTime)
		return nil
	}

	// instantiate each trip from DB individually
	// (instead of using dataobjects.GetTrips)
	// to reduce memory usage
	for _, tripID := range tripIDs {
		trip, err := dataobjects.GetTrip(tx, tripID)
		if err != nil {
			return 0, err
		}

		if err = processTrip(trip); err != nil {
			return 0, err
		}
	}

	km := float64(totalDistance) / 1000
	hours := totalTime.Hours()

	return km / hours, nil
}

// ComputeAverageSpeedCached returns the average service speed in km/h
// based on the trips in the specified time range, with cache in front
// to minimize computational cost
type avgSpeedCacheKey struct {
	From int64
	To   int64
}

var avgSpeedCache map[avgSpeedCacheKey]float64

func ComputeAverageSpeedCached(node sqalx.Node, fromTime time.Time, toTime time.Time) (float64, error) {
	if val, ok := avgSpeedCache[avgSpeedCacheKey{fromTime.Unix(), toTime.Unix()}]; ok {
		return val, nil
	}
	val, err := ComputeAverageSpeed(node, fromTime, toTime)
	if err != nil {
		return val, err
	}
	avgSpeedCache[avgSpeedCacheKey{fromTime.Unix(), toTime.Unix()}] = val
	return val, nil
}

func init() {
	avgSpeedCache = make(map[avgSpeedCacheKey]float64)
}
<commit_msg>Fix path finding with a work-around<commit_after>package main

import (
	"fmt"
	"time"

	"github.com/gbl08ma/disturbancesmlx/dataobjects"
	"github.com/heetch/sqalx"
)

// ComputeTypicalSeconds calculates and updates the TypicalSeconds
// for all the Connections and Transfers where that can be done using the registered Trips
// Current TypicalSeconds are ignored and discarded.
func ComputeTypicalSeconds(node sqalx.Node) error {
	tx, err := node.Beginx()
	if err != nil {
		return err
	}
	defer tx.Rollback()

	tripIDs, err := dataobjects.GetTripIDs(tx)
	if err != nil {
		return err
	}

	fmt.Printf("%d trip IDs\n", len(tripIDs))

	type TransferKey struct {
		Station string
		From    string
		To      string
	}

	transferCache := make(map[TransferKey]*dataobjects.Transfer)
	getTransfer := func(station, from, to string) (*dataobjects.Transfer, error) {
		transfer, cached := transferCache[TransferKey{station, from, to}]
		if !cached {
			var err error
			transfer, err = dataobjects.GetTransfer(tx, station, from, to)
			if err != nil {
				return nil, err
			}
			transferCache[TransferKey{station, from, to}] = transfer
		}
		return transfer, nil
	}
	// we can use pointers as keys in the following maps because getTransfer and transferCache
	// make sure only one instance of each transfer is created throughout the execution of this function
	transferAvgNumerator := make(map[*dataobjects.Transfer]float64)
	transferAvgDenominator := make(map[*dataobjects.Transfer]float64)

	type ConnectionKey struct {
		From string
		To   string
	}

	connectionCache := make(map[ConnectionKey]*dataobjects.Connection)
	getConnection := func(from, to string) (*dataobjects.Connection, error) {
		connection, cached := connectionCache[ConnectionKey{from, to}]
		if !cached {
			var err error
			connection, err = dataobjects.GetConnection(tx, from, to)
			if err != nil {
				return nil, err
			}
			connectionCache[ConnectionKey{from, to}] = connection
		}
		return connection, nil
	}
	// we can use pointers as keys in the following maps because getConnection and connectionCache
	// make sure only one instance of each connection is created throughout the execution of this function
	connectionAvgNumerator := make(map[*dataobjects.Connection]float64)
	connectionAvgDenominator := make(map[*dataobjects.Connection]float64)
	connectionStopAvgNumerator := make(map[*dataobjects.Connection]float64)
	connectionStopAvgDenominator := make(map[*dataobjects.Connection]float64)
	connectionWaitAvgNumerator := make(map[*dataobjects.Connection]float64)
	connectionWaitAvgDenominator := make(map[*dataobjects.Connection]float64)

	processTransfer := func(transfer *dataobjects.Transfer, use *dataobjects.StationUse) error {
		seconds := use.LeaveTime.Sub(use.EntryTime).Seconds()
		transferAvgNumerator[transfer] += seconds - 20
		transferAvgDenominator[transfer]++
		return nil
	}

	processConnection := func(connection *dataobjects.Connection, sourceUse *dataobjects.StationUse, targetUse *dataobjects.StationUse) error {
		seconds := targetUse.EntryTime.Sub(sourceUse.LeaveTime).Seconds()
		connectionAvgNumerator[connection] += seconds + 20
		connectionAvgDenominator[connection]++

		waitSeconds := sourceUse.LeaveTime.Sub(sourceUse.EntryTime).Seconds()
		if sourceUse.Type == dataobjects.NetworkEntry && waitSeconds < 60*3 {
			connectionWaitAvgNumerator[connection] += waitSeconds - 20
			connectionWaitAvgDenominator[connection]++
		} else if sourceUse.Type == dataobjects.GoneThrough && waitSeconds < 60*3 {
			connectionStopAvgNumerator[connection] += waitSeconds - 20
			connectionStopAvgDenominator[connection]++
		}
		return nil
	}

	processTrip := func(trip *dataobjects.Trip) error {
		if len(trip.StationUses) <= 1 {
			// station visit or invalid trip
			// can't extract any data about connections
			return nil
		}

		for useIdx := 0; useIdx < len(trip.StationUses)-1; useIdx++ {
			sourceUse := trip.StationUses[useIdx]

			if sourceUse.Manual {
				// manual path extensions don't contain valid time data
				// skip
				continue
			}

			// if this is a transfer, process it
			if sourceUse.Type == dataobjects.Interchange {
				transfer, err := getTransfer(sourceUse.Station.ID, sourceUse.SourceLine.ID, sourceUse.TargetLine.ID)
				if err != nil {
					// transfer might no longer exist (closed stations, etc.)
					// move on
					fmt.Printf("Transfer on %s from %s to %s skipped\n", sourceUse.Station.ID, sourceUse.SourceLine.ID, sourceUse.TargetLine.ID)
					return nil
				}

				if err = processTransfer(transfer, sourceUse); err != nil {
					return err
				}
			}

			targetUse := trip.StationUses[useIdx+1]

			if targetUse.Manual {
				// manual path extensions don't contain valid time data
				// skip
				continue
			}

			connection, err := getConnection(sourceUse.Station.ID, targetUse.Station.ID)
			if err != nil {
				// connection might no longer exist (closed stations, etc.)
				// move on
				fmt.Printf("Connection from %s to %s skipped\n", sourceUse.Station.ID, targetUse.Station.ID)
				continue
			}
			if useIdx+2 < len(trip.StationUses) && trip.StationUses[useIdx+2].EntryTime.Sub(targetUse.EntryTime) < 1*time.Second {
				// this station use is certainly a forced extension to make up for a station the client did not capture correct times for
				// skip
				continue
			}

			if err = processConnection(connection, sourceUse, targetUse); err != nil {
				return err
			}
		}
		return nil
	}

	// instantiate each trip from DB individually
	// (instead of using dataobjects.GetTrips)
	// to reduce memory usage
	for _, tripID := range tripIDs {
		trip, err := dataobjects.GetTrip(tx, tripID)
		if err != nil {
			return err
		}

		if err = processTrip(trip); err != nil {
			return err
		}
	}

	for connection, denominator := range connectionAvgDenominator {
		if denominator < 2 {
			// data is not significant enough
			continue
		}
		average := connectionAvgNumerator[connection] / denominator
		// TODO: add math.Round to int cast once Go 1.10 is released
		connection.TypicalSeconds = int(average)
		fmt.Printf("Updating connection from %s to %s with %d (%f)\n", connection.From.ID, connection.To.ID, connection.TypicalSeconds, denominator)
		err := connection.Update(tx)
		if err != nil {
			return err
		}
	}

	for connection, denominator := range connectionStopAvgDenominator {
		if denominator < 2 {
			// data is not significant enough
			continue
		}
		average := connectionStopAvgNumerator[connection] / denominator
		// TODO: add math.Round to int cast once Go 1.10 is released
		connection.TypicalStopSeconds = int(average)
		fmt.Printf("Updating connection from %s to %s with stop %d (%f)\n", connection.From.ID, connection.To.ID, connection.TypicalStopSeconds, denominator)
		err := connection.Update(tx)
		if err != nil {
			return err
		}
	}

	for connection, denominator := range connectionWaitAvgDenominator {
		if denominator < 2 {
			// data is not significant enough
			continue
		}
		average := connectionWaitAvgNumerator[connection] / denominator
		// TODO: add math.Round to int cast once Go 1.10 is released
		connection.TypicalWaitingSeconds = int(average)
		fmt.Printf("Updating connection from %s to %s with wait %d (%f)\n", connection.From.ID, connection.To.ID, connection.TypicalWaitingSeconds, denominator)
		err := connection.Update(tx)
		if err != nil {
			return err
		}
	}

	for transfer, denominator := range transferAvgDenominator {
		if denominator < 2 {
			// data is not significant enough
			continue
		}
		average := transferAvgNumerator[transfer] / denominator

		// subtract average of stop times, because the pathfinding algos can't
		// deal with edges that have different weights depending on where one
		// "comes from"

		outgoingConnections := []*dataobjects.Connection{}
		for connection := range connectionStopAvgDenominator {
			if connection.From.ID == transfer.Station.ID {
				outgoingConnections = append(outgoingConnections, connection)
			}
		}

		outgoingDestConnections := []*dataobjects.Connection{}
		for _, connection := range outgoingConnections {
			lines, err := connection.To.Lines(tx)
			if err != nil {
				return err
			}
			for _, line := range lines {
				if line.ID == transfer.To.ID {
					outgoingDestConnections = append(outgoingDestConnections, connection)
					break
				}
			}
		}

		avgStopTime := 0
		for _, connection := range outgoingDestConnections {
			avgStopTime += connection.TypicalStopSeconds
		}
		if len(outgoingDestConnections) > 0 {
			average -= float64(avgStopTime) / float64(len(outgoingDestConnections))
		}

		// TODO: add math.Round to int cast once Go 1.10 is released
		transfer.TypicalSeconds = int(average)
		fmt.Printf("Updating transfer from %s to %s with %d (%f)\n", transfer.From.ID, transfer.To.ID, transfer.TypicalSeconds, denominator)
		err := transfer.Update(tx)
		if err != nil {
			return err
		}
	}

	return tx.Commit()
}

// ComputeAverageSpeed returns the average service speed in km/h
// based on the trips in the specified time range
func ComputeAverageSpeed(node sqalx.Node, fromTime time.Time, toTime time.Time) (float64, error) {
	tx, err := node.Beginx()
	if err != nil {
		return 0, err
	}
	defer tx.Commit() // read-only tx

	tripIDs, err := dataobjects.GetTripIDsBetween(tx, fromTime, toTime)
	if err != nil {
		return 0, err
	}

	if len(tripIDs) == 0 {
		return 0, nil
	}

	type TransferKey struct {
		Station string
		From    string
		To      string
	}

	type ConnectionKey struct {
		From string
		To   string
	}

	connectionCache := make(map[ConnectionKey]*dataobjects.Connection)
	getConnection := func(from, to string) (*dataobjects.Connection, error) {
		connection, cached := connectionCache[ConnectionKey{from, to}]
		if !cached {
			var err error
			connection, err = dataobjects.GetConnection(tx, from, to)
			if err != nil {
				return nil, err
			}
			connectionCache[ConnectionKey{from, to}] = connection
		}
		return connection, nil
	}

	var totalTime time.Duration
	var totalDistance int64

	processTrip := func(trip *dataobjects.Trip) error {
		if len(trip.StationUses) <= 1 {
			// station visit or invalid trip
			// can't extract any data about connections
			return nil
		}

		var startTime, endTime time.Time
		for useIdx := 0; useIdx < len(trip.StationUses)-1; useIdx++ {
			sourceUse := trip.StationUses[useIdx]

			if sourceUse.Manual {
				// manual path extensions don't contain valid time data
				// skip
				continue
			}

			if sourceUse.Type == dataobjects.Interchange ||
				sourceUse.Type == dataobjects.Visit {
				continue
			}

			targetUse := trip.StationUses[useIdx+1]

			if targetUse.Manual {
				// manual path extensions don't contain valid time data
				// skip
				continue
			}

			connection, err := getConnection(sourceUse.Station.ID, targetUse.Station.ID)
			if err != nil {
				// connection might no longer exist (closed stations, etc.)
				// move on
				fmt.Printf("Connection from %s to %s skipped\n", sourceUse.Station.ID, targetUse.Station.ID)
				return nil
			}

			totalDistance += int64(connection.WorldLength)
			if startTime.IsZero() {
				startTime = sourceUse.LeaveTime
			}
			endTime = targetUse.EntryTime
		}
		totalTime += endTime.Sub(startTime)
		return nil
	}

	// instantiate each trip from DB individually
	// (instead of using dataobjects.GetTrips)
	// to reduce memory usage
	for _, tripID := range tripIDs {
		trip, err := dataobjects.GetTrip(tx, tripID)
		if err != nil {
			return 0, err
		}

		if err = processTrip(trip); err != nil {
			return 0, err
		}
	}

	km := float64(totalDistance) / 1000
	hours := totalTime.Hours()

	return km / hours, nil
}

// ComputeAverageSpeedCached returns the average service speed in km/h
// based on the trips in the specified time range, with cache in front
// to minimize computational cost
type avgSpeedCacheKey struct {
	From int64
	To   int64
}

var avgSpeedCache map[avgSpeedCacheKey]float64

func ComputeAverageSpeedCached(node sqalx.Node, fromTime time.Time, toTime time.Time) (float64, error) {
	if val, ok := avgSpeedCache[avgSpeedCacheKey{fromTime.Unix(), toTime.Unix()}]; ok {
		return val, nil
	}
	val, err := ComputeAverageSpeed(node, fromTime, toTime)
	if err != nil {
		return val, err
	}
	avgSpeedCache[avgSpeedCacheKey{fromTime.Unix(), toTime.Unix()}] = val
	return val, nil
}

func init() {
	avgSpeedCache = make(map[avgSpeedCacheKey]float64)
}
<|endoftext|>


<commit_before>package testSuit

import (
	"bytes"
	"image"
	"image/color"
	"image/gif"
	"image/jpeg"
	"image/png"
	"io"
	"math/rand"
	"mime/multipart"
	"net/http"
	"net/http/httptest"
	"net/url"
	"strings"
	"time"

	"github.com/gin-gonic/gin"
	"github.com/google/uuid"
)

// TestSuit http request test form
type TestSuit struct {
	Method      string
	URL         string
	Data        url.Values
	Buffer      io.Reader
	ContentType string
	request     *http.Request
	response    *httptest.ResponseRecorder
	Header      map[string]string
	Router      *gin.Engine
}

// Do start test
func (ts *TestSuit) Do() *httptest.ResponseRecorder {
	if ts.ContentType == "" {
		ts.ContentType = "application/x-www-form-urlencoded"
	}

	addHeader := func() {
		ts.request.Header.Add("Content-Type", ts.ContentType)

		for k, v := range ts.Header {
			ts.request.Header.Add(k, v)
		}

	}

	if ts.Data != nil {
		encodedData := ts.Data.Encode()

		switch ts.Method {
		case "ALL":
			ts.request, _ = http.NewRequest("POST", ts.URL,
				strings.NewReader(encodedData))
			addHeader()
		case "GET":
			ts.request, _ = http.NewRequest(ts.Method, ts.URL, nil)
			ts.request.URL.RawQuery = ts.Data.Encode()
			addHeader()
		case "POST":
			ts.request, _ = http.NewRequest(ts.Method, ts.URL,
				strings.NewReader(ts.Data.Encode()))
			addHeader()
		}

	} else if ts.Buffer != nil {
		ts.request, _ = http.NewRequest(ts.Method, ts.URL, ts.Buffer)
		addHeader()
	} else {
		ts.request, _ = http.NewRequest(ts.Method, ts.URL, nil)
		addHeader()
	}

	ts.response = httptest.NewRecorder()
	ts.Router.ServeHTTP(ts.response, ts.request)
	return ts.response
}

// GetGinEngine create gin.Engine with test mode
func GetGinEngine() *gin.Engine {
	gin.SetMode(gin.TestMode)
	r := gin.New()

	return r
}

// GenImage generate randomize image
func GenImage(ext string, width, height int) (*bytes.Buffer, string, error) {
	rand.Seed(time.Now().UTC().UnixNano())

	img := image.NewRGBA(image.Rect(0, 0, width, height))
	for x := 0; x < width; x++ {
		for y := 0; y < height; y++ {
			img.Set(x, y, color.RGBA{
				uint8(rand.Intn(255)),
				uint8(rand.Intn(255)),
				uint8(rand.Intn(255)),
				uint8(rand.Intn(255)),
			})
		}
	}

	nameFix := uuid.New().URN()
	buffer := &bytes.Buffer{}
	bufferWriter := multipart.NewWriter(buffer)
	formWriter, err := bufferWriter.CreateFormFile("file", nameFix+".png")
	contentType := bufferWriter.FormDataContentType()

	switch ext {
	case "png":
		png.Encode(formWriter, img)
		bufferWriter.Close()
	case "jpg":
		jpeg.Encode(formWriter, img, nil)
		bufferWriter.Close()
	case "gif":
		gif.Encode(formWriter, img, nil)
		bufferWriter.Close()
	}
	return buffer, contentType, err
}
<commit_msg>support rest api<commit_after>package testSuit

import (
	"bytes"
	"image"
	"image/color"
	"image/gif"
	"image/jpeg"
	"image/png"
	"io"
	"math/rand"
	"mime/multipart"
	"net/http"
	"net/http/httptest"
	"net/url"
	"strings"
	"time"

	"github.com/gin-gonic/gin"
	"github.com/google/uuid"
)

// TestSuit http request test form
type TestSuit struct {
	Method      string
	URL         string
	Data        url.Values
	Buffer      io.Reader
	ContentType string
	request     *http.Request
	response    *httptest.ResponseRecorder
	Header      map[string]string
	Router      *gin.Engine
}

// Do start test
func (ts *TestSuit) Do() *httptest.ResponseRecorder {
	if ts.ContentType == "" {
		ts.ContentType = "application/x-www-form-urlencoded"
	}

	addHeader := func() {
		ts.request.Header.Add("Content-Type", ts.ContentType)

		for k, v := range ts.Header {
			ts.request.Header.Add(k, v)
		}

	}

	if ts.Data != nil {
		encodedData := ts.Data.Encode()

		switch ts.Method {
		case "GET":
			ts.request, _ = http.NewRequest(ts.Method, ts.URL, nil)
			ts.request.URL.RawQuery = ts.Data.Encode()
			addHeader()
		case "POST":
			ts.request, _ = http.NewRequest(ts.Method, ts.URL,
				strings.NewReader(ts.Data.Encode()))
			addHeader()
		default:
			ts.request, _ = http.NewRequest(t.Method, ts.URL,
				strings.NewReader(encodedData))
			addHeader()
		}

	} else if ts.Buffer != nil {
		ts.request, _ = http.NewRequest(ts.Method, ts.URL, ts.Buffer)
		addHeader()
	} else {
		ts.request, _ = http.NewRequest(ts.Method, ts.URL, nil)
		addHeader()
	}

	ts.response = httptest.NewRecorder()
	ts.Router.ServeHTTP(ts.response, ts.request)
	return ts.response
}

// GetGinEngine create gin.Engine with test mode
func GetGinEngine() *gin.Engine {
	gin.SetMode(gin.TestMode)
	r := gin.New()

	return r
}

// GenImage generate randomize image
func GenImage(ext string, width, height int) (*bytes.Buffer, string, error) {
	rand.Seed(time.Now().UTC().UnixNano())

	img := image.NewRGBA(image.Rect(0, 0, width, height))
	for x := 0; x < width; x++ {
		for y := 0; y < height; y++ {
			img.Set(x, y, color.RGBA{
				uint8(rand.Intn(255)),
				uint8(rand.Intn(255)),
				uint8(rand.Intn(255)),
				uint8(rand.Intn(255)),
			})
		}
	}

	nameFix := uuid.New().URN()
	buffer := &bytes.Buffer{}
	bufferWriter := multipart.NewWriter(buffer)
	formWriter, err := bufferWriter.CreateFormFile("file", nameFix+".png")
	contentType := bufferWriter.FormDataContentType()

	switch ext {
	case "png":
		png.Encode(formWriter, img)
		bufferWriter.Close()
	case "jpg":
		jpeg.Encode(formWriter, img, nil)
		bufferWriter.Close()
	case "gif":
		gif.Encode(formWriter, img, nil)
		bufferWriter.Close()
	}
	return buffer, contentType, err
}
<|endoftext|>


<commit_before>// +build integration

/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"context"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"testing"
	"time"

	"github.com/google/go-containerregistry/pkg/name"
	"github.com/google/go-containerregistry/pkg/v1/daemon"
	"k8s.io/minikube/pkg/drivers/kic"
	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
	"k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/pkg/minikube/localpath"
)

func TestDownloadOnly(t *testing.T) {
	profile := UniqueProfileName("download")
	ctx, cancel := context.WithTimeout(context.Background(), Minutes(15))
	defer Cleanup(t, profile, cancel)

	// Stores the startup run result for later error messages
	var rrr *RunResult
	var err error

	t.Run("group", func(t *testing.T) {
		versions := []string{
			constants.OldestKubernetesVersion,
			constants.DefaultKubernetesVersion,
			constants.NewestKubernetesVersion,
		}
		for _, v := range versions {
			t.Run(v, func(t *testing.T) {
				// Explicitly does not pass StartArgs() to test driver default
				// --force to avoid uid check
				args := append([]string{"start", "--download-only", "-p", profile, "--force", "--alsologtostderr", fmt.Sprintf("--kubernetes-version=%s", v)}, StartArgs()...)

				// Preserve the initial run-result for debugging
				if rrr == nil {
					rrr, err = Run(t, exec.CommandContext(ctx, Target(), args...))
				} else {
					_, err = Run(t, exec.CommandContext(ctx, Target(), args...))
				}

				if err != nil {
					t.Errorf("%s failed: %v", args, err)
				}

				imgs, err := images.Kubeadm("", v)
				if err != nil {
					t.Errorf("kubeadm images: %v %+v", v, err)
				}

				// skip verify for cache images if --vm-driver=none
				if !NoneDriver() {
					for _, img := range imgs {
						img = strings.Replace(img, ":", "_", 1) // for example kube-scheduler:v1.15.2 --> kube-scheduler_v1.15.2
						fp := filepath.Join(localpath.MiniPath(), "cache", "images", img)
						_, err := os.Stat(fp)
						if err != nil {
							t.Errorf("expected image file exist at %q but got error: %v", fp, err)
						}
					}
				}

				// checking binaries downloaded (kubelet,kubeadm)
				for _, bin := range constants.KubernetesReleaseBinaries {
					fp := filepath.Join(localpath.MiniPath(), "cache", "linux", v, bin)
					_, err := os.Stat(fp)
					if err != nil {
						t.Errorf("expected the file for binary exist at %q but got error %v", fp, err)
					}
				}

				// If we are on darwin/windows, check to make sure OS specific kubectl has been downloaded
				// as well for the `minikube kubectl` command
				if runtime.GOOS == "linux" {
					return
				}
				binary := "kubectl"
				if runtime.GOOS == "windows" {
					binary = "kubectl.exe"
				}
				fp := filepath.Join(localpath.MiniPath(), "cache", runtime.GOOS, v, binary)
				if _, err := os.Stat(fp); err != nil {
					t.Errorf("expected the file for binary exist at %q but got error %v", fp, err)
				}
			})
		}

		// Check that the profile we've created has the expected driver
		t.Run("ExpectedDefaultDriver", func(t *testing.T) {
			if ExpectedDefaultDriver() == "" {
				t.Skipf("--expected-default-driver is unset, skipping test")
				return
			}
			rr, err := Run(t, exec.CommandContext(ctx, Target(), "profile", "list", "--output", "json"))
			if err != nil {
				t.Errorf("%s failed: %v", rr.Args, err)
			}
			var ps map[string][]config.Profile
			err = json.Unmarshal(rr.Stdout.Bytes(), &ps)
			if err != nil {
				t.Errorf("%s failed: %v", rr.Args, err)
			}

			got := ""
			for _, p := range ps["valid"] {
				if p.Name == profile {
					got = p.Config.Driver
				}
			}

			if got != ExpectedDefaultDriver() {
				t.Errorf("got driver %q, expected %q\nstart output: %s", got, ExpectedDefaultDriver(), rrr.Output())
			}
		})

		// This is a weird place to test profile deletion, but this test is serial, and we have a profile to delete!
		t.Run("DeleteAll", func(t *testing.T) {
			if !CanCleanup() {
				t.Skip("skipping, as cleanup is disabled")
			}
			rr, err := Run(t, exec.CommandContext(ctx, Target(), "delete", "--all"))
			if err != nil {
				t.Errorf("%s failed: %v", rr.Args, err)
			}
		})
		// Delete should always succeed, even if previously partially or fully deleted.
		t.Run("DeleteAlwaysSucceeds", func(t *testing.T) {
			if !CanCleanup() {
				t.Skip("skipping, as cleanup is disabled")
			}
			rr, err := Run(t, exec.CommandContext(ctx, Target(), "delete", "-p", profile))
			if err != nil {
				t.Errorf("%s failed: %v", rr.Args, err)
			}
		})
	})

}
func TestDownloadOnlyDocker(t *testing.T) {
	if !runningDockerDriver(StartArgs()) {
		t.Skip("this test only runs with the docker driver")
	}

	profile := UniqueProfileName("download-docker")
	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
	defer Cleanup(t, profile, cancel)

	args := []string{"start", "--download-only", "-p", profile, "--force", "--alsologtostderr", "--vm-driver=docker"}
	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Errorf("%s failed: %v:\n%s", args, err, rr.Output())
	}

	// Make sure the preloaded image tarball exists
	tarball := download.TarballFilepath(constants.DefaultKubernetesVersion)
	contents, err := ioutil.ReadFile(tarball)
	if err != nil {
		t.Errorf("reading tarball: %v", err)
	}
	// Make sure it has the correct checksum
	checksum := md5.Sum(contents)
	remoteChecksum, err := ioutil.ReadFile(preload.ChecksumFilepath(constants.DefaultKubernetesVersion))
	if err != nil {
		t.Errorf("reading checksum file: %v", err)
	}
	if string(remoteChecksum) != string(checksum[:]) {
		t.Errorf("checksum of %s does not match remote checksum (%s != %s)", tarball, string(remoteChecksum), string(checksum[:]))
	}

	// Make sure this image exists in the docker daemon
	ref, err := name.ParseReference(kic.BaseImage)
	if err != nil {
		t.Errorf("parsing reference failed: %v", err)
	}
	if _, err := daemon.Image(ref); err != nil {
		t.Errorf("expected image does not exist in local daemon: %v", err)
	}
}

func runningDockerDriver(startArgs []string) bool {
	for _, s := range startArgs {
		if s == "--vm-driver=docker" {
			return true
		}
	}
	return false
}
<commit_msg>Fix integration test<commit_after>// +build integration

/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"context"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"testing"
	"time"

	"github.com/google/go-containerregistry/pkg/name"
	"github.com/google/go-containerregistry/pkg/v1/daemon"
	"k8s.io/minikube/pkg/drivers/kic"
	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
	"k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/pkg/minikube/download"
	"k8s.io/minikube/pkg/minikube/localpath"
)

func TestDownloadOnly(t *testing.T) {
	profile := UniqueProfileName("download")
	ctx, cancel := context.WithTimeout(context.Background(), Minutes(15))
	defer Cleanup(t, profile, cancel)

	// Stores the startup run result for later error messages
	var rrr *RunResult
	var err error

	t.Run("group", func(t *testing.T) {
		versions := []string{
			constants.OldestKubernetesVersion,
			constants.DefaultKubernetesVersion,
			constants.NewestKubernetesVersion,
		}
		for _, v := range versions {
			t.Run(v, func(t *testing.T) {
				// Explicitly does not pass StartArgs() to test driver default
				// --force to avoid uid check
				args := append([]string{"start", "--download-only", "-p", profile, "--force", "--alsologtostderr", fmt.Sprintf("--kubernetes-version=%s", v)}, StartArgs()...)

				// Preserve the initial run-result for debugging
				if rrr == nil {
					rrr, err = Run(t, exec.CommandContext(ctx, Target(), args...))
				} else {
					_, err = Run(t, exec.CommandContext(ctx, Target(), args...))
				}

				if err != nil {
					t.Errorf("%s failed: %v", args, err)
				}

				imgs, err := images.Kubeadm("", v)
				if err != nil {
					t.Errorf("kubeadm images: %v %+v", v, err)
				}

				// skip verify for cache images if --vm-driver=none
				if !NoneDriver() {
					for _, img := range imgs {
						img = strings.Replace(img, ":", "_", 1) // for example kube-scheduler:v1.15.2 --> kube-scheduler_v1.15.2
						fp := filepath.Join(localpath.MiniPath(), "cache", "images", img)
						_, err := os.Stat(fp)
						if err != nil {
							t.Errorf("expected image file exist at %q but got error: %v", fp, err)
						}
					}
				}

				// checking binaries downloaded (kubelet,kubeadm)
				for _, bin := range constants.KubernetesReleaseBinaries {
					fp := filepath.Join(localpath.MiniPath(), "cache", "linux", v, bin)
					_, err := os.Stat(fp)
					if err != nil {
						t.Errorf("expected the file for binary exist at %q but got error %v", fp, err)
					}
				}

				// If we are on darwin/windows, check to make sure OS specific kubectl has been downloaded
				// as well for the `minikube kubectl` command
				if runtime.GOOS == "linux" {
					return
				}
				binary := "kubectl"
				if runtime.GOOS == "windows" {
					binary = "kubectl.exe"
				}
				fp := filepath.Join(localpath.MiniPath(), "cache", runtime.GOOS, v, binary)
				if _, err := os.Stat(fp); err != nil {
					t.Errorf("expected the file for binary exist at %q but got error %v", fp, err)
				}
			})
		}

		// Check that the profile we've created has the expected driver
		t.Run("ExpectedDefaultDriver", func(t *testing.T) {
			if ExpectedDefaultDriver() == "" {
				t.Skipf("--expected-default-driver is unset, skipping test")
				return
			}
			rr, err := Run(t, exec.CommandContext(ctx, Target(), "profile", "list", "--output", "json"))
			if err != nil {
				t.Errorf("%s failed: %v", rr.Args, err)
			}
			var ps map[string][]config.Profile
			err = json.Unmarshal(rr.Stdout.Bytes(), &ps)
			if err != nil {
				t.Errorf("%s failed: %v", rr.Args, err)
			}

			got := ""
			for _, p := range ps["valid"] {
				if p.Name == profile {
					got = p.Config.Driver
				}
			}

			if got != ExpectedDefaultDriver() {
				t.Errorf("got driver %q, expected %q\nstart output: %s", got, ExpectedDefaultDriver(), rrr.Output())
			}
		})

		// This is a weird place to test profile deletion, but this test is serial, and we have a profile to delete!
		t.Run("DeleteAll", func(t *testing.T) {
			if !CanCleanup() {
				t.Skip("skipping, as cleanup is disabled")
			}
			rr, err := Run(t, exec.CommandContext(ctx, Target(), "delete", "--all"))
			if err != nil {
				t.Errorf("%s failed: %v", rr.Args, err)
			}
		})
		// Delete should always succeed, even if previously partially or fully deleted.
		t.Run("DeleteAlwaysSucceeds", func(t *testing.T) {
			if !CanCleanup() {
				t.Skip("skipping, as cleanup is disabled")
			}
			rr, err := Run(t, exec.CommandContext(ctx, Target(), "delete", "-p", profile))
			if err != nil {
				t.Errorf("%s failed: %v", rr.Args, err)
			}
		})
	})

}
func TestDownloadOnlyDocker(t *testing.T) {
	if !runningDockerDriver(StartArgs()) {
		t.Skip("this test only runs with the docker driver")
	}

	profile := UniqueProfileName("download-docker")
	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
	defer Cleanup(t, profile, cancel)

	args := []string{"start", "--download-only", "-p", profile, "--force", "--alsologtostderr", "--vm-driver=docker"}
	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Errorf("%s failed: %v:\n%s", args, err, rr.Output())
	}

	// Make sure the downloaded image tarball exists
	tarball := download.TarballPath(constants.DefaultKubernetesVersion)
	contents, err := ioutil.ReadFile(tarball)
	if err != nil {
		t.Errorf("reading tarball: %v", err)
	}
	// Make sure it has the correct checksum
	checksum := md5.Sum(contents)
	remoteChecksum, err := ioutil.ReadFile(download.PreloadChecksumPath(constants.DefaultKubernetesVersion))
	if err != nil {
		t.Errorf("reading checksum file: %v", err)
	}
	if string(remoteChecksum) != string(checksum[:]) {
		t.Errorf("checksum of %s does not match remote checksum (%s != %s)", tarball, string(remoteChecksum), string(checksum[:]))
	}

	// Make sure this image exists in the docker daemon
	ref, err := name.ParseReference(kic.BaseImage)
	if err != nil {
		t.Errorf("parsing reference failed: %v", err)
	}
	if _, err := daemon.Image(ref); err != nil {
		t.Errorf("expected image does not exist in local daemon: %v", err)
	}
}

func runningDockerDriver(startArgs []string) bool {
	for _, s := range startArgs {
		if s == "--vm-driver=docker" {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>// +build integration

/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"bufio"
	"bytes"
	"context"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"testing"

	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/pkg/minikube/download"
	"k8s.io/minikube/pkg/minikube/localpath"
)

func TestDownloadOnly(t *testing.T) {
	for _, r := range []string{"crio", "docker", "containerd"} {
		t.Run(r, func(t *testing.T) {
			// Stores the startup run result for later error messages
			var rrr *RunResult

			profile := UniqueProfileName(r)
			ctx, cancel := context.WithTimeout(context.Background(), Minutes(30))
			defer Cleanup(t, profile, cancel)

			versions := []string{
				constants.OldestKubernetesVersion,
				constants.DefaultKubernetesVersion,
				constants.NewestKubernetesVersion,
			}

			for _, v := range versions {
				t.Run(v, func(t *testing.T) {
					defer PostMortemLogs(t, profile)

					// --force to avoid uid check
					args := append([]string{"start", "-o=json", "--download-only", "-p", profile, "--force", "--alsologtostderr", fmt.Sprintf("--kubernetes-version=%s", v), fmt.Sprintf("--container-runtime=%s", r)}, StartArgs()...)

					rt, err := Run(t, exec.CommandContext(ctx, Target(), args...))
					if rrr == nil {
						// Preserve the initial run-result for debugging
						rrr = rt
					}
					if err != nil {
						t.Errorf("failed to download only. args: %q %v", args, err)
					}
					s := bufio.NewScanner(bytes.NewReader(rt.Stdout.Bytes()))
					for s.Scan() {
						var rtObj map[string]interface{}
						err = json.Unmarshal(s.Bytes(), &rtObj)
						if err != nil {
							t.Errorf("failed to parse output: %v", err)
						} else {
							if step, ok := rtObj["data"]; ok {
								if stepMap, ok := step.(map[string]interface{}); ok {
									if stepMap["currentstep"] == "" {
										t.Errorf("Empty step number for %v", stepMap["name"])
									}
								}
							}
						}
					}

					// skip for none, as none driver does not have preload feature.
					if !NoneDriver() {
						if download.PreloadExists(v, r, true) {
							// Just make sure the tarball path exists
							if _, err := os.Stat(download.TarballPath(v, r)); err != nil {
								t.Errorf("failed to verify preloaded tarball file exists: %v", err)
							}
							return
						}
					}
					imgs, err := images.Kubeadm("", v)
					if err != nil {
						t.Errorf("failed to get kubeadm images for %v: %+v", v, err)
					}

					// skip verify for cache images if --driver=none
					if !NoneDriver() {
						for _, img := range imgs {
							img = strings.Replace(img, ":", "_", 1) // for example kube-scheduler:v1.15.2 --> kube-scheduler_v1.15.2
							fp := filepath.Join(localpath.MiniPath(), "cache", "images", img)
							_, err := os.Stat(fp)
							if err != nil {
								t.Errorf("expected image file exist at %q but got error: %v", fp, err)
							}
						}
					}

					// checking binaries downloaded (kubelet,kubeadm)
					for _, bin := range constants.KubernetesReleaseBinaries {
						fp := filepath.Join(localpath.MiniPath(), "cache", "linux", v, bin)
						_, err := os.Stat(fp)
						if err != nil {
							t.Errorf("expected the file for binary exist at %q but got error %v", fp, err)
						}
					}

					// If we are on darwin/windows, check to make sure OS specific kubectl has been downloaded
					// as well for the `minikube kubectl` command
					if runtime.GOOS == "linux" {
						return
					}
					binary := "kubectl"
					if runtime.GOOS == "windows" {
						binary = "kubectl.exe"
					}
					fp := filepath.Join(localpath.MiniPath(), "cache", runtime.GOOS, v, binary)
					if _, err := os.Stat(fp); err != nil {
						t.Errorf("expected the file for binary exist at %q but got error %v", fp, err)
					}
				})
			}

			// This is a weird place to test profile deletion, but this test is serial, and we have a profile to delete!
			t.Run("DeleteAll", func(t *testing.T) {
				defer PostMortemLogs(t, profile)

				if !CanCleanup() {
					t.Skip("skipping, as cleanup is disabled")
				}
				rr, err := Run(t, exec.CommandContext(ctx, Target(), "delete", "--all"))
				if err != nil {
					t.Errorf("failed to delete all. args: %q : %v", rr.Command(), err)
				}
			})
			// Delete should always succeed, even if previously partially or fully deleted.
			t.Run("DeleteAlwaysSucceeds", func(t *testing.T) {
				defer PostMortemLogs(t, profile)

				if !CanCleanup() {
					t.Skip("skipping, as cleanup is disabled")
				}
				rr, err := Run(t, exec.CommandContext(ctx, Target(), "delete", "-p", profile))
				if err != nil {
					t.Errorf("failed to delete. args: %q: %v", rr.Command(), err)
				}
			})
		})
	}
}

func TestDownloadOnlyKic(t *testing.T) {
	if !KicDriver() {
		t.Skip("skipping, only for docker or podman driver")
	}
	profile := UniqueProfileName("download-docker")
	ctx, cancel := context.WithTimeout(context.Background(), Minutes(15))
	defer Cleanup(t, profile, cancel)

	// TODO: #7795 add containerd to download only too
	cRuntime := "docker"

	args := []string{"start", "--download-only", "-p", profile, "--force", "--alsologtostderr"}
	args = append(args, StartArgs()...)

	cmd := exec.CommandContext(ctx, Target(), args...)
	// make sure this works even if docker daemon isn't running
	cmd.Env = append(os.Environ(), "DOCKER_HOST=/does/not/exist")
	if _, err := Run(t, cmd); err != nil {
		t.Errorf("start with download only failed %q : %v", args, err)
	}

	// Make sure the downloaded image tarball exists
	tarball := download.TarballPath(constants.DefaultKubernetesVersion, cRuntime)
	contents, err := ioutil.ReadFile(tarball)
	if err != nil {
		t.Errorf("failed to read tarball file %q: %v", tarball, err)
	}
	// Make sure it has the correct checksum
	checksum := md5.Sum(contents)
	remoteChecksum, err := ioutil.ReadFile(download.PreloadChecksumPath(constants.DefaultKubernetesVersion, cRuntime))
	if err != nil {
		t.Errorf("failed to read checksum file %q : %v", download.PreloadChecksumPath(constants.DefaultKubernetesVersion, cRuntime), err)
	}
	if string(remoteChecksum) != string(checksum[:]) {
		t.Errorf("failed to verify checksum. checksum of %q does not match remote checksum (%q != %q)", tarball, string(remoteChecksum), string(checksum[:]))
	}
}
<commit_msg>fix linter error<commit_after>// +build integration

/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"bufio"
	"bytes"
	"context"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"testing"

	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/pkg/minikube/download"
	"k8s.io/minikube/pkg/minikube/localpath"
)

func TestDownloadOnly(t *testing.T) {
	for _, r := range []string{"crio", "docker", "containerd"} {
		t.Run(r, func(t *testing.T) {
			// Stores the startup run result for later error messages
			var rrr *RunResult

			profile := UniqueProfileName(r)
			ctx, cancel := context.WithTimeout(context.Background(), Minutes(30))
			defer Cleanup(t, profile, cancel)

			versions := []string{
				constants.OldestKubernetesVersion,
				constants.DefaultKubernetesVersion,
				constants.NewestKubernetesVersion,
			}

			for _, v := range versions {
				t.Run(v, func(t *testing.T) {
					defer PostMortemLogs(t, profile)

					// --force to avoid uid check
					args := append([]string{"start", "-o=json", "--download-only", "-p", profile, "--force", "--alsologtostderr", fmt.Sprintf("--kubernetes-version=%s", v), fmt.Sprintf("--container-runtime=%s", r)}, StartArgs()...)

					rt, err := Run(t, exec.CommandContext(ctx, Target(), args...))
					if rrr == nil {
						// Preserve the initial run-result for debugging
						rrr = rt
					}
					if err != nil {
						t.Errorf("failed to download only. args: %q %v", args, err)
					}
					s := bufio.NewScanner(bytes.NewReader(rt.Stdout.Bytes()))
					for s.Scan() {
						var rtObj map[string]interface{}
						err = json.Unmarshal(s.Bytes(), &rtObj)
						if err != nil {
							t.Errorf("failed to parse output: %v", err)
						} else if step, ok := rtObj["data"]; ok {
							if stepMap, ok := step.(map[string]interface{}); ok {
								if stepMap["currentstep"] == "" {
									t.Errorf("Empty step number for %v", stepMap["name"])
								}
							}
						}
					}

					// skip for none, as none driver does not have preload feature.
					if !NoneDriver() {
						if download.PreloadExists(v, r, true) {
							// Just make sure the tarball path exists
							if _, err := os.Stat(download.TarballPath(v, r)); err != nil {
								t.Errorf("failed to verify preloaded tarball file exists: %v", err)
							}
							return
						}
					}
					imgs, err := images.Kubeadm("", v)
					if err != nil {
						t.Errorf("failed to get kubeadm images for %v: %+v", v, err)
					}

					// skip verify for cache images if --driver=none
					if !NoneDriver() {
						for _, img := range imgs {
							img = strings.Replace(img, ":", "_", 1) // for example kube-scheduler:v1.15.2 --> kube-scheduler_v1.15.2
							fp := filepath.Join(localpath.MiniPath(), "cache", "images", img)
							_, err := os.Stat(fp)
							if err != nil {
								t.Errorf("expected image file exist at %q but got error: %v", fp, err)
							}
						}
					}

					// checking binaries downloaded (kubelet,kubeadm)
					for _, bin := range constants.KubernetesReleaseBinaries {
						fp := filepath.Join(localpath.MiniPath(), "cache", "linux", v, bin)
						_, err := os.Stat(fp)
						if err != nil {
							t.Errorf("expected the file for binary exist at %q but got error %v", fp, err)
						}
					}

					// If we are on darwin/windows, check to make sure OS specific kubectl has been downloaded
					// as well for the `minikube kubectl` command
					if runtime.GOOS == "linux" {
						return
					}
					binary := "kubectl"
					if runtime.GOOS == "windows" {
						binary = "kubectl.exe"
					}
					fp := filepath.Join(localpath.MiniPath(), "cache", runtime.GOOS, v, binary)
					if _, err := os.Stat(fp); err != nil {
						t.Errorf("expected the file for binary exist at %q but got error %v", fp, err)
					}
				})
			}

			// This is a weird place to test profile deletion, but this test is serial, and we have a profile to delete!
			t.Run("DeleteAll", func(t *testing.T) {
				defer PostMortemLogs(t, profile)

				if !CanCleanup() {
					t.Skip("skipping, as cleanup is disabled")
				}
				rr, err := Run(t, exec.CommandContext(ctx, Target(), "delete", "--all"))
				if err != nil {
					t.Errorf("failed to delete all. args: %q : %v", rr.Command(), err)
				}
			})
			// Delete should always succeed, even if previously partially or fully deleted.
			t.Run("DeleteAlwaysSucceeds", func(t *testing.T) {
				defer PostMortemLogs(t, profile)

				if !CanCleanup() {
					t.Skip("skipping, as cleanup is disabled")
				}
				rr, err := Run(t, exec.CommandContext(ctx, Target(), "delete", "-p", profile))
				if err != nil {
					t.Errorf("failed to delete. args: %q: %v", rr.Command(), err)
				}
			})
		})
	}
}

func TestDownloadOnlyKic(t *testing.T) {
	if !KicDriver() {
		t.Skip("skipping, only for docker or podman driver")
	}
	profile := UniqueProfileName("download-docker")
	ctx, cancel := context.WithTimeout(context.Background(), Minutes(15))
	defer Cleanup(t, profile, cancel)

	// TODO: #7795 add containerd to download only too
	cRuntime := "docker"

	args := []string{"start", "--download-only", "-p", profile, "--force", "--alsologtostderr"}
	args = append(args, StartArgs()...)

	cmd := exec.CommandContext(ctx, Target(), args...)
	// make sure this works even if docker daemon isn't running
	cmd.Env = append(os.Environ(), "DOCKER_HOST=/does/not/exist")
	if _, err := Run(t, cmd); err != nil {
		t.Errorf("start with download only failed %q : %v", args, err)
	}

	// Make sure the downloaded image tarball exists
	tarball := download.TarballPath(constants.DefaultKubernetesVersion, cRuntime)
	contents, err := ioutil.ReadFile(tarball)
	if err != nil {
		t.Errorf("failed to read tarball file %q: %v", tarball, err)
	}
	// Make sure it has the correct checksum
	checksum := md5.Sum(contents)
	remoteChecksum, err := ioutil.ReadFile(download.PreloadChecksumPath(constants.DefaultKubernetesVersion, cRuntime))
	if err != nil {
		t.Errorf("failed to read checksum file %q : %v", download.PreloadChecksumPath(constants.DefaultKubernetesVersion, cRuntime), err)
	}
	if string(remoteChecksum) != string(checksum[:]) {
		t.Errorf("failed to verify checksum. checksum of %q does not match remote checksum (%q != %q)", tarball, string(remoteChecksum), string(checksum[:]))
	}
}
<|endoftext|>


<commit_before>// +build integration

/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"context"
	"fmt"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/docker/machine/libmachine/state"
	"k8s.io/minikube/pkg/minikube/constants"
)

func TestStartStop(t *testing.T) {
	MaybeParallel(t)

	t.Run("group", func(t *testing.T) {
		tests := []struct {
			name string
			args []string
		}{
			{"docker", []string{
				"--cache-images=false",
				fmt.Sprintf("--kubernetes-version=%s", constants.OldestKubernetesVersion),
				// default is the network created by libvirt, if we change the name minikube won't boot
				// because the given network doesn't exist
				"--kvm-network=default",
				"--kvm-qemu-uri=qemu:///system",
				"--disable-driver-mounts",
				"--keep-context=false",
				"--container-runtime=docker",
			}},
			{"cni", []string{
				"--feature-gates",
				"ServerSideApply=true",
				"--network-plugin=cni",
				"--extra-config=kubelet.network-plugin=cni",
				"--extra-config=kubeadm.pod-network-cidr=192.168.111.111/16",
				fmt.Sprintf("--kubernetes-version=%s", constants.NewestKubernetesVersion),
			}},
			{"containerd", []string{
				"--container-runtime=containerd",
				"--docker-opt",
				"containerd=/var/run/containerd/containerd.sock",
				"--apiserver-port=8444",
			}},
			{"crio", []string{
				"--container-runtime=crio",
				"--disable-driver-mounts",
				"--extra-config=kubeadm.ignore-preflight-errors=SystemVerification",
			}},
		}

		for _, tc := range tests {
			tc := tc
			t.Run(tc.name, func(t *testing.T) {
				MaybeSlowParallel(t)

				if !strings.Contains(tc.name, "docker") && NoneDriver() {
					t.Skipf("skipping %s - incompatible with none driver", t.Name())
				}

				profile := UniqueProfileName(tc.name)
				ctx, cancel := context.WithTimeout(context.Background(), 40*time.Minute)
				defer CleanupWithLogs(t, profile, cancel)

				startArgs := append([]string{"start", "-p", profile, "--alsologtostderr", "-v=3", "--wait=true"}, tc.args...)
				startArgs = append(startArgs, StartArgs()...)
				rr, err := Run(t, exec.CommandContext(ctx, Target(), startArgs...))
				if err != nil {
					// Fatal so that we may collect logs before stop/delete steps
					t.Fatalf("%s failed: %v", rr.Args, err)
				}

				// SADNESS: 0/1 nodes are available: 1 node(s) had taints that the pod didn't tolerate.
				if strings.Contains(tc.name, "cni") {
					t.Logf("WARNING: cni mode requires additional setup before pods can schedule :(")
				} else {
					// schedule a pod to assert persistence
					rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "create", "-f", filepath.Join(*testdataDir, "busybox.yaml")))
					if err != nil {
						t.Fatalf("%s failed: %v", rr.Args, err)
					}

					names, err := PodWait(ctx, t, profile, "default", "integration-test=busybox", 4*time.Minute);
					if err != nil {
						t.Fatalf("wait: %v", err)
					}

					// Use this pod to confirm that the runtime resource limits are sane
					rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "exec", names[0], "--", "/bin/sh", "-c", "ulimit -n"))
					if err != nil {
						t.Fatalf("ulimit: %v", err)
					}

					got, err := strconv.ParseInt(strings.TrimSpace(rr.Stdout.String()), 10, 64)
					if err != nil {
						t.Errorf("ParseInt(%q): %v", rr.Stdout.String(), err)
					}

					// Arbitrary value set by some container runtimes. If higher, apps like MySQL may make bad decisions.
					expected := int64(1048576)
					if got != expected {
						t.Errorf("'ulimit -n' returned %d, expected %d", got, expected)
					}
				}

				rr, err = Run(t, exec.CommandContext(ctx, Target(), "stop", "-p", profile, "--alsologtostderr", "-v=3"))
				if err != nil {
					t.Errorf("%s failed: %v", rr.Args, err)
				}

				got := Status(ctx, t, Target(), profile)
				if got != state.Stopped.String() {
					t.Errorf("status = %q; want = %q", got, state.Stopped)
				}

				rr, err = Run(t, exec.CommandContext(ctx, Target(), startArgs...))
				if err != nil {
					// Explicit fatal so that failures don't move directly to deletion
					t.Fatalf("%s failed: %v", rr.Args, err)
				}

				if strings.Contains(tc.name, "cni") {
					t.Logf("WARNING: cni mode requires additional setup before pods can schedule :(")
				} else if _, err := PodWait(ctx, t, profile, "default", "integration-test=busybox", 2*time.Minute); err != nil {
					t.Fatalf("wait: %v", err)
				}

				got = Status(ctx, t, Target(), profile)
				if got != state.Running.String() {
					t.Errorf("status = %q; want = %q", got, state.Running)
				}

				if !*cleanup {
					// Normally handled by cleanuprofile, but not fatal there
					rr, err = Run(t, exec.CommandContext(ctx, Target(), "delete", "-p", profile))
					if err != nil {
						t.Errorf("%s failed: %v", rr.Args, err)
					}
				}
			})
		}
	})
}
<commit_msg>Remove trailing semicolon<commit_after>// +build integration

/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"context"
	"fmt"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/docker/machine/libmachine/state"
	"k8s.io/minikube/pkg/minikube/constants"
)

func TestStartStop(t *testing.T) {
	MaybeParallel(t)

	t.Run("group", func(t *testing.T) {
		tests := []struct {
			name string
			args []string
		}{
			{"docker", []string{
				"--cache-images=false",
				fmt.Sprintf("--kubernetes-version=%s", constants.OldestKubernetesVersion),
				// default is the network created by libvirt, if we change the name minikube won't boot
				// because the given network doesn't exist
				"--kvm-network=default",
				"--kvm-qemu-uri=qemu:///system",
				"--disable-driver-mounts",
				"--keep-context=false",
				"--container-runtime=docker",
			}},
			{"cni", []string{
				"--feature-gates",
				"ServerSideApply=true",
				"--network-plugin=cni",
				"--extra-config=kubelet.network-plugin=cni",
				"--extra-config=kubeadm.pod-network-cidr=192.168.111.111/16",
				fmt.Sprintf("--kubernetes-version=%s", constants.NewestKubernetesVersion),
			}},
			{"containerd", []string{
				"--container-runtime=containerd",
				"--docker-opt",
				"containerd=/var/run/containerd/containerd.sock",
				"--apiserver-port=8444",
			}},
			{"crio", []string{
				"--container-runtime=crio",
				"--disable-driver-mounts",
				"--extra-config=kubeadm.ignore-preflight-errors=SystemVerification",
			}},
		}

		for _, tc := range tests {
			tc := tc
			t.Run(tc.name, func(t *testing.T) {
				MaybeSlowParallel(t)

				if !strings.Contains(tc.name, "docker") && NoneDriver() {
					t.Skipf("skipping %s - incompatible with none driver", t.Name())
				}

				profile := UniqueProfileName(tc.name)
				ctx, cancel := context.WithTimeout(context.Background(), 40*time.Minute)
				defer CleanupWithLogs(t, profile, cancel)

				startArgs := append([]string{"start", "-p", profile, "--alsologtostderr", "-v=3", "--wait=true"}, tc.args...)
				startArgs = append(startArgs, StartArgs()...)
				rr, err := Run(t, exec.CommandContext(ctx, Target(), startArgs...))
				if err != nil {
					// Fatal so that we may collect logs before stop/delete steps
					t.Fatalf("%s failed: %v", rr.Args, err)
				}

				// SADNESS: 0/1 nodes are available: 1 node(s) had taints that the pod didn't tolerate.
				if strings.Contains(tc.name, "cni") {
					t.Logf("WARNING: cni mode requires additional setup before pods can schedule :(")
				} else {
					// schedule a pod to assert persistence
					rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "create", "-f", filepath.Join(*testdataDir, "busybox.yaml")))
					if err != nil {
						t.Fatalf("%s failed: %v", rr.Args, err)
					}

					names, err := PodWait(ctx, t, profile, "default", "integration-test=busybox", 4*time.Minute)
					if err != nil {
						t.Fatalf("wait: %v", err)
					}

					// Use this pod to confirm that the runtime resource limits are sane
					rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "exec", names[0], "--", "/bin/sh", "-c", "ulimit -n"))
					if err != nil {
						t.Fatalf("ulimit: %v", err)
					}

					got, err := strconv.ParseInt(strings.TrimSpace(rr.Stdout.String()), 10, 64)
					if err != nil {
						t.Errorf("ParseInt(%q): %v", rr.Stdout.String(), err)
					}

					// Arbitrary value set by some container runtimes. If higher, apps like MySQL may make bad decisions.
					expected := int64(1048576)
					if got != expected {
						t.Errorf("'ulimit -n' returned %d, expected %d", got, expected)
					}
				}

				rr, err = Run(t, exec.CommandContext(ctx, Target(), "stop", "-p", profile, "--alsologtostderr", "-v=3"))
				if err != nil {
					t.Errorf("%s failed: %v", rr.Args, err)
				}

				got := Status(ctx, t, Target(), profile)
				if got != state.Stopped.String() {
					t.Errorf("status = %q; want = %q", got, state.Stopped)
				}

				rr, err = Run(t, exec.CommandContext(ctx, Target(), startArgs...))
				if err != nil {
					// Explicit fatal so that failures don't move directly to deletion
					t.Fatalf("%s failed: %v", rr.Args, err)
				}

				if strings.Contains(tc.name, "cni") {
					t.Logf("WARNING: cni mode requires additional setup before pods can schedule :(")
				} else if _, err := PodWait(ctx, t, profile, "default", "integration-test=busybox", 2*time.Minute); err != nil {
					t.Fatalf("wait: %v", err)
				}

				got = Status(ctx, t, Target(), profile)
				if got != state.Running.String() {
					t.Errorf("status = %q; want = %q", got, state.Running)
				}

				if !*cleanup {
					// Normally handled by cleanuprofile, but not fatal there
					rr, err = Run(t, exec.CommandContext(ctx, Target(), "delete", "-p", profile))
					if err != nil {
						t.Errorf("%s failed: %v", rr.Args, err)
					}
				}
			})
		}
	})
}
<|endoftext|>


<commit_before>// Copyright (c) 2015-2016 The btcsuite developers
// Copyright (c) 2016 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package txscript

import (
	"bytes"
	"sync"

	"github.com/decred/dcrd/chaincfg/chainec"
	"github.com/decred/dcrd/chaincfg/chainhash"
)

// sigCacheEntry represents an entry in the SigCache. Entries within the
// SigCache are keyed according to the sigHash of the signature. In the
// scenario of a cache-hit (according to the sigHash), an additional comparison
// of the signature, and public key will be executed in order to ensure a complete
// match. In the occasion that two sigHashes collide, the newer sigHash will
// simply overwrite the existing entry.
type sigCacheEntry struct {
	sig    chainec.Signature
	pubKey chainec.PublicKey
}

// SigCache implements an ECDSA signature verification cache with a randomized
// entry eviction policy. Only valid signatures will be added to the cache. The
// benefits of SigCache are two fold. Firstly, usage of SigCache mitigates a DoS
// attack wherein an attack causes a victim's client to hang due to worst-case
// behavior triggered while processing attacker crafted invalid transactions. A
// detailed description of the mitigated DoS attack can be found here:
// https://bitslog.wordpress.com/2013/01/23/fixed-bitcoin-vulnerability-explanation-why-the-signature-cache-is-a-dos-protection/.
// Secondly, usage of the SigCache introduces a signature verification
// optimization which speeds up the validation of transactions within a block,
// if they've already been seen and verified within the mempool.
type SigCache struct {
	sync.RWMutex
	validSigs  map[chainhash.Hash]sigCacheEntry
	maxEntries uint
}

// NewSigCache creates and initializes a new instance of SigCache. Its sole
// parameter 'maxEntries' represents the maximum number of entries allowed to
// exist in the SigCache at any particular moment. Random entries are evicted
// to make room for new entries that would cause the number of entries in the
// cache to exceed the max.
func NewSigCache(maxEntries uint) *SigCache {
	return &SigCache{
		validSigs:  make(map[chainhash.Hash]sigCacheEntry, maxEntries),
		maxEntries: maxEntries,
	}
}

// Exists returns true if an existing entry of 'sig' over 'sigHash' for public
// key 'pubKey' is found within the SigCache. Otherwise, false is returned.
//
// NOTE: This function is safe for concurrent access. Readers won't be blocked
// unless there exists a writer, adding an entry to the SigCache.
func (s *SigCache) Exists(sigHash chainhash.Hash, sig chainec.Signature, pubKey chainec.PublicKey) bool {
	s.RLock()
	entry, ok := s.validSigs[sigHash]
	s.RUnlock()

	return ok &&
		bytes.Equal(entry.pubKey.SerializeCompressed(),
			pubKey.SerializeCompressed()) &&
		bytes.Equal(entry.sig.Serialize(), sig.Serialize())
}

// Add adds an entry for a signature over 'sigHash' under public key 'pubKey'
// to the signature cache. In the event that the SigCache is 'full', an
// existing entry is randomly chosen to be evicted in order to make space for
// the new entry.
//
// NOTE: This function is safe for concurrent access. Writers will block
// simultaneous readers until function execution has concluded.
func (s *SigCache) Add(sigHash chainhash.Hash, sig chainec.Signature, pubKey chainec.PublicKey) {
	s.Lock()
	defer s.Unlock()

	if s.maxEntries == 0 {
		return
	}

	// If adding this new entry will put us over the max number of allowed
	// entries, then evict an entry.
	if uint(len(s.validSigs)+1) > s.maxEntries {
		// Remove a random entry from the map. Relying on the random
		// starting point of Go's map iteration. It's worth noting that
		// the random iteration starting point is not 100% guaranteed
		// by the spec, however most Go compilers support it.
		// Ultimately, the iteration order isn't important here because
		// in order to manipulate which items are evicted, an adversary
		// would need to be able to execute preimage attacks on the
		// hashing function in order to start eviction at a specific
		// entry.
		for sigEntry := range s.validSigs {
			delete(s.validSigs, sigEntry)
			break
		}
	}
	s.validSigs[sigHash] = sigCacheEntry{sig, pubKey}
}
<commit_msg>txscript: Use secp256k1 types in sig cache.<commit_after>// Copyright (c) 2015-2016 The btcsuite developers
// Copyright (c) 2016-2019 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package txscript

import (
	"bytes"
	"sync"

	"github.com/decred/dcrd/chaincfg/chainhash"
	"github.com/decred/dcrd/dcrec/secp256k1"
)

// sigCacheEntry represents an entry in the SigCache. Entries within the
// SigCache are keyed according to the sigHash of the signature. In the
// scenario of a cache-hit (according to the sigHash), an additional comparison
// of the signature, and public key will be executed in order to ensure a complete
// match. In the occasion that two sigHashes collide, the newer sigHash will
// simply overwrite the existing entry.
type sigCacheEntry struct {
	sig    *secp256k1.Signature
	pubKey *secp256k1.PublicKey
}

// SigCache implements an ECDSA signature verification cache with a randomized
// entry eviction policy. Only valid signatures will be added to the cache. The
// benefits of SigCache are two fold. Firstly, usage of SigCache mitigates a DoS
// attack wherein an attack causes a victim's client to hang due to worst-case
// behavior triggered while processing attacker crafted invalid transactions. A
// detailed description of the mitigated DoS attack can be found here:
// https://bitslog.wordpress.com/2013/01/23/fixed-bitcoin-vulnerability-explanation-why-the-signature-cache-is-a-dos-protection/.
// Secondly, usage of the SigCache introduces a signature verification
// optimization which speeds up the validation of transactions within a block,
// if they've already been seen and verified within the mempool.
type SigCache struct {
	sync.RWMutex
	validSigs  map[chainhash.Hash]sigCacheEntry
	maxEntries uint
}

// NewSigCache creates and initializes a new instance of SigCache. Its sole
// parameter 'maxEntries' represents the maximum number of entries allowed to
// exist in the SigCache at any particular moment. Random entries are evicted
// to make room for new entries that would cause the number of entries in the
// cache to exceed the max.
func NewSigCache(maxEntries uint) *SigCache {
	return &SigCache{
		validSigs:  make(map[chainhash.Hash]sigCacheEntry, maxEntries),
		maxEntries: maxEntries,
	}
}

// Exists returns true if an existing entry of 'sig' over 'sigHash' for public
// key 'pubKey' is found within the SigCache. Otherwise, false is returned.
//
// NOTE: This function is safe for concurrent access. Readers won't be blocked
// unless there exists a writer, adding an entry to the SigCache.
func (s *SigCache) Exists(sigHash chainhash.Hash, sig *secp256k1.Signature, pubKey *secp256k1.PublicKey) bool {
	s.RLock()
	entry, ok := s.validSigs[sigHash]
	s.RUnlock()

	return ok &&
		bytes.Equal(entry.pubKey.SerializeCompressed(),
			pubKey.SerializeCompressed()) &&
		bytes.Equal(entry.sig.Serialize(), sig.Serialize())
}

// Add adds an entry for a signature over 'sigHash' under public key 'pubKey'
// to the signature cache. In the event that the SigCache is 'full', an
// existing entry is randomly chosen to be evicted in order to make space for
// the new entry.
//
// NOTE: This function is safe for concurrent access. Writers will block
// simultaneous readers until function execution has concluded.
func (s *SigCache) Add(sigHash chainhash.Hash, sig *secp256k1.Signature, pubKey *secp256k1.PublicKey) {
	s.Lock()
	defer s.Unlock()

	if s.maxEntries == 0 {
		return
	}

	// If adding this new entry will put us over the max number of allowed
	// entries, then evict an entry.
	if uint(len(s.validSigs)+1) > s.maxEntries {
		// Remove a random entry from the map. Relying on the random
		// starting point of Go's map iteration. It's worth noting that
		// the random iteration starting point is not 100% guaranteed
		// by the spec, however most Go compilers support it.
		// Ultimately, the iteration order isn't important here because
		// in order to manipulate which items are evicted, an adversary
		// would need to be able to execute preimage attacks on the
		// hashing function in order to start eviction at a specific
		// entry.
		for sigEntry := range s.validSigs {
			delete(s.validSigs, sigEntry)
			break
		}
	}
	s.validSigs[sigHash] = sigCacheEntry{sig, pubKey}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package typed_test

import (
	"io/ioutil"
	"path/filepath"
	"testing"

	yaml "gopkg.in/yaml.v2"
	"sigs.k8s.io/structured-merge-diff/typed"
)

func testdata(file string) string {
	return filepath.Join("..", "internal", "testdata", file)
}

func BenchmarkFromUnstructured(b *testing.B) {
	pod, err := ioutil.ReadFile(testdata("pod.yaml"))
	if err != nil {
		b.Fatal(err)
	}

	s, err := ioutil.ReadFile(testdata("k8s-schema.yaml"))
	if err != nil {
		b.Fatal(err)
	}
	parser, err := typed.NewParser(typed.YAMLObject(s))
	if err != nil {
		b.Fatal(err)
	}
	pt := parser.Type("io.k8s.api.core.v1.Pod")

	obj := map[string]interface{}{}
	if err := yaml.Unmarshal([]byte(pod), &obj); err != nil {
		b.Fatal(err)
	}

	b.ReportAllocs()
	b.ResetTimer()
	for n := 0; n < b.N; n++ {
		if _, err := pt.FromUnstructured(obj); err != nil {
			b.Fatal(err)
		}
	}

}
<commit_msg>Benchmark typed.Parser on all available types<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package typed_test

import (
	"io/ioutil"
	"path/filepath"
	"strings"
	"testing"

	yaml "gopkg.in/yaml.v2"
	"sigs.k8s.io/structured-merge-diff/typed"
)

func testdata(file string) string {
	return filepath.Join("..", "internal", "testdata", file)
}

func read(file string) []byte {
	obj, err := ioutil.ReadFile(file)
	if err != nil {
		panic(err)
	}
	return obj
}

func lastPart(s string) string {
	return s[strings.LastIndex(s, ".")+1:]
}

func BenchmarkFromUnstructured(b *testing.B) {
	tests := []struct {
		typename string
		obj      []byte
	}{
		{
			typename: "io.k8s.api.core.v1.Pod",
			obj:      read(testdata("pod.yaml")),
		},
		{
			typename: "io.k8s.api.core.v1.Node",
			obj:      read(testdata("node.yaml")),
		},
		{
			typename: "io.k8s.api.core.v1.Endpoints",
			obj:      read(testdata("endpoints.yaml")),
		},
	}

	s, err := ioutil.ReadFile(testdata("k8s-schema.yaml"))
	if err != nil {
		b.Fatal(err)
	}
	parser, err := typed.NewParser(typed.YAMLObject(s))
	if err != nil {
		b.Fatal(err)
	}

	for _, test := range tests {
		b.Run(lastPart(test.typename), func(b *testing.B) {
			pt := parser.Type(test.typename)

			obj := map[string]interface{}{}
			if err := yaml.Unmarshal(test.obj, &obj); err != nil {
				b.Fatal(err)
			}

			b.ReportAllocs()
			b.ResetTimer()
			for n := 0; n < b.N; n++ {
				if _, err := pt.FromUnstructured(obj); err != nil {
					b.Fatal(err)
				}
			}
		})
	}

}
<|endoftext|>


<commit_before>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A command that consumes the output of a `comeback verify` run (--fast mode
// is okay), assuming that the roots were all backup jobs of interest. Any
// score that is in the bucket but not represented in the verify output is
// cloned to a garbage/ prefix in the bucket, and deleted from the blobs/
// prefix.

package main

import (
	"errors"
	"io"
	"log"
	"runtime"

	"github.com/jacobsa/comeback/blob"
	"golang.org/x/net/context"
)

var cmdGC = &Command{
	Name: "gc",
}

var fInput = cmdGC.Flags.String(
	"input",
	"",
	"Path to a file containing the output of a verify run.")

func init() {
	cmdGC.Run = runGC // Break flag-related dependency loop.
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// Parse the verify output, returning a list of all scores encountered.
func parseInput(
	r io.Reader) (scores []blob.Score, err error) {
	err = errors.New("TODO: parseInput")
	return
}

// Filter out scores that are in the list of non-garbage accessible scores,
// passing on only garbage.
func filterToGarbage(
	ctx context.Context,
	accessible []blob.Score,
	allScores <-chan blob.Score,
	garbageScores chan<- blob.Score) (err error) {
	err = errors.New("TODO: filterToGarbage")
	return
}

// Count the scores that pass through.
func countScores(
	ctx context.Context,
	in <-chan blob.Score,
	out chan<- blob.Score) (count uint64, err error) {
	err = errors.New("TODO: countScores")
	return
}

// Clone garbage blobs into a new location. Pass on the names of the source
// objects that were cloned.
func cloneGarbage(
	ctx context.Context,
	garbageScores <-chan blob.Score,
	garbageObjects chan<- string) (err error) {
	err = errors.New("TODO: cloneGarbage")
	return
}

// Delete all objects whose name come in on the supplied channel.
func deleteObjects(
	ctx context.Context,
	names <-chan string) (err error) {
	err = errors.New("TODO: deleteObjects")
	return
}

////////////////////////////////////////////////////////////////////////
// GC
////////////////////////////////////////////////////////////////////////

func runGC(args []string) {
	// Allow parallelism.
	runtime.GOMAXPROCS(runtime.NumCPU())

	// Die on error.
	var err error
	defer func() {
		if err != nil {
			log.Fatalln(err)
		}
	}()

	err = errors.New("TODO: runGC")
}
<commit_msg>runGC<commit_after>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A command that consumes the output of a `comeback verify` run (--fast mode
// is okay), assuming that the roots were all backup jobs of interest. Any
// score that is in the bucket but not represented in the verify output is
// cloned to a garbage/ prefix in the bucket, and deleted from the blobs/
// prefix.

package main

import (
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"runtime"

	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/wiring"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
)

var cmdGC = &Command{
	Name: "gc",
}

var fInput = cmdGC.Flags.String(
	"input",
	"",
	"Path to a file containing the output of a verify run.")

func init() {
	cmdGC.Run = runGC // Break flag-related dependency loop.
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// Parse the verify output, returning a list of all scores encountered.
func parseInput(
	r io.Reader) (scores []blob.Score, err error) {
	err = errors.New("TODO: parseInput")
	return
}

// Filter out scores that are in the list of non-garbage accessible scores,
// passing on only garbage.
func filterToGarbage(
	ctx context.Context,
	accessible []blob.Score,
	allScores <-chan blob.Score,
	garbageScores chan<- blob.Score) (err error) {
	err = errors.New("TODO: filterToGarbage")
	return
}

// Count the scores that pass through.
func countScores(
	ctx context.Context,
	in <-chan blob.Score,
	out chan<- blob.Score) (count uint64, err error) {
	err = errors.New("TODO: countScores")
	return
}

// Clone garbage blobs into a new location. Pass on the names of the source
// objects that were cloned.
func cloneGarbage(
	ctx context.Context,
	bucket gcs.Bucket,
	garbageScores <-chan blob.Score,
	garbageObjects chan<- string) (err error) {
	err = errors.New("TODO: cloneGarbage")
	return
}

// Delete all objects whose name come in on the supplied channel.
func deleteObjects(
	ctx context.Context,
	bucket gcs.Bucket,
	names <-chan string) (err error) {
	err = errors.New("TODO: deleteObjects")
	return
}

////////////////////////////////////////////////////////////////////////
// GC
////////////////////////////////////////////////////////////////////////

func runGC(args []string) {
	// Allow parallelism.
	runtime.GOMAXPROCS(runtime.NumCPU())

	// Die on error.
	var err error
	defer func() {
		if err != nil {
			log.Fatalln(err)
		}
	}()

	// Grab dependencies.
	bucket := getBucket()

	// Open the input file.
	if *fInput == "" {
		err = fmt.Errorf("You must set --input.")
		return
	}

	inputFile, err := os.Open(*fInput)
	if err != nil {
		err = fmt.Errorf("Open: %v", err)
		return
	}

	// Parse it.
	accessibleScores, err := parseInput(inputFile)
	inputFile.Close()

	if err != nil {
		err = fmt.Errorf("parseInput: %v", err)
		return
	}

	b := syncutil.NewBundle(context.Background())

	// List all extant scores into a channel.
	allScores := make(chan blob.Score, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(allScores)
		err = blob.ListScores(ctx, bucket, wiring.BlobObjectNamePrefix, allScores)
		if err != nil {
			err = fmt.Errorf("ListScores: %v", err)
			return
		}

		return
	})

	// Count the total number of scores.
	var allScoresCount uint64
	allScoresAfterCounting := make(chan blob.Score, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(allScoresAfterCounting)
		allScoresCount, err = countScores(
			ctx,
			allScores,
			allScoresAfterCounting)

		if err != nil {
			err = fmt.Errorf("countScores: %v", err)
			return
		}

		return
	})

	// Filter to garbage scores.
	garbageScores := make(chan blob.Score, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(garbageScores)
		err = filterToGarbage(
			ctx,
			accessibleScores,
			allScoresAfterCounting,
			garbageScores)

		if err != nil {
			err = fmt.Errorf("filterToGarbage: %v", err)
			return
		}

		return
	})

	// Count the number of garbage scores.
	var garbageScoresCount uint64
	garbageScoresAfterCounting := make(chan blob.Score, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(garbageScoresAfterCounting)
		garbageScoresCount, err = countScores(
			ctx,
			garbageScores,
			garbageScoresAfterCounting)

		if err != nil {
			err = fmt.Errorf("countScores: %v", err)
			return
		}

		return
	})

	// Clone garbage blobs into a backup location.
	toDelete := make(chan string, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(toDelete)
		err = cloneGarbage(
			ctx,
			bucket,
			garbageScoresAfterCounting,
			toDelete)
		if err != nil {
			err = fmt.Errorf("cloneGarbage: %v", err)
			return
		}

		return
	})

	// Delete the original objects.
	b.Add(func(ctx context.Context) (err error) {
		err = deleteObjects(ctx, bucket, toDelete)
		if err != nil {
			err = fmt.Errorf("deleteObjects: %v", err)
			return
		}

		return
	})

	err = b.Join()
	if err != nil {
		return
	}

	// Print a summary.
	log.Printf(
		"Deleted %d objects out of %d total.",
		garbageScoresCount,
		allScoresCount)
}
<|endoftext|>