<commit_before>package main_test

import (
	"encoding/json"
	"fmt"
	"net"
	"net/url"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"testing"
	"time"

	"code.cloudfoundry.org/lager/lagertest"
	"code.cloudfoundry.org/route-emitter/diegonats"
	"code.cloudfoundry.org/route-emitter/diegonats/gnatsdrunner"
	"github.com/cloudfoundry/sonde-go/events"
	"github.com/gogo/protobuf/proto"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"

	"code.cloudfoundry.org/bbs"
	bbsconfig "code.cloudfoundry.org/bbs/cmd/bbs/config"
	bbstestrunner "code.cloudfoundry.org/bbs/cmd/bbs/testrunner"
	"code.cloudfoundry.org/bbs/encryption"
	"code.cloudfoundry.org/bbs/test_helpers"
	"code.cloudfoundry.org/bbs/test_helpers/sqlrunner"
	"code.cloudfoundry.org/consuladapter/consulrunner"
)

const heartbeatInterval = 1 * time.Second

var (
	emitterPath   string
	natsPort      int
	dropsondePort int

	bbsPath    string
	bbsURL     *url.URL
	bbsConfig  bbsconfig.BBSConfig
	bbsRunner  *ginkgomon.Runner
	bbsProcess ifrit.Process

	consulRunner         *consulrunner.ClusterRunner
	gnatsdRunner         ifrit.Process
	natsClient           diegonats.NATSClient
	bbsClient            bbs.InternalClient
	logger               *lagertest.TestLogger
	syncInterval         time.Duration
	consulClusterAddress string
	testMetricsListener  net.PacketConn
	testMetricsChan      chan *events.Envelope

	sqlProcess ifrit.Process
	sqlRunner  sqlrunner.SQLRunner
	bbsRunning = false
)

func TestRouteEmitter(t *testing.T) {
	RegisterFailHandler(Fail)
	SetDefaultEventuallyTimeout(2 * time.Second)
	RunSpecs(t, "Route Emitter Suite")
}

func createEmitterRunner(sessionName string, extraArgs ...string) *ginkgomon.Runner {
	args := []string{"-sessionName", sessionName,
		"-dropsondePort", strconv.Itoa(dropsondePort),
		"-natsAddresses", fmt.Sprintf("127.0.0.1:%d", natsPort),
		"-bbsAddress", bbsURL.String(),
		"-communicationTimeout", "100ms",
		"-syncInterval", syncInterval.String(),
		"-lockRetryInterval", "1s",
		"-lockTTL", "5s",
		"-consulCluster", consulClusterAddress,
	}
	args = append(args, extraArgs...)

	return ginkgomon.New(ginkgomon.Config{
		Command: exec.Command(
			string(emitterPath),
			args...,
		),

		StartCheck: "route-emitter.watcher.sync.complete",

		AnsiColorCode: "97m",
	})
}

var _ = SynchronizedBeforeSuite(func() []byte {
	emitter, err := gexec.Build("code.cloudfoundry.org/route-emitter/cmd/route-emitter", "-race")
	Expect(err).NotTo(HaveOccurred())

	bbs, err := gexec.Build("code.cloudfoundry.org/bbs/cmd/bbs", "-race")
	Expect(err).NotTo(HaveOccurred())

	payload, err := json.Marshal(map[string]string{
		"emitter": emitter,
		"bbs":     bbs,
	})

	Expect(err).NotTo(HaveOccurred())

	return payload
}, func(payload []byte) {
	binaries := map[string]string{}

	err := json.Unmarshal(payload, &binaries)
	Expect(err).NotTo(HaveOccurred())

	natsPort = 4001 + GinkgoParallelNode()

	emitterPath = string(binaries["emitter"])

	dbName := fmt.Sprintf("diego_%d", GinkgoParallelNode())
	sqlRunner = test_helpers.NewSQLRunner(dbName)

	consulRunner = consulrunner.NewClusterRunner(
		9001+config.GinkgoConfig.ParallelNode*consulrunner.PortOffsetLength,
		1,
		"http",
	)

	logger = lagertest.NewTestLogger("test")

	syncInterval = 200 * time.Millisecond

	bbsPath = string(binaries["bbs"])
	bbsPort := 13000 + GinkgoParallelNode()*2
	healthPort := bbsPort + 1
	bbsAddress := fmt.Sprintf("127.0.0.1:%d", bbsPort)
	healthAddress := fmt.Sprintf("127.0.0.1:%d", healthPort)

	bbsURL = &url.URL{
		Scheme: "http",
		Host:   bbsAddress,
	}

	bbsClient = bbs.NewClient(bbsURL.String())

	bbsConfig = bbsconfig.BBSConfig{
		ListenAddress:            bbsAddress,
		AdvertiseURL:             bbsURL.String(),
		AuctioneerAddress:        "some-address",
		DatabaseDriver:           sqlRunner.DriverName(),
		DatabaseConnectionString: sqlRunner.ConnectionString(),
		ConsulCluster:            consulRunner.ConsulCluster(),
		HealthAddress:            healthAddress,

		EncryptionConfig: encryption.EncryptionConfig{
			EncryptionKeys: map[string]string{"label": "key"},
			ActiveKeyLabel: "label",
		},
	}
})

var _ = BeforeEach(func() {
	consulRunner.Start()
	consulRunner.WaitUntilReady()
	consulClusterAddress = consulRunner.ConsulCluster()

	sqlProcess = ginkgomon.Invoke(sqlRunner)

	startBBS()

	gnatsdRunner, natsClient = gnatsdrunner.StartGnatsd(natsPort)

	testMetricsListener, _ = net.ListenPacket("udp", "127.0.0.1:0")
	testMetricsChan = make(chan *events.Envelope, 1)
	go func() {
		defer GinkgoRecover()
		for {
			buffer := make([]byte, 1024)
			n, _, err := testMetricsListener.ReadFrom(buffer)
			if err != nil {
				close(testMetricsChan)
				return
			}

			var envelope events.Envelope
			err = proto.Unmarshal(buffer[:n], &envelope)
			Expect(err).NotTo(HaveOccurred())
			testMetricsChan <- &envelope
		}
	}()

	var err error
	dropsondePort, err = strconv.Atoi(strings.TrimPrefix(testMetricsListener.LocalAddr().String(), "127.0.0.1:"))
	Expect(err).NotTo(HaveOccurred())
})

var _ = AfterEach(func() {
	stopBBS()
	consulRunner.Stop()
	gnatsdRunner.Signal(os.Interrupt)
	Eventually(gnatsdRunner.Wait(), 5).Should(Receive())

	testMetricsListener.Close()
	Eventually(testMetricsChan).Should(BeClosed())

	ginkgomon.Kill(sqlProcess)
})

func stopBBS() {
	if !bbsRunning {
		return
	}

	bbsRunning = false
	ginkgomon.Interrupt(bbsProcess)
}

func startBBS() {
	if bbsRunning {
		return
	}

	bbsRunner = bbstestrunner.New(bbsPath, bbsConfig)
	bbsProcess = ginkgomon.Invoke(bbsRunner)
	bbsRunning = true
}

var _ = SynchronizedAfterSuite(func() {
}, func() {
	gexec.CleanupBuildArtifacts()
})
<commit_msg>pass ClusterRunnerConfig to NewClusterRunner<commit_after>package main_test

import (
	"encoding/json"
	"fmt"
	"net"
	"net/url"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"testing"
	"time"

	"code.cloudfoundry.org/lager/lagertest"
	"code.cloudfoundry.org/route-emitter/diegonats"
	"code.cloudfoundry.org/route-emitter/diegonats/gnatsdrunner"
	"github.com/cloudfoundry/sonde-go/events"
	"github.com/gogo/protobuf/proto"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"

	"code.cloudfoundry.org/bbs"
	bbsconfig "code.cloudfoundry.org/bbs/cmd/bbs/config"
	bbstestrunner "code.cloudfoundry.org/bbs/cmd/bbs/testrunner"
	"code.cloudfoundry.org/bbs/encryption"
	"code.cloudfoundry.org/bbs/test_helpers"
	"code.cloudfoundry.org/bbs/test_helpers/sqlrunner"
	"code.cloudfoundry.org/consuladapter/consulrunner"
)

const heartbeatInterval = 1 * time.Second

var (
	emitterPath   string
	natsPort      int
	dropsondePort int

	bbsPath    string
	bbsURL     *url.URL
	bbsConfig  bbsconfig.BBSConfig
	bbsRunner  *ginkgomon.Runner
	bbsProcess ifrit.Process

	consulRunner         *consulrunner.ClusterRunner
	gnatsdRunner         ifrit.Process
	natsClient           diegonats.NATSClient
	bbsClient            bbs.InternalClient
	logger               *lagertest.TestLogger
	syncInterval         time.Duration
	consulClusterAddress string
	testMetricsListener  net.PacketConn
	testMetricsChan      chan *events.Envelope

	sqlProcess ifrit.Process
	sqlRunner  sqlrunner.SQLRunner
	bbsRunning = false
)

func TestRouteEmitter(t *testing.T) {
	RegisterFailHandler(Fail)
	SetDefaultEventuallyTimeout(2 * time.Second)
	RunSpecs(t, "Route Emitter Suite")
}

func createEmitterRunner(sessionName string, extraArgs ...string) *ginkgomon.Runner {
	args := []string{"-sessionName", sessionName,
		"-dropsondePort", strconv.Itoa(dropsondePort),
		"-natsAddresses", fmt.Sprintf("127.0.0.1:%d", natsPort),
		"-bbsAddress", bbsURL.String(),
		"-communicationTimeout", "100ms",
		"-syncInterval", syncInterval.String(),
		"-lockRetryInterval", "1s",
		"-lockTTL", "5s",
		"-consulCluster", consulClusterAddress,
	}
	args = append(args, extraArgs...)

	return ginkgomon.New(ginkgomon.Config{
		Command: exec.Command(
			string(emitterPath),
			args...,
		),

		StartCheck: "route-emitter.watcher.sync.complete",

		AnsiColorCode: "97m",
	})
}

var _ = SynchronizedBeforeSuite(func() []byte {
	emitter, err := gexec.Build("code.cloudfoundry.org/route-emitter/cmd/route-emitter", "-race")
	Expect(err).NotTo(HaveOccurred())

	bbs, err := gexec.Build("code.cloudfoundry.org/bbs/cmd/bbs", "-race")
	Expect(err).NotTo(HaveOccurred())

	payload, err := json.Marshal(map[string]string{
		"emitter": emitter,
		"bbs":     bbs,
	})

	Expect(err).NotTo(HaveOccurred())

	return payload
}, func(payload []byte) {
	binaries := map[string]string{}

	err := json.Unmarshal(payload, &binaries)
	Expect(err).NotTo(HaveOccurred())

	natsPort = 4001 + GinkgoParallelNode()

	emitterPath = string(binaries["emitter"])

	dbName := fmt.Sprintf("diego_%d", GinkgoParallelNode())
	sqlRunner = test_helpers.NewSQLRunner(dbName)

	consulRunner = consulrunner.NewClusterRunner(
		consulrunner.ClusterRunnerConfig{
			StartingPort: 9001 + config.GinkgoConfig.ParallelNode*consulrunner.PortOffsetLength,
			NumNodes:     1,
			Scheme:       "http",
		},
	)

	logger = lagertest.NewTestLogger("test")

	syncInterval = 200 * time.Millisecond

	bbsPath = string(binaries["bbs"])
	bbsPort := 13000 + GinkgoParallelNode()*2
	healthPort := bbsPort + 1
	bbsAddress := fmt.Sprintf("127.0.0.1:%d", bbsPort)
	healthAddress := fmt.Sprintf("127.0.0.1:%d", healthPort)

	bbsURL = &url.URL{
		Scheme: "http",
		Host:   bbsAddress,
	}

	bbsClient = bbs.NewClient(bbsURL.String())

	bbsConfig = bbsconfig.BBSConfig{
		ListenAddress:            bbsAddress,
		AdvertiseURL:             bbsURL.String(),
		AuctioneerAddress:        "some-address",
		DatabaseDriver:           sqlRunner.DriverName(),
		DatabaseConnectionString: sqlRunner.ConnectionString(),
		ConsulCluster:            consulRunner.ConsulCluster(),
		HealthAddress:            healthAddress,

		EncryptionConfig: encryption.EncryptionConfig{
			EncryptionKeys: map[string]string{"label": "key"},
			ActiveKeyLabel: "label",
		},
	}
})

var _ = BeforeEach(func() {
	consulRunner.Start()
	consulRunner.WaitUntilReady()
	consulClusterAddress = consulRunner.ConsulCluster()

	sqlProcess = ginkgomon.Invoke(sqlRunner)

	startBBS()

	gnatsdRunner, natsClient = gnatsdrunner.StartGnatsd(natsPort)

	testMetricsListener, _ = net.ListenPacket("udp", "127.0.0.1:0")
	testMetricsChan = make(chan *events.Envelope, 1)
	go func() {
		defer GinkgoRecover()
		for {
			buffer := make([]byte, 1024)
			n, _, err := testMetricsListener.ReadFrom(buffer)
			if err != nil {
				close(testMetricsChan)
				return
			}

			var envelope events.Envelope
			err = proto.Unmarshal(buffer[:n], &envelope)
			Expect(err).NotTo(HaveOccurred())
			testMetricsChan <- &envelope
		}
	}()

	var err error
	dropsondePort, err = strconv.Atoi(strings.TrimPrefix(testMetricsListener.LocalAddr().String(), "127.0.0.1:"))
	Expect(err).NotTo(HaveOccurred())
})

var _ = AfterEach(func() {
	stopBBS()
	consulRunner.Stop()
	gnatsdRunner.Signal(os.Interrupt)
	Eventually(gnatsdRunner.Wait(), 5).Should(Receive())

	testMetricsListener.Close()
	Eventually(testMetricsChan).Should(BeClosed())

	ginkgomon.Kill(sqlProcess)
})

func stopBBS() {
	if !bbsRunning {
		return
	}

	bbsRunning = false
	ginkgomon.Interrupt(bbsProcess)
}

func startBBS() {
	if bbsRunning {
		return
	}

	bbsRunner = bbstestrunner.New(bbsPath, bbsConfig)
	bbsProcess = ginkgomon.Invoke(bbsRunner)
	bbsRunning = true
}

var _ = SynchronizedAfterSuite(func() {
}, func() {
	gexec.CleanupBuildArtifacts()
})
<|endoftext|>


<commit_before>package partners

import (
	"fmt"
	"log"
	"net/url"
	"time"

	"github.com/brnstz/bus/internal/conf"
	"github.com/brnstz/bus/internal/etc"
	"github.com/brnstz/bus/internal/models"

	"github.com/brnstz/bus/internal/partners/nyct_subway"
	"github.com/brnstz/bus/internal/partners/transit_realtime"

	"github.com/golang/protobuf/proto"
)

var (
	esiURL = "http://datamine.mta.info/mta_esi.php"

	mtaSubwayRouteToFeed = map[string]string{
		"1":  "1",
		"2":  "1",
		"3":  "1",
		"4":  "1",
		"5":  "1",
		"6":  "1",
		"6X": "1",
		"S":  "1",
		"GS": "1",
		"L":  "2",
		"SI": "11",
	}

	// mapping from feed routeIDs to actual routeIDs
	expressMatch = map[string]string{
		// The feed uses 6 to mean 6X. We need to dig deeper to
		// find out that it's express.
		"6": "6X",
	}
)

type mtaNYCSubway struct{}

// getURL returns the URL for getting this routeID's feed. Second return
// value is false if there is no feed to get.
func (p mtaNYCSubway) getURL(routeID string) (string, bool) {
	var u string

	// Get the feed for this route, if there is one. Otherwise, nothing
	// to return.
	feed, exists := mtaSubwayRouteToFeed[routeID]
	if !exists {
		return "", false
	}

	// Construct URL and call external API, possibly getting cached
	// value.
	q := url.Values{}
	q.Set("key", conf.Partner.DatamineAPIKey)
	q.Set("feed_id", feed)
	u = fmt.Sprint(esiURL, "?", q.Encode())

	return u, true
}

func (p mtaNYCSubway) Precache(agencyID, routeID string, directionID int) error {
	k := fmt.Sprintf("%v|%v|%v", agencyID, routeID, directionID)

	u, exists := p.getURL(routeID)
	if !exists {
		return nil
	}

	// Since the URL we call is the same no matter which direction, arbirarily
	// decide to ignore one of the directions.
	if directionID == 1 {
		return nil
	}

	// Also feed ID "1" applies to multiple routes. We only need to cache one.
	// Arbitrarily choose the "1" route.
	if mtaSubwayRouteToFeed[routeID] == "1" && routeID != "1" {
		return nil
	}

	_, err := etc.RedisCacheURL(u)
	if err != nil {
		log.Println("can't cache live subway response", err)
		return err
	}

	// attempt to parse response to ensure it is valid
	_, _, err = p.Live(agencyID, routeID, "", directionID)
	if err != nil {
		log.Println("can't parse response", err)
		return err
	}

	log.Println("succesfully saved", k)

	return nil
}

// matchingRoutes determines if a route from the feed is equivalent to our
// routeID
func (p mtaNYCSubway) matchingRoute(feedRouteID, routeID string) bool {
	// The obvious case
	if feedRouteID == routeID {
		return true
	}

	if expressMatch[feedRouteID] == routeID {
		return true
	}

	return false
}

func (p mtaNYCSubway) Live(agencyID, routeID, stopID string, directionID int) (d []*models.Departure, v []models.Vehicle, err error) {
	now := time.Now()

	u, exists := p.getURL(routeID)
	if !exists {
		return
	}

	b, err := etc.RedisGet(u)
	if err != nil {
		log.Println("can't get live subways", err)
		return
	}

	// Load the protobuf struct
	tr := &transit_realtime.FeedMessage{}
	err = proto.Unmarshal(b, tr)
	if err != nil {
		log.Println("can't unmarshal", err)
		return
	}

	// Look at each message in the feed
	for _, e := range tr.Entity {

		// Get some updates
		tripUpdate := e.GetTripUpdate()
		trip := tripUpdate.GetTrip()
		stopTimeUpdates := tripUpdate.GetStopTimeUpdate()

		if !p.matchingRoute(trip.GetRouteId(), routeID) {
			continue
		}

		// If we have at least one stopTimeUpdate and the trip is non-nil,
		// we can get the NYCT extensions.
		if len(stopTimeUpdates) > 0 && trip != nil {
			var event interface{}

			// Get the NYC extension so we can see if the Trip is "assigned"
			// yet. If it's assigned, we'll put the vehicle on the map.
			event, err = proto.GetExtension(
				trip, nyct_subway.E_NyctTripDescriptor,
			)
			if err != nil {
				log.Println("can't get extension", err)
				return
			}
			nycTrip, ok := event.(*nyct_subway.NyctTripDescriptor)
			if !ok {
				log.Println("can't coerce to nyct_subway.NyctTripDescriptor")
				return
			}

			// The first update in an entity is the stop where the train will
			// next be. Include only "assigned" trips, which are those that
			// are about to start.
			if nycTrip.GetIsAssigned() {
				var vehicle models.Vehicle

				// Get a "vehicle" with the lat/lon of the update's stop
				// (*not* the stop of our request)
				vehicle, err = models.GetVehicle(
					agencyID, routeID,
					stopTimeUpdates[0].GetStopId(),
					directionID,
				)
				if err != nil {
					log.Println("can't get vehicle", err)

				} else {
					vehicle.Live = true
					v = append(v, vehicle)
				}
			}
		}

		// Go through all updates to check for our stop ID's departure time.
		for _, u := range stopTimeUpdates {

			// If this is our stop, then get the departure time.
			if u.GetStopId() == stopID {
				dtime := time.Unix(u.GetDeparture().GetTime(), 0)
				if dtime.After(now) {
					d = append(d,
						&models.Departure{
							Time:   dtime,
							TripID: trip.GetTripId(),
							Live:   true,
						},
					)
				}
			}
		}
	}

	return
}
<commit_msg>fixing 6X issue<commit_after>package partners

import (
	"fmt"
	"log"
	"net/url"
	"time"

	"github.com/brnstz/bus/internal/conf"
	"github.com/brnstz/bus/internal/etc"
	"github.com/brnstz/bus/internal/models"

	"github.com/brnstz/bus/internal/partners/nyct_subway"
	"github.com/brnstz/bus/internal/partners/transit_realtime"

	"github.com/golang/protobuf/proto"
)

var (
	esiURL = "http://datamine.mta.info/mta_esi.php"

	mtaSubwayRouteToFeed = map[string]string{
		"1":  "1",
		"2":  "1",
		"3":  "1",
		"4":  "1",
		"5":  "1",
		"6":  "1",
		"6X": "1",
		"S":  "1",
		"GS": "1",
		"L":  "2",
		"SI": "11",
	}
)

type mtaNYCSubway struct{}

// getURL returns the URL for getting this routeID's feed. Second return
// value is false if there is no feed to get.
func (p mtaNYCSubway) getURL(routeID string) (string, bool) {
	var u string

	// Get the feed for this route, if there is one. Otherwise, nothing
	// to return.
	feed, exists := mtaSubwayRouteToFeed[routeID]
	if !exists {
		return "", false
	}

	// Construct URL and call external API, possibly getting cached
	// value.
	q := url.Values{}
	q.Set("key", conf.Partner.DatamineAPIKey)
	q.Set("feed_id", feed)
	u = fmt.Sprint(esiURL, "?", q.Encode())

	return u, true
}

func (p mtaNYCSubway) Precache(agencyID, routeID string, directionID int) error {
	k := fmt.Sprintf("%v|%v|%v", agencyID, routeID, directionID)

	u, exists := p.getURL(routeID)
	if !exists {
		return nil
	}

	// Since the URL we call is the same no matter which direction, arbirarily
	// decide to ignore one of the directions.
	if directionID == 1 {
		return nil
	}

	// Also feed ID "1" applies to multiple routes. We only need to cache one.
	// Arbitrarily choose the "1" route.
	if mtaSubwayRouteToFeed[routeID] == "1" && routeID != "1" {
		return nil
	}

	_, err := etc.RedisCacheURL(u)
	if err != nil {
		log.Println("can't cache live subway response", err)
		return err
	}

	// attempt to parse response to ensure it is valid
	_, _, err = p.Live(agencyID, routeID, "", directionID)
	if err != nil {
		log.Println("can't parse response", err)
		return err
	}

	log.Println("succesfully saved", k)

	return nil
}

func (p mtaNYCSubway) Live(agencyID, routeID, stopID string, directionID int) (d []*models.Departure, v []models.Vehicle, err error) {
	now := time.Now()

	u, exists := p.getURL(routeID)
	if !exists {
		return
	}

	b, err := etc.RedisGet(u)
	if err != nil {
		log.Println("can't get live subways", err)
		return
	}

	// Load the protobuf struct
	tr := &transit_realtime.FeedMessage{}
	err = proto.Unmarshal(b, tr)
	if err != nil {
		log.Println("can't unmarshal", err)
		return
	}

	// Look at each message in the feed
	for _, e := range tr.Entity {

		// Get some updates
		tripUpdate := e.GetTripUpdate()
		trip := tripUpdate.GetTrip()
		stopTimeUpdates := tripUpdate.GetStopTimeUpdate()

		// Ensure we have at least one stop time update
		if len(stopTimeUpdates) < 1 {
			continue
		}

		// Check the first stop time update to see if it's express or not
		var updateEvent interface{}
		firstUpdate := stopTimeUpdates[0]

		// Get the NYC extension so we can see if the Trip is "assigned"
		// yet. If it's assigned, we'll put the vehicle on the map.
		updateEvent, err = proto.GetExtension(
			firstUpdate, nyct_subway.E_NyctStopTimeUpdate,
		)
		if err != nil {
			log.Println("can't get extension", err)
			return
		}
		nycEvent, ok := updateEvent.(*nyct_subway.NyctStopTimeUpdate)
		if !ok {
			log.Println("can't coerce to nyct_subway.NyctStopTimeUpdate")
			return
		}

		var feedRouteID string

		switch nycEvent.GetScheduledTrack() {

		case "2", "3", "M":
			// Express track. Special case for 6X.
			if trip.GetRouteId() == "6" {
				feedRouteID = trip.GetRouteId() + "X"
			} else {
				feedRouteID = trip.GetRouteId()
			}
		default:
			// not express track
			feedRouteID = trip.GetRouteId()
		}

		if feedRouteID != routeID {
			continue
		}

		// If we have at least one stopTimeUpdate (already checked before) and
		// the trip is non-nil, we can get the NYCT extensions.
		if trip != nil {
			var event interface{}

			// Get the NYC extension so we can see if the Trip is "assigned"
			// yet. If it's assigned, we'll put the vehicle on the map.
			event, err = proto.GetExtension(
				trip, nyct_subway.E_NyctTripDescriptor,
			)
			if err != nil {
				log.Println("can't get extension", err)
				return
			}
			nycTrip, ok := event.(*nyct_subway.NyctTripDescriptor)
			if !ok {
				log.Println("can't coerce to nyct_subway.NyctTripDescriptor")
				return
			}

			// The first update in an entity is the stop where the train will
			// next be. Include only "assigned" trips, which are those that
			// are about to start.
			if nycTrip.GetIsAssigned() {
				var vehicle models.Vehicle

				// Get a "vehicle" with the lat/lon of the update's stop
				// (*not* the stop of our request)
				vehicle, err = models.GetVehicle(
					agencyID, routeID,
					stopTimeUpdates[0].GetStopId(),
					directionID,
				)
				if err != nil {
					// FIXME: why are these showing up again?
					//log.Println("can't get vehicle", err)

				} else {
					vehicle.Live = true
					v = append(v, vehicle)
				}
			}
		}

		// Go through all updates to check for our stop ID's departure time.
		for _, u := range stopTimeUpdates {
			// If this is our stop, then get the departure time.
			if u.GetStopId() == stopID {
				dtime := time.Unix(u.GetDeparture().GetTime(), 0)
				if dtime.After(now) {
					d = append(d,
						&models.Departure{
							Time:   dtime,
							TripID: trip.GetTripId(),
							Live:   true,
						},
					)
				}
			}
		}
	}

	return
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package windows

import (
	"context"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/kubernetes/test/e2e/framework"
	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
	testutils "k8s.io/kubernetes/test/utils"
	imageutils "k8s.io/kubernetes/test/utils/image"

	"github.com/onsi/ginkgo"
)

const runAsUserNameContainerName = "run-as-username-container"

var _ = SIGDescribe("[Feature:Windows] SecurityContext", func() {
	f := framework.NewDefaultFramework("windows-run-as-username")

	ginkgo.It("should be able create pods and run containers with a given username", func() {
		ginkgo.By("Creating 2 pods: 1 with the default user, and one with a custom one.")
		podDefault := runAsUserNamePod(nil)
		f.TestContainerOutput("check default user", podDefault, 0, []string{"ContainerUser"})

		podUserName := runAsUserNamePod(toPtr("ContainerAdministrator"))
		f.TestContainerOutput("check set user", podUserName, 0, []string{"ContainerAdministrator"})
	})

	ginkgo.It("should not be able to create pods with unknown usernames", func() {
		ginkgo.By("Creating a pod with an invalid username")
		podInvalid := f.PodClient().Create(runAsUserNamePod(toPtr("FooLish")))

		framework.Logf("Waiting for pod %s to enter the error state.", podInvalid.Name)
		framework.ExpectNoError(e2epod.WaitForPodTerminatedInNamespace(f.ClientSet, podInvalid.Name, "", f.Namespace.Name))

		podInvalid, _ = f.PodClient().Get(context.TODO(), podInvalid.Name, metav1.GetOptions{})
		podTerminatedReason := testutils.TerminatedContainers(podInvalid)[runAsUserNameContainerName]
		if podTerminatedReason != "ContainerCannotRun" && podTerminatedReason != "StartError" {
			framework.Failf("The container terminated reason was supposed to be: 'ContainerCannotRun' or 'StartError', not: '%q'", podTerminatedReason)
		}
	})

	ginkgo.It("should override SecurityContext username if set", func() {
		ginkgo.By("Creating a pod with 2 containers with different username configurations.")

		pod := runAsUserNamePod(toPtr("ContainerAdministrator"))
		pod.Spec.Containers[0].SecurityContext.WindowsOptions.RunAsUserName = toPtr("ContainerUser")
		pod.Spec.Containers = append(pod.Spec.Containers, v1.Container{
			Name:    "run-as-username-new-container",
			Image:   imageutils.GetE2EImage(imageutils.NonRoot),
			Command: []string{"cmd", "/S", "/C", "echo %username%"},
		})

		f.TestContainerOutput("check overridden username", pod, 0, []string{"ContainerUser"})
		f.TestContainerOutput("check pod SecurityContext username", pod, 1, []string{"ContainerAdministrator"})
	})
	ginkgo.It("should ignore Linux Specific SecurityContext if set", func() {
		ginkgo.By("Creating a pod with SELinux options")
		// It is sufficient to show that the pod comes up here. Since we're stripping the SELinux and other linux
		// security contexts in apiserver and not updating the pod object in the apiserver, we cannot validate the
		// the pod object to not have those security contexts. However the pod coming to running state is a sufficient
		// enough condition for us to validate since prior to https://github.com/kubernetes/kubernetes/pull/93475
		// the pod would have failed to come up.
		windowsPodWithSELinux := createTestPod(f, windowsBusyBoximage, windowsOS)
		windowsPodWithSELinux.Spec.Containers[0].Args = []string{"test-webserver-with-selinux"}
		windowsPodWithSELinux.Spec.SecurityContext = &v1.PodSecurityContext{}
		containerUserName := "ContainerAdministrator"
		windowsPodWithSELinux.Spec.SecurityContext.SELinuxOptions = &v1.SELinuxOptions{Level: "s0:c24,c9"}
		windowsPodWithSELinux.Spec.Containers[0].SecurityContext = &v1.SecurityContext{
			SELinuxOptions: &v1.SELinuxOptions{Level: "s0:c24,c9"},
			WindowsOptions: &v1.WindowsSecurityContextOptions{RunAsUserName: &containerUserName}}
		windowsPodWithSELinux.Spec.Tolerations = []v1.Toleration{{Key: "os", Value: "Windows"}}
		windowsPodWithSELinux, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(),
			windowsPodWithSELinux, metav1.CreateOptions{})
		framework.ExpectNoError(err)
		framework.Logf("Created pod %v", windowsPodWithSELinux)
		framework.ExpectNoError(e2epod.WaitForPodNameRunningInNamespace(f.ClientSet, windowsPodWithSELinux.Name,
			f.Namespace.Name), "failed to wait for pod %s to be running", windowsPodWithSELinux.Name)
	})
})

func runAsUserNamePod(username *string) *v1.Pod {
	podName := "run-as-username-" + string(uuid.NewUUID())
	return &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: podName,
		},
		Spec: v1.PodSpec{
			NodeSelector: map[string]string{"kubernetes.io/os": "windows"},
			Containers: []v1.Container{
				{
					Name:    runAsUserNameContainerName,
					Image:   imageutils.GetE2EImage(imageutils.NonRoot),
					Command: []string{"cmd", "/S", "/C", "echo %username%"},
					SecurityContext: &v1.SecurityContext{
						WindowsOptions: &v1.WindowsSecurityContextOptions{
							RunAsUserName: username,
						},
					},
				},
			},
			SecurityContext: &v1.PodSecurityContext{
				WindowsOptions: &v1.WindowsSecurityContextOptions{
					RunAsUserName: username,
				},
			},
			RestartPolicy: v1.RestartPolicyNever,
		},
	}
}

func toPtr(s string) *string {
	return &s
}
<commit_msg>[windows] Test: Check for failed sandbox pod when testing for RunAsUserName (#105943)<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package windows

import (
	"context"
	"fmt"
	"strings"
	"time"

	"github.com/onsi/ginkgo"
	"github.com/onsi/gomega"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/util/uuid"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/kubernetes/pkg/kubelet/events"
	"k8s.io/kubernetes/test/e2e/framework"
	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
	testutils "k8s.io/kubernetes/test/utils"
	imageutils "k8s.io/kubernetes/test/utils/image"
)

const runAsUserNameContainerName = "run-as-username-container"

var _ = SIGDescribe("[Feature:Windows] SecurityContext", func() {
	f := framework.NewDefaultFramework("windows-run-as-username")

	ginkgo.It("should be able create pods and run containers with a given username", func() {
		ginkgo.By("Creating 2 pods: 1 with the default user, and one with a custom one.")
		podDefault := runAsUserNamePod(nil)
		f.TestContainerOutput("check default user", podDefault, 0, []string{"ContainerUser"})

		podUserName := runAsUserNamePod(toPtr("ContainerAdministrator"))
		f.TestContainerOutput("check set user", podUserName, 0, []string{"ContainerAdministrator"})
	})

	ginkgo.It("should not be able to create pods with unknown usernames at Pod level", func() {
		ginkgo.By("Creating a pod with an invalid username")
		podInvalid := f.PodClient().Create(runAsUserNamePod(toPtr("FooLish")))

		failedSandboxEventSelector := fields.Set{
			"involvedObject.kind":      "Pod",
			"involvedObject.name":      podInvalid.Name,
			"involvedObject.namespace": podInvalid.Namespace,
			"reason":                   events.FailedCreatePodSandBox,
		}.AsSelector().String()
		hcsschimError := "The user name or password is incorrect."

		// Hostprocess updated the cri to pass RunAsUserName to sandbox: https://github.com/kubernetes/kubernetes/pull/99576/commits/51a02fdb80cb7ba042a66362eb76facd2fd82401
		// Some runtimes might use that and set the username on the podsandbox. Containerd 1.6+ is known to do this.
		// If there is an error when creating the pod sandbox then the pod stays in pending state by design
		// See https://github.com/kubernetes/kubernetes/issues/104635
		// Not all runtimes use the sandbox information.  This means the test needs to check if the pod
		// sandbox failed or workload pod failed.
		framework.Logf("Waiting for pod %s to enter the error state.", podInvalid.Name)
		gomega.Eventually(func() bool {
			failedSandbox, err := eventOccurred(f.ClientSet, podInvalid.Namespace, failedSandboxEventSelector, hcsschimError)
			if err != nil {
				framework.Logf("Error retrieving events for pod. Ignoring...")
			}
			if failedSandbox {
				framework.Logf("Found Expected Event 'Failed to Create Pod Sandbox' with message containing: %s", hcsschimError)
				return true
			}

			framework.Logf("No Sandbox error found. Looking for failure in workload pods")
			pod, err := f.PodClient().Get(context.Background(), podInvalid.Name, metav1.GetOptions{})
			if err != nil {
				framework.Logf("Error retrieving pod: %s", err)
				return false
			}

			podTerminatedReason := testutils.TerminatedContainers(pod)[runAsUserNameContainerName]
			podFailedToStart := podTerminatedReason == "ContainerCannotRun" || podTerminatedReason == "StartError"
			if pod.Status.Phase == v1.PodFailed && podFailedToStart {
				framework.Logf("Found terminated workload Pod that could not start")
				return true
			}

			return false
		}, framework.PodStartTimeout, 1*time.Second).Should(gomega.BeTrue())
	})

	ginkgo.It("should not be able to create pods with unknown usernames at Container level", func() {
		ginkgo.By("Creating a pod with an invalid username at container level and pod running as ContainerUser")
		p := runAsUserNamePod(toPtr("FooLish"))
		p.Spec.SecurityContext.WindowsOptions.RunAsUserName = toPtr("ContainerUser")
		podInvalid := f.PodClient().Create(p)

		framework.Logf("Waiting for pod %s to enter the error state.", podInvalid.Name)
		framework.ExpectNoError(e2epod.WaitForPodTerminatedInNamespace(f.ClientSet, podInvalid.Name, "", f.Namespace.Name))

		podInvalid, _ = f.PodClient().Get(context.TODO(), podInvalid.Name, metav1.GetOptions{})
		podTerminatedReason := testutils.TerminatedContainers(podInvalid)[runAsUserNameContainerName]
		if podTerminatedReason != "ContainerCannotRun" && podTerminatedReason != "StartError" {
			framework.Failf("The container terminated reason was supposed to be: 'ContainerCannotRun' or 'StartError', not: '%q'", podTerminatedReason)
		}
	})

	ginkgo.It("should override SecurityContext username if set", func() {
		ginkgo.By("Creating a pod with 2 containers with different username configurations.")

		pod := runAsUserNamePod(toPtr("ContainerAdministrator"))
		pod.Spec.Containers[0].SecurityContext.WindowsOptions.RunAsUserName = toPtr("ContainerUser")
		pod.Spec.Containers = append(pod.Spec.Containers, v1.Container{
			Name:    "run-as-username-new-container",
			Image:   imageutils.GetE2EImage(imageutils.NonRoot),
			Command: []string{"cmd", "/S", "/C", "echo %username%"},
		})

		f.TestContainerOutput("check overridden username", pod, 0, []string{"ContainerUser"})
		f.TestContainerOutput("check pod SecurityContext username", pod, 1, []string{"ContainerAdministrator"})
	})

	ginkgo.It("should ignore Linux Specific SecurityContext if set", func() {
		ginkgo.By("Creating a pod with SELinux options")
		// It is sufficient to show that the pod comes up here. Since we're stripping the SELinux and other linux
		// security contexts in apiserver and not updating the pod object in the apiserver, we cannot validate the
		// the pod object to not have those security contexts. However the pod coming to running state is a sufficient
		// enough condition for us to validate since prior to https://github.com/kubernetes/kubernetes/pull/93475
		// the pod would have failed to come up.
		windowsPodWithSELinux := createTestPod(f, windowsBusyBoximage, windowsOS)
		windowsPodWithSELinux.Spec.Containers[0].Args = []string{"test-webserver-with-selinux"}
		windowsPodWithSELinux.Spec.SecurityContext = &v1.PodSecurityContext{}
		containerUserName := "ContainerAdministrator"
		windowsPodWithSELinux.Spec.SecurityContext.SELinuxOptions = &v1.SELinuxOptions{Level: "s0:c24,c9"}
		windowsPodWithSELinux.Spec.Containers[0].SecurityContext = &v1.SecurityContext{
			SELinuxOptions: &v1.SELinuxOptions{Level: "s0:c24,c9"},
			WindowsOptions: &v1.WindowsSecurityContextOptions{RunAsUserName: &containerUserName}}
		windowsPodWithSELinux.Spec.Tolerations = []v1.Toleration{{Key: "os", Value: "Windows"}}
		windowsPodWithSELinux, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(),
			windowsPodWithSELinux, metav1.CreateOptions{})
		framework.ExpectNoError(err)
		framework.Logf("Created pod %v", windowsPodWithSELinux)
		framework.ExpectNoError(e2epod.WaitForPodNameRunningInNamespace(f.ClientSet, windowsPodWithSELinux.Name,
			f.Namespace.Name), "failed to wait for pod %s to be running", windowsPodWithSELinux.Name)
	})
})

func runAsUserNamePod(username *string) *v1.Pod {
	podName := "run-as-username-" + string(uuid.NewUUID())
	return &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: podName,
		},
		Spec: v1.PodSpec{
			NodeSelector: map[string]string{"kubernetes.io/os": "windows"},
			Containers: []v1.Container{
				{
					Name:    runAsUserNameContainerName,
					Image:   imageutils.GetE2EImage(imageutils.NonRoot),
					Command: []string{"cmd", "/S", "/C", "echo %username%"},
					SecurityContext: &v1.SecurityContext{
						WindowsOptions: &v1.WindowsSecurityContextOptions{
							RunAsUserName: username,
						},
					},
				},
			},
			SecurityContext: &v1.PodSecurityContext{
				WindowsOptions: &v1.WindowsSecurityContextOptions{
					RunAsUserName: username,
				},
			},
			RestartPolicy: v1.RestartPolicyNever,
		},
	}
}

func toPtr(s string) *string {
	return &s
}

func eventOccurred(c clientset.Interface, namespace, eventSelector, msg string) (bool, error) {
	options := metav1.ListOptions{FieldSelector: eventSelector}

	events, err := c.CoreV1().Events(namespace).List(context.TODO(), options)
	if err != nil {
		return false, fmt.Errorf("got error while getting events: %v", err)
	}
	for _, event := range events.Items {
		if strings.Contains(event.Message, msg) {
			return true, nil
		}
	}
	return false, nil
}
<|endoftext|>


<commit_before>package client

import (
	"compress/gzip"
	"crypto/x509"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"reflect"

	"github.com/getlantern/keyman"
	"github.com/getlantern/yaml"

	"github.com/getlantern/flashlight/client"
)

const (
	httpIfNoneMatch = "If-None-Match"
	httpEtag        = "Etag"
)

var lastCloudConfigETag string

type config struct {
	Client           *client.ClientConfig `yaml:"client"`
	TrustedCAs       []*ca                `yaml:"trustedcas"`
	InstanceId       string               `yaml:"instanceid"`
	FireTweetVersion string               `yaml:"firetweetversion"`
}

var (
	// errFailedConfigRequest is returned when the server replies with a non-200
	// status code to our request for a configuration file.
	errFailedConfigRequest = errors.New(`Could not get configuration file.`)

	// errInvalidConfiguration is returned in case the configuration file is
	// downloaded but has no useful data.
	errInvalidConfiguration = errors.New(`Invalid configuration file.`)

	errConfigurationUnchanged = errors.New(`Configuration remain unchanged.`)
)

const (
	cloudConfigCA = ``
	// URL of the configuration file. Remember to use HTTPs.
	remoteConfigURL = `https://config.getiantem.org/cloud.yaml.gz`
	instanceId      = ``
)

// pullConfigFile attempts to retrieve a configuration file over the network,
// then it decompresses it and returns the file's raw bytes.
func pullConfigFile(cli *http.Client) ([]byte, error) {
	var err error
	var req *http.Request
	var res *http.Response

	if cli == nil {
		return nil, errors.New("Missing HTTP client.")
	}

	if req, err = http.NewRequest("GET", remoteConfigURL, nil); err != nil {
		return nil, err
	}

	if lastCloudConfigETag != "" {
		// Don't bother fetching if unchanged.
		req.Header.Set(httpIfNoneMatch, lastCloudConfigETag)
	}

	if res, err = cli.Do(req); err != nil {
		return nil, err
	}

	// Has changed?
	if res.StatusCode == http.StatusNotModified {
		log.Debugf("Configuration file has not changed since last pull.\n")
		return nil, errConfigurationUnchanged
	}

	// Expecting 200 OK
	if res.StatusCode != http.StatusOK {
		return nil, errFailedConfigRequest
	}

	// Saving ETAG
	lastCloudConfigETag = res.Header.Get(httpEtag)

	// Using a gzip reader as we're getting a compressed file.
	var body io.ReadCloser
	if body, err = gzip.NewReader(res.Body); err != nil {
		return nil, err
	}
	defer func() {
		if err := body.Close(); err != nil {
			log.Debugf("Unable to close body: %v", err)
		}
	}()

	// Uncompressing bytes.
	return ioutil.ReadAll(body)
}

// defaultConfig returns the embedded configuration.
func defaultConfig() *config {
	cfg := &config{
		Client: &client.ClientConfig{
			ChainedServers: defaultChainedServers,
			MasqueradeSets: defaultMasqueradeSets,
		},
		TrustedCAs: defaultTrustedCAs,
	}
	return cfg
}

func (c *config) updateFrom(buf []byte) error {
	var err error
	var newCfg config

	// Attempt to parse configuration file.
	if err = yaml.Unmarshal(buf, &newCfg); err != nil {
		return err
	}

	// Making sure we can actually use this configuration.
	if len(newCfg.Client.FrontedServers) > 0 && len(newCfg.Client.MasqueradeSets) > 0 && len(newCfg.TrustedCAs) > 0 {
		if reflect.DeepEqual(newCfg, *c) {
			return errConfigurationUnchanged
		}
		*c = newCfg
		return nil
	}

	return errInvalidConfiguration
}

func (c *config) getTrustedCerts() []string {
	certs := make([]string, 0, len(c.TrustedCAs))

	for _, ca := range c.TrustedCAs {
		certs = append(certs, ca.Cert)
	}

	return certs
}

func (c *config) getTrustedCertPool() (certPool *x509.CertPool, err error) {
	trustedCerts := c.getTrustedCerts()

	if certPool, err = keyman.PoolContainingCerts(trustedCerts...); err != nil {
		return nil, err
	}

	return certPool, nil
}
<commit_msg>removed check for fronted servers existing<commit_after>package client

import (
	"compress/gzip"
	"crypto/x509"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"reflect"

	"github.com/getlantern/keyman"
	"github.com/getlantern/yaml"

	"github.com/getlantern/flashlight/client"
)

const (
	httpIfNoneMatch = "If-None-Match"
	httpEtag        = "Etag"
)

var lastCloudConfigETag string

type config struct {
	Client           *client.ClientConfig `yaml:"client"`
	TrustedCAs       []*ca                `yaml:"trustedcas"`
	InstanceId       string               `yaml:"instanceid"`
	FireTweetVersion string               `yaml:"firetweetversion"`
}

var (
	// errFailedConfigRequest is returned when the server replies with a non-200
	// status code to our request for a configuration file.
	errFailedConfigRequest = errors.New(`Could not get configuration file.`)

	// errInvalidConfiguration is returned in case the configuration file is
	// downloaded but has no useful data.
	errInvalidConfiguration = errors.New(`Invalid configuration file.`)

	errConfigurationUnchanged = errors.New(`Configuration remain unchanged.`)
)

const (
	cloudConfigCA = ``
	// URL of the configuration file. Remember to use HTTPs.
	remoteConfigURL = `https://config.getiantem.org/cloud.yaml.gz`
	instanceId      = ``
)

// pullConfigFile attempts to retrieve a configuration file over the network,
// then it decompresses it and returns the file's raw bytes.
func pullConfigFile(cli *http.Client) ([]byte, error) {
	var err error
	var req *http.Request
	var res *http.Response

	if cli == nil {
		return nil, errors.New("Missing HTTP client.")
	}

	if req, err = http.NewRequest("GET", remoteConfigURL, nil); err != nil {
		return nil, err
	}

	if lastCloudConfigETag != "" {
		// Don't bother fetching if unchanged.
		req.Header.Set(httpIfNoneMatch, lastCloudConfigETag)
	}

	if res, err = cli.Do(req); err != nil {
		return nil, err
	}

	// Has changed?
	if res.StatusCode == http.StatusNotModified {
		log.Debugf("Configuration file has not changed since last pull.\n")
		return nil, errConfigurationUnchanged
	}

	// Expecting 200 OK
	if res.StatusCode != http.StatusOK {
		return nil, errFailedConfigRequest
	}

	// Saving ETAG
	lastCloudConfigETag = res.Header.Get(httpEtag)

	// Using a gzip reader as we're getting a compressed file.
	var body io.ReadCloser
	if body, err = gzip.NewReader(res.Body); err != nil {
		return nil, err
	}
	defer func() {
		if err := body.Close(); err != nil {
			log.Debugf("Unable to close body: %v", err)
		}
	}()

	// Uncompressing bytes.
	return ioutil.ReadAll(body)
}

// defaultConfig returns the embedded configuration.
func defaultConfig() *config {
	cfg := &config{
		Client: &client.ClientConfig{
			ChainedServers: defaultChainedServers,
			MasqueradeSets: defaultMasqueradeSets,
		},
		TrustedCAs: defaultTrustedCAs,
	}
	return cfg
}

func (c *config) updateFrom(buf []byte) error {
	var err error
	var newCfg config

	// Attempt to parse configuration file.
	if err = yaml.Unmarshal(buf, &newCfg); err != nil {
		return err
	}

	// Making sure we can actually use this configuration.
	if len(newCfg.Client.MasqueradeSets) > 0 && len(newCfg.TrustedCAs) > 0 {
		if reflect.DeepEqual(newCfg, *c) {
			return errConfigurationUnchanged
		}
		*c = newCfg
		return nil
	}

	return errInvalidConfiguration
}

func (c *config) getTrustedCerts() []string {
	certs := make([]string, 0, len(c.TrustedCAs))

	for _, ca := range c.TrustedCAs {
		certs = append(certs, ca.Cert)
	}

	return certs
}

func (c *config) getTrustedCertPool() (certPool *x509.CertPool, err error) {
	trustedCerts := c.getTrustedCerts()

	if certPool, err = keyman.PoolContainingCerts(trustedCerts...); err != nil {
		return nil, err
	}

	return certPool, nil
}
<|endoftext|>


<commit_before>// Copyright (C) 2015 Nicolas Lamirault <nicolas.lamirault@gmail.com>

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

//     http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package version

// Version represents the application version using SemVer
const Version string = "0.1.0"
<commit_msg>bump version<commit_after>// Copyright (C) 2015 Nicolas Lamirault <nicolas.lamirault@gmail.com>

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

//     http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package version

// Version represents the application version using SemVer
const Version string = "0.2.0"
<|endoftext|>


<commit_before>package funcgo.clojure_cookbook_test
import(
        test midje.sweet
        fgo funcgo.core
)

func add(x,y) {
        x + y
}
test.fact("Simple example",
        add(1,2),
        =>, 3
)

test.fact("More complex example",
        into([],  \range(1, 20)),
        =>,  [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
)

test.fact("Any function of two arguments can be written infix",
        [] into \range(1, 20),
        =>,  [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
)

<commit_msg>add tests from clojure cookbook<commit_after>package funcgo.clojure_cookbook_test
import(
        test midje.sweet
        fgo funcgo.core
	string clojure.string
)

func add(x,y) {
        x + y
}
test.fact("Simple example",
        add(1,2),
        =>, 3
)

test.fact("More complex example",
        into([],  \range(1, 20)),
        =>,  [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
)

test.fact("Any function of two arguments can be written infix",
        [] into \range(1, 20),
        =>,  [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
)

test.fact("Infix is most convenient for math operators.",
	1 + 2,
	=>, 3
)

test.fact("Dotted identifers are from other packages.",
	// import section includes
	//    string clojure.string
	string.isBlank(""),
	=>, true
)

test.fact("Capitalize first character in a string.",
	string.capitalize("this is a proper sentence."),
	=>,  "This is a proper sentence."
)

test.fact("Capitalize or lower-case all characters.",
	string.upperCase("loud noises!"),
	=>, "LOUD NOISES!",

	string.lowerCase("COLUMN_HEADER_ONE"),
	=>, "column_header_one",

	string.lowerCase("!&$#@#%^[]"),
	=>, "!&$#@#%^[]",

	string.upperCase("Dépêchez-vous, l'ordinateur!"),
	=>, "DÉPÊCHEZ-VOUS, L'ORDINATEUR!"
)

test.fact("Remove whitespace at beginning and end.",
	string.trim(" \tBacon ipsum dolor sit.\n"),
	=>, "Bacon ipsum dolor sit."
)

test.fact("Collapse whitespace into single whitespace",
	string.replace("Who\t\nput  all this\fwhitespace here?", /\s+/, " "),
	=>, "Who put all this whitespace here?"
)

test.fact("Windows to Unix line-endings",
	string.replace("Line 1\r\nLine 2", "\r\n", "\n"),
	=>, "Line 1\nLine 2"
)
<|endoftext|>


<commit_before>// +build integration

/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"fmt"
	"path/filepath"
	"testing"
	"time"

	"k8s.io/kubernetes/pkg/api"
	commonutil "k8s.io/minikube/pkg/util"

	"k8s.io/minikube/test/integration/util"
)

func TestPersistence(t *testing.T) {
	minikubeRunner := util.MinikubeRunner{BinaryPath: *binaryPath, T: t}
	minikubeRunner.EnsureRunning()

	kubectlRunner := util.NewKubectlRunner(t)
	podName := "busybox"
	podPath, _ := filepath.Abs("testdata/busybox.yaml")

	podNamespace := kubectlRunner.CreateRandomNamespace()
	defer kubectlRunner.DeleteNamespace(podNamespace)

	// Create a pod and wait for it to be running.
	if _, err := kubectlRunner.RunCommand([]string{"create", "-f", podPath, "--namespace=" + podNamespace}); err != nil {
		t.Fatalf("Error creating test pod: %s", err)
	}

	checkPod := func() error {
		p := kubectlRunner.GetPod(podName, podNamespace)
		if util.IsPodReady(p) {
			return nil
		}
		return fmt.Errorf("Pod %s is not ready yet.", podName)
	}

	if err := commonutil.RetryAfter(10, checkPod, 6*time.Second); err != nil {
		t.Fatalf("Error checking the status of pod %s. Err: %s", podName, err)
	}

	checkDashboard := func() error {
		pods := api.PodList{}
		cmd := []string{"get", "pods", "--namespace=kube-system", "--selector=app=kubernetes-dashboard"}
		if err := kubectlRunner.RunCommandParseOutput(cmd, &pods); err != nil {
			return err
		}
		if len(pods.Items) < 1 {
			return fmt.Errorf("No pods found matching query: %v", cmd)
		}
		db := pods.Items[0]
		if util.IsPodReady(&db) {
			return nil
		}
		return fmt.Errorf("Dashboard pod is not ready yet.")
	}

	// Make sure the dashboard is running before we stop the VM.
	// On slow networks it can take several minutes to pull the addon-manager then the dashboard image.
	if err := commonutil.RetryAfter(20, checkDashboard, 6*time.Second); err != nil {
		t.Fatalf("Dashboard pod is not healthy: %s", err)
	}

	// Now restart minikube and make sure the pod is still there.
	minikubeRunner.RunCommand("stop", true)
	minikubeRunner.CheckStatus("Stopped")

	minikubeRunner.RunCommand("start", true)
	minikubeRunner.CheckStatus("Running")

	if err := commonutil.RetryAfter(5, checkPod, 3*time.Second); err != nil {
		t.Fatalf("Error checking the status of pod %s. Err: %s", podName, err)
	}

	// Now make sure it's still running after.
	if err := commonutil.RetryAfter(5, checkDashboard, 3*time.Second); err != nil {
		t.Fatalf("Dashboard pod is not healthy: %s", err)
	}
}
<commit_msg>Increase the pull timeout even more.<commit_after>// +build integration

/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"fmt"
	"path/filepath"
	"testing"
	"time"

	"k8s.io/kubernetes/pkg/api"
	commonutil "k8s.io/minikube/pkg/util"

	"k8s.io/minikube/test/integration/util"
)

func TestPersistence(t *testing.T) {
	minikubeRunner := util.MinikubeRunner{BinaryPath: *binaryPath, T: t}
	minikubeRunner.EnsureRunning()

	kubectlRunner := util.NewKubectlRunner(t)
	podName := "busybox"
	podPath, _ := filepath.Abs("testdata/busybox.yaml")

	podNamespace := kubectlRunner.CreateRandomNamespace()
	defer kubectlRunner.DeleteNamespace(podNamespace)

	// Create a pod and wait for it to be running.
	if _, err := kubectlRunner.RunCommand([]string{"create", "-f", podPath, "--namespace=" + podNamespace}); err != nil {
		t.Fatalf("Error creating test pod: %s", err)
	}

	checkPod := func() error {
		p := kubectlRunner.GetPod(podName, podNamespace)
		if util.IsPodReady(p) {
			return nil
		}
		return fmt.Errorf("Pod %s is not ready yet.", podName)
	}

	if err := commonutil.RetryAfter(20, checkPod, 6*time.Second); err != nil {
		t.Fatalf("Error checking the status of pod %s. Err: %s", podName, err)
	}

	checkDashboard := func() error {
		pods := api.PodList{}
		cmd := []string{"get", "pods", "--namespace=kube-system", "--selector=app=kubernetes-dashboard"}
		if err := kubectlRunner.RunCommandParseOutput(cmd, &pods); err != nil {
			return err
		}
		if len(pods.Items) < 1 {
			return fmt.Errorf("No pods found matching query: %v", cmd)
		}
		db := pods.Items[0]
		if util.IsPodReady(&db) {
			return nil
		}
		return fmt.Errorf("Dashboard pod is not ready yet.")
	}

	// Make sure the dashboard is running before we stop the VM.
	// On slow networks it can take several minutes to pull the addon-manager then the dashboard image.
	if err := commonutil.RetryAfter(20, checkDashboard, 6*time.Second); err != nil {
		t.Fatalf("Dashboard pod is not healthy: %s", err)
	}

	// Now restart minikube and make sure the pod is still there.
	minikubeRunner.RunCommand("stop", true)
	minikubeRunner.CheckStatus("Stopped")

	minikubeRunner.RunCommand("start", true)
	minikubeRunner.CheckStatus("Running")

	if err := commonutil.RetryAfter(5, checkPod, 3*time.Second); err != nil {
		t.Fatalf("Error checking the status of pod %s. Err: %s", podName, err)
	}

	// Now make sure it's still running after.
	if err := commonutil.RetryAfter(5, checkDashboard, 3*time.Second); err != nil {
		t.Fatalf("Dashboard pod is not healthy: %s", err)
	}
}
<|endoftext|>


<commit_before>// (C) Copyright 2021 Hewlett Packard Enterprise Development LP
//
// Licensed under the Apache License, Version 2.0 (the "License");
// You may not use this file except in compliance with the License.
// You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.

package oneview

import (
	"strings"

	"github.com/HewlettPackard/oneview-golang/ov"
	"github.com/HewlettPackard/oneview-golang/utils"
	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
)

func resourceFirmwareDrivers() *schema.Resource {
	return &schema.Resource{
		Read:   resourceFirmwareDriversRead,
		Create: resourceFirmwareDriversCreate,
		Update: resourceConnectionTemplatesUpdate,
		Delete: resourceFirmwareDriversDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"category": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"created": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"modified": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"description": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"etag": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"name": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"state": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"uri": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"status": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"type": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"baseline_short_name": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"bundle_size": {
				Type:     schema.TypeInt,
				Computed: true,
			},
			"bundle_type": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"esxi_os_driver_meta_data": {
				Computed: true,
				Type:     schema.TypeSet,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
				Set: schema.HashString,
			},
			"fw_components": {
				Computed: true,
				Type:     schema.TypeSet,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"component_version": {
							Type:     schema.TypeString,
							Computed: true,
						},
						"file_name": {
							Type:     schema.TypeString,
							Computed: true,
						},
						"name": {
							Type:     schema.TypeString,
							Computed: true,
						},
						"sw_key_name_list": {
							Computed: true,
							Type:     schema.TypeSet,
							Elem: &schema.Schema{
								Type: schema.TypeString,
							},
							Set: schema.HashString,
						},
					},
				},
			},
			"hotfixes": {
				Computed: true,
				Type:     schema.TypeSet,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"hotfix_name": {
							Type:     schema.TypeString,
							Computed: true,
						},
						"release_data": {
							Type:     schema.TypeString,
							Computed: true,
						},
						"resource_id": {
							Type:     schema.TypeString,
							Computed: true,
						},
					},
				},
			},
			"hpsum_version": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"iso_file_name": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"last_task_uri": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"mirror_list": {
				Computed: true,
				Type:     schema.TypeMap,
				Elem: &schema.Schema{
					Type: schema.TypeList,
				},
			},
			"locations": {
				Computed: true,
				Type:     schema.TypeMap,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
			},
			"parent_bundle": {
				Computed: true,
				Type:     schema.TypeSet,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"parent_bundle_name": {
							Type:     schema.TypeString,
							Computed: true,
						},
						"release_data": {
							Type:     schema.TypeString,
							Computed: true,
						},
						"version": {
							Type:     schema.TypeString,
							Computed: true,
						},
					},
				},
			},
			"release_data": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"resource_id": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"resource_state": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"scope_uri": {
				Computed: true,
				Type:     schema.TypeString,
			},
			"signature_file_name": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"signature_file_required": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"supported_languages": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"supported_os_list": {
				Computed: true,
				Type:     schema.TypeSet,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
				Set: schema.HashString,
			},
			"sw_packages_full_path": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"uuid": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"version": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"xml_key_name": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"baseline_uri": {
				Type:     schema.TypeString,
				Required: true,
			},
			"hotfix_uris": {
				Type: schema.TypeSet,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
				Set:      schema.HashString,
				Required: true,
			},
			"custom_baseline_name": {
				Type:     schema.TypeString,
				Required: true,
			},
			"initial_scope_uris": {
				Optional: true,
				Type:     schema.TypeSet,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
				Set: schema.HashString,
			},
			"force": {
				Type:     schema.TypeString,
				Optional: true,
			},
		},
	}
}

func resourceFirmwareDriversCreate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	customBundle := ov.CustomServicePack{
		CustomBaselineName: d.Get("custom_baseline_name").(string),
		BaselineUri:        d.Get("baseline_uri").(string),
	}
	force := "false"
	if _, ok := d.GetOk("force"); ok {
		force = d.Get("force").(string)
	}
	if val, ok := d.GetOk("initial_scope_uris"); ok {
		rawInitialScopeUris := val.(*schema.Set).List()
		initialScopeUris := make([]utils.Nstring, len(rawInitialScopeUris))
		for i, raw := range rawInitialScopeUris {
			initialScopeUris[i] = utils.Nstring(raw.(string))
		}
		customBundle.InitialScopeUris = initialScopeUris
	}

	rawHotflixURI := d.Get("hotfix_uris").(*schema.Set).List()
	HotflixURI := make([]utils.Nstring, len(rawHotflixURI))
	for i, raw := range rawHotflixURI {
		HotflixURI[i] = utils.Nstring(raw.(string))
	}
	customBundle.HotfixUris = HotflixURI

	err := config.ovClient.CreateCustomServicePack(customBundle, force)
	if err != nil {
		return err
	}

	return resourceFirmwareDriversRead(d, meta)
}

func resourceFirmwareDriversRead(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	firmwareAll, err := config.ovClient.GetFirmwareBaselineList("", "", "")
	if err != nil || firmwareAll.Uri.IsNil() {
		d.SetId("")
		return nil
	}
	firmware := ov.FirmwareDrivers{}
	for i := range firmwareAll.Members {
		if firmwareAll.Members[i].Name != d.Get("custom_baseline_name").(string) {
			continue
		} else {
			firmware = firmwareAll.Members[i]
			d.Set("name", firmware.Name)
			d.Set("type", firmware.Type)
			d.Set("created", firmware.Created)
			d.Set("modified", firmware.Modified)
			d.Set("uri", firmware.Uri.String())
			d.Set("status", firmware.Status)
			d.Set("category", firmware.Category)
			d.Set("state", firmware.State)
			d.Set("etag", firmware.ETAG)
			d.Set("description", firmware.Description)
			d.Set("baseline_short_name", firmware.BaselineShortName)
			d.Set("bundle_size", firmware.BundleSize)
			d.Set("bundle_type", firmware.BundleType)
			d.Set("esxi_os_driver_meta_data", firmware.EsxiOsDriverMetaData)
			d.Set("hpsum_version", firmware.HpsumVersion)
			d.Set("iso_file_name", firmware.IsoFileName)
			d.Set("last_task_uri", firmware.LastTaskUri)
			d.Set("release_data", firmware.ReleaseDate)
			d.Set("resource_id", firmware.ResourceId)
			d.Set("resource_state", firmware.ResourceState)
			d.Set("scope_uri", firmware.ScopesUri)
			d.Set("signature_file_name", firmware.SignatureFileName)
			d.Set("signature_file_required", firmware.SignatureFileRequired)
			d.Set("supported_languages", firmware.SupportedLanguages)
			d.Set("supported_os_list", firmware.SupportedOSList)
			d.Set("sw_packages_full_path", firmware.SwPackagesFullPath)
			d.Set("uuid", firmware.Uuid)
			d.Set("version", firmware.Version)
			d.Set("xml_key_name", firmware.XmlKeyName)

			fwcomponent := make([]map[string]interface{}, 0, len(firmware.FwComponents))
			for _, component := range firmware.FwComponents {
				fwcomponent = append(fwcomponent, map[string]interface{}{
					"component_version": component.ComponentVersion,
					"file_name":         component.FileName,
					"name":              component.Name,
					"sw_key_name_list":  component.SwKeyNameList,
				})
			}
			d.Set("fw_components", fwcomponent)

			hotFixes := make([]map[string]interface{}, 0, len(firmware.Hotfixes))
			for _, hotfix := range firmware.Hotfixes {
				hotFixes = append(hotFixes, map[string]interface{}{
					"hotfix_name":  hotfix.HotfixName,
					"release_data": hotfix.ReleaseDate,
					"resource_id":  hotfix.ResourceId,
				})
			}
			d.Set("hotfixes", hotFixes)

			parentBundle := make([]map[string]interface{}, 0, 1)
			parentBundle = append(parentBundle, map[string]interface{}{
				"parent_bundle_name": firmware.ParentBundle.ParentBundleName,
				"release_data":       firmware.ParentBundle.ReleaseDate,
				"version":            firmware.ParentBundle.Version,
			})

			d.Set("parent_bundle", parentBundle)

			d.Set("locations", firmware.Locations)
			d.Set("mirror_list", firmware.Mirrorlist)
			id := strings.Split(firmware.Uri.String(), "/")[3]

			d.SetId(id)
			return nil

		}
	}
	d.SetId("")
	return nil
}

func resourceFirmwareDriversUpdate(d *schema.ResourceData, meta interface{}) error {
	return nil
}

func resourceFirmwareDriversDelete(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	err := config.ovClient.DeleteFirmwareBaseline(d.Id(), "false")
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>removed-forcenew<commit_after>// (C) Copyright 2021 Hewlett Packard Enterprise Development LP
//
// Licensed under the Apache License, Version 2.0 (the "License");
// You may not use this file except in compliance with the License.
// You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.

package oneview

import (
	"errors"
	"strings"

	"github.com/HewlettPackard/oneview-golang/ov"
	"github.com/HewlettPackard/oneview-golang/utils"
	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
)

func resourceFirmwareDrivers() *schema.Resource {
	return &schema.Resource{
		Read:   resourceFirmwareDriversRead,
		Create: resourceFirmwareDriversCreate,
		Update: resourceConnectionTemplatesUpdate,
		Delete: resourceFirmwareDriversDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"category": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"created": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"modified": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"description": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"etag": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"name": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"state": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"uri": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"status": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"type": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"baseline_short_name": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"bundle_size": {
				Type:     schema.TypeInt,
				Computed: true,
			},
			"bundle_type": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"esxi_os_driver_meta_data": {
				Computed: true,
				Type:     schema.TypeSet,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
			},
			"fw_components": {
				Computed: true,
				Type:     schema.TypeSet,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"component_version": {
							Type:     schema.TypeString,
							Computed: true,
						},
						"file_name": {
							Type:     schema.TypeString,
							Computed: true,
						},
						"name": {
							Type:     schema.TypeString,
							Computed: true,
						},
						"sw_key_name_list": {
							Computed: true,
							Type:     schema.TypeSet,
							Elem: &schema.Schema{
								Type: schema.TypeString,
							},
						},
					},
				},
			},
			"hotfixes": {
				Computed: true,
				Type:     schema.TypeSet,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"hotfix_name": {
							Type:     schema.TypeString,
							Computed: true,
						},
						"release_data": {
							Type:     schema.TypeString,
							Computed: true,
						},
						"resource_id": {
							Type:     schema.TypeString,
							Computed: true,
						},
					},
				},
			},
			"hpsum_version": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"iso_file_name": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"last_task_uri": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"mirror_list": {
				Computed: true,
				Type:     schema.TypeMap,
				Elem: &schema.Schema{
					Type: schema.TypeList,
				},
			},
			"locations": {
				Computed: true,
				Type:     schema.TypeMap,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
			},
			"parent_bundle": {
				Computed: true,
				Type:     schema.TypeSet,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"parent_bundle_name": {
							Type:     schema.TypeString,
							Computed: true,
						},
						"release_data": {
							Type:     schema.TypeString,
							Computed: true,
						},
						"version": {
							Type:     schema.TypeString,
							Computed: true,
						},
					},
				},
			},
			"release_data": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"resource_id": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"resource_state": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"scope_uri": {
				Computed: true,
				Type:     schema.TypeString,
			},
			"signature_file_name": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"signature_file_required": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"supported_languages": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"supported_os_list": {
				Computed: true,
				Type:     schema.TypeSet,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
			},
			"sw_packages_full_path": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"uuid": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"version": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"xml_key_name": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"baseline_uri": {
				Type:     schema.TypeString,
				Required: true,
			},
			"hotfix_uris": {
				Type: schema.TypeSet,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
				Set:      schema.HashString,
				Required: true,
			},
			"custom_baseline_name": {
				Type:     schema.TypeString,
				Required: true,
			},
			"initial_scope_uris": {
				Optional: true,
				Type:     schema.TypeSet,
				Elem: &schema.Schema{
					Type: schema.TypeString,
				},
			},
			"force": {
				Type:     schema.TypeString,
				Optional: true,
			},
		},
	}
}

func resourceFirmwareDriversCreate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	customBundle := ov.CustomServicePack{
		CustomBaselineName: d.Get("custom_baseline_name").(string),
		BaselineUri:        d.Get("baseline_uri").(string),
	}
	force := "false"
	if _, ok := d.GetOk("force"); ok {
		force = d.Get("force").(string)
	}
	if val, ok := d.GetOk("initial_scope_uris"); ok {
		rawInitialScopeUris := val.(*schema.Set).List()
		initialScopeUris := make([]utils.Nstring, len(rawInitialScopeUris))
		for i, raw := range rawInitialScopeUris {
			initialScopeUris[i] = utils.Nstring(raw.(string))
		}
		customBundle.InitialScopeUris = initialScopeUris
	}

	rawHotflixURI := d.Get("hotfix_uris").(*schema.Set).List()
	HotflixURI := make([]utils.Nstring, len(rawHotflixURI))
	for i, raw := range rawHotflixURI {
		HotflixURI[i] = utils.Nstring(raw.(string))
	}
	customBundle.HotfixUris = HotflixURI

	err := config.ovClient.CreateCustomServicePack(customBundle, force)
	if err != nil {
		return err
	}

	return resourceFirmwareDriversRead(d, meta)
}

func resourceFirmwareDriversRead(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	firmwareAll, err := config.ovClient.GetFirmwareBaselineList("", "", "")
	if err != nil || firmwareAll.Uri.IsNil() {
		d.SetId("")
		return nil
	}
	firmware := ov.FirmwareDrivers{}
	for i := range firmwareAll.Members {
		if firmwareAll.Members[i].Name != d.Get("custom_baseline_name").(string) {
			continue
		} else {
			firmware = firmwareAll.Members[i]
			d.Set("name", firmware.Name)
			d.Set("type", firmware.Type)
			d.Set("created", firmware.Created)
			d.Set("modified", firmware.Modified)
			d.Set("uri", firmware.Uri.String())
			d.Set("status", firmware.Status)
			d.Set("category", firmware.Category)
			d.Set("state", firmware.State)
			d.Set("etag", firmware.ETAG)
			d.Set("description", firmware.Description)
			d.Set("baseline_short_name", firmware.BaselineShortName)
			d.Set("bundle_size", firmware.BundleSize)
			d.Set("bundle_type", firmware.BundleType)
			d.Set("esxi_os_driver_meta_data", firmware.EsxiOsDriverMetaData)
			d.Set("hpsum_version", firmware.HpsumVersion)
			d.Set("iso_file_name", firmware.IsoFileName)
			d.Set("last_task_uri", firmware.LastTaskUri)
			d.Set("release_data", firmware.ReleaseDate)
			d.Set("resource_id", firmware.ResourceId)
			d.Set("resource_state", firmware.ResourceState)
			d.Set("scope_uri", firmware.ScopesUri)
			d.Set("signature_file_name", firmware.SignatureFileName)
			d.Set("signature_file_required", firmware.SignatureFileRequired)
			d.Set("supported_languages", firmware.SupportedLanguages)
			d.Set("supported_os_list", firmware.SupportedOSList)
			d.Set("sw_packages_full_path", firmware.SwPackagesFullPath)
			d.Set("uuid", firmware.Uuid)
			d.Set("version", firmware.Version)
			d.Set("xml_key_name", firmware.XmlKeyName)

			fwcomponent := make([]map[string]interface{}, 0, len(firmware.FwComponents))
			for _, component := range firmware.FwComponents {
				fwcomponent = append(fwcomponent, map[string]interface{}{
					"component_version": component.ComponentVersion,
					"file_name":         component.FileName,
					"name":              component.Name,
					"sw_key_name_list":  component.SwKeyNameList,
				})
			}
			d.Set("fw_components", fwcomponent)

			hotFixes := make([]map[string]interface{}, 0, len(firmware.Hotfixes))
			for _, hotfix := range firmware.Hotfixes {
				hotFixes = append(hotFixes, map[string]interface{}{
					"hotfix_name":  hotfix.HotfixName,
					"release_data": hotfix.ReleaseDate,
					"resource_id":  hotfix.ResourceId,
				})
			}
			d.Set("hotfixes", hotFixes)

			parentBundle := make([]map[string]interface{}, 0, 1)
			parentBundle = append(parentBundle, map[string]interface{}{
				"parent_bundle_name": firmware.ParentBundle.ParentBundleName,
				"release_data":       firmware.ParentBundle.ReleaseDate,
				"version":            firmware.ParentBundle.Version,
			})

			d.Set("parent_bundle", parentBundle)

			d.Set("locations", firmware.Locations)
			d.Set("mirror_list", firmware.Mirrorlist)
			id := strings.Split(firmware.Uri.String(), "/")[3]

			d.SetId(id)
			return nil

		}
	}
	d.SetId("")
	return nil
}

func resourceFirmwareDriversUpdate(d *schema.ResourceData, meta interface{}) error {
	err := errors.New("this resource do not support update request")
	return err
}

func resourceFirmwareDriversDelete(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	err := config.ovClient.DeleteFirmwareBaseline(d.Id(), "false")
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package sparse_test

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"testing"

	. "github.com/rancher/sparse-tools/sparse"

	"time"

	"github.com/rancher/sparse-tools/log"
)

const batch = 32 // blocks for read/write

type TestFileInterval struct {
	FileInterval
	dataMask byte // XORed with other generated data bytes
}

func (i TestFileInterval) String() string {
	return fmt.Sprintf("{%v %2X}", i.FileInterval, i.dataMask)
}

func TestRandomLayout10MB(t *testing.T) {
	const seed = 0
	const size = 10 /*MB*/ << 20
	prefix := "ssync"
	name := tempFileName(prefix)
	layoutStream := generateLayout(prefix, size, seed)
	layout1, layout2 := teeLayout(layoutStream)

	done := createTestSparseFileLayout(name, size, layout1)
	layoutTmp := unstreamLayout(layout2)
	<-done
	log.Info("Done writing layout of ", len(layoutTmp), "items")

	layout := streamLayout(layoutTmp)
	err := checkTestSparseFileLayout(name, layout)
	if err != nil {
		t.Fatal(err)
	}
	os.Remove(name)
}

func TestRandomLayout100MB(t *testing.T) {
	const seed = 0
	const size = 100 /*MB*/ << 20
	prefix := "ssync"
	name := tempFileName(prefix)
	layoutStream := generateLayout(prefix, size, seed)
	layout1, layout2 := teeLayout(layoutStream)

	done := createTestSparseFileLayout(name, size, layout1)
	layoutTmp := unstreamLayout(layout2)
	<-done
	log.Info("Done writing layout of ", len(layoutTmp), "items")

	layout := streamLayout(layoutTmp)
	err := checkTestSparseFileLayout(name, layout)
	if err != nil {
		t.Fatal(err)
	}
	os.Remove(name)
}

func TestRandomSync100MB(t *testing.T) {
	const seed = 1
	const size = 100 /*MB*/ << 20
	RandomSync(t, size, seed)
}

func TestRandomSync1GB(t *testing.T) {
	seed := time.Now().UnixNano()
	const size = 1 /*GB*/ << 30
	if testing.Short() {
		t.Skip("skipped 1GB random sync")
	}
	log.Info("seed=", seed)

	log.LevelPush(log.LevelInfo)
	defer log.LevelPop()

	RandomSync(t, size, seed)
}

func RandomSync(t *testing.T, size, seed int64) {
	const localhost = "127.0.0.1"
	const timeout = 10 //seconds
	var remoteAddr = TCPEndPoint{localhost, 5000}
	const srcPrefix = "ssync-src"
	const dstPrefix = "ssync-dst"

	srcName := tempFileName(srcPrefix)
	dstName := tempFileName(dstPrefix)
	srcLayoutStream1, srcLayoutStream2 := teeLayout(generateLayout(srcPrefix, size, seed))
	dstLayoutStream := generateLayout(dstPrefix, size, seed+1)

	srcDone := createTestSparseFileLayout(srcName, size, srcLayoutStream1)
	dstDone := createTestSparseFileLayout(dstName, size, dstLayoutStream)
	srcLayout := unstreamLayout(srcLayoutStream2)
	<-srcDone
	<-dstDone
	log.Info("Done writing layout of ", len(srcLayout), "items")

	log.Info("Syncing...")

	go TestServer(remoteAddr, timeout)
	_, err := SyncFile(srcName, remoteAddr, dstName, timeout)

	if err != nil {
		t.Fatal("sync error")
	}
	log.Info("...syncing done")

	log.Info("Checking...")
	layoutStream := streamLayout(srcLayout)
	err = checkTestSparseFileLayout(dstName, layoutStream)
	if err != nil {
		t.Fatal(err)
	}
	os.Remove(srcName)
	os.Remove(dstName)
}

func tempFileName(prefix string) string {
	// Make a temporary file name
	f, err := ioutil.TempFile(".", prefix)
	if err != nil {
		log.Fatal("Failed to make temp file", err)
	}
	defer f.Close()
	return f.Name()
}

func unstreamLayout(in <-chan TestFileInterval) []TestFileInterval {
	layout := make([]TestFileInterval, 0, 4096)
	for i := range in {
		log.Trace("unstream", i)
		layout = append(layout, i)
	}
	return layout
}

func streamLayout(in []TestFileInterval) (out chan TestFileInterval) {
	out = make(chan TestFileInterval, 128)

	go func() {
		for _, i := range in {
			log.Trace("stream", i)
			out <- i
		}
		close(out)
	}()

	return out
}

func teeLayout(in <-chan TestFileInterval) (out1 chan TestFileInterval, out2 chan TestFileInterval) {
	out1 = make(chan TestFileInterval, 128)
	out2 = make(chan TestFileInterval, 128)

	go func() {
		for i := range in {
			log.Trace("Tee1...")
			out1 <- i
			log.Trace("Tee2...")
			out2 <- i
		}
		close(out1)
		close(out2)
	}()

	return out1, out2
}

func generateLayout(prefix string, size, seed int64) <-chan TestFileInterval {
	const maxInterval = 256 // Blocks
	layoutStream := make(chan TestFileInterval, 128)
	r := rand.New(rand.NewSource(seed))

	go func() {
		offset := int64(0)
		for offset < size {
			blocks := int64(r.Intn(maxInterval)) + 1 // 1..maxInterval
			length := blocks * Blocks
			if offset+length > size {
				// don't overshoot size
				length = size - offset
			}

			interval := Interval{offset, offset + length}
			offset += interval.Len()

			kind := SparseHole
			var mask byte
			if r.Intn(2) == 0 {
				// Data
				kind = SparseData
				mask = 0xAA * byte(r.Intn(10)/9) // 10%
			}
			t := TestFileInterval{FileInterval{kind, interval}, mask}
			log.Debug(prefix, t)
			layoutStream <- t
		}
		close(layoutStream)
	}()

	return layoutStream
}

func makeIntervalData(interval TestFileInterval) []byte {
	data := make([]byte, interval.Len())
	if SparseData == interval.Kind {
		for i := range data {
            value := byte((interval.Begin+int64(i))/Blocks)
			data[i] = interval.dataMask ^ value
		}
	}
	return data
}

func createTestSparseFileLayout(name string, fileSize int64, layout <-chan TestFileInterval) (done chan struct{}) {
	done = make(chan struct{})

	// Fill up file with layout data
	go func() {
		f, err := os.Create(name)
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()
		err = f.Truncate(fileSize)
		if err != nil {
			log.Fatal(err)
		}

		for interval := range layout {
			log.Debug("writing...", interval)
			if SparseData == interval.Kind {
				size := batch * Blocks
				for offset := interval.Begin; offset < interval.End; {
					if offset+size > interval.End {
						size = interval.End - offset
					}
					chunkInterval := TestFileInterval{FileInterval{SparseData, Interval{offset, offset + size}}, interval.dataMask}
					data := makeIntervalData(chunkInterval)
					_, err = f.WriteAt(data, offset)
					if err != nil {
						log.Fatal(err)
					}
					offset += size
				}
			}
		}
		f.Sync()
		close(done)
	}()

	return done
}

func checkTestSparseFileLayout(name string, layout <-chan TestFileInterval) error {
	f, err := os.Open(name)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	// Read and check data
	for interval := range layout {
		log.Debug("checking...", interval)
		if SparseData == interval.Kind {
			size := batch * Blocks
			for offset := interval.Begin; offset < interval.End; {
				if offset+size > interval.End {
					size = interval.End - offset
				}
				dataModel := makeIntervalData(TestFileInterval{FileInterval{SparseData, Interval{offset, offset + size}}, interval.dataMask})
				data := make([]byte, size)
				f.ReadAt(data, offset)
				offset += size

				if !bytes.Equal(data, dataModel) {
					return errors.New(fmt.Sprint("data equality check failure at", interval))
				}
			}
		} else if SparseHole == interval.Kind {
			layoutActual, err := RetrieveLayout(f, interval.Interval)
			if err != nil {
				return errors.New(fmt.Sprint("hole retrieval failure at", interval, err))
			}
			if len(layoutActual) != 1 {
				return errors.New(fmt.Sprint("hole check failure at", interval))
			}
			if layoutActual[0] != interval.FileInterval {
				return errors.New(fmt.Sprint("hole equality check failure at", interval))
			}
		}
	}
	return nil // success
}
<commit_msg>test: added optional parameter to set custom file size for TestRandomSyncCustomGB<commit_after>package sparse_test

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"testing"

	. "github.com/rancher/sparse-tools/sparse"

	"time"

	"strconv"

	"github.com/rancher/sparse-tools/log"
)

const batch = 32 // blocks for read/write

type TestFileInterval struct {
	FileInterval
	dataMask byte // XORed with other generated data bytes
}

func (i TestFileInterval) String() string {
	return fmt.Sprintf("{%v %2X}", i.FileInterval, i.dataMask)
}

func TestRandomLayout10MB(t *testing.T) {
	const seed = 0
	const size = 10 /*MB*/ << 20
	prefix := "ssync"
	name := tempFileName(prefix)
	layoutStream := generateLayout(prefix, size, seed)
	layout1, layout2 := teeLayout(layoutStream)

	done := createTestSparseFileLayout(name, size, layout1)
	layoutTmp := unstreamLayout(layout2)
	<-done
	log.Info("Done writing layout of ", len(layoutTmp), "items")

	layout := streamLayout(layoutTmp)
	err := checkTestSparseFileLayout(name, layout)
	if err != nil {
		t.Fatal(err)
	}
	os.Remove(name)
}

func TestRandomLayout100MB(t *testing.T) {
	const seed = 0
	const size = 100 /*MB*/ << 20
	prefix := "ssync"
	name := tempFileName(prefix)
	layoutStream := generateLayout(prefix, size, seed)
	layout1, layout2 := teeLayout(layoutStream)

	done := createTestSparseFileLayout(name, size, layout1)
	layoutTmp := unstreamLayout(layout2)
	<-done
	log.Info("Done writing layout of ", len(layoutTmp), "items")

	layout := streamLayout(layoutTmp)
	err := checkTestSparseFileLayout(name, layout)
	if err != nil {
		t.Fatal(err)
	}
	os.Remove(name)
}

func TestRandomSync100MB(t *testing.T) {
	const seed = 1
	const size = 100 /*MB*/ << 20
	RandomSync(t, size, seed)
}

func TestRandomSyncCustomGB(t *testing.T) {
	if testing.Short() {
		t.Skip("skipped custom random sync")
	}

    // random seed
	seed := time.Now().UnixNano()
	log.LevelPush(log.LevelInfo)
	defer log.LevelPop()
	log.Info("seed=", seed)

	// default size
	var size = int64(100) /*MB*/ << 20
	arg := os.Args[len(os.Args)-1]
	sizeGB, err := strconv.Atoi(arg)
	if err != nil {
		log.Info("")
		log.Info("Using default 100MB size for random seed test")
		log.Info("For alternative size in GB use -timeout 10m -args <GB>")
		log.Info("Increase the optional -timeout value for 20GB and larger sizes")
		log.Info("")
	} else {
		log.Info("Using ", sizeGB, "(GB) size for random seed test")
		size = int64(sizeGB) << 30
	}

	RandomSync(t, size, seed)
}

func RandomSync(t *testing.T, size, seed int64) {
	const localhost = "127.0.0.1"
	const timeout = 10 //seconds
	var remoteAddr = TCPEndPoint{localhost, 5000}
	const srcPrefix = "ssync-src"
	const dstPrefix = "ssync-dst"

	srcName := tempFileName(srcPrefix)
	dstName := tempFileName(dstPrefix)
	srcLayoutStream1, srcLayoutStream2 := teeLayout(generateLayout(srcPrefix, size, seed))
	dstLayoutStream := generateLayout(dstPrefix, size, seed+1)

	srcDone := createTestSparseFileLayout(srcName, size, srcLayoutStream1)
	dstDone := createTestSparseFileLayout(dstName, size, dstLayoutStream)
	srcLayout := unstreamLayout(srcLayoutStream2)
	<-srcDone
	<-dstDone
	log.Info("Done writing layout of ", len(srcLayout), "items")

	log.Info("Syncing...")

	go TestServer(remoteAddr, timeout)
	_, err := SyncFile(srcName, remoteAddr, dstName, timeout)

	if err != nil {
		t.Fatal("sync error")
	}
	log.Info("...syncing done")

	log.Info("Checking...")
	layoutStream := streamLayout(srcLayout)
	err = checkTestSparseFileLayout(dstName, layoutStream)
	if err != nil {
		t.Fatal(err)
	}
	os.Remove(srcName)
	os.Remove(dstName)
}

func tempFileName(prefix string) string {
	// Make a temporary file name
	f, err := ioutil.TempFile(".", prefix)
	if err != nil {
		log.Fatal("Failed to make temp file", err)
	}
	defer f.Close()
	return f.Name()
}

func unstreamLayout(in <-chan TestFileInterval) []TestFileInterval {
	layout := make([]TestFileInterval, 0, 4096)
	for i := range in {
		log.Trace("unstream", i)
		layout = append(layout, i)
	}
	return layout
}

func streamLayout(in []TestFileInterval) (out chan TestFileInterval) {
	out = make(chan TestFileInterval, 128)

	go func() {
		for _, i := range in {
			log.Trace("stream", i)
			out <- i
		}
		close(out)
	}()

	return out
}

func teeLayout(in <-chan TestFileInterval) (out1 chan TestFileInterval, out2 chan TestFileInterval) {
	out1 = make(chan TestFileInterval, 128)
	out2 = make(chan TestFileInterval, 128)

	go func() {
		for i := range in {
			log.Trace("Tee1...")
			out1 <- i
			log.Trace("Tee2...")
			out2 <- i
		}
		close(out1)
		close(out2)
	}()

	return out1, out2
}

func generateLayout(prefix string, size, seed int64) <-chan TestFileInterval {
	const maxInterval = 256 // Blocks
	layoutStream := make(chan TestFileInterval, 128)
	r := rand.New(rand.NewSource(seed))

	go func() {
		offset := int64(0)
		for offset < size {
			blocks := int64(r.Intn(maxInterval)) + 1 // 1..maxInterval
			length := blocks * Blocks
			if offset+length > size {
				// don't overshoot size
				length = size - offset
			}

			interval := Interval{offset, offset + length}
			offset += interval.Len()

			kind := SparseHole
			var mask byte
			if r.Intn(2) == 0 {
				// Data
				kind = SparseData
				mask = 0xAA * byte(r.Intn(10)/9) // 10%
			}
			t := TestFileInterval{FileInterval{kind, interval}, mask}
			log.Debug(prefix, t)
			layoutStream <- t
		}
		close(layoutStream)
	}()

	return layoutStream
}

func makeIntervalData(interval TestFileInterval) []byte {
	data := make([]byte, interval.Len())
	if SparseData == interval.Kind {
		for i := range data {
			value := byte((interval.Begin + int64(i)) / Blocks)
			data[i] = interval.dataMask ^ value
		}
	}
	return data
}

func createTestSparseFileLayout(name string, fileSize int64, layout <-chan TestFileInterval) (done chan struct{}) {
	done = make(chan struct{})

	// Fill up file with layout data
	go func() {
		f, err := os.Create(name)
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()
		err = f.Truncate(fileSize)
		if err != nil {
			log.Fatal(err)
		}

		for interval := range layout {
			log.Debug("writing...", interval)
			if SparseData == interval.Kind {
				size := batch * Blocks
				for offset := interval.Begin; offset < interval.End; {
					if offset+size > interval.End {
						size = interval.End - offset
					}
					chunkInterval := TestFileInterval{FileInterval{SparseData, Interval{offset, offset + size}}, interval.dataMask}
					data := makeIntervalData(chunkInterval)
					_, err = f.WriteAt(data, offset)
					if err != nil {
						log.Fatal(err)
					}
					offset += size
				}
			}
		}
		f.Sync()
		close(done)
	}()

	return done
}

func checkTestSparseFileLayout(name string, layout <-chan TestFileInterval) error {
	f, err := os.Open(name)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	// Read and check data
	for interval := range layout {
		log.Debug("checking...", interval)
		if SparseData == interval.Kind {
			size := batch * Blocks
			for offset := interval.Begin; offset < interval.End; {
				if offset+size > interval.End {
					size = interval.End - offset
				}
				dataModel := makeIntervalData(TestFileInterval{FileInterval{SparseData, Interval{offset, offset + size}}, interval.dataMask})
				data := make([]byte, size)
				f.ReadAt(data, offset)
				offset += size

				if !bytes.Equal(data, dataModel) {
					return errors.New(fmt.Sprint("data equality check failure at", interval))
				}
			}
		} else if SparseHole == interval.Kind {
			layoutActual, err := RetrieveLayout(f, interval.Interval)
			if err != nil {
				return errors.New(fmt.Sprint("hole retrieval failure at", interval, err))
			}
			if len(layoutActual) != 1 {
				return errors.New(fmt.Sprint("hole check failure at", interval))
			}
			if layoutActual[0] != interval.FileInterval {
				return errors.New(fmt.Sprint("hole equality check failure at", interval))
			}
		}
	}
	return nil // success
}
<|endoftext|>