<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package export defines the handlers for managing exposure key exporting.
package export

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"time"

	"github.com/google/exposure-notifications-server/internal/database"
	"github.com/google/exposure-notifications-server/internal/logging"
	"github.com/google/exposure-notifications-server/internal/model"
)

// CreateBatchesHandler is a handler to iterate the rows of ExportConfig and
// create entries in ExportBatchJob as appropriate.
func (s *Server) CreateBatchesHandler(w http.ResponseWriter, r *http.Request) {
	ctx, cancel := context.WithTimeout(r.Context(), s.config.CreateTimeout)
	defer cancel()
	logger := logging.FromContext(ctx)
	metrics := s.env.MetricsExporter(ctx)

	// Obtain lock to make sure there are no other processes working to create batches.
	lock := "create_batches"
	unlockFn, err := s.db.Lock(ctx, lock, s.config.CreateTimeout)
	if err != nil {
		if errors.Is(err, database.ErrAlreadyLocked) {
			metrics.WriteInt("export-batcher-lock-contention", true, 1)
			msg := fmt.Sprintf("Lock %s already in use, no work will be performed", lock)
			logger.Infof(msg)
			w.Write([]byte(msg)) // We return status 200 here so that Cloud Scheduler does not retry.
			return
		}
		logger.Errorf("Could not acquire lock %s: %v", lock, err)
		http.Error(w, fmt.Sprintf("Could not acquire lock %s, check logs.", lock), http.StatusInternalServerError)
		return
	}
	defer unlockFn()

	now := time.Now()
	err = s.db.IterateExportConfigs(ctx, now, func(ec *model.ExportConfig) error {
		if err := s.maybeCreateBatches(ctx, ec, now); err != nil {
			logger.Errorf("Failed to create batches for config %d: %v, continuing to next config", ec.ConfigID, err)
		}
		return nil
	})
	if err != nil {
		// some specific error handling below, but just need one metric.
		metrics.WriteInt("export-batcher-failed", true, 1)
	}
	switch {
	case err == nil:
		return
	case errors.Is(err, context.DeadlineExceeded):
		logger.Infof("Timed out creating batches, batch creation will continue on next invocation")
	case errors.Is(err, context.Canceled):
		logger.Infof("Canceled while creating batches, batch creation will continue on next invocation")
	default:
		logger.Errorf("creating batches: %v", err)
		http.Error(w, "Failed to create batches, check logs.", http.StatusInternalServerError)
	}
}

func (s *Server) maybeCreateBatches(ctx context.Context, ec *model.ExportConfig, now time.Time) error {
	logger := logging.FromContext(ctx)
	metrics := s.env.MetricsExporter(ctx)

	latestEnd, err := s.db.LatestExportBatchEnd(ctx, ec)
	if err != nil {
		return fmt.Errorf("fetching most recent batch for config %d: %w", ec.ConfigID, err)
	}

	ranges := makeBatchRanges(ec.Period, latestEnd, now)
	if len(ranges) == 0 {
		metrics.WriteInt("export-batcher-no-work", true, 1)
		logger.Infof("Batch creation for config %d is not required, skipping", ec.ConfigID)
		return nil
	}

	var batches []*model.ExportBatch
	for _, br := range ranges {
		batches = append(batches, &model.ExportBatch{
			ConfigID:       ec.ConfigID,
			BucketName:     ec.BucketName,
			FilenameRoot:   ec.FilenameRoot,
			StartTimestamp: br.start,
			EndTimestamp:   br.end,
			Region:         ec.Region,
			Status:         model.ExportBatchOpen,
			SigningKey:     ec.SigningKey,
		})
	}

	if err := s.db.AddExportBatches(ctx, batches); err != nil {
		return fmt.Errorf("creating export batches for config %d: %w", ec.ConfigID, err)
	}

	metrics.WriteInt("export-batcher-created", true, len(batches))
	logger.Infof("Created %d batch(es) for config %d", len(batches), ec.ConfigID)
	return nil
}

type batchRange struct {
	start, end time.Time
}

var sanityDate = time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC)

func makeBatchRanges(period time.Duration, latestEnd, now time.Time) []batchRange {

	// Compute the end of the exposure publish window; we don't want any batches with an end date greater than this time.
	publishEnd := model.TruncateWindow(now)

	// Special case: if there have not been batches before, return only a single one.
	// We use sanityDate here because the loop below will happily create batch ranges
	// until the beginning of time otherwise.
	if latestEnd.Before(sanityDate) {
		// We want to create a batch aligned on the period, but not overlapping the current publish window.
		// To do this, we use the publishEnd and truncate it to the period; this becomes the end date.
		// Then we just subtract the period to get the start date.
		end := publishEnd.Truncate(period)
		start := end.Add(-period)
		return []batchRange{{start: start, end: end}}
	}

	// Truncate now to align with period; use this as the end date.
	end := now.Truncate(period)

	// If the end date < latest end date, we already have a batch that covers this period, so return no batches.
	if end.Before(latestEnd) {
		return nil
	}

	// Subtract period to get the start date.
	start := end.Add(-period)

	// Build up a list of batches until we reach that latestEnd.
	// Allow for overlap so we don't miss keys; this might happen in the event that
	// an ExportConfig was edited and the new settings don't quite align.
	ranges := []batchRange{}
	for end.After(latestEnd) {
		// If the batch's end is after the publish window, don't add this range.
		if !end.After(publishEnd) {
			ranges = append([]batchRange{{start: start, end: end}}, ranges...)
		}
		start = start.Add(-period)
		end = end.Add(-period)
	}
	return ranges
}
<commit_msg>Improved export batcher logging. (#216)<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package export defines the handlers for managing exposure key exporting.
package export

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"time"

	"github.com/google/exposure-notifications-server/internal/database"
	"github.com/google/exposure-notifications-server/internal/logging"
	"github.com/google/exposure-notifications-server/internal/model"
)

// CreateBatchesHandler is a handler to iterate the rows of ExportConfig and
// create entries in ExportBatchJob as appropriate.
func (s *Server) CreateBatchesHandler(w http.ResponseWriter, r *http.Request) {
	ctx, cancel := context.WithTimeout(r.Context(), s.config.CreateTimeout)
	defer cancel()
	logger := logging.FromContext(ctx)
	metrics := s.env.MetricsExporter(ctx)

	// Obtain lock to make sure there are no other processes working to create batches.
	lock := "create_batches"
	unlockFn, err := s.db.Lock(ctx, lock, s.config.CreateTimeout)
	if err != nil {
		if errors.Is(err, database.ErrAlreadyLocked) {
			metrics.WriteInt("export-batcher-lock-contention", true, 1)
			msg := fmt.Sprintf("Lock %s already in use, no work will be performed", lock)
			logger.Infof(msg)
			w.Write([]byte(msg)) // We return status 200 here so that Cloud Scheduler does not retry.
			return
		}
		logger.Errorf("Could not acquire lock %s: %v", lock, err)
		http.Error(w, fmt.Sprintf("Could not acquire lock %s, check logs.", lock), http.StatusInternalServerError)
		return
	}
	defer unlockFn()

	total := 0
	totalWithBatches := 0
	defer func() {
		logger.Infof("Processed %d export configs, creating batches for %d", total, totalWithBatches)
	}()

	now := time.Now()
	err = s.db.IterateExportConfigs(ctx, now, func(ec *model.ExportConfig) error {
		if createdBatches, err := s.maybeCreateBatches(ctx, ec, now); err != nil {
			logger.Errorf("Failed to create batches for config %d: %v, continuing to next config", ec.ConfigID, err)
		} else {
			total++
			if createdBatches {
				totalWithBatches++
			}
		}
		return nil
	})
	if err != nil {
		// some specific error handling below, but just need one metric.
		metrics.WriteInt("export-batcher-failed", true, 1)
	}
	switch {
	case err == nil:
		return
	case errors.Is(err, context.DeadlineExceeded):
		logger.Infof("Timed out creating batches, batch creation will continue on next invocation")
	case errors.Is(err, context.Canceled):
		logger.Infof("Canceled while creating batches, batch creation will continue on next invocation")
	default:
		logger.Errorf("creating batches: %v", err)
		http.Error(w, "Failed to create batches, check logs.", http.StatusInternalServerError)
	}
}

func (s *Server) maybeCreateBatches(ctx context.Context, ec *model.ExportConfig, now time.Time) (bool, error) {
	logger := logging.FromContext(ctx)
	metrics := s.env.MetricsExporter(ctx)

	latestEnd, err := s.db.LatestExportBatchEnd(ctx, ec)
	if err != nil {
		return false, fmt.Errorf("fetching most recent batch for config %d: %w", ec.ConfigID, err)
	}

	ranges := makeBatchRanges(ec.Period, latestEnd, now)
	if len(ranges) == 0 {
		metrics.WriteInt("export-batcher-no-work", true, 1)
		logger.Debugf("Batch creation for config %d is not required, skipping", ec.ConfigID)
		return false, nil
	}

	var batches []*model.ExportBatch
	for _, br := range ranges {
		batches = append(batches, &model.ExportBatch{
			ConfigID:       ec.ConfigID,
			BucketName:     ec.BucketName,
			FilenameRoot:   ec.FilenameRoot,
			StartTimestamp: br.start,
			EndTimestamp:   br.end,
			Region:         ec.Region,
			Status:         model.ExportBatchOpen,
			SigningKey:     ec.SigningKey,
		})
	}

	if err := s.db.AddExportBatches(ctx, batches); err != nil {
		return false, fmt.Errorf("creating export batches for config %d: %w", ec.ConfigID, err)
	}

	metrics.WriteInt("export-batcher-created", true, len(batches))
	logger.Infof("Created %d batch(es) for config %d", len(batches), ec.ConfigID)
	return true, nil
}

type batchRange struct {
	start, end time.Time
}

var sanityDate = time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC)

func makeBatchRanges(period time.Duration, latestEnd, now time.Time) []batchRange {

	// Compute the end of the exposure publish window; we don't want any batches with an end date greater than this time.
	publishEnd := model.TruncateWindow(now)

	// Special case: if there have not been batches before, return only a single one.
	// We use sanityDate here because the loop below will happily create batch ranges
	// until the beginning of time otherwise.
	if latestEnd.Before(sanityDate) {
		// We want to create a batch aligned on the period, but not overlapping the current publish window.
		// To do this, we use the publishEnd and truncate it to the period; this becomes the end date.
		// Then we just subtract the period to get the start date.
		end := publishEnd.Truncate(period)
		start := end.Add(-period)
		return []batchRange{{start: start, end: end}}
	}

	// Truncate now to align with period; use this as the end date.
	end := now.Truncate(period)

	// If the end date < latest end date, we already have a batch that covers this period, so return no batches.
	if end.Before(latestEnd) {
		return nil
	}

	// Subtract period to get the start date.
	start := end.Add(-period)

	// Build up a list of batches until we reach that latestEnd.
	// Allow for overlap so we don't miss keys; this might happen in the event that
	// an ExportConfig was edited and the new settings don't quite align.
	ranges := []batchRange{}
	for end.After(latestEnd) {
		// If the batch's end is after the publish window, don't add this range.
		if !end.After(publishEnd) {
			ranges = append([]batchRange{{start: start, end: end}}, ranges...)
		}
		start = start.Add(-period)
		end = end.Add(-period)
	}
	return ranges
}
<|endoftext|>


<commit_before>package models

import (
	"fmt"
	"log"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/brnstz/bus/internal/conf"
	"github.com/brnstz/bus/internal/etc"
	"github.com/jmoiron/sqlx"
)

const (
	maxStops = 15
)

type HereResult struct {
	AgencyID  string `db:"agency_id"`
	RouteID   string `db:"route_id"`
	StopID    string `db:"stop_id"`
	ServiceID string `db:"service_id"`

	TripIDs       string `db:"trip_ids"`
	ArrivalSecs   string `db:"arrival_secs"`
	DepartureSecs string `db:"departure_secs"`
	StopSequences string `db:"stop_sequences"`
	NextStopLats  string `db:"next_stop_lats"`
	NextStopLons  string `db:"next_stop_lons"`

	TripHeadsign string `db:"trip_headsign"`

	StopName     string  `db:"stop_name"`
	StopHeadsign string  `db:"stop_headsign"`
	DirectionID  int     `db:"direction_id"`
	Lat          float64 `db:"lat"`
	Lon          float64 `db:"lon"`
	Dist         float64 `db:"dist"`

	RouteType      int    `db:"route_type"`
	RouteColor     string `db:"route_color"`
	RouteTextColor string `db:"route_text_color"`
	RouteShortName string `db:"route_short_name"`
	RouteLongName  string `db:"route_long_name"`

	HQ *HereQuery

	Stop  *Stop
	Route *Route

	Departures []*Departure
}

func (h *HereResult) createStop() (stop *Stop, err error) {

	stop = &Stop{
		StopID:      h.StopID,
		RouteID:     h.RouteID,
		AgencyID:    h.AgencyID,
		Name:        h.StopName,
		DirectionID: h.DirectionID,
		Headsign:    h.StopHeadsign,
		Lat:         h.Lat,
		Lon:         h.Lon,
		Dist:        h.Dist,

		RouteType:      h.RouteType,
		RouteColor:     h.RouteColor,
		RouteTextColor: h.RouteTextColor,

		RouteShortName: h.RouteShortName,
		RouteLongName:  h.RouteLongName,
		TripHeadsign:   h.TripHeadsign,
	}
	err = stop.Initialize()
	if err != nil {
		log.Println("can't init stop", err)
		return
	}

	return
}

func (h *HereResult) createRoute() (route *Route, err error) {
	route = &Route{
		RouteID:   h.RouteID,
		AgencyID:  h.AgencyID,
		Type:      h.RouteType,
		Color:     h.RouteColor,
		TextColor: h.RouteTextColor,
	}

	err = route.Initialize()
	if err != nil {
		log.Println("can't init route", err)
		return
	}

	return
}

func (h *HereResult) Initialize() error {
	var err error

	h.Stop, err = h.createStop()
	if err != nil {
		log.Println("can't init here stop", err)
		return err
	}

	h.Route, err = h.createRoute()
	if err != nil {
		log.Println("can't init here route", err)
		return err
	}

	h.Departures, err = h.createDepartures()
	if err != nil {
		log.Println("can't init departure", err)
		return err
	}

	return nil
}

func (h *HereResult) createDepartures() (departures []*Departure, err error) {
	var (
		departureSec  int
		compassDir    float64
		nextLat       float64
		nextLon       float64
		tripID        string
		departureBase time.Time
	)

	departureSecs := strings.Split(h.DepartureSecs, ",")
	tripIDs := strings.Split(h.TripIDs, ",")
	nextLats := strings.Split(h.NextStopLats, ",")
	nextLons := strings.Split(h.NextStopLons, ",")

	if len(departureSecs) < 1 {
		err = fmt.Errorf("invalid departureSecs: %v", h.DepartureSecs)
		return
	}
	if departureSecs[0] == "" {
		err = fmt.Errorf("empty departure secs")
		return
	}
	if len(departureSecs) != len(tripIDs) {
		err = fmt.Errorf("mismatch between departureSecs length (%v) and tripIDs length (%v)", len(departureSecs), len(tripIDs))
		return
	}

	if len(departureSecs) != len(nextLats) {
		err = fmt.Errorf("mismatch between departureSecs length (%v) and nextLats length (%v)", len(departureSecs), len(nextLats))
		return
	}

	if len(departureSecs) != len(nextLons) {
		err = fmt.Errorf("mismatch between departureSecs length (%v) and nextLons length (%v)", len(departureSecs), len(nextLons))
		return
	}

	for i := range departureSecs {
		relID := h.AgencyID + "|" + h.RouteID + "|" + h.ServiceID

		departureSec, err = strconv.Atoi(strings.TrimSpace(departureSecs[i]))
		if err != nil {
			log.Println("can't parse departure sec", err)
			return
		}
		tripID = strings.TrimSpace(tripIDs[i])
		nextLat, err = strconv.ParseFloat(nextLats[i], 64)
		if err != nil {
			log.Println("can't parse next lat dir", err)
			return
		}

		nextLon, err = strconv.ParseFloat(nextLons[i], 64)
		if err != nil {
			log.Println("can't parse next lon dir", err)
			return
		}

		compassDir = etc.Bearing(h.Lat, h.Lon, nextLat, nextLon)

		// We have up to three non-overlapping ranges of departure sec,
		// that could be yesterday, today or tomorrow. We're able to do this
		// because the range is only 3 hours.

		if departureSec >= h.HQ.YesterdayDepartureMin &&
			departureSec <= h.HQ.YesterdayDepartureMax &&
			h.HQ.YesterdayRelevantIDs[relID] {
			departureBase = h.HQ.YesterdayDepartureBase

		} else if departureSec >= h.HQ.TodayDepartureMin &&
			departureSec <= h.HQ.TodayDepartureMax &&
			h.HQ.TodayRelevantIDs[relID] {
			departureBase = h.HQ.TodayDepartureBase

		} else if departureSec >= h.HQ.TomorrowDepartureMin &&
			departureSec <= h.HQ.TomorrowDepartureMax &&
			h.HQ.TomorrowRelevantIDs[relID] {
			departureBase = h.HQ.TomorrowDepartureBase

		} else {
			// If it's not in our range, then we ignore it
			continue
		}

		departure := &Departure{
			DepartureSec: departureSec,
			TripID:       tripID,
			ServiceID:    h.ServiceID,
			baseTime:     departureBase,
			CompassDir:   compassDir,
		}

		err = departure.Initialize()
		if err != nil {
			log.Println("can't init departure", err)
			return
		}

		departures = append(departures, departure)
	}

	return
}

func GetHereResults(db sqlx.Ext, hq *HereQuery) (stops []*Stop, stopRoutes map[string]*Route, err error) {

	// mapping of stop.UniqueID to route
	stopRoutes = map[string]*Route{}

	// mapping of stop.UniqueID to stop
	sm := map[string]*Stop{}

	// mapping of route.UniqueID + DirectionID to route
	rm := map[string]*Route{}

	// overall function timing
	if conf.API.LogTiming {
		t1 := time.Now()
		defer func() { log.Println(time.Now().Sub(t1)) }()
	}

	//t3 := time.Now()
	rows, err := sqlx.NamedQuery(db, hq.Query, hq)
	if err != nil {
		log.Println("can't get stops", err)
		log.Printf("%s %+v", hq.Query, hq)
		return
	}
	/*
		queryDur := time.Now().Sub(t3)
		if conf.API.LogTiming && queryDur > time.Duration(1)*time.Second {
			log.Printf("long here query (%v): %+v", queryDur, hq)
		}
	*/

	defer rows.Close()

	count := 0
	for rows.Next() {
		here := HereResult{HQ: hq}

		err = rows.StructScan(&here)
		if err != nil {
			log.Println("can't scan row", err)
			continue
		}

		err = here.Initialize()
		if err != nil {
			log.Println("can't initialize here", err)
			continue
		}

		for _, departure := range here.Departures {

			routeHead := fmt.Sprintf("%v|%v", here.Route.UniqueID, here.Stop.TripHeadsign)

			_, stopExists := sm[here.Stop.UniqueID]
			_, routeExists := rm[routeHead]

			// Ignore when the route / direction already exists, but stop is not
			// the same
			if routeExists && !stopExists {
				continue
			}

			// If we didn't have stop or route, put them in our map
			if !stopExists {
				sm[here.Stop.UniqueID] = here.Stop
			}
			if !routeExists {
				rm[routeHead] = here.Route
			}

			// Get the stop and append the current departure
			stop := sm[here.Stop.UniqueID]
			stop.Departures = append(stop.Departures, departure)
			stopRoutes[here.Stop.UniqueID] = here.Route

			count++
		}

	}

	// Add all stops to sortableStops list
	for _, s := range sm {
		stops = append(stops, s)
	}

	ss := newSortableStops(stops)

	// Our goal is to sort
	// - Trains before buses
	// - Then by distance
	// - Grouped by route_id
	// - With consistent direction id 0/1 ordering
	ss.by = byDir
	sort.Sort(ss)

	ss.by = byRoute
	sort.Stable(ss)

	ss.by = byDist
	sort.Stable(ss)

	ss.by = byType
	sort.Stable(ss)

	// Assign stops to our return value
	if len(ss.stops) > maxStops {
		stops = []*Stop(ss.stops[0:maxStops])
	} else {
		stops = []*Stop(ss.stops)
	}

	// For each stop, sort its departures and limit to max number of departures
	for _, s := range stops {
		d := SortableDepartures(s.Departures)
		sort.Sort(d)

		if len(d) > MaxDepartures {
			s.Departures = []*Departure(d[0:MaxDepartures])
		} else {
			s.Departures = []*Departure(d)
		}

		// FIXME: is this check necesssary? where do we filter stops with no departures (oy)
		if len(s.Departures) > 0 {
			s.FallbackTripID = s.Departures[0].TripID
		}
	}

	return
}
<commit_msg>back to up 30<commit_after>package models

import (
	"fmt"
	"log"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/brnstz/bus/internal/conf"
	"github.com/brnstz/bus/internal/etc"
	"github.com/jmoiron/sqlx"
)

const (
	maxStops = 30
)

type HereResult struct {
	AgencyID  string `db:"agency_id"`
	RouteID   string `db:"route_id"`
	StopID    string `db:"stop_id"`
	ServiceID string `db:"service_id"`

	TripIDs       string `db:"trip_ids"`
	ArrivalSecs   string `db:"arrival_secs"`
	DepartureSecs string `db:"departure_secs"`
	StopSequences string `db:"stop_sequences"`
	NextStopLats  string `db:"next_stop_lats"`
	NextStopLons  string `db:"next_stop_lons"`

	TripHeadsign string `db:"trip_headsign"`

	StopName     string  `db:"stop_name"`
	StopHeadsign string  `db:"stop_headsign"`
	DirectionID  int     `db:"direction_id"`
	Lat          float64 `db:"lat"`
	Lon          float64 `db:"lon"`
	Dist         float64 `db:"dist"`

	RouteType      int    `db:"route_type"`
	RouteColor     string `db:"route_color"`
	RouteTextColor string `db:"route_text_color"`
	RouteShortName string `db:"route_short_name"`
	RouteLongName  string `db:"route_long_name"`

	HQ *HereQuery

	Stop  *Stop
	Route *Route

	Departures []*Departure
}

func (h *HereResult) createStop() (stop *Stop, err error) {

	stop = &Stop{
		StopID:      h.StopID,
		RouteID:     h.RouteID,
		AgencyID:    h.AgencyID,
		Name:        h.StopName,
		DirectionID: h.DirectionID,
		Headsign:    h.StopHeadsign,
		Lat:         h.Lat,
		Lon:         h.Lon,
		Dist:        h.Dist,

		RouteType:      h.RouteType,
		RouteColor:     h.RouteColor,
		RouteTextColor: h.RouteTextColor,

		RouteShortName: h.RouteShortName,
		RouteLongName:  h.RouteLongName,
		TripHeadsign:   h.TripHeadsign,
	}
	err = stop.Initialize()
	if err != nil {
		log.Println("can't init stop", err)
		return
	}

	return
}

func (h *HereResult) createRoute() (route *Route, err error) {
	route = &Route{
		RouteID:   h.RouteID,
		AgencyID:  h.AgencyID,
		Type:      h.RouteType,
		Color:     h.RouteColor,
		TextColor: h.RouteTextColor,
	}

	err = route.Initialize()
	if err != nil {
		log.Println("can't init route", err)
		return
	}

	return
}

func (h *HereResult) Initialize() error {
	var err error

	h.Stop, err = h.createStop()
	if err != nil {
		log.Println("can't init here stop", err)
		return err
	}

	h.Route, err = h.createRoute()
	if err != nil {
		log.Println("can't init here route", err)
		return err
	}

	h.Departures, err = h.createDepartures()
	if err != nil {
		log.Println("can't init departure", err)
		return err
	}

	return nil
}

func (h *HereResult) createDepartures() (departures []*Departure, err error) {
	var (
		departureSec  int
		compassDir    float64
		nextLat       float64
		nextLon       float64
		tripID        string
		departureBase time.Time
	)

	departureSecs := strings.Split(h.DepartureSecs, ",")
	tripIDs := strings.Split(h.TripIDs, ",")
	nextLats := strings.Split(h.NextStopLats, ",")
	nextLons := strings.Split(h.NextStopLons, ",")

	if len(departureSecs) < 1 {
		err = fmt.Errorf("invalid departureSecs: %v", h.DepartureSecs)
		return
	}
	if departureSecs[0] == "" {
		err = fmt.Errorf("empty departure secs")
		return
	}
	if len(departureSecs) != len(tripIDs) {
		err = fmt.Errorf("mismatch between departureSecs length (%v) and tripIDs length (%v)", len(departureSecs), len(tripIDs))
		return
	}

	if len(departureSecs) != len(nextLats) {
		err = fmt.Errorf("mismatch between departureSecs length (%v) and nextLats length (%v)", len(departureSecs), len(nextLats))
		return
	}

	if len(departureSecs) != len(nextLons) {
		err = fmt.Errorf("mismatch between departureSecs length (%v) and nextLons length (%v)", len(departureSecs), len(nextLons))
		return
	}

	for i := range departureSecs {
		relID := h.AgencyID + "|" + h.RouteID + "|" + h.ServiceID

		departureSec, err = strconv.Atoi(strings.TrimSpace(departureSecs[i]))
		if err != nil {
			log.Println("can't parse departure sec", err)
			return
		}
		tripID = strings.TrimSpace(tripIDs[i])
		nextLat, err = strconv.ParseFloat(nextLats[i], 64)
		if err != nil {
			log.Println("can't parse next lat dir", err)
			return
		}

		nextLon, err = strconv.ParseFloat(nextLons[i], 64)
		if err != nil {
			log.Println("can't parse next lon dir", err)
			return
		}

		compassDir = etc.Bearing(h.Lat, h.Lon, nextLat, nextLon)

		// We have up to three non-overlapping ranges of departure sec,
		// that could be yesterday, today or tomorrow. We're able to do this
		// because the range is only 3 hours.

		if departureSec >= h.HQ.YesterdayDepartureMin &&
			departureSec <= h.HQ.YesterdayDepartureMax &&
			h.HQ.YesterdayRelevantIDs[relID] {
			departureBase = h.HQ.YesterdayDepartureBase

		} else if departureSec >= h.HQ.TodayDepartureMin &&
			departureSec <= h.HQ.TodayDepartureMax &&
			h.HQ.TodayRelevantIDs[relID] {
			departureBase = h.HQ.TodayDepartureBase

		} else if departureSec >= h.HQ.TomorrowDepartureMin &&
			departureSec <= h.HQ.TomorrowDepartureMax &&
			h.HQ.TomorrowRelevantIDs[relID] {
			departureBase = h.HQ.TomorrowDepartureBase

		} else {
			// If it's not in our range, then we ignore it
			continue
		}

		departure := &Departure{
			DepartureSec: departureSec,
			TripID:       tripID,
			ServiceID:    h.ServiceID,
			baseTime:     departureBase,
			CompassDir:   compassDir,
		}

		err = departure.Initialize()
		if err != nil {
			log.Println("can't init departure", err)
			return
		}

		departures = append(departures, departure)
	}

	return
}

func GetHereResults(db sqlx.Ext, hq *HereQuery) (stops []*Stop, stopRoutes map[string]*Route, err error) {

	// mapping of stop.UniqueID to route
	stopRoutes = map[string]*Route{}

	// mapping of stop.UniqueID to stop
	sm := map[string]*Stop{}

	// mapping of route.UniqueID + DirectionID to route
	rm := map[string]*Route{}

	// overall function timing
	if conf.API.LogTiming {
		t1 := time.Now()
		defer func() { log.Println(time.Now().Sub(t1)) }()
	}

	//t3 := time.Now()
	rows, err := sqlx.NamedQuery(db, hq.Query, hq)
	if err != nil {
		log.Println("can't get stops", err)
		log.Printf("%s %+v", hq.Query, hq)
		return
	}
	/*
		queryDur := time.Now().Sub(t3)
		if conf.API.LogTiming && queryDur > time.Duration(1)*time.Second {
			log.Printf("long here query (%v): %+v", queryDur, hq)
		}
	*/

	defer rows.Close()

	count := 0
	for rows.Next() {
		here := HereResult{HQ: hq}

		err = rows.StructScan(&here)
		if err != nil {
			log.Println("can't scan row", err)
			continue
		}

		err = here.Initialize()
		if err != nil {
			log.Println("can't initialize here", err)
			continue
		}

		for _, departure := range here.Departures {

			routeHead := fmt.Sprintf("%v|%v", here.Route.UniqueID, here.Stop.TripHeadsign)

			_, stopExists := sm[here.Stop.UniqueID]
			_, routeExists := rm[routeHead]

			// Ignore when the route / direction already exists, but stop is not
			// the same
			if routeExists && !stopExists {
				continue
			}

			// If we didn't have stop or route, put them in our map
			if !stopExists {
				sm[here.Stop.UniqueID] = here.Stop
			}
			if !routeExists {
				rm[routeHead] = here.Route
			}

			// Get the stop and append the current departure
			stop := sm[here.Stop.UniqueID]
			stop.Departures = append(stop.Departures, departure)
			stopRoutes[here.Stop.UniqueID] = here.Route

			count++
		}

	}

	// Add all stops to sortableStops list
	for _, s := range sm {
		stops = append(stops, s)
	}

	ss := newSortableStops(stops)

	// Our goal is to sort
	// - Trains before buses
	// - Then by distance
	// - Grouped by route_id
	// - With consistent direction id 0/1 ordering
	ss.by = byDir
	sort.Sort(ss)

	ss.by = byRoute
	sort.Stable(ss)

	ss.by = byDist
	sort.Stable(ss)

	ss.by = byType
	sort.Stable(ss)

	// Assign stops to our return value
	if len(ss.stops) > maxStops {
		stops = []*Stop(ss.stops[0:maxStops])
	} else {
		stops = []*Stop(ss.stops)
	}

	// For each stop, sort its departures and limit to max number of departures
	for _, s := range stops {
		d := SortableDepartures(s.Departures)
		sort.Sort(d)

		if len(d) > MaxDepartures {
			s.Departures = []*Departure(d[0:MaxDepartures])
		} else {
			s.Departures = []*Departure(d)
		}

		// FIXME: is this check necesssary? where do we filter stops with no departures (oy)
		if len(s.Departures) > 0 {
			s.FallbackTripID = s.Departures[0].TripID
		}
	}

	return
}
<|endoftext|>


<commit_before><commit_msg>bagpack test<commit_after>package mnemosyned

import "testing"

func TestBagpack(t *testing.T) {
	b := bagpack{}

	b.Set("key", "value")

	if !b.Has("key") {
		t.Errorf("bagpack should have specified key")
	}

	if b.Get("key") != "value" {
		t.Errorf("bagpack should have specified key")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"strings"

	"github.com/foursquare/gohfile"
)

type CollectionSpecList struct {
	Collections []SingleCollectionSpec
}

type SingleCollectionSpec struct {
	Capacity      int
	Collection    string
	Function      string
	LockNamespace string
	Partition     int
	Path          string
}

func LoadFromUrl(url string) ([]hfile.CollectionConfig, error) {
	configs, err := ConfigsFromJsonUrl(url)
	if err != nil {
		return nil, err
	}
	log.Printf("Found %d collections in config:", len(configs))
	for _, cfg := range configs {
		if Settings.debug {
			log.Printf("\t%s (%s)", cfg.Name, cfg.Path)
		} else {
			log.Printf("\t%s", cfg.Name)
		}
	}
	return FetchCollections(configs)
}

func ConfigsFromJsonUrl(url string) ([]hfile.CollectionConfig, error) {
	if Settings.debug {
		log.Printf("[ConfigsFromJsonUrl] Fetching config from %s...\n", url)
	}
	res, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	if Settings.debug {
		log.Printf("[ConfigsFromJsonUrl] Fetched. Parsing...\n")
	}
	defer res.Body.Close()

	var specs CollectionSpecList

	if err := json.NewDecoder(res.Body).Decode(&specs); err != nil {
		return nil, err
	}

	if Settings.debug {
		log.Printf("[ConfigsFromJsonUrl] Found %d collections.\n", len(specs.Collections))
	}

	ret := make([]hfile.CollectionConfig, len(specs.Collections))
	for i, spec := range specs.Collections {
		name := fmt.Sprintf("%s/%d", spec.Collection, spec.Partition)
		ret[i] = hfile.CollectionConfig{name, spec.Path, false}
	}
	return ret, nil
}

func FetchCollections(unfetched []hfile.CollectionConfig) ([]hfile.CollectionConfig, error) {
	if Settings.debug {
		log.Printf("[FetchCollections] Checking for non-local collections...")
	}

	fetched := make([]hfile.CollectionConfig, len(unfetched))
	for i, cfg := range unfetched {
		if isRemote, remote := IsRemote(cfg.Path); isRemote {

			if Settings.debug {
				log.Printf("[FetchCollections] %s (%s) is remote path on (%s)", cfg.Name, cfg.Path, remote)
			}

			if local, err := FetchRemote(cfg.Name, remote); err != nil {
				return nil, err
			} else {
				cfg.Path = local
			}
		} else if Settings.debug {
			log.Printf("[FetchCollections] %s (%s) is local path.", cfg.Name, cfg.Path)
		}
		fetched[i] = cfg
	}
	return fetched, nil
}

func IsRemote(p string) (bool, string) {
	for prefix, format := range Settings.remotePrefixes.prefixes {
		if strings.HasPrefix(p, prefix) {
			trimmed := strings.TrimPrefix(p, prefix)
			full := fmt.Sprintf(format, trimmed)
			if Settings.debug {
				log.Printf("[IsRmote] path %s is remote: %s", trimmed, full)
			}
			return true, full
		}
	}
	return false, p
}

func FetchRemote(name, remote string) (string, error) {
	h := md5.Sum([]byte(remote))

	base := hex.EncodeToString(h[:]) + ".hfile"

	local := path.Join(Settings.cachePath, base)

	if _, err := os.Stat(local); err == nil {
		if Settings.debug {
			log.Printf("[FetchRemote] %s (%s) already exists at %s.", name, remote, local)
		}
		return local, nil
	} else if !os.IsNotExist(err) {
		if Settings.debug {
			log.Printf("[FetchRemote] %s Error checking local file %s: %v.", name, local, err)
		}
		return "", err
	}

	log.Printf("[FetchRemote] Fetching %s: %s -> %s.", name, remote, local)
	fp, err := os.Create(local)

	if err != nil {
		return "", err
	}
	defer fp.Close()

	resp, err := http.Get(remote)

	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	_, err = io.Copy(fp, resp.Body)
	if err != nil {
		return "", err
	}

	if Settings.debug {
		log.Printf("[FetchRemote] Fetched %s.", name)
	}
	return local, nil
}
<commit_msg>check webhdfs status<commit_after>package main

import (
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"strings"

	"github.com/foursquare/gohfile"
)

type CollectionSpecList struct {
	Collections []SingleCollectionSpec
}

type SingleCollectionSpec struct {
	Capacity      int
	Collection    string
	Function      string
	LockNamespace string
	Partition     int
	Path          string
}

func LoadFromUrl(url string) ([]hfile.CollectionConfig, error) {
	configs, err := ConfigsFromJsonUrl(url)
	if err != nil {
		return nil, err
	}
	log.Printf("Found %d collections in config:", len(configs))
	for _, cfg := range configs {
		if Settings.debug {
			log.Printf("\t%s (%s)", cfg.Name, cfg.Path)
		} else {
			log.Printf("\t%s", cfg.Name)
		}
	}
	return FetchCollections(configs)
}

func ConfigsFromJsonUrl(url string) ([]hfile.CollectionConfig, error) {
	if Settings.debug {
		log.Printf("[ConfigsFromJsonUrl] Fetching config from %s...\n", url)
	}
	res, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	if Settings.debug {
		log.Printf("[ConfigsFromJsonUrl] Fetched. Parsing...\n")
	}
	defer res.Body.Close()

	var specs CollectionSpecList

	if err := json.NewDecoder(res.Body).Decode(&specs); err != nil {
		return nil, err
	}

	if Settings.debug {
		log.Printf("[ConfigsFromJsonUrl] Found %d collections.\n", len(specs.Collections))
	}

	ret := make([]hfile.CollectionConfig, len(specs.Collections))
	for i, spec := range specs.Collections {
		name := fmt.Sprintf("%s/%d", spec.Collection, spec.Partition)
		ret[i] = hfile.CollectionConfig{name, spec.Path, false}
	}
	return ret, nil
}

func FetchCollections(unfetched []hfile.CollectionConfig) ([]hfile.CollectionConfig, error) {
	if Settings.debug {
		log.Printf("[FetchCollections] Checking for non-local collections...")
	}

	fetched := make([]hfile.CollectionConfig, len(unfetched))
	for i, cfg := range unfetched {
		if isRemote, remote := IsRemote(cfg.Path); isRemote {

			if Settings.debug {
				log.Printf("[FetchCollections] %s (%s) is remote path on (%s)", cfg.Name, cfg.Path, remote)
			}

			if local, err := FetchRemote(cfg.Name, remote); err != nil {
				return nil, err
			} else {
				cfg.Path = local
			}
		} else if Settings.debug {
			log.Printf("[FetchCollections] %s (%s) is local path.", cfg.Name, cfg.Path)
		}
		fetched[i] = cfg
	}
	return fetched, nil
}

func IsRemote(p string) (bool, string) {
	for prefix, format := range Settings.remotePrefixes.prefixes {
		if strings.HasPrefix(p, prefix) {
			trimmed := strings.TrimPrefix(p, prefix)
			full := fmt.Sprintf(format, trimmed)
			if Settings.debug {
				log.Printf("[IsRmote] path %s is remote: %s", trimmed, full)
			}
			return true, full
		}
	}
	return false, p
}

func FetchRemote(name, remote string) (string, error) {
	h := md5.Sum([]byte(remote))

	base := hex.EncodeToString(h[:]) + ".hfile"

	local := path.Join(Settings.cachePath, base)

	if _, err := os.Stat(local); err == nil {
		if Settings.debug {
			log.Printf("[FetchRemote] %s (%s) already exists at %s.", name, remote, local)
		}
		return local, nil
	} else if !os.IsNotExist(err) {
		if Settings.debug {
			log.Printf("[FetchRemote] %s Error checking local file %s: %v.", name, local, err)
		}
		return "", err
	}

	log.Printf("[FetchRemote] Fetching %s: %s -> %s.", name, remote, local)
	fp, err := os.Create(local)

	if err != nil {
		return "", err
	}
	defer fp.Close()

	resp, err := http.Get(remote)

	if err != nil {
		return "", err
	}
	if resp.StatusCode >= 400 {
		buf := new(bytes.Buffer)
		buf.ReadFrom(resp.Body)
		return "", fmt.Errorf("HTTP error fetching (%s): %s\n", resp.Status, buf.String())
	}

	defer resp.Body.Close()

	_, err = io.Copy(fp, resp.Body)
	if err != nil {
		return "", err
	}

	if Settings.debug {
		log.Printf("[FetchRemote] Fetched %s.", name)
	}
	return local, nil
}
<|endoftext|>


<commit_before>package ioc_test

import (
	"fmt"
	"log/syslog"
	"testing"
	"time"

	"github.com/itpkg/ioc"
)

func Hello(mod *Model, logger *syslog.Writer, version string) string {
	s := fmt.Sprintf("Model=>%v, Version=>%s", mod, version)
	logger.Info(s)
	return s
}

type Model struct {
	Now     *time.Time `inject:""`
	Version int        `inject:"version"`
}

func TestInjector(t *testing.T) {
	inj := ioc.New()

	wrt, _ := syslog.New(syslog.LOG_DEBUG, "itpkg")
	now := time.Now()
	inj.Provide(
		&ioc.Object{Value: &Model{}},
		&ioc.Object{Value: wrt},
		&ioc.Object{Value: &now},
		&ioc.Object{Name: "version", Value: 20150922},
		&ioc.Object{Value: 1.1},
		&ioc.Object{Name: "hello", Value: "Hello, it-package!"},
	)

	if err := inj.Populate(); err == nil {
		t.Logf(inj.String())
	} else {
		t.Errorf("error on populate: %v", err)
	}

	if vls, err := inj.Run(Hello, "v20150923"); err == nil {
		t.Logf(vls[0].(string))
	} else {
		t.Errorf("error on run: %v", err)
	}

}
<commit_msg>fix test logger name bug<commit_after>package ioc_test

import (
	"fmt"
	"log/syslog"
	"testing"
	"time"

	"github.com/itpkg/ioc"
)

func Hello(mod *Model, logger *syslog.Writer, version string) string {
	s := fmt.Sprintf("Model=>%v, Version=>%s", mod, version)
	logger.Info(s)
	return s
}

type Model struct {
	Now     *time.Time `inject:""`
	Version int        `inject:"version"`
}

func TestInjector(t *testing.T) {
	inj := ioc.New()

	wrt, _ := syslog.New(syslog.LOG_DEBUG, "test")
	now := time.Now()
	inj.Provide(
		&ioc.Object{Value: &Model{}},
		&ioc.Object{Value: wrt},
		&ioc.Object{Value: &now},
		&ioc.Object{Name: "version", Value: 20150922},
		&ioc.Object{Value: 1.1},
		&ioc.Object{Name: "hello", Value: "Hello, it-package!"},
	)

	if err := inj.Populate(); err == nil {
		t.Logf(inj.String())
	} else {
		t.Errorf("error on populate: %v", err)
	}

	if vls, err := inj.Run(Hello, "v20150923"); err == nil {
		t.Logf(vls[0].(string))
	} else {
		t.Errorf("error on run: %v", err)
	}

}
<|endoftext|>


<commit_before>//  Copyright (c) 2016 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package moss

import (
	"bytes"
	"container/heap"
)

// An iterator tracks a min-heap "scan-line" of cursors through a
// segmentStack.  Iterator implements the sort.Interface and
// heap.Interface on its cursors.
type iterator struct {
	ss *segmentStack

	cursors []*cursor // The len(cursors) <= len(ss.a).

	startKeyInclusive []byte
	endKeyExclusive   []byte

	iteratorOptions IteratorOptions

	lowerLevelIter Iterator // May be nil.
}

// A cursor rerpresents a logical entry position inside a segment in a
// segmentStack.  An ssIndex < 0 and pos < 0 mean that the op/k/v came
// from the lowerLevelIter.
type cursor struct {
	ssIndex int // Index into Iterator.ss.a.
	posEnd  int // Found via endKeyExclusive, or the segment length.
	pos     int // Logical entry position into Iterator.ss.a[ssIndex].

	op uint64
	k  []byte
	v  []byte
}

// StartIterator returns a new iterator on the given segmentStack.
//
// On success, the returned Iterator will be positioned so that
// Iterator.Current() will either provide the first entry in the
// iteration range or ErrIteratorDone.
//
// A startKeyInclusive of nil means the logical "bottom-most" possible
// key and an endKeyExclusive of nil means the logical "top-most"
// possible key.
//
// StartIterator can optionally include deletion operations in the
// enumeration via the IteratorOptions.IncludeDeletions flag.
//
// StartIterator can skip lower segments, via the
// IteratorOptions.MinSegmentLevel parameter.  For example, to ignore
// the lowest, 0th segment, use MinSegmentLevel of 1.
func (ss *segmentStack) StartIterator(
	startKeyInclusive, endKeyExclusive []byte,
	iteratorOptions IteratorOptions) (Iterator, error) {
	if iteratorOptions.MaxSegmentHeight <= 0 {
		iteratorOptions.MaxSegmentHeight = len(ss.a)
	}

	return ss.startIterator(startKeyInclusive, endKeyExclusive, iteratorOptions)
}

// startIterator() returns a new iterator on the given segmentStack.
//
// On success, the returned Iterator will be positioned so that
// Iterator.Current() will either provide the first entry in the
// iteration range or ErrIteratorDone.
//
// A startKeyInclusive of nil means the logical "bottom-most" possible
// key and an endKeyExclusive of nil means the logical "top-most"
// possible key.
//
// startIterator() can optionally include deletion operations in the
// enumeration via the IteratorOptions.IncludeDeletions flag.
//
// startIterator() can skip lower segments, via the
// IteratorOptions.MinSegmentLevel parameter.  For example, to ignore
// the lowest, 0th segment, use MinSegmentLevel of 1.
func (ss *segmentStack) startIterator(
	startKeyInclusive, endKeyExclusive []byte,
	iteratorOptions IteratorOptions) (*iterator, error) {
	iter := &iterator{
		ss:      ss,
		cursors: make([]*cursor, 0, len(ss.a)+1),

		startKeyInclusive: startKeyInclusive,
		endKeyExclusive:   endKeyExclusive,

		iteratorOptions: iteratorOptions,
	}

	// ----------------------------------------------
	// Add cursors for our allowed segments.

	minSegmentLevel := iteratorOptions.MinSegmentLevel
	maxSegmentLevel := iteratorOptions.MaxSegmentHeight - 1

	ss.ensureSorted(minSegmentLevel, maxSegmentLevel)

	for ssIndex := minSegmentLevel; ssIndex <= maxSegmentLevel; ssIndex++ {
		b := ss.a[ssIndex]

		posEnd := b.Len()
		if endKeyExclusive != nil {
			posEnd = b.FindStartKeyInclusivePos(endKeyExclusive)
		}

		pos := b.FindStartKeyInclusivePos(startKeyInclusive)
		if pos >= posEnd {
			continue
		}

		op, k, v := b.GetOperationKeyVal(pos)
		if op == 0 && k == nil && v == nil {
			continue
		}

		iter.cursors = append(iter.cursors, &cursor{
			ssIndex: ssIndex,
			posEnd:  posEnd,
			pos:     pos,
			op:      op,
			k:       k,
			v:       v,
		})
	}

	// ----------------------------------------------
	// Add cursor for the lower level, if wanted.

	if !iteratorOptions.SkipLowerLevel &&
		ss.lowerLevelSnapshot != nil {
		llss := ss.lowerLevelSnapshot.addRef()
		if llss != nil {
			lowerLevelIter, err := llss.StartIterator(
				startKeyInclusive, endKeyExclusive, IteratorOptions{})

			llss.decRef()

			if err != nil {
				return nil, err
			}

			k, v, err := lowerLevelIter.Current()
			if err != nil && err != ErrIteratorDone {
				return nil, err
			}
			if err == ErrIteratorDone {
				lowerLevelIter.Close()
			}
			if err == nil {
				iter.cursors = append(iter.cursors, &cursor{
					ssIndex: -1,
					pos:     -1,
					op:      OperationSet,
					k:       k,
					v:       v,
				})

				iter.lowerLevelIter = lowerLevelIter
			}
		}
	}

	// ----------------------------------------------
	// Heap-ify the cursors.

	heap.Init(iter)

	if !iteratorOptions.IncludeDeletions {
		entryEx, _, _, _ := iter.CurrentEx()
		if entryEx.Operation == OperationDel {
			iter.Next()
		}
	}

	return iter, nil
}

// Close must be invoked to release resources.
func (iter *iterator) Close() error {
	if iter.lowerLevelIter != nil {
		iter.lowerLevelIter.Close()
		iter.lowerLevelIter = nil
	}

	return nil
}

// Next returns ErrIteratorDone if the iterator is done.
func (iter *iterator) Next() error {
	if len(iter.cursors) <= 0 {
		return ErrIteratorDone
	}

	lastK := iter.cursors[0].k

	for len(iter.cursors) > 0 {
		next := iter.cursors[0]

		if next.ssIndex < 0 && next.pos < 0 {
			err := iter.lowerLevelIter.Next()
			if err == nil {
				next.k, next.v, err = iter.lowerLevelIter.Current()
				if err == nil {
					heap.Fix(iter, 0)
				}
			}

			if err != nil {
				iter.lowerLevelIter.Close()
				iter.lowerLevelIter = nil

				heap.Pop(iter)
			}
		} else {
			next.pos++
			if next.pos >= next.posEnd {
				heap.Pop(iter)
			} else {
				next.op, next.k, next.v =
					iter.ss.a[next.ssIndex].GetOperationKeyVal(next.pos)
				if next.op == 0 {
					heap.Pop(iter)
				} else {
					heap.Fix(iter, 0)
				}
			}
		}

		if len(iter.cursors) <= 0 {
			return ErrIteratorDone
		}

		if !bytes.Equal(iter.cursors[0].k, lastK) {
			if !iter.iteratorOptions.IncludeDeletions &&
				iter.cursors[0].op == OperationDel {
				return iter.Next()
			}

			return nil
		}
	}

	return ErrIteratorDone
}

// Current returns ErrIteratorDone if the iterator is done.
// Otherwise, Current() returns the current key and val, which should
// be treated as immutable or read-only.  The key and val bytes will
// remain available until the next call to Next() or Close().
func (iter *iterator) Current() ([]byte, []byte, error) {
	entryEx, key, val, err := iter.CurrentEx()
	if err != nil {
		return nil, nil, err
	}

	op := entryEx.Operation
	if op == OperationDel {
		return nil, nil, nil
	}

	if op == OperationMerge {
		valMerged, err := iter.ss.getMerged(key, val, iter.cursors[0].ssIndex-1,
			iter.iteratorOptions.base)
		if err != nil {
			return nil, nil, err
		}

		return key, valMerged, nil
	}

	return key, val, err
}

// CurrentEx is a more advanced form of Current() that returns more
// metadata.  It is used when IteratorOptions.IncludeDeletions is
// true.  It returns ErrIteratorDone if the iterator is done.
// Otherwise, the current operation, key, val are returned.
func (iter *iterator) CurrentEx() (
	entryEx EntryEx, key, val []byte, err error) {
	if len(iter.cursors) <= 0 {
		return EntryEx{}, nil, nil, ErrIteratorDone
	}

	cursor := iter.cursors[0]

	return EntryEx{Operation: cursor.op}, cursor.k, cursor.v, nil
}

func (iter *iterator) Len() int {
	return len(iter.cursors)
}

func (iter *iterator) Less(i, j int) bool {
	c := bytes.Compare(iter.cursors[i].k, iter.cursors[j].k)
	if c < 0 {
		return true
	}
	if c > 0 {
		return false
	}

	return iter.cursors[i].ssIndex > iter.cursors[j].ssIndex
}

func (iter *iterator) Swap(i, j int) {
	iter.cursors[i], iter.cursors[j] = iter.cursors[j], iter.cursors[i]
}

func (iter *iterator) Push(x interface{}) {
	// Push and Pop use pointer receivers because they modify the slice's length,
	// not just its contents.
	iter.cursors = append(iter.cursors, x.(*cursor))
}

func (iter *iterator) Pop() interface{} {
	n := len(iter.cursors)
	x := iter.cursors[n-1]
	iter.cursors = iter.cursors[0 : n-1]
	return x
}
<commit_msg>optimize when heap of cursors is size <= 1<commit_after>//  Copyright (c) 2016 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package moss

import (
	"bytes"
	"container/heap"
)

// An iterator tracks a min-heap "scan-line" of cursors through a
// segmentStack.  Iterator implements the sort.Interface and
// heap.Interface on its cursors.
type iterator struct {
	ss *segmentStack

	cursors []*cursor // The len(cursors) <= len(ss.a).

	startKeyInclusive []byte
	endKeyExclusive   []byte

	iteratorOptions IteratorOptions

	lowerLevelIter Iterator // May be nil.
}

// A cursor rerpresents a logical entry position inside a segment in a
// segmentStack.  An ssIndex < 0 and pos < 0 mean that the op/k/v came
// from the lowerLevelIter.
type cursor struct {
	ssIndex int // Index into Iterator.ss.a.
	posEnd  int // Found via endKeyExclusive, or the segment length.
	pos     int // Logical entry position into Iterator.ss.a[ssIndex].

	op uint64
	k  []byte
	v  []byte
}

// StartIterator returns a new iterator on the given segmentStack.
//
// On success, the returned Iterator will be positioned so that
// Iterator.Current() will either provide the first entry in the
// iteration range or ErrIteratorDone.
//
// A startKeyInclusive of nil means the logical "bottom-most" possible
// key and an endKeyExclusive of nil means the logical "top-most"
// possible key.
//
// StartIterator can optionally include deletion operations in the
// enumeration via the IteratorOptions.IncludeDeletions flag.
//
// StartIterator can skip lower segments, via the
// IteratorOptions.MinSegmentLevel parameter.  For example, to ignore
// the lowest, 0th segment, use MinSegmentLevel of 1.
func (ss *segmentStack) StartIterator(
	startKeyInclusive, endKeyExclusive []byte,
	iteratorOptions IteratorOptions) (Iterator, error) {
	if iteratorOptions.MaxSegmentHeight <= 0 {
		iteratorOptions.MaxSegmentHeight = len(ss.a)
	}

	return ss.startIterator(startKeyInclusive, endKeyExclusive, iteratorOptions)
}

// startIterator() returns a new iterator on the given segmentStack.
//
// On success, the returned Iterator will be positioned so that
// Iterator.Current() will either provide the first entry in the
// iteration range or ErrIteratorDone.
//
// A startKeyInclusive of nil means the logical "bottom-most" possible
// key and an endKeyExclusive of nil means the logical "top-most"
// possible key.
//
// startIterator() can optionally include deletion operations in the
// enumeration via the IteratorOptions.IncludeDeletions flag.
//
// startIterator() can skip lower segments, via the
// IteratorOptions.MinSegmentLevel parameter.  For example, to ignore
// the lowest, 0th segment, use MinSegmentLevel of 1.
func (ss *segmentStack) startIterator(
	startKeyInclusive, endKeyExclusive []byte,
	iteratorOptions IteratorOptions) (*iterator, error) {
	iter := &iterator{
		ss:      ss,
		cursors: make([]*cursor, 0, len(ss.a)+1),

		startKeyInclusive: startKeyInclusive,
		endKeyExclusive:   endKeyExclusive,

		iteratorOptions: iteratorOptions,
	}

	// ----------------------------------------------
	// Add cursors for our allowed segments.

	minSegmentLevel := iteratorOptions.MinSegmentLevel
	maxSegmentLevel := iteratorOptions.MaxSegmentHeight - 1

	ss.ensureSorted(minSegmentLevel, maxSegmentLevel)

	for ssIndex := minSegmentLevel; ssIndex <= maxSegmentLevel; ssIndex++ {
		b := ss.a[ssIndex]

		posEnd := b.Len()
		if endKeyExclusive != nil {
			posEnd = b.FindStartKeyInclusivePos(endKeyExclusive)
		}

		pos := b.FindStartKeyInclusivePos(startKeyInclusive)
		if pos >= posEnd {
			continue
		}

		op, k, v := b.GetOperationKeyVal(pos)
		if op == 0 && k == nil && v == nil {
			continue
		}

		iter.cursors = append(iter.cursors, &cursor{
			ssIndex: ssIndex,
			posEnd:  posEnd,
			pos:     pos,
			op:      op,
			k:       k,
			v:       v,
		})
	}

	// ----------------------------------------------
	// Add cursor for the lower level, if wanted.

	if !iteratorOptions.SkipLowerLevel &&
		ss.lowerLevelSnapshot != nil {
		llss := ss.lowerLevelSnapshot.addRef()
		if llss != nil {
			lowerLevelIter, err := llss.StartIterator(
				startKeyInclusive, endKeyExclusive, IteratorOptions{})

			llss.decRef()

			if err != nil {
				return nil, err
			}

			k, v, err := lowerLevelIter.Current()
			if err != nil && err != ErrIteratorDone {
				return nil, err
			}
			if err == ErrIteratorDone {
				lowerLevelIter.Close()
			}
			if err == nil {
				iter.cursors = append(iter.cursors, &cursor{
					ssIndex: -1,
					pos:     -1,
					op:      OperationSet,
					k:       k,
					v:       v,
				})

				iter.lowerLevelIter = lowerLevelIter
			}
		}
	}

	// ----------------------------------------------
	// Heap-ify the cursors.

	heap.Init(iter)

	if !iteratorOptions.IncludeDeletions {
		entryEx, _, _, _ := iter.CurrentEx()
		if entryEx.Operation == OperationDel {
			iter.Next()
		}
	}

	return iter, nil
}

// Close must be invoked to release resources.
func (iter *iterator) Close() error {
	if iter.lowerLevelIter != nil {
		iter.lowerLevelIter.Close()
		iter.lowerLevelIter = nil
	}

	return nil
}

// Next returns ErrIteratorDone if the iterator is done.
func (iter *iterator) Next() error {
	if len(iter.cursors) <= 0 {
		return ErrIteratorDone
	}

	lastK := iter.cursors[0].k

	for len(iter.cursors) > 0 {
		next := iter.cursors[0]

		if next.ssIndex < 0 && next.pos < 0 {
			err := iter.lowerLevelIter.Next()
			if err == nil {
				next.k, next.v, err = iter.lowerLevelIter.Current()
				if err == nil && len(iter.cursors) > 1 {
					heap.Fix(iter, 0)
				}
			}

			if err != nil {
				iter.lowerLevelIter.Close()
				iter.lowerLevelIter = nil

				heap.Pop(iter)
			}
		} else {
			next.pos++
			if next.pos >= next.posEnd {
				heap.Pop(iter)
			} else {
				next.op, next.k, next.v =
					iter.ss.a[next.ssIndex].GetOperationKeyVal(next.pos)
				if next.op == 0 {
					heap.Pop(iter)
				} else if len(iter.cursors) > 1 {
					heap.Fix(iter, 0)
				}
			}
		}

		if len(iter.cursors) <= 0 {
			return ErrIteratorDone
		}

		if !bytes.Equal(iter.cursors[0].k, lastK) {
			if !iter.iteratorOptions.IncludeDeletions &&
				iter.cursors[0].op == OperationDel {
				return iter.Next()
			}

			return nil
		}
	}

	return ErrIteratorDone
}

// Current returns ErrIteratorDone if the iterator is done.
// Otherwise, Current() returns the current key and val, which should
// be treated as immutable or read-only.  The key and val bytes will
// remain available until the next call to Next() or Close().
func (iter *iterator) Current() ([]byte, []byte, error) {
	entryEx, key, val, err := iter.CurrentEx()
	if err != nil {
		return nil, nil, err
	}

	op := entryEx.Operation
	if op == OperationDel {
		return nil, nil, nil
	}

	if op == OperationMerge {
		valMerged, err := iter.ss.getMerged(key, val, iter.cursors[0].ssIndex-1,
			iter.iteratorOptions.base)
		if err != nil {
			return nil, nil, err
		}

		return key, valMerged, nil
	}

	return key, val, err
}

// CurrentEx is a more advanced form of Current() that returns more
// metadata.  It is used when IteratorOptions.IncludeDeletions is
// true.  It returns ErrIteratorDone if the iterator is done.
// Otherwise, the current operation, key, val are returned.
func (iter *iterator) CurrentEx() (
	entryEx EntryEx, key, val []byte, err error) {
	if len(iter.cursors) <= 0 {
		return EntryEx{}, nil, nil, ErrIteratorDone
	}

	cursor := iter.cursors[0]

	return EntryEx{Operation: cursor.op}, cursor.k, cursor.v, nil
}

func (iter *iterator) Len() int {
	return len(iter.cursors)
}

func (iter *iterator) Less(i, j int) bool {
	c := bytes.Compare(iter.cursors[i].k, iter.cursors[j].k)
	if c < 0 {
		return true
	}
	if c > 0 {
		return false
	}

	return iter.cursors[i].ssIndex > iter.cursors[j].ssIndex
}

func (iter *iterator) Swap(i, j int) {
	iter.cursors[i], iter.cursors[j] = iter.cursors[j], iter.cursors[i]
}

func (iter *iterator) Push(x interface{}) {
	// Push and Pop use pointer receivers because they modify the slice's length,
	// not just its contents.
	iter.cursors = append(iter.cursors, x.(*cursor))
}

func (iter *iterator) Pop() interface{} {
	n := len(iter.cursors)
	x := iter.cursors[n-1]
	iter.cursors = iter.cursors[0 : n-1]
	return x
}
<|endoftext|>


<commit_before>package gorocksdb

// #include <stdlib.h>
// #include "rocksdb/c.h"
import "C"
import (
	"bytes"
	"errors"
	"unsafe"
)

// Iterator provides a way to seek to specific keys and iterate through
// the keyspace from that point, as well as access the values of those keys.
//
// For example:
//
//      it := db.NewIterator(readOpts)
//      defer it.Close()
//
//      it.Seek([]byte("foo"))
//		for ; it.Valid(); it.Next() {
//          fmt.Printf("Key: %v Value: %v\n", it.Key().Data(), it.Value().Data())
// 		}
//
//      if err := it.Err(); err != nil {
//          return err
//      }
//
type Iterator struct {
	c *C.rocksdb_iterator_t
}

// NewNativeIterator creates a Iterator object.
func NewNativeIterator(c unsafe.Pointer) *Iterator {
	return &Iterator{(*C.rocksdb_iterator_t)(c)}
}

// Valid returns false only when an Iterator has iterated past either the
// first or the last key in the database.
func (iter *Iterator) Valid() bool {
	return C.rocksdb_iter_valid(iter.c) != 0
}

// ValidForPrefix returns false only when an Iterator has iterated past the
// first or the last key in the database or the specified prefix.
func (iter *Iterator) ValidForPrefix(prefix []byte) bool {
	if C.rocksdb_iter_valid(iter.c) == 0 {
		return false
	}

	key := iter.Key()
	result := bytes.HasPrefix(key.Data(), prefix)
	key.Free()
	return result
}

// Key returns the key the iterator currently holds.
func (iter *Iterator) Key() *Slice {
	var cLen C.size_t
	cKey := C.rocksdb_iter_key(iter.c, &cLen)
	if cKey == nil {
		return nil
	}
	return &Slice{cKey, cLen, true}
}

// Value returns the value in the database the iterator currently holds.
func (iter *Iterator) Value() *Slice {
	var cLen C.size_t
	cVal := C.rocksdb_iter_value(iter.c, &cLen)
	if cVal == nil {
		return nil
	}
	return &Slice{cVal, cLen, true}
}

// Next moves the iterator to the next sequential key in the database.
func (iter *Iterator) Next() {
	C.rocksdb_iter_next(iter.c)
}

// Prev moves the iterator to the previous sequential key in the database.
func (iter *Iterator) Prev() {
	C.rocksdb_iter_prev(iter.c)
}

// SeekToFirst moves the iterator to the first key in the database.
func (iter *Iterator) SeekToFirst() {
	C.rocksdb_iter_seek_to_first(iter.c)
}

// SeekToLast moves the iterator to the last key in the database.
func (iter *Iterator) SeekToLast() {
	C.rocksdb_iter_seek_to_last(iter.c)
}

// Seek moves the iterator to the position greater than or equal to the key.
func (iter *Iterator) Seek(key []byte) {
	cKey := byteToChar(key)
	C.rocksdb_iter_seek(iter.c, cKey, C.size_t(len(key)))
}

// Seek moves the iterator to the last key that less than or equal to the target key, in contrast with Seek().
func (iter *Iterator) SeekForPrev(key []byte) {
	cKey := byteToChar(key)
	C.rocksdb_iter_seek_for_prev(iter.c, cKey, C.size_t(len(key)))
}

// Err returns nil if no errors happened during iteration, or the actual
// error otherwise.
func (iter *Iterator) Err() error {
	var cErr *C.char
	C.rocksdb_iter_get_error(iter.c, &cErr)
	if cErr != nil {
		defer C.free(unsafe.Pointer(cErr))
		return errors.New(C.GoString(cErr))
	}
	return nil
}

// Close closes the iterator.
func (iter *Iterator) Close() {
	C.rocksdb_iter_destroy(iter.c)
	iter.c = nil
}
<commit_msg>update SeekForPrev comment<commit_after>package gorocksdb

// #include <stdlib.h>
// #include "rocksdb/c.h"
import "C"
import (
	"bytes"
	"errors"
	"unsafe"
)

// Iterator provides a way to seek to specific keys and iterate through
// the keyspace from that point, as well as access the values of those keys.
//
// For example:
//
//      it := db.NewIterator(readOpts)
//      defer it.Close()
//
//      it.Seek([]byte("foo"))
//		for ; it.Valid(); it.Next() {
//          fmt.Printf("Key: %v Value: %v\n", it.Key().Data(), it.Value().Data())
// 		}
//
//      if err := it.Err(); err != nil {
//          return err
//      }
//
type Iterator struct {
	c *C.rocksdb_iterator_t
}

// NewNativeIterator creates a Iterator object.
func NewNativeIterator(c unsafe.Pointer) *Iterator {
	return &Iterator{(*C.rocksdb_iterator_t)(c)}
}

// Valid returns false only when an Iterator has iterated past either the
// first or the last key in the database.
func (iter *Iterator) Valid() bool {
	return C.rocksdb_iter_valid(iter.c) != 0
}

// ValidForPrefix returns false only when an Iterator has iterated past the
// first or the last key in the database or the specified prefix.
func (iter *Iterator) ValidForPrefix(prefix []byte) bool {
	if C.rocksdb_iter_valid(iter.c) == 0 {
		return false
	}

	key := iter.Key()
	result := bytes.HasPrefix(key.Data(), prefix)
	key.Free()
	return result
}

// Key returns the key the iterator currently holds.
func (iter *Iterator) Key() *Slice {
	var cLen C.size_t
	cKey := C.rocksdb_iter_key(iter.c, &cLen)
	if cKey == nil {
		return nil
	}
	return &Slice{cKey, cLen, true}
}

// Value returns the value in the database the iterator currently holds.
func (iter *Iterator) Value() *Slice {
	var cLen C.size_t
	cVal := C.rocksdb_iter_value(iter.c, &cLen)
	if cVal == nil {
		return nil
	}
	return &Slice{cVal, cLen, true}
}

// Next moves the iterator to the next sequential key in the database.
func (iter *Iterator) Next() {
	C.rocksdb_iter_next(iter.c)
}

// Prev moves the iterator to the previous sequential key in the database.
func (iter *Iterator) Prev() {
	C.rocksdb_iter_prev(iter.c)
}

// SeekToFirst moves the iterator to the first key in the database.
func (iter *Iterator) SeekToFirst() {
	C.rocksdb_iter_seek_to_first(iter.c)
}

// SeekToLast moves the iterator to the last key in the database.
func (iter *Iterator) SeekToLast() {
	C.rocksdb_iter_seek_to_last(iter.c)
}

// Seek moves the iterator to the position greater than or equal to the key.
func (iter *Iterator) Seek(key []byte) {
	cKey := byteToChar(key)
	C.rocksdb_iter_seek(iter.c, cKey, C.size_t(len(key)))
}

// SeekForPrev moves the iterator to the last key that less than or equal
// to the target key, in contrast with Seek.
func (iter *Iterator) SeekForPrev(key []byte) {
	cKey := byteToChar(key)
	C.rocksdb_iter_seek_for_prev(iter.c, cKey, C.size_t(len(key)))
}

// Err returns nil if no errors happened during iteration, or the actual
// error otherwise.
func (iter *Iterator) Err() error {
	var cErr *C.char
	C.rocksdb_iter_get_error(iter.c, &cErr)
	if cErr != nil {
		defer C.free(unsafe.Pointer(cErr))
		return errors.New(C.GoString(cErr))
	}
	return nil
}

// Close closes the iterator.
func (iter *Iterator) Close() {
	C.rocksdb_iter_destroy(iter.c)
	iter.c = nil
}
<|endoftext|>


<commit_before>package gorocksdb

// #include <stdlib.h>
// #include "rocksdb/c.h"
import "C"
import (
	"bytes"
	"errors"
	"unsafe"
)

// Iterator provides a way to seek to specific keys and iterate through
// the keyspace from that point, as well as access the values of those keys.
//
// For example:
//
//      it := db.NewIterator(readOpts)
//      defer it.Close()
//
//      it.Seek([]byte("foo"))
//		for ; it.Valid(); it.Next() {
//          fmt.Printf("Key: %v Value: %v\n", it.Key().Data(), it.Value().Data())
// 		}
//
//      if err := it.Err(); err != nil {
//          return err
//      }
//
type Iterator struct {
	c *C.rocksdb_iterator_t
}

// NewNativeIterator creates a Iterator object.
func NewNativeIterator(c unsafe.Pointer) *Iterator {
	return &Iterator{(*C.rocksdb_iterator_t)(c)}
}

// Valid returns false only when an Iterator has iterated past either the
// first or the last key in the database.
func (iter *Iterator) Valid() bool {
	return C.rocksdb_iter_valid(iter.c) != 0
}

// ValidForPrefix returns false only when an Iterator has iterated past the
// first or the last key in the database or the specified prefix.
func (iter *Iterator) ValidForPrefix(prefix []byte) bool {
	if C.rocksdb_iter_valid(iter.c) == 0 {
		return false
	}

	key := iter.Key()
	result := bytes.HasPrefix(key.Data(), prefix)
	key.Free()
	return result
}

// Key returns the key the iterator currently holds.
func (iter *Iterator) Key() *Slice {
	var cLen C.size_t
	cKey := C.rocksdb_iter_key(iter.c, &cLen)
	if cKey == nil {
		return nil
	}
	return &Slice{cKey, cLen, true}
}

// Value returns the value in the database the iterator currently holds.
func (iter *Iterator) Value() *Slice {
	var cLen C.size_t
	cVal := C.rocksdb_iter_value(iter.c, &cLen)
	if cVal == nil {
		return nil
	}
	return &Slice{cVal, cLen, true}
}

// Next moves the iterator to the next sequential key in the database.
func (iter *Iterator) Next() {
	C.rocksdb_iter_next(iter.c)
}

// Prev moves the iterator to the previous sequential key in the database.
func (iter *Iterator) Prev() {
	C.rocksdb_iter_prev(iter.c)
}

// SeekToFirst moves the iterator to the first key in the database.
func (iter *Iterator) SeekToFirst() {
	C.rocksdb_iter_seek_to_first(iter.c)
}

// SeekToLast moves the iterator to the last key in the database.
func (iter *Iterator) SeekToLast() {
	C.rocksdb_iter_seek_to_last(iter.c)
}

// Seek moves the iterator to the position greater than or equal to the key.
func (iter *Iterator) Seek(key []byte) {
	cKey := byteToChar(key)
	C.rocksdb_iter_seek(iter.c, cKey, C.size_t(len(key)))
}

// Err returns nil if no errors happened during iteration, or the actual
// error otherwise.
func (iter *Iterator) Err() error {
	var cErr *C.char
	C.rocksdb_iter_get_error(iter.c, &cErr)
	if cErr != nil {
		defer C.free(unsafe.Pointer(cErr))
		return errors.New(C.GoString(cErr))
	}
	return nil
}

// Close closes the iterator.
func (iter *Iterator) Close() {
	C.rocksdb_iter_destroy(iter.c)
	iter.c = nil
}
<commit_msg>adding SeekForPrev() support<commit_after>package gorocksdb

// #include <stdlib.h>
// #include "rocksdb/c.h"
import "C"
import (
	"bytes"
	"errors"
	"unsafe"
)

// Iterator provides a way to seek to specific keys and iterate through
// the keyspace from that point, as well as access the values of those keys.
//
// For example:
//
//      it := db.NewIterator(readOpts)
//      defer it.Close()
//
//      it.Seek([]byte("foo"))
//		for ; it.Valid(); it.Next() {
//          fmt.Printf("Key: %v Value: %v\n", it.Key().Data(), it.Value().Data())
// 		}
//
//      if err := it.Err(); err != nil {
//          return err
//      }
//
type Iterator struct {
	c *C.rocksdb_iterator_t
}

// NewNativeIterator creates a Iterator object.
func NewNativeIterator(c unsafe.Pointer) *Iterator {
	return &Iterator{(*C.rocksdb_iterator_t)(c)}
}

// Valid returns false only when an Iterator has iterated past either the
// first or the last key in the database.
func (iter *Iterator) Valid() bool {
	return C.rocksdb_iter_valid(iter.c) != 0
}

// ValidForPrefix returns false only when an Iterator has iterated past the
// first or the last key in the database or the specified prefix.
func (iter *Iterator) ValidForPrefix(prefix []byte) bool {
	if C.rocksdb_iter_valid(iter.c) == 0 {
		return false
	}

	key := iter.Key()
	result := bytes.HasPrefix(key.Data(), prefix)
	key.Free()
	return result
}

// Key returns the key the iterator currently holds.
func (iter *Iterator) Key() *Slice {
	var cLen C.size_t
	cKey := C.rocksdb_iter_key(iter.c, &cLen)
	if cKey == nil {
		return nil
	}
	return &Slice{cKey, cLen, true}
}

// Value returns the value in the database the iterator currently holds.
func (iter *Iterator) Value() *Slice {
	var cLen C.size_t
	cVal := C.rocksdb_iter_value(iter.c, &cLen)
	if cVal == nil {
		return nil
	}
	return &Slice{cVal, cLen, true}
}

// Next moves the iterator to the next sequential key in the database.
func (iter *Iterator) Next() {
	C.rocksdb_iter_next(iter.c)
}

// Prev moves the iterator to the previous sequential key in the database.
func (iter *Iterator) Prev() {
	C.rocksdb_iter_prev(iter.c)
}

// SeekToFirst moves the iterator to the first key in the database.
func (iter *Iterator) SeekToFirst() {
	C.rocksdb_iter_seek_to_first(iter.c)
}

// SeekToLast moves the iterator to the last key in the database.
func (iter *Iterator) SeekToLast() {
	C.rocksdb_iter_seek_to_last(iter.c)
}

// Seek moves the iterator to the position greater than or equal to the key.
func (iter *Iterator) Seek(key []byte) {
	cKey := byteToChar(key)
	C.rocksdb_iter_seek(iter.c, cKey, C.size_t(len(key)))
}

// Seek moves the iterator to the last key that less than or equal to the target key, in contrast with Seek().
func (iter *Iterator) SeekForPrev(key []byte) {
	cKey := byteToChar(key)
	C.rocksdb_iter_seek_for_prev(iter.c, cKey, C.size_t(len(key)))
}

// Err returns nil if no errors happened during iteration, or the actual
// error otherwise.
func (iter *Iterator) Err() error {
	var cErr *C.char
	C.rocksdb_iter_get_error(iter.c, &cErr)
	if cErr != nil {
		defer C.free(unsafe.Pointer(cErr))
		return errors.New(C.GoString(cErr))
	}
	return nil
}

// Close closes the iterator.
func (iter *Iterator) Close() {
	C.rocksdb_iter_destroy(iter.c)
	iter.c = nil
}
<|endoftext|>


<commit_before>package models_test

import (
	// "encoding/json"
	// "fmt"
	"github.com/APTrust/exchange/constants"
	"github.com/APTrust/exchange/models"
	"github.com/APTrust/exchange/util"
	"github.com/stretchr/testify/assert"
	"io/ioutil"
	"os"
	"os/exec"
	// "path"
	"path/filepath"
	"runtime"
	"testing"
	// "time"
)

func TestNewVirtualBag(t *testing.T) {
	tarFilePath := vbagGetPath("example.edu.tagsample_good.tar")
	vbag := models.NewVirtualBag(tarFilePath, nil, false, false)
	assert.NotNil(t, vbag)
}

func TestVirtualBagRead_FromDirectory(t *testing.T) {
	tempDir, bagPath := untarTestBag(t)
	defer os.RemoveAll(tempDir)
	files := []string {"bagit.txt", "bag-info.txt", "aptrust-info.txt"}
	vbag := models.NewVirtualBag(bagPath, files, true, true)
	assert.NotNil(t, vbag)
	obj, summary := vbag.Read()
	runAssertions(t, obj, summary, "TestVirtualBagRead_FromDirectory")
}

func TestVirtualBagRead_FromTarFile(t *testing.T) {
	tarFilePath := vbagGetPath("example.edu.tagsample_good.tar")
	files := []string {"bagit.txt", "bag-info.txt", "aptrust-info.txt"}
	vbag := models.NewVirtualBag(tarFilePath, files, true, true)
	assert.NotNil(t, vbag)
	obj, summary := vbag.Read()
	runAssertions(t, obj, summary, "TestVirtualBagRead_FromTarFile")
}

func TestVirtualBagRead_ChecksumOptions(t *testing.T) {
	tempDir, bagPath := untarTestBag(t)
	defer os.RemoveAll(tempDir)
	files := []string {"bagit.txt", "bag-info.txt", "aptrust-info.txt"}
	vbag := models.NewVirtualBag(bagPath, files, true, false)
	assert.NotNil(t, vbag)
	obj, _ := vbag.Read()

	// Should calculate md5 only
	for _, gf := range obj.GenericFiles {
		assert.NotEmpty(t, gf.IngestMd5)
		assert.Empty(t, gf.IngestSha256)
	}

	vbag = models.NewVirtualBag(bagPath, files, false, true)
	assert.NotNil(t, vbag)
	obj, _ = vbag.Read()

	// Should calculate sha256 only
	for _, gf := range obj.GenericFiles {
		assert.Empty(t, gf.IngestMd5)
		assert.NotEmpty(t, gf.IngestSha256)
	}

}

// With md5 manifest only, sha256 only, and both
func TestVirtualBagRead_ManifestOptions(t *testing.T) {

}

func runAssertions(t *testing.T, obj *models.IntellectualObject, summary *models.WorkSummary, caller string) {
	// WorkSummary
	assert.False(t, summary.StartedAt.IsZero(), caller)
	assert.False(t, summary.FinishedAt.IsZero(), caller)
	assert.Empty(t, summary.Errors, caller)

	// IntelObj properties
	assert.Equal(t, 0, obj.Id, caller)
	assert.Equal(t, "example.edu.tagsample_good", obj.Identifier, caller)

	// TODO: Is BagName necessary? It should be the same as obj.Identifier
	assert.Equal(t, "", obj.BagName, caller)

	assert.Equal(t, "virginia.edu", obj.Institution, caller)
	assert.Equal(t, 0, obj.InstitutionId, caller)
	assert.Equal(t, "Thirteen Ways of Looking at a Blackbird", obj.Title, caller)
	assert.Equal(t, "so much depends upon a red wheel barrow glazed with rain water beside the white chickens", obj.Description, caller)
	assert.Equal(t, "Institution", obj.Access, caller)
	assert.Equal(t, "uva-internal-id-0001", obj.AltIdentifier, caller)
	assert.Empty(t, obj.IngestErrorMessage, caller)
	if caller == "TestVirtualBagRead_FromTarFile" {
		assert.NotEmpty(t, obj.IngestTarFilePath, caller)
	} else if caller == "TestVirtualBagRead_FromDirectory" {
		assert.NotEmpty(t, obj.IngestUntarredPath, caller)
	}
	assert.Equal(t, 2, len(obj.IngestManifests), caller)
	assert.True(t, util.StringListContains(obj.IngestManifests, "manifest-md5.txt"), caller)
	assert.True(t, util.StringListContains(obj.IngestManifests, "manifest-sha256.txt"), caller)
	assert.Equal(t, 2, len(obj.IngestTagManifests), caller)
	assert.True(t, util.StringListContains(obj.IngestTagManifests, "tagmanifest-md5.txt"), caller)
	assert.True(t, util.StringListContains(obj.IngestTagManifests, "tagmanifest-sha256.txt"), caller)
	assert.Empty(t, obj.IngestFilesIgnored, caller)

	// Tags
	assert.Equal(t, 10, len(obj.IngestTags))
	for _, tag := range obj.IngestTags {
		assert.NotEmpty(t, tag.SourceFile, caller)
		assert.NotEmpty(t, tag.Label, caller)
		assert.NotEmpty(t, tag.Value, caller)
	}

	// Spot check one tag
	tag := obj.IngestTags[4]
	assert.Equal(t, "bag-info.txt", tag.SourceFile, caller)
	assert.Equal(t, "Bag-Count", tag.Label, caller)
	assert.Equal(t, "1 of 1", tag.Value, caller)

	// Generic Files
	tagFileCount := 0
	payloadFileCount := 0
	manifestCount := 0
	tagManifestCount := 0
	assert.Equal(t, 16, len(obj.GenericFiles), caller)
	for _, gf := range obj.GenericFiles {
		assert.NotEmpty(t, gf.Identifier, caller)
		assert.NotEmpty(t, gf.IntellectualObjectIdentifier, caller)
		assert.NotEmpty(t, gf.FileFormat, caller)
		assert.NotEmpty(t, gf.IngestFileType, caller)
		assert.NotEmpty(t, gf.IngestMd5, caller)
		assert.NotEmpty(t, gf.IngestSha256, caller)

		if caller == "TestVirtualBagRead_FromTarFile" {
			assert.Empty(t, gf.IngestLocalPath, caller)
		} else if caller == "TestVirtualBagRead_FromDirectory" {
			assert.NotEmpty(t, gf.IngestLocalPath, caller)
		}

		assert.Empty(t, gf.IngestStorageURL, caller)
		assert.Empty(t, gf.IngestReplicationURL, caller)
		assert.True(t, gf.Size > 0, caller)
		switch gf.IngestFileType {
		case constants.PAYLOAD_FILE: payloadFileCount++
		case constants.PAYLOAD_MANIFEST: manifestCount++
		case constants.TAG_MANIFEST: tagManifestCount++
		case constants.TAG_FILE: tagFileCount++
		}
	}

	// Make sure file types were all tagged correctly
	assert.Equal(t, 4, payloadFileCount, caller)
	assert.Equal(t, 2, manifestCount, caller)
	assert.Equal(t, 2, tagManifestCount, caller)
	assert.Equal(t, 8, tagFileCount, caller)

	// Spot check generic file aptrust-info.txt
	gf := obj.FindGenericFileByPath("aptrust-info.txt")
	if gf == nil {
		assert.Fail(t, "Could not find aptrust-info.txt", caller)
	}
	assert.Equal(t, "example.edu.tagsample_good/aptrust-info.txt", gf.Identifier, caller)
	assert.Equal(t, 0, gf.IntellectualObjectId, caller)
	assert.Equal(t, "example.edu.tagsample_good", gf.IntellectualObjectIdentifier, caller)
	assert.Equal(t, "application/binary", gf.FileFormat, caller)
	assert.Empty(t, gf.URI, caller)
	assert.EqualValues(t, 67, gf.Size, caller)
	assert.False(t, gf.FileModified.IsZero(), caller)
	assert.Equal(t, constants.TAG_FILE, gf.IngestFileType, caller)
	assert.Equal(t, "300e936e622605f9f7a846d261d53093", gf.IngestManifestMd5, caller)
	assert.Equal(t, "300e936e622605f9f7a846d261d53093", gf.IngestMd5, caller)
	assert.False(t, gf.IngestMd5GeneratedAt.IsZero(), caller)
	assert.True(t, gf.IngestMd5VerifiedAt.IsZero(), caller)
	assert.Equal(t, "a2b6c5a713af771c5e4edde8d5be25fbcad86e45ea338f43a5bb769347e7c8bb", gf.IngestManifestSha256, caller)
	assert.Equal(t, "a2b6c5a713af771c5e4edde8d5be25fbcad86e45ea338f43a5bb769347e7c8bb", gf.IngestSha256, caller)
	assert.False(t, gf.IngestSha256GeneratedAt.IsZero(), caller)
	assert.True(t, gf.IngestSha256VerifiedAt.IsZero(), caller)
	assert.NotEmpty(t, gf.IngestUUID, caller)
	assert.False(t, gf.IngestUUIDGeneratedAt.IsZero(), caller)
	assert.Empty(t, gf.IngestStorageURL, caller)
	assert.True(t, gf.IngestStoredAt.IsZero(), caller)
	assert.Empty(t, gf.IngestReplicationURL, caller)
	assert.True(t, gf.IngestReplicatedAt.IsZero(), caller)
	assert.False(t, gf.IngestPreviousVersionExists, caller)
	assert.True(t, gf.IngestNeedsSave, caller)
	assert.Empty(t, gf.IngestErrorMessage, caller)
}

func vbagGetPath(fileName string) (string) {
	_, filename, _, _ := runtime.Caller(0)
	dir, _ := filepath.Abs(filepath.Dir(filename))
	testDataPath := filepath.Join(dir, "..", "testdata", fileName)
	return testDataPath
}

func untarTestBag(t *testing.T) (tempDir string, bagPath string) {
	tarFilePath := vbagGetPath("example.edu.tagsample_good.tar")
	tempDir, err := ioutil.TempDir("", "test")
	if err != nil {
		assert.Fail(t, "Cannot create temp dir: %v", err)
	}
	cmd := exec.Command("tar", "xf", tarFilePath, "--directory", tempDir)
	err = cmd.Run()
	if err != nil {
		assert.Fail(t, "Cannot untar test bag into temp dir: %v", err)
	}
	pathToUntarredBag := filepath.Join(tempDir, "example.edu.tagsample_good")
	return tempDir, pathToUntarredBag
}
<commit_msg>Test manifest options in VirtualBag<commit_after>package models_test

import (
	// "encoding/json"
	// "fmt"
	"github.com/APTrust/exchange/constants"
	"github.com/APTrust/exchange/models"
	"github.com/APTrust/exchange/util"
	"github.com/stretchr/testify/assert"
	"io/ioutil"
	"os"
	"os/exec"
	// "path"
	"path/filepath"
	"runtime"
	"testing"
	// "time"
)

func TestNewVirtualBag(t *testing.T) {
	tarFilePath := vbagGetPath("example.edu.tagsample_good.tar")
	vbag := models.NewVirtualBag(tarFilePath, nil, false, false)
	assert.NotNil(t, vbag)
}

func TestVirtualBagRead_FromDirectory(t *testing.T) {
	tempDir, bagPath := untarTestBag(t)
	defer os.RemoveAll(tempDir)
	files := []string {"bagit.txt", "bag-info.txt", "aptrust-info.txt"}
	vbag := models.NewVirtualBag(bagPath, files, true, true)
	assert.NotNil(t, vbag)
	obj, summary := vbag.Read()
	runAssertions(t, obj, summary, "TestVirtualBagRead_FromDirectory")
}

func TestVirtualBagRead_FromTarFile(t *testing.T) {
	tarFilePath := vbagGetPath("example.edu.tagsample_good.tar")
	files := []string {"bagit.txt", "bag-info.txt", "aptrust-info.txt"}
	vbag := models.NewVirtualBag(tarFilePath, files, true, true)
	assert.NotNil(t, vbag)
	obj, summary := vbag.Read()
	runAssertions(t, obj, summary, "TestVirtualBagRead_FromTarFile")
}

func TestVirtualBagRead_ChecksumOptions(t *testing.T) {
	tempDir, bagPath := untarTestBag(t)
	defer os.RemoveAll(tempDir)
	files := []string {"bagit.txt", "bag-info.txt", "aptrust-info.txt"}
	vbag := models.NewVirtualBag(bagPath, files, true, false)
	assert.NotNil(t, vbag)
	obj, _ := vbag.Read()

	// Should calculate md5 only
	for _, gf := range obj.GenericFiles {
		assert.NotEmpty(t, gf.IngestMd5)
		assert.Empty(t, gf.IngestSha256)
	}

	vbag = models.NewVirtualBag(bagPath, files, false, true)
	assert.NotNil(t, vbag)
	obj, _ = vbag.Read()

	// Should calculate sha256 only
	for _, gf := range obj.GenericFiles {
		assert.Empty(t, gf.IngestMd5)
		assert.NotEmpty(t, gf.IngestSha256)
	}

}

// With md5 manifest only, sha256 only, and both
func TestVirtualBagRead_ManifestOptions(t *testing.T) {
	tempDir, bagPath := untarTestBag(t)
	defer os.RemoveAll(tempDir)

	// Delete the md5 manifest
	os.Remove(filepath.Join(bagPath, "manifest-md5.txt"))

	files := []string {"bagit.txt", "bag-info.txt", "aptrust-info.txt"}
	vbag := models.NewVirtualBag(bagPath, files, true, true)
	assert.NotNil(t, vbag)
	obj, _ := vbag.Read()

	// Should have manifest values for sha256
	for _, gf := range obj.GenericFiles {
		if gf.IngestFileType == constants.PAYLOAD_FILE {
			assert.Empty(t, gf.IngestManifestMd5)
			assert.NotEmpty(t, gf.IngestManifestSha256)
		}
	}

	tempDir, bagPath = untarTestBag(t)
	defer os.RemoveAll(tempDir)

	// Delete the sha256 manifest
	os.Remove(filepath.Join(bagPath, "manifest-sha256.txt"))

	files = []string {"bagit.txt", "bag-info.txt", "aptrust-info.txt"}
	vbag = models.NewVirtualBag(bagPath, files, true, true)
	assert.NotNil(t, vbag)
	obj, _ = vbag.Read()

	// Should have manifest values for sha256
	for _, gf := range obj.GenericFiles {
		if gf.IngestFileType == constants.PAYLOAD_FILE {
			assert.NotEmpty(t, gf.IngestManifestMd5)
			assert.Empty(t, gf.IngestManifestSha256)
		}
	}
}

func runAssertions(t *testing.T, obj *models.IntellectualObject, summary *models.WorkSummary, caller string) {
	// WorkSummary
	assert.False(t, summary.StartedAt.IsZero(), caller)
	assert.False(t, summary.FinishedAt.IsZero(), caller)
	assert.Empty(t, summary.Errors, caller)

	// IntelObj properties
	assert.Equal(t, 0, obj.Id, caller)
	assert.Equal(t, "example.edu.tagsample_good", obj.Identifier, caller)

	// TODO: Is BagName necessary? It should be the same as obj.Identifier
	assert.Equal(t, "", obj.BagName, caller)

	assert.Equal(t, "virginia.edu", obj.Institution, caller)
	assert.Equal(t, 0, obj.InstitutionId, caller)
	assert.Equal(t, "Thirteen Ways of Looking at a Blackbird", obj.Title, caller)
	assert.Equal(t, "so much depends upon a red wheel barrow glazed with rain water beside the white chickens", obj.Description, caller)
	assert.Equal(t, "Institution", obj.Access, caller)
	assert.Equal(t, "uva-internal-id-0001", obj.AltIdentifier, caller)
	assert.Empty(t, obj.IngestErrorMessage, caller)
	if caller == "TestVirtualBagRead_FromTarFile" {
		assert.NotEmpty(t, obj.IngestTarFilePath, caller)
	} else if caller == "TestVirtualBagRead_FromDirectory" {
		assert.NotEmpty(t, obj.IngestUntarredPath, caller)
	}
	assert.Equal(t, 2, len(obj.IngestManifests), caller)
	assert.True(t, util.StringListContains(obj.IngestManifests, "manifest-md5.txt"), caller)
	assert.True(t, util.StringListContains(obj.IngestManifests, "manifest-sha256.txt"), caller)
	assert.Equal(t, 2, len(obj.IngestTagManifests), caller)
	assert.True(t, util.StringListContains(obj.IngestTagManifests, "tagmanifest-md5.txt"), caller)
	assert.True(t, util.StringListContains(obj.IngestTagManifests, "tagmanifest-sha256.txt"), caller)
	assert.Empty(t, obj.IngestFilesIgnored, caller)

	// Tags
	assert.Equal(t, 10, len(obj.IngestTags))
	for _, tag := range obj.IngestTags {
		assert.NotEmpty(t, tag.SourceFile, caller)
		assert.NotEmpty(t, tag.Label, caller)
		assert.NotEmpty(t, tag.Value, caller)
	}

	// Spot check one tag
	tag := obj.IngestTags[4]
	assert.Equal(t, "bag-info.txt", tag.SourceFile, caller)
	assert.Equal(t, "Bag-Count", tag.Label, caller)
	assert.Equal(t, "1 of 1", tag.Value, caller)

	// Generic Files
	tagFileCount := 0
	payloadFileCount := 0
	manifestCount := 0
	tagManifestCount := 0
	assert.Equal(t, 16, len(obj.GenericFiles), caller)
	for _, gf := range obj.GenericFiles {
		assert.NotEmpty(t, gf.Identifier, caller)
		assert.NotEmpty(t, gf.IntellectualObjectIdentifier, caller)
		assert.NotEmpty(t, gf.FileFormat, caller)
		assert.NotEmpty(t, gf.IngestFileType, caller)
		assert.NotEmpty(t, gf.IngestMd5, caller)
		assert.NotEmpty(t, gf.IngestSha256, caller)

		if caller == "TestVirtualBagRead_FromTarFile" {
			assert.Empty(t, gf.IngestLocalPath, caller)
		} else if caller == "TestVirtualBagRead_FromDirectory" {
			assert.NotEmpty(t, gf.IngestLocalPath, caller)
		}

		assert.Empty(t, gf.IngestStorageURL, caller)
		assert.Empty(t, gf.IngestReplicationURL, caller)
		assert.True(t, gf.Size > 0, caller)
		switch gf.IngestFileType {
		case constants.PAYLOAD_FILE: payloadFileCount++
		case constants.PAYLOAD_MANIFEST: manifestCount++
		case constants.TAG_MANIFEST: tagManifestCount++
		case constants.TAG_FILE: tagFileCount++
		}
	}

	// Make sure file types were all tagged correctly
	assert.Equal(t, 4, payloadFileCount, caller)
	assert.Equal(t, 2, manifestCount, caller)
	assert.Equal(t, 2, tagManifestCount, caller)
	assert.Equal(t, 8, tagFileCount, caller)

	// Spot check generic file aptrust-info.txt
	gf := obj.FindGenericFileByPath("aptrust-info.txt")
	if gf == nil {
		assert.Fail(t, "Could not find aptrust-info.txt", caller)
	}
	assert.Equal(t, "example.edu.tagsample_good/aptrust-info.txt", gf.Identifier, caller)
	assert.Equal(t, 0, gf.IntellectualObjectId, caller)
	assert.Equal(t, "example.edu.tagsample_good", gf.IntellectualObjectIdentifier, caller)
	assert.Equal(t, "application/binary", gf.FileFormat, caller)
	assert.Empty(t, gf.URI, caller)
	assert.EqualValues(t, 67, gf.Size, caller)
	assert.False(t, gf.FileModified.IsZero(), caller)
	assert.Equal(t, constants.TAG_FILE, gf.IngestFileType, caller)
	assert.Equal(t, "300e936e622605f9f7a846d261d53093", gf.IngestManifestMd5, caller)
	assert.Equal(t, "300e936e622605f9f7a846d261d53093", gf.IngestMd5, caller)
	assert.False(t, gf.IngestMd5GeneratedAt.IsZero(), caller)
	assert.True(t, gf.IngestMd5VerifiedAt.IsZero(), caller)
	assert.Equal(t, "a2b6c5a713af771c5e4edde8d5be25fbcad86e45ea338f43a5bb769347e7c8bb", gf.IngestManifestSha256, caller)
	assert.Equal(t, "a2b6c5a713af771c5e4edde8d5be25fbcad86e45ea338f43a5bb769347e7c8bb", gf.IngestSha256, caller)
	assert.False(t, gf.IngestSha256GeneratedAt.IsZero(), caller)
	assert.True(t, gf.IngestSha256VerifiedAt.IsZero(), caller)
	assert.NotEmpty(t, gf.IngestUUID, caller)
	assert.False(t, gf.IngestUUIDGeneratedAt.IsZero(), caller)
	assert.Empty(t, gf.IngestStorageURL, caller)
	assert.True(t, gf.IngestStoredAt.IsZero(), caller)
	assert.Empty(t, gf.IngestReplicationURL, caller)
	assert.True(t, gf.IngestReplicatedAt.IsZero(), caller)
	assert.False(t, gf.IngestPreviousVersionExists, caller)
	assert.True(t, gf.IngestNeedsSave, caller)
	assert.Empty(t, gf.IngestErrorMessage, caller)
}

func vbagGetPath(fileName string) (string) {
	_, filename, _, _ := runtime.Caller(0)
	dir, _ := filepath.Abs(filepath.Dir(filename))
	testDataPath := filepath.Join(dir, "..", "testdata", fileName)
	return testDataPath
}

func untarTestBag(t *testing.T) (tempDir string, bagPath string) {
	tarFilePath := vbagGetPath("example.edu.tagsample_good.tar")
	tempDir, err := ioutil.TempDir("", "test")
	if err != nil {
		assert.Fail(t, "Cannot create temp dir: %v", err)
	}
	cmd := exec.Command("tar", "xf", tarFilePath, "--directory", tempDir)
	err = cmd.Run()
	if err != nil {
		assert.Fail(t, "Cannot untar test bag into temp dir: %v", err)
	}
	pathToUntarredBag := filepath.Join(tempDir, "example.edu.tagsample_good")
	return tempDir, pathToUntarredBag
}
<|endoftext|>


<commit_before>package weedharvester

import (
	"bytes"
	"fmt"
	"io"
	"net/http"
)

// Filer represents a filer for SeaweedFS
type Filer struct {
	url string
}

// Directory represents a directory mapped from the seaweed filer
type Directory struct {
	Directory      string         `json:"Directory"`
	Files          []file         `json:"Files"`
	Subdirectories []subdirectory `json:"Subdirectories"`
}

type subdirectory struct {
	Name string `json:"Name"`
	ID   int    `json:"Id"`
}

type file struct {
	Name string `json:"name"`
	Fid  string `json:"fid"`
}

// Create creates a new file with the given content name and under the given path
func (f *Filer) Create(content io.Reader, filename string, path string) error {
	var b bytes.Buffer
	writer, err := createMultipartForm(&content, &b)

	if err != nil {
		return err
	}

	var url string
	if len(path) == 0 {
		url = fmt.Sprintf("%s/%s", f.url, filename)
	} else {
		url = fmt.Sprintf("%s/%s/%s", f.url, path, filename)
	}

	return sendMultipartFormData(writer, &b, url)
}

func (f *Filer) Read(filename string, path string) io.Reader {
	var url string
	if len(path) == 0 {
		url = fmt.Sprintf("%s/%s", f.url, filename)
	} else {
		url = fmt.Sprintf("%s/%s/%s", f.url, path, filename)
	}

	resp, err := http.Get(url)
	if err != nil {
		panic(err)
	}

	if resp.StatusCode >= 300 {
		panic(fmt.Sprintln("Bad status code reading"))
	}

	return resp.Body
}

// ReadDirectory returns all files contained in a given directory
func (f *Filer) ReadDirectory(path string, lastFileName string) Directory {
	var url string
	if len(lastFileName) != 0 {
		url = fmt.Sprintf("%s/%s/?lastFileName=%s", f.url, path, lastFileName)
	} else {
		url = fmt.Sprintf("%s/%s/", f.url, path)
	}
	resp, err := http.Get(url)

	if err != nil {
		panic(err)
	}

	directory := Directory{}
	// data, err := ioutil.ReadAll(resp.Body)
	// fmt.Println(string(data))
	fmt.Println(url)
	err = decodeJSON(resp.Body, &directory)

	if err != nil {
		panic(err)
	}
	return directory
}
<commit_msg>Remove debug fmt<commit_after>package weedharvester

import (
	"bytes"
	"fmt"
	"io"
	"net/http"
)

// Filer represents a filer for SeaweedFS
type Filer struct {
	url string
}

// Directory represents a directory mapped from the seaweed filer
type Directory struct {
	Directory      string         `json:"Directory"`
	Files          []file         `json:"Files"`
	Subdirectories []subdirectory `json:"Subdirectories"`
}

type subdirectory struct {
	Name string `json:"Name"`
	ID   int    `json:"Id"`
}

type file struct {
	Name string `json:"name"`
	Fid  string `json:"fid"`
}

// Create creates a new file with the given content name and under the given path
func (f *Filer) Create(content io.Reader, filename string, path string) error {
	var b bytes.Buffer
	writer, err := createMultipartForm(&content, &b)

	if err != nil {
		return err
	}

	var url string
	if len(path) == 0 {
		url = fmt.Sprintf("%s/%s", f.url, filename)
	} else {
		url = fmt.Sprintf("%s/%s/%s", f.url, path, filename)
	}

	return sendMultipartFormData(writer, &b, url)
}

func (f *Filer) Read(filename string, path string) io.Reader {
	var url string
	if len(path) == 0 {
		url = fmt.Sprintf("%s/%s", f.url, filename)
	} else {
		url = fmt.Sprintf("%s/%s/%s", f.url, path, filename)
	}

	resp, err := http.Get(url)
	if err != nil {
		panic(err)
	}

	if resp.StatusCode >= 300 {
		panic(fmt.Sprintln("Bad status code reading"))
	}

	return resp.Body
}

// ReadDirectory returns all files contained in a given directory
func (f *Filer) ReadDirectory(path string, lastFileName string) Directory {
	var url string
	if len(lastFileName) != 0 {
		url = fmt.Sprintf("%s/%s/?lastFileName=%s", f.url, path, lastFileName)
	} else {
		url = fmt.Sprintf("%s/%s/", f.url, path)
	}
	resp, err := http.Get(url)

	if err != nil {
		panic(err)
	}

	directory := Directory{}
	err = decodeJSON(resp.Body, &directory)

	if err != nil {
		panic(err)
	}
	return directory
}
<|endoftext|>