<commit_before>package announcer

import (
	"context"
	"math"
	"net"
	"reflect"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/cenkalti/backoff"
	"github.com/cenkalti/rain/internal/logger"
	"github.com/cenkalti/rain/internal/tracker"
	"github.com/cenkalti/rain/internal/tracker/httptracker"
)

type Status int

const (
	NotContactedYet Status = iota
	Contacting
	Working
	NotWorking
)

type PeriodicalAnnouncer struct {
	Tracker       tracker.Tracker
	status        Status
	statsCommandC chan statsRequest
	numWant       int
	interval      time.Duration
	minInterval   time.Duration
	seeders       int
	leechers      int
	lastError     *AnnounceError
	log           logger.Logger
	completedC    chan struct{}
	newPeers      chan []*net.TCPAddr
	backoff       backoff.BackOff
	getTorrent    func() tracker.Torrent
	lastAnnounce  time.Time
	HasAnnounced  bool
	responseC     chan *tracker.AnnounceResponse
	errC          chan error
	closeC        chan struct{}
	doneC         chan struct{}

	needMorePeers  bool
	mNeedMorePeers sync.RWMutex
	needMorePeersC chan struct{}
}

func NewPeriodicalAnnouncer(trk tracker.Tracker, numWant int, minInterval time.Duration, getTorrent func() tracker.Torrent, completedC chan struct{}, newPeers chan []*net.TCPAddr, l logger.Logger) *PeriodicalAnnouncer {
	return &PeriodicalAnnouncer{
		Tracker:        trk,
		status:         NotContactedYet,
		statsCommandC:  make(chan statsRequest),
		numWant:        numWant,
		minInterval:    minInterval,
		log:            l,
		completedC:     completedC,
		newPeers:       newPeers,
		getTorrent:     getTorrent,
		needMorePeersC: make(chan struct{}, 1),
		responseC:      make(chan *tracker.AnnounceResponse),
		errC:           make(chan error),
		closeC:         make(chan struct{}),
		doneC:          make(chan struct{}),
		backoff: &backoff.ExponentialBackOff{
			InitialInterval:     5 * time.Second,
			RandomizationFactor: 0.5,
			Multiplier:          2,
			MaxInterval:         30 * time.Minute,
			MaxElapsedTime:      0, // never stop
			Clock:               backoff.SystemClock,
		},
	}
}

func (a *PeriodicalAnnouncer) Close() {
	close(a.closeC)
	<-a.doneC
}

type statsRequest struct {
	Response chan Stats
}

func (a *PeriodicalAnnouncer) Stats() Stats {
	var stats Stats
	req := statsRequest{Response: make(chan Stats, 1)}
	select {
	case a.statsCommandC <- req:
	case <-a.closeC:
	}
	select {
	case stats = <-req.Response:
	case <-a.closeC:
	}
	return stats
}

func (a *PeriodicalAnnouncer) NeedMorePeers(val bool) {
	a.mNeedMorePeers.Lock()
	a.needMorePeers = val
	a.mNeedMorePeers.Unlock()
	select {
	case a.needMorePeersC <- struct{}{}:
	case <-a.doneC:
	default:
	}
}

func (a *PeriodicalAnnouncer) Run() {
	defer close(a.doneC)
	a.backoff.Reset()

	timer := time.NewTimer(math.MaxInt64)
	defer timer.Stop()

	// BEP 0003: No completed is sent if the file was complete when started.
	select {
	case <-a.completedC:
		a.completedC = nil
	default:
	}

	ctx, cancel := context.WithCancel(context.Background())
	go a.announce(ctx, tracker.EventStarted, a.numWant)
	a.status = Contacting
	for {
		select {
		case <-timer.C:
			if a.status == Contacting {
				break
			}
			go a.announce(ctx, tracker.EventNone, a.numWant)
			a.status = Contacting
		case resp := <-a.responseC:
			a.status = Working
			a.lastAnnounce = time.Now()
			a.seeders = int(resp.Seeders)
			a.leechers = int(resp.Leechers)
			a.interval = resp.Interval
			if resp.MinInterval > 0 {
				a.minInterval = resp.MinInterval
			}
			a.HasAnnounced = true
			a.lastError = nil
			a.backoff.Reset()
			a.mNeedMorePeers.RLock()
			needMorePeers := a.needMorePeers
			a.mNeedMorePeers.RUnlock()
			if needMorePeers {
				timer.Reset(a.minInterval)
			} else {
				timer.Reset(a.interval)
			}
		case err := <-a.errC:
			a.status = NotWorking
			a.lastAnnounce = time.Now()
			// Give more friendly error to the user
			a.lastError = newAnnounceError(err)
			if a.lastError.Unknown {
				a.log.Errorln("announce error:", a.lastError.ErrorWithType())
			} else {
				a.log.Debugln("announce error:", a.lastError.Err.Error())
			}
			if terr, ok := a.lastError.Err.(*tracker.Error); ok && terr.RetryIn > 0 {
				timer.Reset(terr.RetryIn)
			} else {
				timer.Reset(a.backoff.NextBackOff())
			}
		case <-a.needMorePeersC:
			a.mNeedMorePeers.RLock()
			needMorePeers := a.needMorePeers
			a.mNeedMorePeers.RUnlock()
			if a.status == Contacting {
				break
			}
			if needMorePeers {
				timer.Reset(time.Until(a.lastAnnounce.Add(a.minInterval)))
			} else {
				timer.Reset(time.Until(a.lastAnnounce.Add(a.interval)))
			}
		case <-a.completedC:
			if a.status == Contacting {
				cancel()
				ctx, cancel = context.WithCancel(context.Background())
			}
			go a.announce(ctx, tracker.EventCompleted, 0)
			a.status = Contacting
			a.completedC = nil // do not send more than one "completed" event
		case req := <-a.statsCommandC:
			req.Response <- a.stats()
		case <-a.closeC:
			cancel()
			return
		}
	}
}

func (a *PeriodicalAnnouncer) announce(ctx context.Context, event tracker.Event, numWant int) {
	announce(ctx, a.Tracker, event, numWant, a.getTorrent(), a.responseC, a.errC)
}

type Stats struct {
	Status   Status
	Error    *AnnounceError
	Seeders  int
	Leechers int
}

func (a *PeriodicalAnnouncer) stats() Stats {
	return Stats{
		Status:   a.status,
		Error:    a.lastError,
		Seeders:  a.seeders,
		Leechers: a.leechers,
	}
}

type AnnounceError struct {
	Err     error
	Message string
	Unknown bool
}

func newAnnounceError(err error) (e *AnnounceError) {
	e = &AnnounceError{Err: err}
	switch err := err.(type) {
	case *net.DNSError:
		s := err.Error()
		if strings.HasSuffix(s, "no such host") {
			e.Message = "host not found: " + err.Name
			return
		}
	case net.Error:
		if err.Timeout() {
			e.Message = "timeout contacting tracker"
			return
		}
	case *httptracker.StatusError:
		if err.Code == 403 || err.Code == 404 {
			e.Message = "tracker returned http status: " + strconv.Itoa(err.Code)
			return
		}
	case *tracker.Error:
		e.Message = "announce error: " + err.FailureReason
		return
	}
	e.Message = "unknown error in announce"
	e.Unknown = true
	return
}

func (e *AnnounceError) ErrorWithType() string {
	return reflect.TypeOf(e.Err).String() + ": " + e.Err.Error()
}
<commit_msg>handle connection refused errors<commit_after>package announcer

import (
	"context"
	"math"
	"net"
	"net/url"
	"reflect"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/cenkalti/backoff"
	"github.com/cenkalti/rain/internal/logger"
	"github.com/cenkalti/rain/internal/tracker"
	"github.com/cenkalti/rain/internal/tracker/httptracker"
)

type Status int

const (
	NotContactedYet Status = iota
	Contacting
	Working
	NotWorking
)

type PeriodicalAnnouncer struct {
	Tracker       tracker.Tracker
	status        Status
	statsCommandC chan statsRequest
	numWant       int
	interval      time.Duration
	minInterval   time.Duration
	seeders       int
	leechers      int
	lastError     *AnnounceError
	log           logger.Logger
	completedC    chan struct{}
	newPeers      chan []*net.TCPAddr
	backoff       backoff.BackOff
	getTorrent    func() tracker.Torrent
	lastAnnounce  time.Time
	HasAnnounced  bool
	responseC     chan *tracker.AnnounceResponse
	errC          chan error
	closeC        chan struct{}
	doneC         chan struct{}

	needMorePeers  bool
	mNeedMorePeers sync.RWMutex
	needMorePeersC chan struct{}
}

func NewPeriodicalAnnouncer(trk tracker.Tracker, numWant int, minInterval time.Duration, getTorrent func() tracker.Torrent, completedC chan struct{}, newPeers chan []*net.TCPAddr, l logger.Logger) *PeriodicalAnnouncer {
	return &PeriodicalAnnouncer{
		Tracker:        trk,
		status:         NotContactedYet,
		statsCommandC:  make(chan statsRequest),
		numWant:        numWant,
		minInterval:    minInterval,
		log:            l,
		completedC:     completedC,
		newPeers:       newPeers,
		getTorrent:     getTorrent,
		needMorePeersC: make(chan struct{}, 1),
		responseC:      make(chan *tracker.AnnounceResponse),
		errC:           make(chan error),
		closeC:         make(chan struct{}),
		doneC:          make(chan struct{}),
		backoff: &backoff.ExponentialBackOff{
			InitialInterval:     5 * time.Second,
			RandomizationFactor: 0.5,
			Multiplier:          2,
			MaxInterval:         30 * time.Minute,
			MaxElapsedTime:      0, // never stop
			Clock:               backoff.SystemClock,
		},
	}
}

func (a *PeriodicalAnnouncer) Close() {
	close(a.closeC)
	<-a.doneC
}

type statsRequest struct {
	Response chan Stats
}

func (a *PeriodicalAnnouncer) Stats() Stats {
	var stats Stats
	req := statsRequest{Response: make(chan Stats, 1)}
	select {
	case a.statsCommandC <- req:
	case <-a.closeC:
	}
	select {
	case stats = <-req.Response:
	case <-a.closeC:
	}
	return stats
}

func (a *PeriodicalAnnouncer) NeedMorePeers(val bool) {
	a.mNeedMorePeers.Lock()
	a.needMorePeers = val
	a.mNeedMorePeers.Unlock()
	select {
	case a.needMorePeersC <- struct{}{}:
	case <-a.doneC:
	default:
	}
}

func (a *PeriodicalAnnouncer) Run() {
	defer close(a.doneC)
	a.backoff.Reset()

	timer := time.NewTimer(math.MaxInt64)
	defer timer.Stop()

	// BEP 0003: No completed is sent if the file was complete when started.
	select {
	case <-a.completedC:
		a.completedC = nil
	default:
	}

	ctx, cancel := context.WithCancel(context.Background())
	go a.announce(ctx, tracker.EventStarted, a.numWant)
	a.status = Contacting
	for {
		select {
		case <-timer.C:
			if a.status == Contacting {
				break
			}
			go a.announce(ctx, tracker.EventNone, a.numWant)
			a.status = Contacting
		case resp := <-a.responseC:
			a.status = Working
			a.lastAnnounce = time.Now()
			a.seeders = int(resp.Seeders)
			a.leechers = int(resp.Leechers)
			a.interval = resp.Interval
			if resp.MinInterval > 0 {
				a.minInterval = resp.MinInterval
			}
			a.HasAnnounced = true
			a.lastError = nil
			a.backoff.Reset()
			a.mNeedMorePeers.RLock()
			needMorePeers := a.needMorePeers
			a.mNeedMorePeers.RUnlock()
			if needMorePeers {
				timer.Reset(a.minInterval)
			} else {
				timer.Reset(a.interval)
			}
		case err := <-a.errC:
			a.status = NotWorking
			a.lastAnnounce = time.Now()
			// Give more friendly error to the user
			a.lastError = newAnnounceError(err)
			if a.lastError.Unknown {
				a.log.Errorln("announce error:", a.lastError.ErrorWithType())
			} else {
				a.log.Debugln("announce error:", a.lastError.Err.Error())
			}
			if terr, ok := a.lastError.Err.(*tracker.Error); ok && terr.RetryIn > 0 {
				timer.Reset(terr.RetryIn)
			} else {
				timer.Reset(a.backoff.NextBackOff())
			}
		case <-a.needMorePeersC:
			a.mNeedMorePeers.RLock()
			needMorePeers := a.needMorePeers
			a.mNeedMorePeers.RUnlock()
			if a.status == Contacting {
				break
			}
			if needMorePeers {
				timer.Reset(time.Until(a.lastAnnounce.Add(a.minInterval)))
			} else {
				timer.Reset(time.Until(a.lastAnnounce.Add(a.interval)))
			}
		case <-a.completedC:
			if a.status == Contacting {
				cancel()
				ctx, cancel = context.WithCancel(context.Background())
			}
			go a.announce(ctx, tracker.EventCompleted, 0)
			a.status = Contacting
			a.completedC = nil // do not send more than one "completed" event
		case req := <-a.statsCommandC:
			req.Response <- a.stats()
		case <-a.closeC:
			cancel()
			return
		}
	}
}

func (a *PeriodicalAnnouncer) announce(ctx context.Context, event tracker.Event, numWant int) {
	announce(ctx, a.Tracker, event, numWant, a.getTorrent(), a.responseC, a.errC)
}

type Stats struct {
	Status   Status
	Error    *AnnounceError
	Seeders  int
	Leechers int
}

func (a *PeriodicalAnnouncer) stats() Stats {
	return Stats{
		Status:   a.status,
		Error:    a.lastError,
		Seeders:  a.seeders,
		Leechers: a.leechers,
	}
}

type AnnounceError struct {
	Err     error
	Message string
	Unknown bool
}

func newAnnounceError(err error) (e *AnnounceError) {
	e = &AnnounceError{Err: err}
	switch err := err.(type) {
	case *net.DNSError:
		s := err.Error()
		if strings.HasSuffix(s, "no such host") {
			e.Message = "host not found: " + err.Name
			return
		}
	case *url.Error:
		s := err.Error()
		if strings.HasSuffix(s, "connection refused") {
			e.Message = "tracker refused the connection"
			return
		}
	case net.Error:
		if err.Timeout() {
			e.Message = "timeout contacting tracker"
			return
		}
	case *httptracker.StatusError:
		if err.Code == 403 || err.Code == 404 {
			e.Message = "tracker returned http status: " + strconv.Itoa(err.Code)
			return
		}
	case *tracker.Error:
		e.Message = "announce error: " + err.FailureReason
		return
	}
	e.Message = "unknown error in announce"
	e.Unknown = true
	return
}

func (e *AnnounceError) ErrorWithType() string {
	return reflect.TypeOf(e.Err).String() + ": " + e.Err.Error()
}
<|endoftext|>


<commit_before>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package frontend

import (
	"context"
	"database/sql"
	"errors"
	"fmt"
	"math"
	"sort"
	"strconv"
	"strings"

	"github.com/google/safehtml"
	"github.com/google/safehtml/template"
	"golang.org/x/pkgsite/internal"
	"golang.org/x/pkgsite/internal/derrors"
	"golang.org/x/pkgsite/internal/experiment"
	"golang.org/x/pkgsite/internal/godoc"
	"golang.org/x/pkgsite/internal/godoc/dochtml"
	"golang.org/x/pkgsite/internal/log"
	"golang.org/x/pkgsite/internal/middleware"
	"golang.org/x/pkgsite/internal/postgres"
)

// MainDetails contains data needed to render the unit template.
type MainDetails struct {
	// NestedModules are nested modules relative to the path for the unit.
	NestedModules []*NestedModule

	// Subdirectories are packages in subdirectories relative to the path for
	// the unit.
	Subdirectories []*Subdirectory

	// Licenses contains license metadata used in the header.
	Licenses []LicenseMetadata

	// NumImports is the number of imports for the package.
	NumImports int

	// CommitTime is time that this version was published, or the time that
	// has elapsed since this version was committed if it was done so recently.
	CommitTime string

	// Readme is the rendered readme HTML.
	Readme safehtml.HTML

	// ReadmeOutline is a collection of headings from the readme file
	// used to render the readme outline in the sidebar.
	ReadmeOutline []*Heading

	// ImportedByCount is the number of packages that import this path.
	// When the count is > limit it will read as 'limit+'. This field
	// is not supported when using a datasource proxy.
	ImportedByCount string

	DocBody       safehtml.HTML
	DocOutline    safehtml.HTML
	MobileOutline safehtml.HTML
	IsPackage     bool

	// SourceFiles contains .go files for the package.
	SourceFiles []*File

	// RepositoryURL is the URL to the repository containing the package.
	RepositoryURL string

	// SourceURL is the URL to the source of the package.
	SourceURL string

	// ExpandReadme is holds the expandable readme state.
	ExpandReadme bool
}

// File is a source file for a package.
type File struct {
	Name string
	URL  string
}

// NestedModule is a nested module relative to the path of a given unit.
// This content is used in the Directories section of the unit page.
type NestedModule struct {
	Suffix string // suffix after the unit path
	URL    string
}

// Subdirectory is a package in a subdirectory relative to the path of a given
// unit. This content is used in the Directories section of the unit page.
type Subdirectory struct {
	Suffix   string
	URL      string
	Synopsis string
}

func fetchMainDetails(ctx context.Context, ds internal.DataSource, um *internal.UnitMeta, expandReadme bool) (_ *MainDetails, err error) {
	defer middleware.ElapsedStat(ctx, "fetchMainDetails")()

	unit, err := ds.GetUnit(ctx, um, internal.WithReadme|internal.WithDocumentation|internal.WithSubdirectories|internal.WithImports)
	if err != nil {
		return nil, err
	}

	nestedModules, err := getNestedModules(ctx, ds, um)
	if err != nil {
		return nil, err
	}
	subdirectories := getSubdirectories(um, unit.Subdirectories)
	if err != nil {
		return nil, err
	}
	readme, readmeOutline, err := readmeContent(ctx, unit)
	if err != nil {
		return nil, err
	}
	importedByCount, err := getImportedByCount(ctx, ds, unit)
	if err != nil {
		return nil, err
	}
	var (
		docBody, docOutline, mobileOutline safehtml.HTML
		files                              []*File
	)
	if unit.Documentation != nil {
		end := middleware.ElapsedStat(ctx, "DecodePackage")
		docPkg, err := godoc.DecodePackage(unit.Documentation.Source)
		end()
		if err != nil {
			if errors.Is(err, godoc.ErrInvalidEncodingType) {
				// Instead of returning a 500, return a 404 so the user can
				// reprocess the documentation.
				log.Errorf(ctx, "fetchMainDetails(%q, %q, %q): %v", um.Path, um.ModulePath, um.Version, err)
				return nil, errUnitNotFoundWithoutFetch
			}
			return nil, err
		}
		docBody, docOutline, mobileOutline, err = getHTML(ctx, unit, docPkg)
		// If err  is ErrTooLarge, then docBody will have an appropriate message.
		if err != nil && !errors.Is(err, dochtml.ErrTooLarge) {
			return nil, err
		}
		end = middleware.ElapsedStat(ctx, "sourceFiles")
		files = sourceFiles(unit, docPkg)
		end()
	}
	return &MainDetails{
		ExpandReadme:    expandReadme,
		NestedModules:   nestedModules,
		Subdirectories:  subdirectories,
		Licenses:        transformLicenseMetadata(um.Licenses),
		CommitTime:      absoluteTime(um.CommitTime),
		Readme:          readme,
		ReadmeOutline:   readmeOutline,
		DocOutline:      docOutline,
		DocBody:         docBody,
		SourceFiles:     files,
		RepositoryURL:   um.SourceInfo.RepoURL(),
		SourceURL:       um.SourceInfo.DirectoryURL(internal.Suffix(um.Path, um.ModulePath)),
		MobileOutline:   mobileOutline,
		NumImports:      unit.NumImports,
		ImportedByCount: importedByCount,
		IsPackage:       unit.IsPackage(),
	}, nil
}

// moduleInfo extracts module info from a unit. This is a shim
// for functions ReadmeHTML and createDirectory that will be removed
// when we complete the switch to units.
func moduleInfo(um *internal.UnitMeta) *internal.ModuleInfo {
	return &internal.ModuleInfo{
		ModulePath:        um.ModulePath,
		Version:           um.Version,
		CommitTime:        um.CommitTime,
		IsRedistributable: um.IsRedistributable,
		SourceInfo:        um.SourceInfo,
	}
}

// readmeContent renders the readme to html and collects the headings
// into an outline when the goldmark experiment active.
func readmeContent(ctx context.Context, u *internal.Unit) (_ safehtml.HTML, _ []*Heading, err error) {
	defer derrors.Wrap(&err, "readmeContent(%q, %q, %q)", u.Path, u.ModulePath, u.Version)
	defer middleware.ElapsedStat(ctx, "readmeContent")()
	if !u.IsRedistributable {
		return safehtml.HTML{}, nil, nil
	}
	mi := moduleInfo(&u.UnitMeta)
	var (
		readmeHTML    safehtml.HTML
		readmeOutline []*Heading
	)
	if experiment.IsActive(ctx, internal.ExperimentGoldmark) {
		readmeHTML, readmeOutline, err = Readme(ctx, u)
	} else {
		readmeHTML, err = LegacyReadmeHTML(ctx, mi, u.Readme)
	}
	if err != nil {
		return safehtml.HTML{}, nil, err
	}
	return readmeHTML, readmeOutline, nil
}

func getNestedModules(ctx context.Context, ds internal.DataSource, um *internal.UnitMeta) ([]*NestedModule, error) {
	nestedModules, err := ds.GetNestedModules(ctx, um.ModulePath)
	if err != nil {
		return nil, err
	}
	var mods []*NestedModule
	for _, m := range nestedModules {
		if m.SeriesPath() == internal.SeriesPathForModule(um.ModulePath) {
			continue
		}
		if !strings.HasPrefix(m.ModulePath, um.Path+"/") {
			continue
		}
		mods = append(mods, &NestedModule{
			URL:    constructUnitURL(m.ModulePath, m.ModulePath, internal.LatestVersion),
			Suffix: internal.Suffix(m.SeriesPath(), um.Path),
		})
	}
	return mods, nil
}

func getSubdirectories(um *internal.UnitMeta, pkgs []*internal.PackageMeta) []*Subdirectory {
	var sdirs []*Subdirectory
	for _, pm := range pkgs {
		if um.Path == pm.Path {
			continue
		}
		sdirs = append(sdirs, &Subdirectory{
			URL:      constructUnitURL(pm.Path, um.ModulePath, linkVersion(um.Version, um.ModulePath)),
			Suffix:   internal.Suffix(pm.Path, um.Path),
			Synopsis: pm.Synopsis,
		})
	}
	sort.Slice(sdirs, func(i, j int) bool { return sdirs[i].Suffix < sdirs[j].Suffix })
	return sdirs
}

const missingDocReplacement = `<p>Documentation is missing.</p>`

func getHTML(ctx context.Context, u *internal.Unit, docPkg *godoc.Package) (body, outline, mobileOutline safehtml.HTML, err error) {
	defer derrors.Wrap(&err, "getHTML(%s)", u.Path)

	if len(u.Documentation.Source) > 0 {
		return renderDocParts(ctx, u, docPkg)
	}
	return template.MustParseAndExecuteToHTML(missingDocReplacement), safehtml.HTML{}, safehtml.HTML{}, nil
}

// getImportedByCount fetches the imported by count for the unit and returns a
// string to be displayed. If the datasource does not support imported by, it
// will return N/A.
func getImportedByCount(ctx context.Context, ds internal.DataSource, unit *internal.Unit) (_ string, err error) {
	defer derrors.Wrap(&err, "getImportedByCount(%q, %q, %q)", unit.Path, unit.ModulePath, unit.Version)
	defer middleware.ElapsedStat(ctx, "getImportedByCount")()

	db, ok := ds.(*postgres.DB)
	if !ok {
		return "N/A", nil
	}

	// Get an exact number for a small limit, to determine whether we should
	// fetch data from search_documents and display an approximate count, or
	// just use the exact count.
	importedBy, err := db.GetImportedBy(ctx, unit.Path, unit.ModulePath, mainPageImportedByLimit)
	if err != nil {
		return "", err
	}
	if len(importedBy) < mainPageImportedByLimit {
		// Exact number is less than the limit, so just return that.
		return strconv.Itoa(len(importedBy)), nil
	}

	// Exact number is greater than the limit, so fetch an approximate value
	// from search_documents.num_imported_by. This number might be different
	// than the result of GetImportedBy because alternative modules and internal
	// packages are excluded.
	var count int
	if experiment.IsActive(ctx, internal.ExperimentGetUnitWithOneQuery) {
		count = unit.NumImportedBy
	} else {
		count, err = db.GetImportedByCount(ctx, unit.Path, unit.ModulePath)
		if err != nil {
			if err == sql.ErrNoRows {
				log.Errorf(ctx, "missing search_documents row for path %s, module path %s", unit.Path, unit.ModulePath)
				return "", nil
			}
			return "", err
		}
		if count < mainPageImportedByLimit {
			count = mainPageImportedByLimit
		}
	}
	// Treat the result as approximate.
	return fmt.Sprintf("%d+", approximateLowerBound(count)), nil
}

// approximateLowerBound rounds n down to a multiple of a power of 10.
// See the test for examples.
func approximateLowerBound(n int) int {
	if n == 0 {
		return 0
	}
	f := float64(n)
	powerOf10 := math.Pow(10, math.Floor(math.Log10(f)))
	return int(powerOf10 * math.Floor(f/powerOf10))
}
<commit_msg>internal/frontend: log missing doc source<commit_after>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package frontend

import (
	"context"
	"database/sql"
	"errors"
	"fmt"
	"math"
	"sort"
	"strconv"
	"strings"

	"github.com/google/safehtml"
	"github.com/google/safehtml/template"
	"golang.org/x/pkgsite/internal"
	"golang.org/x/pkgsite/internal/derrors"
	"golang.org/x/pkgsite/internal/experiment"
	"golang.org/x/pkgsite/internal/godoc"
	"golang.org/x/pkgsite/internal/godoc/dochtml"
	"golang.org/x/pkgsite/internal/log"
	"golang.org/x/pkgsite/internal/middleware"
	"golang.org/x/pkgsite/internal/postgres"
)

// MainDetails contains data needed to render the unit template.
type MainDetails struct {
	// NestedModules are nested modules relative to the path for the unit.
	NestedModules []*NestedModule

	// Subdirectories are packages in subdirectories relative to the path for
	// the unit.
	Subdirectories []*Subdirectory

	// Licenses contains license metadata used in the header.
	Licenses []LicenseMetadata

	// NumImports is the number of imports for the package.
	NumImports int

	// CommitTime is time that this version was published, or the time that
	// has elapsed since this version was committed if it was done so recently.
	CommitTime string

	// Readme is the rendered readme HTML.
	Readme safehtml.HTML

	// ReadmeOutline is a collection of headings from the readme file
	// used to render the readme outline in the sidebar.
	ReadmeOutline []*Heading

	// ImportedByCount is the number of packages that import this path.
	// When the count is > limit it will read as 'limit+'. This field
	// is not supported when using a datasource proxy.
	ImportedByCount string

	DocBody       safehtml.HTML
	DocOutline    safehtml.HTML
	MobileOutline safehtml.HTML
	IsPackage     bool

	// SourceFiles contains .go files for the package.
	SourceFiles []*File

	// RepositoryURL is the URL to the repository containing the package.
	RepositoryURL string

	// SourceURL is the URL to the source of the package.
	SourceURL string

	// ExpandReadme is holds the expandable readme state.
	ExpandReadme bool
}

// File is a source file for a package.
type File struct {
	Name string
	URL  string
}

// NestedModule is a nested module relative to the path of a given unit.
// This content is used in the Directories section of the unit page.
type NestedModule struct {
	Suffix string // suffix after the unit path
	URL    string
}

// Subdirectory is a package in a subdirectory relative to the path of a given
// unit. This content is used in the Directories section of the unit page.
type Subdirectory struct {
	Suffix   string
	URL      string
	Synopsis string
}

func fetchMainDetails(ctx context.Context, ds internal.DataSource, um *internal.UnitMeta, expandReadme bool) (_ *MainDetails, err error) {
	defer middleware.ElapsedStat(ctx, "fetchMainDetails")()

	unit, err := ds.GetUnit(ctx, um, internal.WithReadme|internal.WithDocumentation|internal.WithSubdirectories|internal.WithImports)
	if err != nil {
		return nil, err
	}

	nestedModules, err := getNestedModules(ctx, ds, um)
	if err != nil {
		return nil, err
	}
	subdirectories := getSubdirectories(um, unit.Subdirectories)
	if err != nil {
		return nil, err
	}
	readme, readmeOutline, err := readmeContent(ctx, unit)
	if err != nil {
		return nil, err
	}
	importedByCount, err := getImportedByCount(ctx, ds, unit)
	if err != nil {
		return nil, err
	}
	var (
		docBody, docOutline, mobileOutline safehtml.HTML
		files                              []*File
	)
	if unit.Documentation != nil {
		end := middleware.ElapsedStat(ctx, "DecodePackage")
		docPkg, err := godoc.DecodePackage(unit.Documentation.Source)
		end()
		if err != nil {
			if errors.Is(err, godoc.ErrInvalidEncodingType) {
				// Instead of returning a 500, return a 404 so the user can
				// reprocess the documentation.
				log.Errorf(ctx, "fetchMainDetails(%q, %q, %q): %v", um.Path, um.ModulePath, um.Version, err)
				return nil, errUnitNotFoundWithoutFetch
			}
			return nil, err
		}
		docBody, docOutline, mobileOutline, err = getHTML(ctx, unit, docPkg)
		// If err  is ErrTooLarge, then docBody will have an appropriate message.
		if err != nil && !errors.Is(err, dochtml.ErrTooLarge) {
			return nil, err
		}
		end = middleware.ElapsedStat(ctx, "sourceFiles")
		files = sourceFiles(unit, docPkg)
		end()
	}
	return &MainDetails{
		ExpandReadme:    expandReadme,
		NestedModules:   nestedModules,
		Subdirectories:  subdirectories,
		Licenses:        transformLicenseMetadata(um.Licenses),
		CommitTime:      absoluteTime(um.CommitTime),
		Readme:          readme,
		ReadmeOutline:   readmeOutline,
		DocOutline:      docOutline,
		DocBody:         docBody,
		SourceFiles:     files,
		RepositoryURL:   um.SourceInfo.RepoURL(),
		SourceURL:       um.SourceInfo.DirectoryURL(internal.Suffix(um.Path, um.ModulePath)),
		MobileOutline:   mobileOutline,
		NumImports:      unit.NumImports,
		ImportedByCount: importedByCount,
		IsPackage:       unit.IsPackage(),
	}, nil
}

// moduleInfo extracts module info from a unit. This is a shim
// for functions ReadmeHTML and createDirectory that will be removed
// when we complete the switch to units.
func moduleInfo(um *internal.UnitMeta) *internal.ModuleInfo {
	return &internal.ModuleInfo{
		ModulePath:        um.ModulePath,
		Version:           um.Version,
		CommitTime:        um.CommitTime,
		IsRedistributable: um.IsRedistributable,
		SourceInfo:        um.SourceInfo,
	}
}

// readmeContent renders the readme to html and collects the headings
// into an outline when the goldmark experiment active.
func readmeContent(ctx context.Context, u *internal.Unit) (_ safehtml.HTML, _ []*Heading, err error) {
	defer derrors.Wrap(&err, "readmeContent(%q, %q, %q)", u.Path, u.ModulePath, u.Version)
	defer middleware.ElapsedStat(ctx, "readmeContent")()
	if !u.IsRedistributable {
		return safehtml.HTML{}, nil, nil
	}
	mi := moduleInfo(&u.UnitMeta)
	var (
		readmeHTML    safehtml.HTML
		readmeOutline []*Heading
	)
	if experiment.IsActive(ctx, internal.ExperimentGoldmark) {
		readmeHTML, readmeOutline, err = Readme(ctx, u)
	} else {
		readmeHTML, err = LegacyReadmeHTML(ctx, mi, u.Readme)
	}
	if err != nil {
		return safehtml.HTML{}, nil, err
	}
	return readmeHTML, readmeOutline, nil
}

func getNestedModules(ctx context.Context, ds internal.DataSource, um *internal.UnitMeta) ([]*NestedModule, error) {
	nestedModules, err := ds.GetNestedModules(ctx, um.ModulePath)
	if err != nil {
		return nil, err
	}
	var mods []*NestedModule
	for _, m := range nestedModules {
		if m.SeriesPath() == internal.SeriesPathForModule(um.ModulePath) {
			continue
		}
		if !strings.HasPrefix(m.ModulePath, um.Path+"/") {
			continue
		}
		mods = append(mods, &NestedModule{
			URL:    constructUnitURL(m.ModulePath, m.ModulePath, internal.LatestVersion),
			Suffix: internal.Suffix(m.SeriesPath(), um.Path),
		})
	}
	return mods, nil
}

func getSubdirectories(um *internal.UnitMeta, pkgs []*internal.PackageMeta) []*Subdirectory {
	var sdirs []*Subdirectory
	for _, pm := range pkgs {
		if um.Path == pm.Path {
			continue
		}
		sdirs = append(sdirs, &Subdirectory{
			URL:      constructUnitURL(pm.Path, um.ModulePath, linkVersion(um.Version, um.ModulePath)),
			Suffix:   internal.Suffix(pm.Path, um.Path),
			Synopsis: pm.Synopsis,
		})
	}
	sort.Slice(sdirs, func(i, j int) bool { return sdirs[i].Suffix < sdirs[j].Suffix })
	return sdirs
}

const missingDocReplacement = `<p>Documentation is missing.</p>`

func getHTML(ctx context.Context, u *internal.Unit, docPkg *godoc.Package) (body, outline, mobileOutline safehtml.HTML, err error) {
	defer derrors.Wrap(&err, "getHTML(%s)", u.Path)

	if len(u.Documentation.Source) > 0 {
		return renderDocParts(ctx, u, docPkg)
	}
	log.Errorf(ctx, "unit %s (%s@%s) missing documentation source", u.Path, u.ModulePath, u.Version)
	return template.MustParseAndExecuteToHTML(missingDocReplacement), safehtml.HTML{}, safehtml.HTML{}, nil
}

// getImportedByCount fetches the imported by count for the unit and returns a
// string to be displayed. If the datasource does not support imported by, it
// will return N/A.
func getImportedByCount(ctx context.Context, ds internal.DataSource, unit *internal.Unit) (_ string, err error) {
	defer derrors.Wrap(&err, "getImportedByCount(%q, %q, %q)", unit.Path, unit.ModulePath, unit.Version)
	defer middleware.ElapsedStat(ctx, "getImportedByCount")()

	db, ok := ds.(*postgres.DB)
	if !ok {
		return "N/A", nil
	}

	// Get an exact number for a small limit, to determine whether we should
	// fetch data from search_documents and display an approximate count, or
	// just use the exact count.
	importedBy, err := db.GetImportedBy(ctx, unit.Path, unit.ModulePath, mainPageImportedByLimit)
	if err != nil {
		return "", err
	}
	if len(importedBy) < mainPageImportedByLimit {
		// Exact number is less than the limit, so just return that.
		return strconv.Itoa(len(importedBy)), nil
	}

	// Exact number is greater than the limit, so fetch an approximate value
	// from search_documents.num_imported_by. This number might be different
	// than the result of GetImportedBy because alternative modules and internal
	// packages are excluded.
	var count int
	if experiment.IsActive(ctx, internal.ExperimentGetUnitWithOneQuery) {
		count = unit.NumImportedBy
	} else {
		count, err = db.GetImportedByCount(ctx, unit.Path, unit.ModulePath)
		if err != nil {
			if err == sql.ErrNoRows {
				log.Errorf(ctx, "missing search_documents row for path %s, module path %s", unit.Path, unit.ModulePath)
				return "", nil
			}
			return "", err
		}
		if count < mainPageImportedByLimit {
			count = mainPageImportedByLimit
		}
	}
	// Treat the result as approximate.
	return fmt.Sprintf("%d+", approximateLowerBound(count)), nil
}

// approximateLowerBound rounds n down to a multiple of a power of 10.
// See the test for examples.
func approximateLowerBound(n int) int {
	if n == 0 {
		return 0
	}
	f := float64(n)
	powerOf10 := math.Pow(10, math.Floor(math.Log10(f)))
	return int(powerOf10 * math.Floor(f/powerOf10))
}
<|endoftext|>


<commit_before>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package frontend

import (
	"context"
	"database/sql"
	"errors"
	"sort"
	"strconv"
	"strings"

	"github.com/google/safehtml"
	"golang.org/x/pkgsite/internal"
	"golang.org/x/pkgsite/internal/derrors"
	"golang.org/x/pkgsite/internal/experiment"
	"golang.org/x/pkgsite/internal/godoc"
	"golang.org/x/pkgsite/internal/log"
	"golang.org/x/pkgsite/internal/middleware"
	"golang.org/x/pkgsite/internal/postgres"
)

// MainDetails contains data needed to render the unit template.
type MainDetails struct {
	// NestedModules are nested modules relative to the path for the unit.
	NestedModules []*NestedModule

	// Subdirectories are packages in subdirectories relative to the path for
	// the unit.
	Subdirectories []*Subdirectory

	// Licenses contains license metadata used in the header.
	Licenses []LicenseMetadata

	// NumImports is the number of imports for the package.
	NumImports int

	// CommitTime is time that this version was published, or the time that
	// has elapsed since this version was committed if it was done so recently.
	CommitTime string

	// Readme is the rendered readme HTML.
	Readme safehtml.HTML

	// ImportedByCount is the number of packages that import this path.
	// When the count is > limit it will read as 'limit+'. This field
	// is not supported when using a datasource proxy.
	ImportedByCount string

	DocBody       safehtml.HTML
	DocOutline    safehtml.HTML
	MobileOutline safehtml.HTML
	IsPackage     bool

	// SourceFiles contains .go files for the package.
	SourceFiles []*File

	// RepositoryURL is the URL to the repository containing the package.
	RepositoryURL string

	// SourceURL is the URL to the source of the package.
	SourceURL string

	// ExpandReadme is holds the expandable readme state.
	ExpandReadme bool
}

// File is a source file for a package.
type File struct {
	Name string
	URL  string
}

// NestedModule is a nested module relative to the path of a given unit.
// This content is used in the Directories section of the unit page.
type NestedModule struct {
	Suffix string // suffix after the unit path
	URL    string
}

// Subdirectory is a package in a subdirectory relative to the path of a given
// unit. This content is used in the Directories section of the unit page.
type Subdirectory struct {
	Suffix   string
	URL      string
	Synopsis string
}

func fetchMainDetails(ctx context.Context, ds internal.DataSource, um *internal.UnitMeta, expandReadme bool) (_ *MainDetails, err error) {
	defer middleware.ElapsedStat(ctx, "fetchMainDetails")()

	unit, err := ds.GetUnit(ctx, um, internal.WithReadme|internal.WithDocumentation|internal.WithSubdirectories|internal.WithImports)
	if err != nil {
		return nil, err
	}

	nestedModules, err := getNestedModules(ctx, ds, um)
	if err != nil {
		return nil, err
	}
	subdirectories := getSubdirectories(um, unit.Subdirectories)
	if err != nil {
		return nil, err
	}
	readme, err := readmeContent(ctx, um, unit.Readme)
	if err != nil {
		return nil, err
	}
	importedByCount, err := getImportedByCount(ctx, ds, unit)
	if err != nil {
		return nil, err
	}

	var (
		docBody, docOutline, mobileOutline safehtml.HTML
		files                              []*File
	)
	if unit.Documentation != nil {
		end := middleware.ElapsedStat(ctx, "DecodePackage")
		docPkg, err := godoc.DecodePackage(unit.Documentation.Source)
		end()
		if err != nil {
			if errors.Is(err, godoc.ErrInvalidEncodingType) {
				// Instead of returning a 500, return a 404 so the user can
				// reprocess the documentation.
				log.Errorf(ctx, "fetchMainDetails(%q, %q, %q): %v", um.Path, um.ModulePath, um.Version, err)
				return nil, pathNotFoundError(um.Path, um.Version)
			}
			return nil, err
		}
		docHTML := getHTML(ctx, unit, docPkg)
		// TODO: Deprecate godoc.Parse. The sidenav and body can
		// either be rendered using separate functions, or all this content can
		// be passed to the template via the UnitPage struct.
		end = middleware.ElapsedStat(ctx, "godoc Parses")
		b, err := godoc.Parse(docHTML, godoc.BodySection)
		if err != nil {
			return nil, err
		}
		docBody = b
		o, err := godoc.Parse(docHTML, godoc.SidenavSection)
		if err != nil {
			return nil, err
		}
		docOutline = o
		m, err := godoc.Parse(docHTML, godoc.SidenavMobileSection)
		if err != nil {
			return nil, err
		}
		mobileOutline = m
		end()

		end = middleware.ElapsedStat(ctx, "sourceFiles")
		files = sourceFiles(unit, docPkg)
		end()
		if err != nil {
			return nil, err
		}
	}
	return &MainDetails{
		ExpandReadme:    expandReadme,
		NestedModules:   nestedModules,
		Subdirectories:  subdirectories,
		Licenses:        transformLicenseMetadata(um.Licenses),
		CommitTime:      absoluteTime(um.CommitTime),
		Readme:          readme,
		DocOutline:      docOutline,
		DocBody:         docBody,
		SourceFiles:     files,
		RepositoryURL:   um.SourceInfo.RepoURL(),
		SourceURL:       um.SourceInfo.DirectoryURL(internal.Suffix(um.Path, um.ModulePath)),
		MobileOutline:   mobileOutline,
		NumImports:      unit.NumImports,
		ImportedByCount: importedByCount,
		IsPackage:       unit.IsPackage(),
	}, nil
}

// moduleInfo extracts module info from a unit. This is a shim
// for functions ReadmeHTML and createDirectory that will be removed
// when we complete the switch to units.
func moduleInfo(um *internal.UnitMeta) *internal.ModuleInfo {
	return &internal.ModuleInfo{
		ModulePath:        um.ModulePath,
		Version:           um.Version,
		CommitTime:        um.CommitTime,
		IsRedistributable: um.IsRedistributable,
		SourceInfo:        um.SourceInfo,
	}
}

// readmeContent renders the readme to html.
func readmeContent(ctx context.Context, um *internal.UnitMeta, readme *internal.Readme) (safehtml.HTML, error) {
	defer middleware.ElapsedStat(ctx, "readmeContent")()

	if um.IsRedistributable && readme != nil {
		mi := moduleInfo(um)
		readme, err := ReadmeHTML(ctx, mi, readme)
		if err != nil {
			return safehtml.HTML{}, err
		}
		return readme, nil
	}
	return safehtml.HTML{}, nil
}

func getNestedModules(ctx context.Context, ds internal.DataSource, um *internal.UnitMeta) ([]*NestedModule, error) {
	nestedModules, err := ds.GetNestedModules(ctx, um.ModulePath)
	if err != nil {
		return nil, err
	}
	var mods []*NestedModule
	for _, m := range nestedModules {
		if m.SeriesPath() == internal.SeriesPathForModule(um.ModulePath) {
			continue
		}
		if !strings.HasPrefix(m.ModulePath, um.Path+"/") {
			continue
		}
		mods = append(mods, &NestedModule{
			URL:    constructPackageURL(m.ModulePath, m.ModulePath, internal.LatestVersion),
			Suffix: internal.Suffix(m.SeriesPath(), um.Path),
		})
	}
	return mods, nil
}

func getSubdirectories(um *internal.UnitMeta, pkgs []*internal.PackageMeta) []*Subdirectory {
	var sdirs []*Subdirectory
	for _, pm := range pkgs {
		if um.Path == pm.Path {
			continue
		}
		sdirs = append(sdirs, &Subdirectory{
			URL:      constructPackageURL(pm.Path, um.ModulePath, linkVersion(um.Version, um.ModulePath)),
			Suffix:   internal.Suffix(pm.Path, um.Path),
			Synopsis: pm.Synopsis,
		})
	}
	sort.Slice(sdirs, func(i, j int) bool { return sdirs[i].Suffix < sdirs[j].Suffix })
	return sdirs
}

func getHTML(ctx context.Context, u *internal.Unit, docPkg *godoc.Package) safehtml.HTML {
	if experiment.IsActive(ctx, internal.ExperimentFrontendRenderDoc) && len(u.Documentation.Source) > 0 {
		dd, err := renderDoc(ctx, u, docPkg)
		if err != nil {
			log.Errorf(ctx, "render doc failed: %v", err)
			// Fall through to use stored doc.
		} else {
			return dd.Documentation
		}
	}
	return u.Documentation.HTML
}

// getImportedByCount fetches the imported by count for the unit and returns a
// string to be displayed. If the datasource does not support imported by, it
// will return N/A.
func getImportedByCount(ctx context.Context, ds internal.DataSource, unit *internal.Unit) (_ string, err error) {
	defer derrors.Wrap(&err, "getImportedByCount(%q, %q, %q)", unit.Path, unit.ModulePath, unit.Version)
	defer middleware.ElapsedStat(ctx, "getImportedByCount")()

	db, ok := ds.(*postgres.DB)
	if !ok {
		return "N/A", nil
	}

	var count int
	if experiment.IsActive(ctx, internal.ExperimentGetUnitWithOneQuery) {
		count = unit.NumImportedBy
	} else {
		count, err = db.GetImportedByCount(ctx, unit.Path, unit.ModulePath, importedByLimit)
		if err != nil {
			if err == sql.ErrNoRows {
				return "N/A", nil
			}
			return "", err
		}
	}
	// If we reached the query limit, then we might know the total, but we
	// won't display past importedByLimit results. Indicate that with a '+'.
	// For example, if the limit is 101 and we get 101 results, then we'll show
	// '100+ Imported by'.
	if count >= importedByLimit {
		return strconv.Itoa(importedByLimit-1) + "+", nil
	}
	return strconv.Itoa(count), nil
}
<commit_msg>internal/frontend: refactor to move godoc.Parse down<commit_after>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package frontend

import (
	"context"
	"database/sql"
	"errors"
	"sort"
	"strconv"
	"strings"

	"github.com/google/safehtml"
	"golang.org/x/pkgsite/internal"
	"golang.org/x/pkgsite/internal/derrors"
	"golang.org/x/pkgsite/internal/experiment"
	"golang.org/x/pkgsite/internal/godoc"
	"golang.org/x/pkgsite/internal/log"
	"golang.org/x/pkgsite/internal/middleware"
	"golang.org/x/pkgsite/internal/postgres"
)

// MainDetails contains data needed to render the unit template.
type MainDetails struct {
	// NestedModules are nested modules relative to the path for the unit.
	NestedModules []*NestedModule

	// Subdirectories are packages in subdirectories relative to the path for
	// the unit.
	Subdirectories []*Subdirectory

	// Licenses contains license metadata used in the header.
	Licenses []LicenseMetadata

	// NumImports is the number of imports for the package.
	NumImports int

	// CommitTime is time that this version was published, or the time that
	// has elapsed since this version was committed if it was done so recently.
	CommitTime string

	// Readme is the rendered readme HTML.
	Readme safehtml.HTML

	// ImportedByCount is the number of packages that import this path.
	// When the count is > limit it will read as 'limit+'. This field
	// is not supported when using a datasource proxy.
	ImportedByCount string

	DocBody       safehtml.HTML
	DocOutline    safehtml.HTML
	MobileOutline safehtml.HTML
	IsPackage     bool

	// SourceFiles contains .go files for the package.
	SourceFiles []*File

	// RepositoryURL is the URL to the repository containing the package.
	RepositoryURL string

	// SourceURL is the URL to the source of the package.
	SourceURL string

	// ExpandReadme is holds the expandable readme state.
	ExpandReadme bool
}

// File is a source file for a package.
type File struct {
	Name string
	URL  string
}

// NestedModule is a nested module relative to the path of a given unit.
// This content is used in the Directories section of the unit page.
type NestedModule struct {
	Suffix string // suffix after the unit path
	URL    string
}

// Subdirectory is a package in a subdirectory relative to the path of a given
// unit. This content is used in the Directories section of the unit page.
type Subdirectory struct {
	Suffix   string
	URL      string
	Synopsis string
}

func fetchMainDetails(ctx context.Context, ds internal.DataSource, um *internal.UnitMeta, expandReadme bool) (_ *MainDetails, err error) {
	defer middleware.ElapsedStat(ctx, "fetchMainDetails")()

	unit, err := ds.GetUnit(ctx, um, internal.WithReadme|internal.WithDocumentation|internal.WithSubdirectories|internal.WithImports)
	if err != nil {
		return nil, err
	}

	nestedModules, err := getNestedModules(ctx, ds, um)
	if err != nil {
		return nil, err
	}
	subdirectories := getSubdirectories(um, unit.Subdirectories)
	if err != nil {
		return nil, err
	}
	readme, err := readmeContent(ctx, um, unit.Readme)
	if err != nil {
		return nil, err
	}
	importedByCount, err := getImportedByCount(ctx, ds, unit)
	if err != nil {
		return nil, err
	}
	var (
		docBody, docOutline, mobileOutline safehtml.HTML
		files                              []*File
	)
	if unit.Documentation != nil {
		end := middleware.ElapsedStat(ctx, "DecodePackage")
		docPkg, err := godoc.DecodePackage(unit.Documentation.Source)
		end()
		if err != nil {
			if errors.Is(err, godoc.ErrInvalidEncodingType) {
				// Instead of returning a 500, return a 404 so the user can
				// reprocess the documentation.
				log.Errorf(ctx, "fetchMainDetails(%q, %q, %q): %v", um.Path, um.ModulePath, um.Version, err)
				return nil, pathNotFoundError(um.Path, um.Version)
			}
			return nil, err
		}
		docBody, docOutline, mobileOutline, err = getHTML(ctx, unit, docPkg)
		if err != nil {
			return nil, err
		}
		end = middleware.ElapsedStat(ctx, "sourceFiles")
		files = sourceFiles(unit, docPkg)
		end()
	}
	return &MainDetails{
		ExpandReadme:    expandReadme,
		NestedModules:   nestedModules,
		Subdirectories:  subdirectories,
		Licenses:        transformLicenseMetadata(um.Licenses),
		CommitTime:      absoluteTime(um.CommitTime),
		Readme:          readme,
		DocOutline:      docOutline,
		DocBody:         docBody,
		SourceFiles:     files,
		RepositoryURL:   um.SourceInfo.RepoURL(),
		SourceURL:       um.SourceInfo.DirectoryURL(internal.Suffix(um.Path, um.ModulePath)),
		MobileOutline:   mobileOutline,
		NumImports:      unit.NumImports,
		ImportedByCount: importedByCount,
		IsPackage:       unit.IsPackage(),
	}, nil
}

// moduleInfo extracts module info from a unit. This is a shim
// for functions ReadmeHTML and createDirectory that will be removed
// when we complete the switch to units.
func moduleInfo(um *internal.UnitMeta) *internal.ModuleInfo {
	return &internal.ModuleInfo{
		ModulePath:        um.ModulePath,
		Version:           um.Version,
		CommitTime:        um.CommitTime,
		IsRedistributable: um.IsRedistributable,
		SourceInfo:        um.SourceInfo,
	}
}

// readmeContent renders the readme to html.
func readmeContent(ctx context.Context, um *internal.UnitMeta, readme *internal.Readme) (safehtml.HTML, error) {
	defer middleware.ElapsedStat(ctx, "readmeContent")()

	if um.IsRedistributable && readme != nil {
		mi := moduleInfo(um)
		readme, err := ReadmeHTML(ctx, mi, readme)
		if err != nil {
			return safehtml.HTML{}, err
		}
		return readme, nil
	}
	return safehtml.HTML{}, nil
}

func getNestedModules(ctx context.Context, ds internal.DataSource, um *internal.UnitMeta) ([]*NestedModule, error) {
	nestedModules, err := ds.GetNestedModules(ctx, um.ModulePath)
	if err != nil {
		return nil, err
	}
	var mods []*NestedModule
	for _, m := range nestedModules {
		if m.SeriesPath() == internal.SeriesPathForModule(um.ModulePath) {
			continue
		}
		if !strings.HasPrefix(m.ModulePath, um.Path+"/") {
			continue
		}
		mods = append(mods, &NestedModule{
			URL:    constructPackageURL(m.ModulePath, m.ModulePath, internal.LatestVersion),
			Suffix: internal.Suffix(m.SeriesPath(), um.Path),
		})
	}
	return mods, nil
}

func getSubdirectories(um *internal.UnitMeta, pkgs []*internal.PackageMeta) []*Subdirectory {
	var sdirs []*Subdirectory
	for _, pm := range pkgs {
		if um.Path == pm.Path {
			continue
		}
		sdirs = append(sdirs, &Subdirectory{
			URL:      constructPackageURL(pm.Path, um.ModulePath, linkVersion(um.Version, um.ModulePath)),
			Suffix:   internal.Suffix(pm.Path, um.Path),
			Synopsis: pm.Synopsis,
		})
	}
	sort.Slice(sdirs, func(i, j int) bool { return sdirs[i].Suffix < sdirs[j].Suffix })
	return sdirs
}

func getHTML(ctx context.Context, u *internal.Unit, docPkg *godoc.Package) (body, outline, mobileOutline safehtml.HTML, err error) {
	defer derrors.Wrap(&err, "getHTML %s", u.Path)

	docHTML := u.Documentation.HTML
	if experiment.IsActive(ctx, internal.ExperimentFrontendRenderDoc) && len(u.Documentation.Source) > 0 {
		dd, err := renderDoc(ctx, u, docPkg)
		if err != nil {
			log.Errorf(ctx, "render doc failed: %v", err)
		} else {
			docHTML = dd.Documentation
		}
	}
	// TODO: Deprecate godoc.Parse. The sidenav and body can
	// either be rendered using separate functions, or all this content can
	// be passed to the template via the UnitPage struct.
	end := middleware.ElapsedStat(ctx, "godoc Parses")
	b, err := godoc.Parse(docHTML, godoc.BodySection)
	if err != nil {
		return safehtml.HTML{}, safehtml.HTML{}, safehtml.HTML{}, err
	}
	o, err := godoc.Parse(docHTML, godoc.SidenavSection)
	if err != nil {
		return safehtml.HTML{}, safehtml.HTML{}, safehtml.HTML{}, err
	}
	m, err := godoc.Parse(docHTML, godoc.SidenavMobileSection)
	if err != nil {
		return safehtml.HTML{}, safehtml.HTML{}, safehtml.HTML{}, err
	}
	end()
	return b, o, m, nil
}

// getImportedByCount fetches the imported by count for the unit and returns a
// string to be displayed. If the datasource does not support imported by, it
// will return N/A.
func getImportedByCount(ctx context.Context, ds internal.DataSource, unit *internal.Unit) (_ string, err error) {
	defer derrors.Wrap(&err, "getImportedByCount(%q, %q, %q)", unit.Path, unit.ModulePath, unit.Version)
	defer middleware.ElapsedStat(ctx, "getImportedByCount")()

	db, ok := ds.(*postgres.DB)
	if !ok {
		return "N/A", nil
	}

	var count int
	if experiment.IsActive(ctx, internal.ExperimentGetUnitWithOneQuery) {
		count = unit.NumImportedBy
	} else {
		count, err = db.GetImportedByCount(ctx, unit.Path, unit.ModulePath, importedByLimit)
		if err != nil {
			if err == sql.ErrNoRows {
				return "N/A", nil
			}
			return "", err
		}
	}
	// If we reached the query limit, then we might know the total, but we
	// won't display past importedByLimit results. Indicate that with a '+'.
	// For example, if the limit is 101 and we get 101 results, then we'll show
	// '100+ Imported by'.
	if count >= importedByLimit {
		return strconv.Itoa(importedByLimit-1) + "+", nil
	}
	return strconv.Itoa(count), nil
}
<|endoftext|>


<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package impl

import (
	"fmt"
	"reflect"
	"sort"

	"google.golang.org/protobuf/encoding/protowire"
	"google.golang.org/protobuf/internal/encoding/messageset"
	"google.golang.org/protobuf/internal/fieldsort"
	pref "google.golang.org/protobuf/reflect/protoreflect"
	piface "google.golang.org/protobuf/runtime/protoiface"
)

// coderMessageInfo contains per-message information used by the fast-path functions.
// This is a different type from MessageInfo to keep MessageInfo as general-purpose as
// possible.
type coderMessageInfo struct {
	methods piface.Methods

	orderedCoderFields []*coderFieldInfo
	denseCoderFields   []*coderFieldInfo
	coderFields        map[protowire.Number]*coderFieldInfo
	sizecacheOffset    offset
	unknownOffset      offset
	extensionOffset    offset
	needsInitCheck     bool
	isMessageSet       bool
	numRequiredFields  uint8

	// Include space for a number of coderFieldInfos to improve cache locality.
	// The number of entries is chosen through a combination of guesswork and
	// empirical testing.
	coderFieldBuf [32]coderFieldInfo
}

type coderFieldInfo struct {
	funcs      pointerCoderFuncs // fast-path per-field functions
	mi         *MessageInfo      // field's message
	ft         reflect.Type
	validation validationInfo   // information used by message validation
	num        pref.FieldNumber // field number
	offset     offset           // struct field offset
	wiretag    uint64           // field tag (number + wire type)
	tagsize    int              // size of the varint-encoded tag
	isPointer  bool             // true if IsNil may be called on the struct field
	isRequired bool             // true if field is required
}

func (mi *MessageInfo) makeCoderMethods(t reflect.Type, si structInfo) {
	mi.sizecacheOffset = si.sizecacheOffset
	mi.unknownOffset = si.unknownOffset
	mi.extensionOffset = si.extensionOffset

	mi.coderFields = make(map[protowire.Number]*coderFieldInfo)
	fields := mi.Desc.Fields()
	preallocFields := mi.coderFieldBuf[:]
	for i := 0; i < fields.Len(); i++ {
		fd := fields.Get(i)

		fs := si.fieldsByNumber[fd.Number()]
		isOneof := fd.ContainingOneof() != nil && !fd.ContainingOneof().IsSynthetic()
		if isOneof {
			fs = si.oneofsByName[fd.ContainingOneof().Name()]
		}
		ft := fs.Type
		var wiretag uint64
		if !fd.IsPacked() {
			wiretag = protowire.EncodeTag(fd.Number(), wireTypes[fd.Kind()])
		} else {
			wiretag = protowire.EncodeTag(fd.Number(), protowire.BytesType)
		}
		var fieldOffset offset
		var funcs pointerCoderFuncs
		var childMessage *MessageInfo
		switch {
		case isOneof:
			fieldOffset = offsetOf(fs, mi.Exporter)
		case fd.IsWeak():
			fieldOffset = si.weakOffset
			funcs = makeWeakMessageFieldCoder(fd)
		default:
			fieldOffset = offsetOf(fs, mi.Exporter)
			childMessage, funcs = fieldCoder(fd, ft)
		}
		var cf *coderFieldInfo
		if len(preallocFields) > 0 {
			cf = &preallocFields[0]
			preallocFields = preallocFields[1:]
		} else {
			cf = new(coderFieldInfo)
		}
		*cf = coderFieldInfo{
			num:        fd.Number(),
			offset:     fieldOffset,
			wiretag:    wiretag,
			ft:         ft,
			tagsize:    protowire.SizeVarint(wiretag),
			funcs:      funcs,
			mi:         childMessage,
			validation: newFieldValidationInfo(mi, si, fd, ft),
			isPointer:  fd.Cardinality() == pref.Repeated || fd.HasPresence(),
			isRequired: fd.Cardinality() == pref.Required,
		}
		mi.orderedCoderFields = append(mi.orderedCoderFields, cf)
		mi.coderFields[cf.num] = cf
	}
	for i, oneofs := 0, mi.Desc.Oneofs(); i < oneofs.Len(); i++ {
		if od := oneofs.Get(i); !od.IsSynthetic() {
			mi.initOneofFieldCoders(od, si)
		}
	}
	if messageset.IsMessageSet(mi.Desc) {
		if !mi.extensionOffset.IsValid() {
			panic(fmt.Sprintf("%v: MessageSet with no extensions field", mi.Desc.FullName()))
		}
		if !mi.unknownOffset.IsValid() {
			panic(fmt.Sprintf("%v: MessageSet with no unknown field", mi.Desc.FullName()))
		}
		mi.isMessageSet = true
	}
	sort.Slice(mi.orderedCoderFields, func(i, j int) bool {
		return mi.orderedCoderFields[i].num < mi.orderedCoderFields[j].num
	})

	var maxDense pref.FieldNumber
	for _, cf := range mi.orderedCoderFields {
		if cf.num >= 16 && cf.num >= 2*maxDense {
			break
		}
		maxDense = cf.num
	}
	mi.denseCoderFields = make([]*coderFieldInfo, maxDense+1)
	for _, cf := range mi.orderedCoderFields {
		if int(cf.num) >= len(mi.denseCoderFields) {
			break
		}
		mi.denseCoderFields[cf.num] = cf
	}

	// To preserve compatibility with historic wire output, marshal oneofs last.
	if mi.Desc.Oneofs().Len() > 0 {
		sort.Slice(mi.orderedCoderFields, func(i, j int) bool {
			fi := fields.ByNumber(mi.orderedCoderFields[i].num)
			fj := fields.ByNumber(mi.orderedCoderFields[j].num)
			return fieldsort.Less(fi, fj)
		})
	}

	mi.needsInitCheck = needsInitCheck(mi.Desc)
	if mi.methods.Marshal == nil && mi.methods.Size == nil {
		mi.methods.Flags |= piface.SupportMarshalDeterministic
		mi.methods.Marshal = mi.marshal
		mi.methods.Size = mi.size
	}
	if mi.methods.Unmarshal == nil {
		mi.methods.Flags |= piface.SupportUnmarshalDiscardUnknown
		mi.methods.Unmarshal = mi.unmarshal
	}
	if mi.methods.CheckInitialized == nil {
		mi.methods.CheckInitialized = mi.checkInitialized
	}
	if mi.methods.Merge == nil {
		mi.methods.Merge = mi.merge
	}
}
<commit_msg>internal/impl: avoid inlining fixed coderFieldInfo array<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package impl

import (
	"fmt"
	"reflect"
	"sort"

	"google.golang.org/protobuf/encoding/protowire"
	"google.golang.org/protobuf/internal/encoding/messageset"
	"google.golang.org/protobuf/internal/fieldsort"
	pref "google.golang.org/protobuf/reflect/protoreflect"
	piface "google.golang.org/protobuf/runtime/protoiface"
)

// coderMessageInfo contains per-message information used by the fast-path functions.
// This is a different type from MessageInfo to keep MessageInfo as general-purpose as
// possible.
type coderMessageInfo struct {
	methods piface.Methods

	orderedCoderFields []*coderFieldInfo
	denseCoderFields   []*coderFieldInfo
	coderFields        map[protowire.Number]*coderFieldInfo
	sizecacheOffset    offset
	unknownOffset      offset
	extensionOffset    offset
	needsInitCheck     bool
	isMessageSet       bool
	numRequiredFields  uint8
}

type coderFieldInfo struct {
	funcs      pointerCoderFuncs // fast-path per-field functions
	mi         *MessageInfo      // field's message
	ft         reflect.Type
	validation validationInfo   // information used by message validation
	num        pref.FieldNumber // field number
	offset     offset           // struct field offset
	wiretag    uint64           // field tag (number + wire type)
	tagsize    int              // size of the varint-encoded tag
	isPointer  bool             // true if IsNil may be called on the struct field
	isRequired bool             // true if field is required
}

func (mi *MessageInfo) makeCoderMethods(t reflect.Type, si structInfo) {
	mi.sizecacheOffset = si.sizecacheOffset
	mi.unknownOffset = si.unknownOffset
	mi.extensionOffset = si.extensionOffset

	mi.coderFields = make(map[protowire.Number]*coderFieldInfo)
	fields := mi.Desc.Fields()
	preallocFields := make([]coderFieldInfo, fields.Len())
	for i := 0; i < fields.Len(); i++ {
		fd := fields.Get(i)

		fs := si.fieldsByNumber[fd.Number()]
		isOneof := fd.ContainingOneof() != nil && !fd.ContainingOneof().IsSynthetic()
		if isOneof {
			fs = si.oneofsByName[fd.ContainingOneof().Name()]
		}
		ft := fs.Type
		var wiretag uint64
		if !fd.IsPacked() {
			wiretag = protowire.EncodeTag(fd.Number(), wireTypes[fd.Kind()])
		} else {
			wiretag = protowire.EncodeTag(fd.Number(), protowire.BytesType)
		}
		var fieldOffset offset
		var funcs pointerCoderFuncs
		var childMessage *MessageInfo
		switch {
		case isOneof:
			fieldOffset = offsetOf(fs, mi.Exporter)
		case fd.IsWeak():
			fieldOffset = si.weakOffset
			funcs = makeWeakMessageFieldCoder(fd)
		default:
			fieldOffset = offsetOf(fs, mi.Exporter)
			childMessage, funcs = fieldCoder(fd, ft)
		}
		cf := &preallocFields[i]
		*cf = coderFieldInfo{
			num:        fd.Number(),
			offset:     fieldOffset,
			wiretag:    wiretag,
			ft:         ft,
			tagsize:    protowire.SizeVarint(wiretag),
			funcs:      funcs,
			mi:         childMessage,
			validation: newFieldValidationInfo(mi, si, fd, ft),
			isPointer:  fd.Cardinality() == pref.Repeated || fd.HasPresence(),
			isRequired: fd.Cardinality() == pref.Required,
		}
		mi.orderedCoderFields = append(mi.orderedCoderFields, cf)
		mi.coderFields[cf.num] = cf
	}
	for i, oneofs := 0, mi.Desc.Oneofs(); i < oneofs.Len(); i++ {
		if od := oneofs.Get(i); !od.IsSynthetic() {
			mi.initOneofFieldCoders(od, si)
		}
	}
	if messageset.IsMessageSet(mi.Desc) {
		if !mi.extensionOffset.IsValid() {
			panic(fmt.Sprintf("%v: MessageSet with no extensions field", mi.Desc.FullName()))
		}
		if !mi.unknownOffset.IsValid() {
			panic(fmt.Sprintf("%v: MessageSet with no unknown field", mi.Desc.FullName()))
		}
		mi.isMessageSet = true
	}
	sort.Slice(mi.orderedCoderFields, func(i, j int) bool {
		return mi.orderedCoderFields[i].num < mi.orderedCoderFields[j].num
	})

	var maxDense pref.FieldNumber
	for _, cf := range mi.orderedCoderFields {
		if cf.num >= 16 && cf.num >= 2*maxDense {
			break
		}
		maxDense = cf.num
	}
	mi.denseCoderFields = make([]*coderFieldInfo, maxDense+1)
	for _, cf := range mi.orderedCoderFields {
		if int(cf.num) >= len(mi.denseCoderFields) {
			break
		}
		mi.denseCoderFields[cf.num] = cf
	}

	// To preserve compatibility with historic wire output, marshal oneofs last.
	if mi.Desc.Oneofs().Len() > 0 {
		sort.Slice(mi.orderedCoderFields, func(i, j int) bool {
			fi := fields.ByNumber(mi.orderedCoderFields[i].num)
			fj := fields.ByNumber(mi.orderedCoderFields[j].num)
			return fieldsort.Less(fi, fj)
		})
	}

	mi.needsInitCheck = needsInitCheck(mi.Desc)
	if mi.methods.Marshal == nil && mi.methods.Size == nil {
		mi.methods.Flags |= piface.SupportMarshalDeterministic
		mi.methods.Marshal = mi.marshal
		mi.methods.Size = mi.size
	}
	if mi.methods.Unmarshal == nil {
		mi.methods.Flags |= piface.SupportUnmarshalDiscardUnknown
		mi.methods.Unmarshal = mi.unmarshal
	}
	if mi.methods.CheckInitialized == nil {
		mi.methods.CheckInitialized = mi.checkInitialized
	}
	if mi.methods.Merge == nil {
		mi.methods.Merge = mi.merge
	}
}
<|endoftext|>


<commit_before>package org

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"strings"
)

type Document struct {
	Path                string
	tokens              []token
	Nodes               []Node
	Footnotes           *Footnotes
	StatusKeywords      []string
	MaxEmphasisNewLines int
	AutoLink            bool
	BufferSettings      map[string]string
	DefaultSettings     map[string]string
	Error               error
}

type Writer interface {
	before(*Document)
	after(*Document)
	writeNodes(...Node)
	String() string
}

type Node interface{}

type lexFn = func(line string) (t token, ok bool)
type parseFn = func(*Document, int, stopFn) (int, Node)
type stopFn = func(*Document, int) bool

type token struct {
	kind    string
	lvl     int
	content string
	matches []string
}

var lexFns = []lexFn{
	lexHeadline,
	lexDrawer,
	lexBlock,
	lexList,
	lexTable,
	lexHorizontalRule,
	lexKeywordOrComment,
	lexFootnoteDefinition,
	lexExample,
	lexText,
}

var nilToken = token{"nil", -1, "", nil}

var DefaultFrontMatterHandler = func(k, v string) interface{} {
	switch k {
	case "TAGS":
		return strings.Fields(v)
	default:
		return v
	}
}

func NewDocument() *Document {
	return &Document{
		Footnotes: &Footnotes{
			Title:       "Footnotes",
			Definitions: map[string]*FootnoteDefinition{},
		},
		AutoLink:            true,
		MaxEmphasisNewLines: 1,
		BufferSettings:      map[string]string{},
		DefaultSettings: map[string]string{
			"TODO": "TODO | DONE",
		},
	}
}

func (d *Document) Write(w Writer) (out string, err error) {
	defer func() {
		if recovered := recover(); recovered != nil {
			err = fmt.Errorf("could not write output: %s", recovered)
		}
	}()
	if d.Error != nil {
		return "", d.Error
	} else if d.Nodes == nil {
		return "", fmt.Errorf("could not write output: parse was not called")
	}
	w.before(d)
	w.writeNodes(d.Nodes...)
	w.after(d)
	return w.String(), err
}

func (d *Document) Parse(input io.Reader) *Document {
	defer func() {
		if err := recover(); err != nil {
			d.Error = fmt.Errorf("could not parse input: %s", err)
		}
	}()
	if d.tokens != nil {
		d.Error = fmt.Errorf("parse was called multiple times")
	}
	d.tokenize(input)
	_, nodes := d.parseMany(0, func(d *Document, i int) bool { return !(i < len(d.tokens)) })
	d.Nodes = nodes
	return d
}

func (d *Document) SetPath(path string) *Document {
	d.Path = path
	return d
}

func (d *Document) FrontMatter(input io.Reader, f func(string, string) interface{}) (_ map[string]interface{}, err error) {
	defer func() {
		d.tokens = nil
		if recovered := recover(); recovered != nil {
			err = fmt.Errorf("could not parse input: %s", recovered)
		}
	}()
	d.tokenize(input)
	d.parseMany(0, func(d *Document, i int) bool {
		if !(i < len(d.tokens)) {
			return true
		}
		t := d.tokens[i]
		return t.kind != "keyword" && !(t.kind == "text" && t.content == "")
	})
	frontMatter := make(map[string]interface{}, len(d.BufferSettings))
	for k, v := range d.BufferSettings {
		frontMatter[k] = f(k, v)
	}
	return frontMatter, err
}

func (d *Document) tokenize(input io.Reader) {
	d.tokens = []token{}
	scanner := bufio.NewScanner(input)
	for scanner.Scan() {
		d.tokens = append(d.tokens, tokenize(scanner.Text()))
	}
	if err := scanner.Err(); err != nil {
		d.Error = fmt.Errorf("could not tokenize input: %s", err)
	}
}

func (d *Document) Get(key string) string {
	if v, ok := d.BufferSettings[key]; ok {
		return v
	}
	if v, ok := d.DefaultSettings[key]; ok {
		return v
	}
	return ""
}

func (d *Document) parseOne(i int, stop stopFn) (consumed int, node Node) {
	switch d.tokens[i].kind {
	case "unorderedList", "orderedList":
		consumed, node = d.parseList(i, stop)
	case "tableRow", "tableSeparator":
		consumed, node = d.parseTable(i, stop)
	case "beginBlock":
		consumed, node = d.parseBlock(i, stop)
	case "beginDrawer":
		consumed, node = d.parseDrawer(i, stop)
	case "text":
		consumed, node = d.parseParagraph(i, stop)
	case "example":
		consumed, node = d.parseExample(i, stop)
	case "horizontalRule":
		consumed, node = d.parseHorizontalRule(i, stop)
	case "comment":
		consumed, node = d.parseComment(i, stop)
	case "keyword":
		consumed, node = d.parseKeyword(i, stop)
	case "headline":
		consumed, node = d.parseHeadline(i, stop)
	case "footnoteDefinition":
		consumed, node = d.parseFootnoteDefinition(i, stop)
	}

	if consumed != 0 {
		return consumed, node
	}
	log.Printf("Could not parse token %#v: Falling back to treating it as plain text.", d.tokens[i])
	m := plainTextRegexp.FindStringSubmatch(d.tokens[i].matches[0])
	d.tokens[i] = token{"text", len(m[1]), m[2], m}
	return d.parseOne(i, stop)
}

func (d *Document) parseMany(i int, stop stopFn) (int, []Node) {
	start, nodes := i, []Node{}
	for i < len(d.tokens) && !stop(d, i) {
		consumed, node := d.parseOne(i, stop)
		i += consumed
		nodes = append(nodes, node)
	}
	return i - start, nodes
}

func tokenize(line string) token {
	for _, lexFn := range lexFns {
		if token, ok := lexFn(line); ok {
			return token
		}
	}
	panic(fmt.Sprintf("could not lex line: %s", line))
}
<commit_msg>Hide logs behind document.Debug flag<commit_after>package org

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"strings"
)

type Document struct {
	Path                string
	tokens              []token
	Nodes               []Node
	Footnotes           *Footnotes
	StatusKeywords      []string
	MaxEmphasisNewLines int
	AutoLink            bool
	BufferSettings      map[string]string
	DefaultSettings     map[string]string
	Error               error
	Debug               bool
}

type Writer interface {
	before(*Document)
	after(*Document)
	writeNodes(...Node)
	String() string
}

type Node interface{}

type lexFn = func(line string) (t token, ok bool)
type parseFn = func(*Document, int, stopFn) (int, Node)
type stopFn = func(*Document, int) bool

type token struct {
	kind    string
	lvl     int
	content string
	matches []string
}

var lexFns = []lexFn{
	lexHeadline,
	lexDrawer,
	lexBlock,
	lexList,
	lexTable,
	lexHorizontalRule,
	lexKeywordOrComment,
	lexFootnoteDefinition,
	lexExample,
	lexText,
}

var nilToken = token{"nil", -1, "", nil}

var DefaultFrontMatterHandler = func(k, v string) interface{} {
	switch k {
	case "TAGS":
		return strings.Fields(v)
	default:
		return v
	}
}

func NewDocument() *Document {
	return &Document{
		Footnotes: &Footnotes{
			Title:       "Footnotes",
			Definitions: map[string]*FootnoteDefinition{},
		},
		AutoLink:            true,
		MaxEmphasisNewLines: 1,
		BufferSettings:      map[string]string{},
		DefaultSettings: map[string]string{
			"TODO": "TODO | DONE",
		},
	}
}

func (d *Document) Write(w Writer) (out string, err error) {
	defer func() {
		if recovered := recover(); recovered != nil {
			err = fmt.Errorf("could not write output: %s", recovered)
		}
	}()
	if d.Error != nil {
		return "", d.Error
	} else if d.Nodes == nil {
		return "", fmt.Errorf("could not write output: parse was not called")
	}
	w.before(d)
	w.writeNodes(d.Nodes...)
	w.after(d)
	return w.String(), err
}

func (d *Document) Parse(input io.Reader) *Document {
	defer func() {
		if err := recover(); err != nil {
			d.Error = fmt.Errorf("could not parse input: %s", err)
		}
	}()
	if d.tokens != nil {
		d.Error = fmt.Errorf("parse was called multiple times")
	}
	d.tokenize(input)
	_, nodes := d.parseMany(0, func(d *Document, i int) bool { return !(i < len(d.tokens)) })
	d.Nodes = nodes
	return d
}

func (d *Document) SetPath(path string) *Document {
	d.Path = path
	return d
}

func (d *Document) FrontMatter(input io.Reader, f func(string, string) interface{}) (_ map[string]interface{}, err error) {
	defer func() {
		d.tokens = nil
		if recovered := recover(); recovered != nil {
			err = fmt.Errorf("could not parse input: %s", recovered)
		}
	}()
	d.tokenize(input)
	d.parseMany(0, func(d *Document, i int) bool {
		if !(i < len(d.tokens)) {
			return true
		}
		t := d.tokens[i]
		return t.kind != "keyword" && !(t.kind == "text" && t.content == "")
	})
	frontMatter := make(map[string]interface{}, len(d.BufferSettings))
	for k, v := range d.BufferSettings {
		frontMatter[k] = f(k, v)
	}
	return frontMatter, err
}

func (d *Document) tokenize(input io.Reader) {
	d.tokens = []token{}
	scanner := bufio.NewScanner(input)
	for scanner.Scan() {
		d.tokens = append(d.tokens, tokenize(scanner.Text()))
	}
	if err := scanner.Err(); err != nil {
		d.Error = fmt.Errorf("could not tokenize input: %s", err)
	}
}

func (d *Document) Get(key string) string {
	if v, ok := d.BufferSettings[key]; ok {
		return v
	}
	if v, ok := d.DefaultSettings[key]; ok {
		return v
	}
	return ""
}

func (d *Document) parseOne(i int, stop stopFn) (consumed int, node Node) {
	switch d.tokens[i].kind {
	case "unorderedList", "orderedList":
		consumed, node = d.parseList(i, stop)
	case "tableRow", "tableSeparator":
		consumed, node = d.parseTable(i, stop)
	case "beginBlock":
		consumed, node = d.parseBlock(i, stop)
	case "beginDrawer":
		consumed, node = d.parseDrawer(i, stop)
	case "text":
		consumed, node = d.parseParagraph(i, stop)
	case "example":
		consumed, node = d.parseExample(i, stop)
	case "horizontalRule":
		consumed, node = d.parseHorizontalRule(i, stop)
	case "comment":
		consumed, node = d.parseComment(i, stop)
	case "keyword":
		consumed, node = d.parseKeyword(i, stop)
	case "headline":
		consumed, node = d.parseHeadline(i, stop)
	case "footnoteDefinition":
		consumed, node = d.parseFootnoteDefinition(i, stop)
	}

	if consumed != 0 {
		return consumed, node
	}
	if d.Debug {
		log.Printf("Could not parse token %#v: Falling back to treating it as plain text.", d.tokens[i])
	}
	m := plainTextRegexp.FindStringSubmatch(d.tokens[i].matches[0])
	d.tokens[i] = token{"text", len(m[1]), m[2], m}
	return d.parseOne(i, stop)
}

func (d *Document) parseMany(i int, stop stopFn) (int, []Node) {
	start, nodes := i, []Node{}
	for i < len(d.tokens) && !stop(d, i) {
		consumed, node := d.parseOne(i, stop)
		i += consumed
		nodes = append(nodes, node)
	}
	return i - start, nodes
}

func tokenize(line string) token {
	for _, lexFn := range lexFns {
		if token, ok := lexFn(line); ok {
			return token
		}
	}
	panic(fmt.Sprintf("could not lex line: %s", line))
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"os/exec"
	"testing"
)

const apigeecli = "./apigeecli"

var token = os.Getenv("APIGEE_TOKEN")
var env = os.Getenv("APIGEE_ENV")
var org = os.Getenv("APIGEE_ORG")

func TestMain(t *testing.T) {
	cmd := exec.Command(apigeecli)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

// orgs test
func TestListOrgs(t *testing.T) {
	cmd := exec.Command(apigeecli, "orgs", "list", "-t", token)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

func TestGetOrg(t *testing.T) {
	cmd := exec.Command(apigeecli, "orgs", "get", "-o", org, "-t", token)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

func TestSetMart(t *testing.T) {
	mart := os.Getenv("MART")
	cmd := exec.Command(apigeecli, "orgs", "setmart", "-o", org, "-m", mart, "-t", token)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

func TestSetMartWhiteList(t *testing.T) {
	mart := os.Getenv("MART")
	cmd := exec.Command(apigeecli, "orgs", "setmart", "-o", org, "-m", mart, "-w", "false", "-t", token)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

// env tests
func TestListEnvs(t *testing.T) {
	cmd := exec.Command(apigeecli, "envs", "list", "-o", org, "-t", token)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

func TestGetEnv(t *testing.T) {
	cmd := exec.Command(apigeecli, "envs", "get", "-o", org, "-e", env, "-t", token)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

// developers test
func TestCreateDeveloper(t *testing.T) {
	email := "test@example.com"
	first := "frstname"
	last := "lastname"
	user := "username"

	cmd := exec.Command(apigeecli, "developers", "create", "-o", org, "-n", email, "-f", first, "-s", last, "-u", user, "-t", token)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

func TestGetDeveloper(t *testing.T) {
	email := "test@example.com"

	cmd := exec.Command(apigeecli, "developers", "get", "-o", org, "-n", email, "-t", token)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

func TestListDeveloper(t *testing.T) {

	cmd := exec.Command(apigeecli, "developers", "list", "-o", org, "-t", token)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

func TestListExpandDeveloper(t *testing.T) {
	expand := "true"

	cmd := exec.Command(apigeecli, "developers", "list", "-o", org, "-x", expand, "-t", token)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

// kvm test
func TestCreateKVM(t *testing.T) {
	name := "test"

	cmd := exec.Command(apigeecli, "kvms", "create", "-o", org, "-e", env, "-n", name, "-t", token)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

func TestCreateEncKVM(t *testing.T) {
	name := "testEnc"
	enc := "true"

	cmd := exec.Command(apigeecli, "kvms", "create", "-o", org, "-e", env, "-n", name, "-c", enc, "-t", token)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

func TestListKVM(t *testing.T) {

	cmd := exec.Command(apigeecli, "kvms", "list", "-o", org, "-e", env, "-t", token)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

func TestDeleteKVM(t *testing.T) {
	name := "test"

	cmd := exec.Command(apigeecli, "kvms", "create", "-o", org, "-e", env, "-n", name, "-t", token)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

func TestDeleteEnvKVM(t *testing.T) {
	name := "testEnc"

	cmd := exec.Command(apigeecli, "kvms", "create", "-o", org, "-e", env, "-n", name, "-t", token)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

// sync tests
func TestGetSync(t *testing.T) {

	cmd := exec.Command(apigeecli, "sync", "get", "-o", org, "-t", token)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

func TestSetSync(t *testing.T) {
	ity := "test@gmail.com"
	cmd := exec.Command(apigeecli, "sync", "set", "-o", org, "-i", ity, "-t", token)
	err := cmd.Run()
	if err == nil {
		t.Fatal("Invalid identity test should have failed")
	}
}

func TestCreateProxy(t *testing.T) {
	name := "test"

	cmd := exec.Command(apigeecli, "apis", "create", "-o", org, "-n", name, "-t", token)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}

}

func TestCreateProduct(t *testing.T) {
	name := "test"
	envs := "test"
	proxy := "test"
	approval := "auto"

	cmd := exec.Command(apigeecli, "products", "create", "-o", org, "-n", name, "-e", envs, "-p", proxy, "-f", approval, "-t", token)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

func TestCreateApp(t *testing.T) {
	name := "test"
	email := "test@example.com"
	product := "test"

	cmd := exec.Command(apigeecli, "apps", "create", "-o", org, "-n", name, "-e", email, "-p", product, "-t", token)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}

}

func TestGetApp(t *testing.T) {
	name := "test"

	cmd := exec.Command(apigeecli, "apps", "get", "-o", org, "-n", name, "-t", token)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}

}

func TestListApp(t *testing.T) {

	cmd := exec.Command(apigeecli, "apps", "get", "-o", org, "-t", token)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}

}

func TestDeleteApp(t *testing.T) {
	name := "test"

	cmd := exec.Command(apigeecli, "apps", "delete", "-o", org, "-n", name, "-t", token)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}

}

func TestGetProduct(t *testing.T) {
	name := "test"

	cmd := exec.Command(apigeecli, "products", "list", "-o", org, "-n", name, "-t", token)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

func TestListProduct(t *testing.T) {

	cmd := exec.Command(apigeecli, "products", "list", "-o", org, "-t", token)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

func TestDeleteProduct(t *testing.T) {
	name := "test"

	cmd := exec.Command(apigeecli, "kvms", "create", "-o", org, "-n", name, "-t", token)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

func TestDeleteDeveloper(t *testing.T) {
	email := "test@example.com"

	cmd := exec.Command(apigeecli, "developers", "delete", "-o", org, "-n", email, "-t", token)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

func TestCreateTS(t *testing.T) {
	name := "test"
	host := "example.com"

	cmd := exec.Command(apigeecli, "targetservers", "create", "-o", org, "-n", name, "-e", env, "-s", host, "-t", token)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}

}

func TestGetTS(t *testing.T) {
	name := "test"

	cmd := exec.Command(apigeecli, "targetservers", "get", "-o", org, "-e", env, "-n", name, "-t", token)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}

}

func TestListTS(t *testing.T) {

	cmd := exec.Command(apigeecli, "targetservers", "get", "-o", org, "-e", env, "-t", token)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}

}

func TestDeleteTS(t *testing.T) {
	name := "test"

	cmd := exec.Command(apigeecli, "targetservers", "delete", "-o", org, "-e", env, "-n", name, "-t", token)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}

}
<commit_msg>fixes to tests<commit_after>package main

import (
	"os"
	"os/exec"
	"testing"
)

const apigeecli = "./apigeecli"

var token = os.Getenv("APIGEE_TOKEN")
var env = os.Getenv("APIGEE_ENV")
var org = os.Getenv("APIGEE_ORG")
var email = "test@gmail.com"

func TestMain(t *testing.T) {
	cmd := exec.Command(apigeecli)
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

// orgs test
func TestListOrgs(t *testing.T) {
	cmd := exec.Command(apigeecli, "orgs", "list", "-t", token, "-l")
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

func TestGetOrg(t *testing.T) {
	cmd := exec.Command(apigeecli, "orgs", "get", "-o", org, "-t", token, "-l")
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

// env tests
func TestListEnvs(t *testing.T) {
	cmd := exec.Command(apigeecli, "envs", "list", "-o", org, "-t", token, "-l")
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

func TestGetEnv(t *testing.T) {
	cmd := exec.Command(apigeecli, "envs", "get", "-o", org, "-e", env, "-t", token, "-l")
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

// developers test
func TestCreateDeveloper(t *testing.T) {
	first := "frstname"
	last := "lastname"
	user := "username"

	cmd := exec.Command(apigeecli, "developers", "create", "-o", org, "-n", email, "-f", first, "-s", last, "-u", user, "-t", token, "-l")
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

func TestGetDeveloper(t *testing.T) {

	cmd := exec.Command(apigeecli, "developers", "get", "-o", org, "-n", email, "-t", token, "-l")
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

func TestListDeveloper(t *testing.T) {

	cmd := exec.Command(apigeecli, "developers", "list", "-o", org, "-t", token, "-l")
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

func TestListExpandDeveloper(t *testing.T) {
	expand := "true"

	cmd := exec.Command(apigeecli, "developers", "list", "-o", org, "-x", expand, "-t", token, "-l")
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

// kvm test
func TestCreateKVM(t *testing.T) {
	name := "test"

	cmd := exec.Command(apigeecli, "kvms", "create", "-o", org, "-e", env, "-n", name, "-t", token, "-l")
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

/*func TestCreateEncKVM(t *testing.T) {
	name := "testEnc"
	enc := "true"

	cmd := exec.Command(apigeecli, "kvms", "create", "-o", org, "-e", env, "-n", name, "-c", enc, "-t", token, "-l")
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}*/

func TestListKVM(t *testing.T) {

	cmd := exec.Command(apigeecli, "kvms", "list", "-o", org, "-e", env, "-t", token, "-l")
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

/*func TestDeleteKVM(t *testing.T) {
	name := "test"

	cmd := exec.Command(apigeecli, "kvms", "create", "-o", org, "-e", env, "-n", name, "-t", token, "-l")
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

func TestDeleteEncKVM(t *testing.T) {
	name := "testEnc"

	cmd := exec.Command(apigeecli, "kvms", "create", "-o", org, "-e", env, "-n", name, "-t", token, "-l")
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}*/

// sync tests
func TestGetSync(t *testing.T) {

	cmd := exec.Command(apigeecli, "sync", "get", "-o", org, "-t", token, "-l")
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

func TestSetSync(t *testing.T) {
	ity := "test@gmail.com"
	cmd := exec.Command(apigeecli, "sync", "set", "-o", org, "-i", ity, "-t", token, "-l")
	err := cmd.Run()
	if err == nil {
		t.Fatal("Invalid identity test should have failed")
	}
}

func TestCreateProxy(t *testing.T) {
	name := "test"

	cmd := exec.Command(apigeecli, "apis", "create", "-o", org, "-n", name, "-t", token, "-l")
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}

}

func TestCreateProduct(t *testing.T) {
	name := "test"
	envs := "test"
	proxy := "test"
	approval := "auto"

	cmd := exec.Command(apigeecli, "products", "create", "-o", org, "-n", name, "-e", envs, "-p", proxy, "-f", approval, "-t", token, "-l")
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

func TestCreateApp(t *testing.T) {
	name := "test"
	product := "test"

	cmd := exec.Command(apigeecli, "apps", "create", "-o", org, "-n", name, "-e", email, "-p", product, "-t", token, "-l")
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}

}

//get requires an app id. run list and get an app id ir get it from create
/*func TestGetApp(t *testing.T) {
	name := "test"

	cmd := exec.Command(apigeecli, "apps", "get", "-o", org, "-n", name, "-t", token, "-l")
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}

}*/

func TestListApp(t *testing.T) {

	cmd := exec.Command(apigeecli, "apps", "list", "-o", org, "-t", token, "-l")
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}

}

/*func TestDeleteApp(t *testing.T) {
	name := "test"

	cmd := exec.Command(apigeecli, "apps", "delete", "-o", org, "-n", name, "-t", token, "-l")
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}

}*/

func TestGetProduct(t *testing.T) {
	name := "test"

	cmd := exec.Command(apigeecli, "products", "get", "-o", org, "-n", name, "-t", token, "-l")
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

func TestListProduct(t *testing.T) {

	cmd := exec.Command(apigeecli, "products", "list", "-o", org, "-t", token, "-l")
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

/*func TestDeleteProduct(t *testing.T) {
	name := "test"

	cmd := exec.Command(apigeecli, "kvms", "create", "-o", org, "-n", name, "-t", token, "-l")
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}*/

func TestDeleteDeveloper(t *testing.T) {

	cmd := exec.Command(apigeecli, "developers", "delete", "-o", org, "-n", email, "-t", token, "-l")
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}
}

func TestCreateTS(t *testing.T) {
	name := "test"
	host := "example.com"

	cmd := exec.Command(apigeecli, "targetservers", "create", "-o", org, "-n", name, "-e", env, "-s", host, "-t", token, "-l")
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}

}

func TestGetTS(t *testing.T) {
	name := "test"

	cmd := exec.Command(apigeecli, "targetservers", "get", "-o", org, "-e", env, "-n", name, "-t", token, "-l")
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}

}

func TestListTS(t *testing.T) {

	cmd := exec.Command(apigeecli, "targetservers", "list", "-o", org, "-e", env, "-t", token, "-l")
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}

}

func TestDeleteTS(t *testing.T) {
	name := "test"

	cmd := exec.Command(apigeecli, "targetservers", "delete", "-o", org, "-e", env, "-n", name, "-t", token, "-l")
	err := cmd.Run()
	if err != nil {
		t.Fatal(err)
	}

}

/*func TestSetMart(t *testing.T) {
	mart := os.Getenv("MART")
	if mart == "" {
		t.Log("MART not set, skipping")
	} else {
		cmd := exec.Command(apigeecli, "orgs", "setmart", "-o", org, "-m", mart, "-t", token, "-l")
		err := cmd.Run()
		if err != nil {
			t.Fatal(err)
		}
	}
}

func TestSetMartWhiteList(t *testing.T) {
	mart := os.Getenv("MART")
	if mart == "" {
		t.Log("MART not set, skipping")
	} else {
		cmd := exec.Command(apigeecli, "orgs", "setmart", "-o", org, "-m", mart, "-w", "false", "-t", token, "-l")
		err := cmd.Run()
		if err != nil {
			t.Fatal(err)
		}
	}
}*/
<|endoftext|>


<commit_before>package main_test

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"syscall"
	"time"

	"github.com/apcera/nats"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	"github.com/cloudfoundry/yagnats"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/rata"

	"github.com/cloudfoundry-incubator/tps/api"
	"github.com/cloudfoundry-incubator/tps/heartbeat"
)

var _ = Describe("TPS", func() {

	var httpClient *http.Client
	var requestGenerator *rata.RequestGenerator
	var natsClient yagnats.NATSConn

	BeforeEach(func() {
		natsClient = natsRunner.MessageBus
		requestGenerator = rata.NewRequestGenerator(fmt.Sprintf("http://%s", tpsAddr), api.Routes)
		httpClient = &http.Client{
			Transport: &http.Transport{},
		}
	})

	JustBeforeEach(func() {
		tps = ifrit.Envoke(runner)
	})

	AfterEach(func() {
		if tps != nil {
			tps.Signal(os.Kill)
			Eventually(tps.Wait()).Should(Receive())
		}
	})
	Describe("GET /lrps/:guid", func() {
		Context("when etcd is running", func() {
			BeforeEach(func() {
				bbs.ReportActualLRPAsStarting("some-process-guid", "some-instance-guid-1", "executor-id", 0)

				bbs.ReportActualLRPAsRunning(models.ActualLRP{
					ProcessGuid:  "some-process-guid",
					InstanceGuid: "some-instance-guid-2",

					Index: 1,

					State: models.ActualLRPStateRunning,
				}, "executor-id")

				bbs.ReportActualLRPAsRunning(models.ActualLRP{
					ProcessGuid:  "some-process-guid",
					InstanceGuid: "some-instance-guid-3",

					Index: 2,

					State: models.ActualLRPStateRunning,
				}, "executor-id")

				bbs.ReportActualLRPAsRunning(models.ActualLRP{
					ProcessGuid:  "some-other-process-guid",
					InstanceGuid: "some-instance-guid-3",

					Index: 0,

					State: models.ActualLRPStateRunning,
				}, "executor-id")
			})

			It("reports the state of the given process guid's instances", func() {
				getLRPs, err := requestGenerator.CreateRequest(
					api.LRPStatus,
					rata.Params{"guid": "some-process-guid"},
					nil,
				)
				Ω(err).ShouldNot(HaveOccurred())

				response, err := httpClient.Do(getLRPs)
				Ω(err).ShouldNot(HaveOccurred())

				var lrpInstances []api.LRPInstance
				err = json.NewDecoder(response.Body).Decode(&lrpInstances)
				Ω(err).ShouldNot(HaveOccurred())

				Ω(lrpInstances).Should(HaveLen(3))

				Ω(lrpInstances).Should(ContainElement(api.LRPInstance{
					ProcessGuid:  "some-process-guid",
					InstanceGuid: "some-instance-guid-1",

					Index: 0,
					Since: timeProvider.Time().UnixNano(),

					State: "starting",
				}))

				Ω(lrpInstances).Should(ContainElement(api.LRPInstance{
					ProcessGuid:  "some-process-guid",
					InstanceGuid: "some-instance-guid-2",

					Index: 1,
					Since: timeProvider.Time().UnixNano(),

					State: "running",
				}))

				Ω(lrpInstances).Should(ContainElement(api.LRPInstance{
					ProcessGuid:  "some-process-guid",
					InstanceGuid: "some-instance-guid-3",

					Index: 2,
					Since: timeProvider.Time().UnixNano(),

					State: "running",
				}))
			})
		})

		Context("when etcd is not running", func() {
			BeforeEach(func() {
				etcdRunner.Stop()
			})

			It("returns 500", func() {
				getLRPs, err := requestGenerator.CreateRequest(
					api.LRPStatus,
					rata.Params{"guid": "some-process-guid"},
					nil,
				)
				Ω(err).ShouldNot(HaveOccurred())

				response, err := httpClient.Do(getLRPs)
				Ω(err).ShouldNot(HaveOccurred())

				Ω(response.StatusCode).Should(Equal(http.StatusInternalServerError))
			})
		})
	})

	Context("when the NATS server is running", func() {
		var tpsNatsSubject = "service.announce.tps"
		var announceMsg chan *nats.Msg
		var subscription *nats.Subscription

		BeforeEach(func() {
			announceMsg = make(chan *nats.Msg)
			var err error
			subscription, err = natsClient.Subscribe(tpsNatsSubject, func(msg *nats.Msg) {
				announceMsg <- msg
			})
			Ω(err).ShouldNot(HaveOccurred())
		})

		AfterEach(func() {
			err := natsClient.Unsubscribe(subscription)
			Ω(err).ShouldNot(HaveOccurred())
		})

		It("heartbeats announcement messages at the predefined interval", func() {
			Eventually(announceMsg, heartbeatInterval+time.Second).Should(Receive())
			Eventually(announceMsg, heartbeatInterval+time.Second).Should(Receive())
		})

		Describe("published HeartbeatMessage", func() {
			var heartbeatMsg heartbeat.HeartbeatMessage

			JustBeforeEach(func(done Done) {
				heartbeatMsg = heartbeat.HeartbeatMessage{}
				msg := <-announceMsg
				err := json.Unmarshal(msg.Data, &heartbeatMsg)
				Ω(err).ShouldNot(HaveOccurred())
				close(done)
			})

			It("contains the correct tps address", func() {
				Ω(heartbeatMsg.Addr).Should(Equal(fmt.Sprintf("http://%s", tpsAddr)))
			})

			It("a ttl 3 times longer than the heartbeatInterval, in seconds", func() {
				Ω(heartbeatMsg.TTL).Should(Equal(uint(3)))
			})
		})
	})

	Context("when the NATS server is down while starting up", func() {
		BeforeEach(func() {
			runner.StartCheck = ""
			natsRunner.KillWithFire()
		})

		It("does not exit", func() {
			Consistently(tps.Wait()).ShouldNot(Receive())
		})

		It("exits when we send a signal", func() {
			tps.Signal(syscall.SIGINT)
			Eventually(tps.Wait()).Should(Receive())
		})

	})

	Context("when the NATS server goes down after startup", func() {
		JustBeforeEach(func() {
			natsRunner.KillWithFire()
			time.Sleep(50 * time.Millisecond)
		})

		It("does not exit", func() {
			Consistently(tps.Wait()).ShouldNot(Receive())
		})

		It("exits when we send a signal", func() {
			tps.Signal(syscall.SIGINT)
			Eventually(tps.Wait()).Should(Receive())
		})
	})
})
<commit_msg>use ginkgomon.Invoke to check for start failure<commit_after>package main_test

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"syscall"
	"time"

	"github.com/apcera/nats"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	"github.com/cloudfoundry/yagnats"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/tedsuo/ifrit/ginkgomon"
	"github.com/tedsuo/rata"

	"github.com/cloudfoundry-incubator/tps/api"
	"github.com/cloudfoundry-incubator/tps/heartbeat"
)

var _ = Describe("TPS", func() {

	var httpClient *http.Client
	var requestGenerator *rata.RequestGenerator
	var natsClient yagnats.NATSConn

	BeforeEach(func() {
		natsClient = natsRunner.MessageBus
		requestGenerator = rata.NewRequestGenerator(fmt.Sprintf("http://%s", tpsAddr), api.Routes)
		httpClient = &http.Client{
			Transport: &http.Transport{},
		}
	})

	JustBeforeEach(func() {
		tps = ginkgomon.Invoke(runner)
	})

	AfterEach(func() {
		if tps != nil {
			tps.Signal(os.Kill)
			Eventually(tps.Wait()).Should(Receive())
		}
	})
	Describe("GET /lrps/:guid", func() {
		Context("when etcd is running", func() {
			BeforeEach(func() {
				bbs.ReportActualLRPAsStarting("some-process-guid", "some-instance-guid-1", "executor-id", 0)

				bbs.ReportActualLRPAsRunning(models.ActualLRP{
					ProcessGuid:  "some-process-guid",
					InstanceGuid: "some-instance-guid-2",

					Index: 1,

					State: models.ActualLRPStateRunning,
				}, "executor-id")

				bbs.ReportActualLRPAsRunning(models.ActualLRP{
					ProcessGuid:  "some-process-guid",
					InstanceGuid: "some-instance-guid-3",

					Index: 2,

					State: models.ActualLRPStateRunning,
				}, "executor-id")

				bbs.ReportActualLRPAsRunning(models.ActualLRP{
					ProcessGuid:  "some-other-process-guid",
					InstanceGuid: "some-instance-guid-3",

					Index: 0,

					State: models.ActualLRPStateRunning,
				}, "executor-id")
			})

			It("reports the state of the given process guid's instances", func() {
				getLRPs, err := requestGenerator.CreateRequest(
					api.LRPStatus,
					rata.Params{"guid": "some-process-guid"},
					nil,
				)
				Ω(err).ShouldNot(HaveOccurred())

				response, err := httpClient.Do(getLRPs)
				Ω(err).ShouldNot(HaveOccurred())

				var lrpInstances []api.LRPInstance
				err = json.NewDecoder(response.Body).Decode(&lrpInstances)
				Ω(err).ShouldNot(HaveOccurred())

				Ω(lrpInstances).Should(HaveLen(3))

				Ω(lrpInstances).Should(ContainElement(api.LRPInstance{
					ProcessGuid:  "some-process-guid",
					InstanceGuid: "some-instance-guid-1",

					Index: 0,
					Since: timeProvider.Time().UnixNano(),

					State: "starting",
				}))

				Ω(lrpInstances).Should(ContainElement(api.LRPInstance{
					ProcessGuid:  "some-process-guid",
					InstanceGuid: "some-instance-guid-2",

					Index: 1,
					Since: timeProvider.Time().UnixNano(),

					State: "running",
				}))

				Ω(lrpInstances).Should(ContainElement(api.LRPInstance{
					ProcessGuid:  "some-process-guid",
					InstanceGuid: "some-instance-guid-3",

					Index: 2,
					Since: timeProvider.Time().UnixNano(),

					State: "running",
				}))
			})
		})

		Context("when etcd is not running", func() {
			BeforeEach(func() {
				etcdRunner.Stop()
			})

			It("returns 500", func() {
				getLRPs, err := requestGenerator.CreateRequest(
					api.LRPStatus,
					rata.Params{"guid": "some-process-guid"},
					nil,
				)
				Ω(err).ShouldNot(HaveOccurred())

				response, err := httpClient.Do(getLRPs)
				Ω(err).ShouldNot(HaveOccurred())

				Ω(response.StatusCode).Should(Equal(http.StatusInternalServerError))
			})
		})
	})

	Context("when the NATS server is running", func() {
		var tpsNatsSubject = "service.announce.tps"
		var announceMsg chan *nats.Msg
		var subscription *nats.Subscription

		BeforeEach(func() {
			announceMsg = make(chan *nats.Msg)
			var err error
			subscription, err = natsClient.Subscribe(tpsNatsSubject, func(msg *nats.Msg) {
				announceMsg <- msg
			})
			Ω(err).ShouldNot(HaveOccurred())
		})

		AfterEach(func() {
			err := natsClient.Unsubscribe(subscription)
			Ω(err).ShouldNot(HaveOccurred())
		})

		It("heartbeats announcement messages at the predefined interval", func() {
			Eventually(announceMsg, heartbeatInterval+time.Second).Should(Receive())
			Eventually(announceMsg, heartbeatInterval+time.Second).Should(Receive())
		})

		Describe("published HeartbeatMessage", func() {
			var heartbeatMsg heartbeat.HeartbeatMessage

			JustBeforeEach(func(done Done) {
				heartbeatMsg = heartbeat.HeartbeatMessage{}
				msg := <-announceMsg
				err := json.Unmarshal(msg.Data, &heartbeatMsg)
				Ω(err).ShouldNot(HaveOccurred())
				close(done)
			})

			It("contains the correct tps address", func() {
				Ω(heartbeatMsg.Addr).Should(Equal(fmt.Sprintf("http://%s", tpsAddr)))
			})

			It("a ttl 3 times longer than the heartbeatInterval, in seconds", func() {
				Ω(heartbeatMsg.TTL).Should(Equal(uint(3)))
			})
		})
	})

	Context("when the NATS server is down while starting up", func() {
		BeforeEach(func() {
			runner.StartCheck = ""
			natsRunner.KillWithFire()
		})

		It("does not exit", func() {
			Consistently(tps.Wait()).ShouldNot(Receive())
		})

		It("exits when we send a signal", func() {
			tps.Signal(syscall.SIGINT)
			Eventually(tps.Wait()).Should(Receive())
		})

	})

	Context("when the NATS server goes down after startup", func() {
		JustBeforeEach(func() {
			natsRunner.KillWithFire()
			time.Sleep(50 * time.Millisecond)
		})

		It("does not exit", func() {
			Consistently(tps.Wait()).ShouldNot(Receive())
		})

		It("exits when we send a signal", func() {
			tps.Signal(syscall.SIGINT)
			Eventually(tps.Wait()).Should(Receive())
		})
	})
})
<|endoftext|>


<commit_before>package mogilefs

import (
	"os"
	"os/exec"
	"testing"
)

func TestMain(m *testing.M) {
	var exitCode = 0
	defer os.Exit(exitCode)

	go exec.Command("mogilefsd", "-c", "mogilefs/mogilefsd.conf").Run()
	go exec.Command("mogstored", "-c", "mogilefs/mogstored.conf").Run()

	defer exec.Command("pkill", "-f", "mogilefsd").Run()
	defer exec.Command("pkill", "-f", "mogstored").Run()

	exitCode = m.Run()
}
<commit_msg>Wait port for starting<commit_after>package mogilefs

import (
	"github.com/lestrrat/go-tcputil"
	"log"
	"os"
	"os/exec"
	"testing"
	"time"
)

func TestMain(m *testing.M) {
	var exitCode = 0
	defer os.Exit(exitCode)

	go exec.Command("mogilefsd", "-c", "mogilefs/mogilefsd.conf").Run()
	go exec.Command("mogstored", "-c", "mogilefs/mogstored.conf").Run()

	defer exec.Command("pkill", "-f", "mogilefsd").Run()
	defer exec.Command("pkill", "-f", "mogstored").Run()

	var err error
	err = tcputil.WaitLocalPort(7001, 30*time.Second)
	if err != nil {
		log.Fatal(err)
	}
	err = tcputil.WaitLocalPort(7500, 30*time.Second)
	if err != nil {
		log.Fatal(err)
	}
	err = tcputil.WaitLocalPort(7501, 30*time.Second)
	if err != nil {
		log.Fatal(err)
	}

	exitCode = m.Run()
}
<|endoftext|>


<commit_before>package randpics

import (
	"io/ioutil"
	"os"
	"testing"
)

func createTempPicFile(sourceDir string) {
	tempFile, _ := ioutil.TempFile(sourceDir, "")
	tempFile.Close()
	os.Rename(tempFile.Name(), tempFile.Name()+".jpg")
}

func TestMain_CopyAllNoDelete(t *testing.T) {
	dest, _ := ioutil.TempDir("", "")
	source, _ := ioutil.TempDir("", "")

	createTempPicFile(source)

	buildFakeArgs(source, dest, 1)

	main()

	os.Remove(source)
	os.Remove(dest)
}
<commit_msg>Added check on whether file was copied.<commit_after>package randpics

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"
)

func createTempPicFile(sourceDir string) string {
	tempFile, _ := ioutil.TempFile(sourceDir, "")
	tempFile.Close()
	os.Rename(tempFile.Name(), tempFile.Name()+".jpg")

	return filepath.Base(tempFile.Name() + ".jpg")
}

func TestMain_CopyAllNoDelete(t *testing.T) {
	dest, _ := ioutil.TempDir("", "")
	source, _ := ioutil.TempDir("", "")

	file1 := createTempPicFile(source)

	buildFakeArgs(source, dest, 1)
	main()

	if _, err := os.Stat(filepath.Join(dest, file1)); os.IsNotExist(err) {
		t.Errorf("Couldn't find copied file %v in destination path %v", file1, dest)
	}

	os.Remove(source)
	os.Remove(dest)
}
<|endoftext|>