<commit_before>/*
 * Copyright ©1998-2022 by Richard A. Wilkes. All rights reserved.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, version 2.0. If a copy of the MPL was not distributed with
 * this file, You can obtain one at http://mozilla.org/MPL/2.0/.
 *
 * This Source Code Form is "Incompatible With Secondary Licenses", as
 * defined by the Mozilla Public License, version 2.0.
 */

package editors

import (
	"github.com/richardwilkes/gcs/v5/model/gurps"
	"github.com/richardwilkes/gcs/v5/model/gurps/weapon"
	"github.com/richardwilkes/gcs/v5/ui/widget"
	"github.com/richardwilkes/gcs/v5/ui/widget/ntable"
	"github.com/richardwilkes/toolbox/errs"
	"github.com/richardwilkes/toolbox/i18n"
	"github.com/richardwilkes/toolbox/log/jot"
	"github.com/richardwilkes/unison"
)

var (
	meleeWeaponColMap = map[int]int{
		0: gurps.WeaponUsageColumn,
		1: gurps.WeaponSLColumn,
		2: gurps.WeaponParryColumn,
		3: gurps.WeaponBlockColumn,
		4: gurps.WeaponDamageColumn,
		5: gurps.WeaponReachColumn,
		6: gurps.WeaponSTColumn,
	}
	meleeWeaponForPageColMap = map[int]int{
		0: gurps.WeaponDescriptionColumn,
		1: gurps.WeaponUsageColumn,
		2: gurps.WeaponSLColumn,
		3: gurps.WeaponParryColumn,
		4: gurps.WeaponBlockColumn,
		5: gurps.WeaponDamageColumn,
		6: gurps.WeaponReachColumn,
		7: gurps.WeaponSTColumn,
	}
	rangedWeaponColMap = map[int]int{
		0: gurps.WeaponUsageColumn,
		1: gurps.WeaponSLColumn,
		2: gurps.WeaponAccColumn,
		3: gurps.WeaponDamageColumn,
		4: gurps.WeaponRangeColumn,
		5: gurps.WeaponRoFColumn,
		6: gurps.WeaponShotsColumn,
		7: gurps.WeaponBulkColumn,
		8: gurps.WeaponRecoilColumn,
		9: gurps.WeaponSTColumn,
	}
	rangedWeaponforPageColMap = map[int]int{
		0:  gurps.WeaponDescriptionColumn,
		1:  gurps.WeaponUsageColumn,
		2:  gurps.WeaponSLColumn,
		3:  gurps.WeaponAccColumn,
		4:  gurps.WeaponDamageColumn,
		5:  gurps.WeaponRangeColumn,
		6:  gurps.WeaponRoFColumn,
		7:  gurps.WeaponShotsColumn,
		8:  gurps.WeaponBulkColumn,
		9:  gurps.WeaponRecoilColumn,
		10: gurps.WeaponSTColumn,
	}
	_ ntable.TableProvider[*gurps.Weapon] = &weaponsProvider{}
)

type weaponsProvider struct {
	table      *unison.Table[*ntable.Node[*gurps.Weapon]]
	colMap     map[int]int
	provider   gurps.WeaponListProvider
	weaponType weapon.Type
	forPage    bool
}

// NewWeaponsProvider creates a new table provider for weapons.
func NewWeaponsProvider(provider gurps.WeaponListProvider, weaponType weapon.Type, forPage bool) ntable.TableProvider[*gurps.Weapon] {
	p := &weaponsProvider{
		provider:   provider,
		weaponType: weaponType,
		forPage:    forPage,
	}
	switch {
	case weaponType == weapon.Melee && forPage:
		p.colMap = meleeWeaponForPageColMap
	case weaponType == weapon.Melee:
		p.colMap = meleeWeaponColMap
	case weaponType == weapon.Ranged && forPage:
		p.colMap = rangedWeaponforPageColMap
	case weaponType == weapon.Ranged:
		p.colMap = rangedWeaponColMap
	default:
		jot.Fatalf(1, "unknown weapon type: %d", weaponType)
	}
	return p
}

func (p *weaponsProvider) SetTable(table *unison.Table[*ntable.Node[*gurps.Weapon]]) {
	p.table = table
}

func (p *weaponsProvider) RootRowCount() int {
	return len(p.provider.Weapons(p.weaponType))
}

func (p *weaponsProvider) RootRows() []*ntable.Node[*gurps.Weapon] {
	data := p.provider.Weapons(p.weaponType)
	rows := make([]*ntable.Node[*gurps.Weapon], 0, len(data))
	for _, one := range data {
		rows = append(rows, ntable.NewNode[*gurps.Weapon](p.table, nil, p.colMap, one, p.forPage))
	}
	return rows
}

func (p *weaponsProvider) SetRootRows(rows []*ntable.Node[*gurps.Weapon]) {
	p.provider.SetWeapons(p.weaponType, ntable.ExtractNodeDataFromList(rows))
}

func (p *weaponsProvider) RootData() []*gurps.Weapon {
	return p.provider.Weapons(p.weaponType)
}

func (p *weaponsProvider) SetRootData(data []*gurps.Weapon) {
	p.provider.SetWeapons(p.weaponType, data)
}

func (p *weaponsProvider) Entity() *gurps.Entity {
	return p.provider.Entity()
}

func (p *weaponsProvider) DragKey() string {
	return p.weaponType.Key()
}

func (p *weaponsProvider) DragSVG() *unison.SVG {
	return p.weaponType.SVG()
}

func (p *weaponsProvider) DropShouldMoveData(_, _ *unison.Table[*ntable.Node[*gurps.Weapon]]) bool {
	// Not used
	return false
}

func (p *weaponsProvider) ItemNames() (singular, plural string) {
	return p.weaponType.String(), p.weaponType.AltString()
}

func (p *weaponsProvider) Headers() []unison.TableColumnHeader[*ntable.Node[*gurps.Weapon]] {
	var headers []unison.TableColumnHeader[*ntable.Node[*gurps.Weapon]]
	for i := 0; i < len(p.colMap); i++ {
		switch p.colMap[i] {
		case gurps.WeaponDescriptionColumn:
			headers = append(headers, NewHeader[*gurps.Weapon](p.weaponType.String(), "", p.forPage))
		case gurps.WeaponUsageColumn:
			var title string
			switch {
			case p.forPage:
				title = i18n.Text("Usage")
			case p.weaponType == weapon.Melee:
				title = i18n.Text("Melee Weapon Usage")
			default:
				title = i18n.Text("Ranged Weapon Usage")
			}
			headers = append(headers, NewHeader[*gurps.Weapon](title, "", p.forPage))
		case gurps.WeaponSLColumn:
			headers = append(headers, NewHeader[*gurps.Weapon](i18n.Text("SL"), i18n.Text("Skill Level"), p.forPage))
		case gurps.WeaponParryColumn:
			headers = append(headers, NewHeader[*gurps.Weapon](i18n.Text("Parry"), "", p.forPage))
		case gurps.WeaponBlockColumn:
			headers = append(headers, NewHeader[*gurps.Weapon](i18n.Text("Block"), "", p.forPage))
		case gurps.WeaponDamageColumn:
			headers = append(headers, NewHeader[*gurps.Weapon](i18n.Text("Damage"), "", p.forPage))
		case gurps.WeaponReachColumn:
			headers = append(headers, NewHeader[*gurps.Weapon](i18n.Text("Reach"), "", p.forPage))
		case gurps.WeaponSTColumn:
			headers = append(headers, NewHeader[*gurps.Weapon](i18n.Text("ST"), i18n.Text("Minimum Strength"), p.forPage))
		case gurps.WeaponAccColumn:
			headers = append(headers, NewHeader[*gurps.Weapon](i18n.Text("Acc"), i18n.Text("Accuracy Bonus"), p.forPage))
		case gurps.WeaponRangeColumn:
			headers = append(headers, NewHeader[*gurps.Weapon](i18n.Text("Range"), "", p.forPage))
		case gurps.WeaponRoFColumn:
			headers = append(headers, NewHeader[*gurps.Weapon](i18n.Text("RoF"), i18n.Text("Rate of Fire"), p.forPage))
		case gurps.WeaponShotsColumn:
			headers = append(headers, NewHeader[*gurps.Weapon](i18n.Text("Shots"), "", p.forPage))
		case gurps.WeaponBulkColumn:
			headers = append(headers, NewHeader[*gurps.Weapon](i18n.Text("Bulk"), "", p.forPage))
		case gurps.WeaponRecoilColumn:
			headers = append(headers, NewHeader[*gurps.Weapon](i18n.Text("Recoil"), "", p.forPage))
		default:
			jot.Fatalf(1, "invalid weapon column: %d", p.colMap[i])
		}
	}
	return headers
}

func (p *weaponsProvider) SyncHeader(_ []unison.TableColumnHeader[*ntable.Node[*gurps.Weapon]]) {
}

func (p *weaponsProvider) HierarchyColumnIndex() int {
	return -1
}

func (p *weaponsProvider) ExcessWidthColumnIndex() int {
	for k, v := range p.colMap {
		if v == gurps.WeaponDescriptionColumn {
			return k
		}
	}
	return 0
}

func (p *weaponsProvider) OpenEditor(owner widget.Rebuildable, table *unison.Table[*ntable.Node[*gurps.Weapon]]) {
	if !p.forPage {
		ntable.OpenEditor[*gurps.Weapon](table, func(item *gurps.Weapon) { EditWeapon(owner, item) })
	}
}

func (p *weaponsProvider) CreateItem(owner widget.Rebuildable, table *unison.Table[*ntable.Node[*gurps.Weapon]], _ ntable.ItemVariant) {
	if !p.forPage {
		wpn := gurps.NewWeapon(p.provider.WeaponOwner(), p.weaponType)
		ntable.InsertItem[*gurps.Weapon](owner, table, wpn,
			func() []*gurps.Weapon { return p.provider.Weapons(p.weaponType) },
			func(list []*gurps.Weapon) { p.provider.SetWeapons(p.weaponType, list) },
			func(_ *unison.Table[*ntable.Node[*gurps.Weapon]]) []*ntable.Node[*gurps.Weapon] { return p.RootRows() })
		EditWeapon(owner, wpn)
	}
}

func (p *weaponsProvider) Serialize() ([]byte, error) {
	return nil, errs.New("not allowed")
}

func (p *weaponsProvider) Deserialize(_ []byte) error {
	return errs.New("not allowed")
}
<commit_msg>Fix error when dragging weapons into the weapon area of an editor<commit_after>/*
 * Copyright ©1998-2022 by Richard A. Wilkes. All rights reserved.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, version 2.0. If a copy of the MPL was not distributed with
 * this file, You can obtain one at http://mozilla.org/MPL/2.0/.
 *
 * This Source Code Form is "Incompatible With Secondary Licenses", as
 * defined by the Mozilla Public License, version 2.0.
 */

package editors

import (
	"github.com/richardwilkes/gcs/v5/model/gurps"
	"github.com/richardwilkes/gcs/v5/model/gurps/weapon"
	"github.com/richardwilkes/gcs/v5/model/jio"
	"github.com/richardwilkes/gcs/v5/ui/widget"
	"github.com/richardwilkes/gcs/v5/ui/widget/ntable"
	"github.com/richardwilkes/toolbox/errs"
	"github.com/richardwilkes/toolbox/i18n"
	"github.com/richardwilkes/toolbox/log/jot"
	"github.com/richardwilkes/unison"
)

var (
	meleeWeaponColMap = map[int]int{
		0: gurps.WeaponUsageColumn,
		1: gurps.WeaponSLColumn,
		2: gurps.WeaponParryColumn,
		3: gurps.WeaponBlockColumn,
		4: gurps.WeaponDamageColumn,
		5: gurps.WeaponReachColumn,
		6: gurps.WeaponSTColumn,
	}
	meleeWeaponForPageColMap = map[int]int{
		0: gurps.WeaponDescriptionColumn,
		1: gurps.WeaponUsageColumn,
		2: gurps.WeaponSLColumn,
		3: gurps.WeaponParryColumn,
		4: gurps.WeaponBlockColumn,
		5: gurps.WeaponDamageColumn,
		6: gurps.WeaponReachColumn,
		7: gurps.WeaponSTColumn,
	}
	rangedWeaponColMap = map[int]int{
		0: gurps.WeaponUsageColumn,
		1: gurps.WeaponSLColumn,
		2: gurps.WeaponAccColumn,
		3: gurps.WeaponDamageColumn,
		4: gurps.WeaponRangeColumn,
		5: gurps.WeaponRoFColumn,
		6: gurps.WeaponShotsColumn,
		7: gurps.WeaponBulkColumn,
		8: gurps.WeaponRecoilColumn,
		9: gurps.WeaponSTColumn,
	}
	rangedWeaponforPageColMap = map[int]int{
		0:  gurps.WeaponDescriptionColumn,
		1:  gurps.WeaponUsageColumn,
		2:  gurps.WeaponSLColumn,
		3:  gurps.WeaponAccColumn,
		4:  gurps.WeaponDamageColumn,
		5:  gurps.WeaponRangeColumn,
		6:  gurps.WeaponRoFColumn,
		7:  gurps.WeaponShotsColumn,
		8:  gurps.WeaponBulkColumn,
		9:  gurps.WeaponRecoilColumn,
		10: gurps.WeaponSTColumn,
	}
	_ ntable.TableProvider[*gurps.Weapon] = &weaponsProvider{}
)

type weaponsProvider struct {
	table      *unison.Table[*ntable.Node[*gurps.Weapon]]
	colMap     map[int]int
	provider   gurps.WeaponListProvider
	weaponType weapon.Type
	forPage    bool
}

// NewWeaponsProvider creates a new table provider for weapons.
func NewWeaponsProvider(provider gurps.WeaponListProvider, weaponType weapon.Type, forPage bool) ntable.TableProvider[*gurps.Weapon] {
	p := &weaponsProvider{
		provider:   provider,
		weaponType: weaponType,
		forPage:    forPage,
	}
	switch {
	case weaponType == weapon.Melee && forPage:
		p.colMap = meleeWeaponForPageColMap
	case weaponType == weapon.Melee:
		p.colMap = meleeWeaponColMap
	case weaponType == weapon.Ranged && forPage:
		p.colMap = rangedWeaponforPageColMap
	case weaponType == weapon.Ranged:
		p.colMap = rangedWeaponColMap
	default:
		jot.Fatalf(1, "unknown weapon type: %d", weaponType)
	}
	return p
}

func (p *weaponsProvider) SetTable(table *unison.Table[*ntable.Node[*gurps.Weapon]]) {
	p.table = table
}

func (p *weaponsProvider) RootRowCount() int {
	return len(p.provider.Weapons(p.weaponType))
}

func (p *weaponsProvider) RootRows() []*ntable.Node[*gurps.Weapon] {
	data := p.provider.Weapons(p.weaponType)
	rows := make([]*ntable.Node[*gurps.Weapon], 0, len(data))
	for _, one := range data {
		rows = append(rows, ntable.NewNode[*gurps.Weapon](p.table, nil, p.colMap, one, p.forPage))
	}
	return rows
}

func (p *weaponsProvider) SetRootRows(rows []*ntable.Node[*gurps.Weapon]) {
	p.provider.SetWeapons(p.weaponType, ntable.ExtractNodeDataFromList(rows))
}

func (p *weaponsProvider) RootData() []*gurps.Weapon {
	return p.provider.Weapons(p.weaponType)
}

func (p *weaponsProvider) SetRootData(data []*gurps.Weapon) {
	p.provider.SetWeapons(p.weaponType, data)
}

func (p *weaponsProvider) Entity() *gurps.Entity {
	return p.provider.Entity()
}

func (p *weaponsProvider) DragKey() string {
	return p.weaponType.Key()
}

func (p *weaponsProvider) DragSVG() *unison.SVG {
	return p.weaponType.SVG()
}

func (p *weaponsProvider) DropShouldMoveData(from, to *unison.Table[*ntable.Node[*gurps.Weapon]]) bool {
	return from == to
}

func (p *weaponsProvider) ItemNames() (singular, plural string) {
	return p.weaponType.String(), p.weaponType.AltString()
}

func (p *weaponsProvider) Headers() []unison.TableColumnHeader[*ntable.Node[*gurps.Weapon]] {
	var headers []unison.TableColumnHeader[*ntable.Node[*gurps.Weapon]]
	for i := 0; i < len(p.colMap); i++ {
		switch p.colMap[i] {
		case gurps.WeaponDescriptionColumn:
			headers = append(headers, NewHeader[*gurps.Weapon](p.weaponType.String(), "", p.forPage))
		case gurps.WeaponUsageColumn:
			var title string
			switch {
			case p.forPage:
				title = i18n.Text("Usage")
			case p.weaponType == weapon.Melee:
				title = i18n.Text("Melee Weapon Usage")
			default:
				title = i18n.Text("Ranged Weapon Usage")
			}
			headers = append(headers, NewHeader[*gurps.Weapon](title, "", p.forPage))
		case gurps.WeaponSLColumn:
			headers = append(headers, NewHeader[*gurps.Weapon](i18n.Text("SL"), i18n.Text("Skill Level"), p.forPage))
		case gurps.WeaponParryColumn:
			headers = append(headers, NewHeader[*gurps.Weapon](i18n.Text("Parry"), "", p.forPage))
		case gurps.WeaponBlockColumn:
			headers = append(headers, NewHeader[*gurps.Weapon](i18n.Text("Block"), "", p.forPage))
		case gurps.WeaponDamageColumn:
			headers = append(headers, NewHeader[*gurps.Weapon](i18n.Text("Damage"), "", p.forPage))
		case gurps.WeaponReachColumn:
			headers = append(headers, NewHeader[*gurps.Weapon](i18n.Text("Reach"), "", p.forPage))
		case gurps.WeaponSTColumn:
			headers = append(headers, NewHeader[*gurps.Weapon](i18n.Text("ST"), i18n.Text("Minimum Strength"), p.forPage))
		case gurps.WeaponAccColumn:
			headers = append(headers, NewHeader[*gurps.Weapon](i18n.Text("Acc"), i18n.Text("Accuracy Bonus"), p.forPage))
		case gurps.WeaponRangeColumn:
			headers = append(headers, NewHeader[*gurps.Weapon](i18n.Text("Range"), "", p.forPage))
		case gurps.WeaponRoFColumn:
			headers = append(headers, NewHeader[*gurps.Weapon](i18n.Text("RoF"), i18n.Text("Rate of Fire"), p.forPage))
		case gurps.WeaponShotsColumn:
			headers = append(headers, NewHeader[*gurps.Weapon](i18n.Text("Shots"), "", p.forPage))
		case gurps.WeaponBulkColumn:
			headers = append(headers, NewHeader[*gurps.Weapon](i18n.Text("Bulk"), "", p.forPage))
		case gurps.WeaponRecoilColumn:
			headers = append(headers, NewHeader[*gurps.Weapon](i18n.Text("Recoil"), "", p.forPage))
		default:
			jot.Fatalf(1, "invalid weapon column: %d", p.colMap[i])
		}
	}
	return headers
}

func (p *weaponsProvider) SyncHeader(_ []unison.TableColumnHeader[*ntable.Node[*gurps.Weapon]]) {
}

func (p *weaponsProvider) HierarchyColumnIndex() int {
	return -1
}

func (p *weaponsProvider) ExcessWidthColumnIndex() int {
	for k, v := range p.colMap {
		if v == gurps.WeaponDescriptionColumn {
			return k
		}
	}
	return 0
}

func (p *weaponsProvider) OpenEditor(owner widget.Rebuildable, table *unison.Table[*ntable.Node[*gurps.Weapon]]) {
	if !p.forPage {
		ntable.OpenEditor[*gurps.Weapon](table, func(item *gurps.Weapon) { EditWeapon(owner, item) })
	}
}

func (p *weaponsProvider) CreateItem(owner widget.Rebuildable, table *unison.Table[*ntable.Node[*gurps.Weapon]], _ ntable.ItemVariant) {
	if !p.forPage {
		wpn := gurps.NewWeapon(p.provider.WeaponOwner(), p.weaponType)
		ntable.InsertItem[*gurps.Weapon](owner, table, wpn,
			func() []*gurps.Weapon { return p.provider.Weapons(p.weaponType) },
			func(list []*gurps.Weapon) { p.provider.SetWeapons(p.weaponType, list) },
			func(_ *unison.Table[*ntable.Node[*gurps.Weapon]]) []*ntable.Node[*gurps.Weapon] { return p.RootRows() })
		EditWeapon(owner, wpn)
	}
}

func (p *weaponsProvider) Serialize() ([]byte, error) {
	if p.forPage {
		return nil, errs.New("not allowed")
	}
	return jio.SerializeAndCompress(p.provider.Weapons(p.weaponType))
}

func (p *weaponsProvider) Deserialize(data []byte) error {
	if p.forPage {
		return errs.New("not allowed")
	}
	var rows []*gurps.Weapon
	if err := jio.DecompressAndDeserialize(data, &rows); err != nil {
		return err
	}
	p.provider.SetWeapons(p.weaponType, rows)
	return nil
}
<|endoftext|>


<commit_before>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2016 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package dummy

import (
	"context"

	"github.com/sirupsen/logrus"

	"github.com/loadimpact/k6/lib"
	"github.com/loadimpact/k6/stats"
)

// Collector implements the lib.Collector interface and should be used only for testing
type Collector struct {
	RunStatus lib.RunStatus

	SampleContainers []stats.SampleContainer
	Samples          []stats.Sample
}

// Verify that Collector implements lib.Collector
var _ lib.Collector = &Collector{}

// Init does nothing, it's only included to satisfy the lib.Collector interface
func (c *Collector) Init() error { return nil }

// MakeConfig does nothing, it's only included to satisfy the lib.Collector interface
func (c *Collector) MakeConfig() interface{} { return nil }

// Run just blocks until the context is done
func (c *Collector) Run(ctx context.Context) {
	<-ctx.Done()
	logrus.Debugf("finished status: %d", c.RunStatus)
}

// Collect just appends all of the samples passed to it to the internal sample slice.
// According to the the lib.Collector interface, it should never be called concurrently,
// so there's no locking on purpose - that way Go's race condition detector can actually
// detect incorrect usage.
// Also, theoretically the collector doesn't have to actually Run() before samples start
// being collected, it only has to be initialized.
func (c *Collector) Collect(scs []stats.SampleContainer) {
	for _, sc := range scs {
		c.SampleContainers = append(c.SampleContainers, sc)
		c.Samples = append(c.Samples, sc.GetSamples()...)
	}
}

// Link returns a dummy string, it's only included to satisfy the lib.Collector interface
func (c *Collector) Link() string {
	return "http://example.com/"
}

// GetRequiredSystemTags returns which sample tags are needed by this collector
func (c *Collector) GetRequiredSystemTags() stats.SystemTagSet {
	return stats.SystemTagSet(0) // There are no required tags for this collector
}

// SetRunStatus just saves the passed status for later inspection
func (c *Collector) SetRunStatus(status lib.RunStatus) {
	c.RunStatus = status
}
<commit_msg>Fix another data race...<commit_after>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2016 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package dummy

import (
	"context"

	"github.com/loadimpact/k6/lib"
	"github.com/loadimpact/k6/stats"
)

// Collector implements the lib.Collector interface and should be used only for testing
type Collector struct {
	RunStatus lib.RunStatus

	SampleContainers []stats.SampleContainer
	Samples          []stats.Sample
}

// Verify that Collector implements lib.Collector
var _ lib.Collector = &Collector{}

// Init does nothing, it's only included to satisfy the lib.Collector interface
func (c *Collector) Init() error { return nil }

// MakeConfig does nothing, it's only included to satisfy the lib.Collector interface
func (c *Collector) MakeConfig() interface{} { return nil }

// Run just blocks until the context is done
func (c *Collector) Run(ctx context.Context) {
	<-ctx.Done()
}

// Collect just appends all of the samples passed to it to the internal sample slice.
// According to the the lib.Collector interface, it should never be called concurrently,
// so there's no locking on purpose - that way Go's race condition detector can actually
// detect incorrect usage.
// Also, theoretically the collector doesn't have to actually Run() before samples start
// being collected, it only has to be initialized.
func (c *Collector) Collect(scs []stats.SampleContainer) {
	for _, sc := range scs {
		c.SampleContainers = append(c.SampleContainers, sc)
		c.Samples = append(c.Samples, sc.GetSamples()...)
	}
}

// Link returns a dummy string, it's only included to satisfy the lib.Collector interface
func (c *Collector) Link() string {
	return "http://example.com/"
}

// GetRequiredSystemTags returns which sample tags are needed by this collector
func (c *Collector) GetRequiredSystemTags() stats.SystemTagSet {
	return stats.SystemTagSet(0) // There are no required tags for this collector
}

// SetRunStatus just saves the passed status for later inspection
func (c *Collector) SetRunStatus(status lib.RunStatus) {
	c.RunStatus = status
}
<|endoftext|>


<commit_before>package services

import (
	"encoding/json"
	"net/http"
	"strconv"
	"strings"

	h "github.com/ernestio/api-gateway/helpers"
	"github.com/ernestio/api-gateway/models"
)

// Delete : Deletes a service by name
func Delete(au models.User, name string) (int, []byte) {
	var raw []byte
	var err error
	var def models.Definition

	if raw, err = getServiceRaw(name, au.GroupID); err != nil {
		return 404, []byte(err.Error())
	}

	s := models.Service{}
	if err := json.Unmarshal(raw, &s); err != nil {
		h.L.Error(err.Error())
		return http.StatusBadRequest, []byte(err.Error())
	}

	if s.Status == "in_progress" {
		return 400, []byte(`"Service is already applying some changes, please wait until they are done"`)
	}

	dID := strconv.Itoa(s.DatacenterID)
	body, err := def.MapDeletion(s.ID, s.Type, dID)
	if err != nil {
		return 500, []byte(`"Couldn't map the service"`)
	}
	if err := s.RequestDeletion(body); err != nil {
		return 500, []byte(`"Couldn't call service.delete"`)
	}

	parts := strings.Split(s.ID, "-")
	stream := parts[len(parts)-1]

	return http.StatusOK, []byte(`{"id":"` + s.ID + `","stream_id":"` + stream + `"}`)
}
<commit_msg>removing stream id from delete handler<commit_after>package services

import (
	"encoding/json"
	"net/http"
	"strconv"

	h "github.com/ernestio/api-gateway/helpers"
	"github.com/ernestio/api-gateway/models"
)

// Delete : Deletes a service by name
func Delete(au models.User, name string) (int, []byte) {
	var raw []byte
	var err error
	var def models.Definition

	if raw, err = getServiceRaw(name, au.GroupID); err != nil {
		return 404, []byte(err.Error())
	}

	s := models.Service{}
	if err := json.Unmarshal(raw, &s); err != nil {
		h.L.Error(err.Error())
		return http.StatusBadRequest, []byte(err.Error())
	}

	if s.Status == "in_progress" {
		return 400, []byte(`"Service is already applying some changes, please wait until they are done"`)
	}

	dID := strconv.Itoa(s.DatacenterID)
	body, err := def.MapDeletion(s.ID, s.Type, dID)
	if err != nil {
		return 500, []byte(`"Couldn't map the service"`)
	}
	if err := s.RequestDeletion(body); err != nil {
		return 500, []byte(`"Couldn't call service.delete"`)
	}

	return http.StatusOK, []byte(`{"id":"` + s.ID + `"}`)
}
<|endoftext|>


<commit_before>// Copyright 2020 Tomas Machalek <tomas.machalek@gmail.com>
// Copyright 2020 Institute of the Czech National Corpus,
//                Faculty of Arts, Charles University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mapka

import (
	"crypto/sha1"
	"encoding/hex"
	"strconv"
	"time"

	"github.com/czcorpus/klogproc/conversion"
)

// createID creates an idempotent ID of rec based on its properties.
func createID(rec *OutputRecord) string {
	str := rec.Type + rec.Path + rec.Datetime + rec.IPAddress + rec.UserID
	sum := sha1.Sum([]byte(str))
	return hex.EncodeToString(sum[:])
}

// Transformer converts a source log object into a destination one
type Transformer struct {
	prevReqs   *PrevReqPool
	numSimilar int
}

// Transform creates a new OutputRecord out of an existing InputRecord
func (t *Transformer) Transform(logRecord *InputRecord, recType string, tzShiftMin int, anonymousUsers []int) (*OutputRecord, error) {
	userID := -1

	r := &OutputRecord{
		Type:        recType,
		time:        logRecord.GetTime(),
		Datetime:    logRecord.GetTime().Add(time.Minute * time.Duration(tzShiftMin)).Format(time.RFC3339),
		IPAddress:   logRecord.Request.RemoteAddr,
		UserAgent:   logRecord.Request.HTTPUserAgent,
		IsAnonymous: userID == -1 || conversion.UserBelongsToList(userID, anonymousUsers),
		IsQuery:     false,
		UserID:      strconv.Itoa(userID),
		Action:      logRecord.Action,
		Path:        logRecord.Path,
		ProcTime:    logRecord.ProcTime,
		Params:      logRecord.Params,
	}
	r.ID = createID(r)
	if t.prevReqs.ContainsSimilar(r) && r.Action == "overlay" || r.Action == "text" {
		r.IsQuery = true
	}
	t.prevReqs.AddItem(r)
	return r, nil
}

// NewTransformer is a default constructor for the Transformer.
// It also loads user ID map from a configured file (if exists).
func NewTransformer() *Transformer {
	return &Transformer{
		prevReqs: NewPrevReqPool(5),
	}
}
<commit_msg>Treat two subsequent 'text' action records and one "true" query<commit_after>// Copyright 2020 Tomas Machalek <tomas.machalek@gmail.com>
// Copyright 2020 Institute of the Czech National Corpus,
//                Faculty of Arts, Charles University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mapka

import (
	"crypto/sha1"
	"encoding/hex"
	"strconv"
	"time"

	"github.com/czcorpus/klogproc/conversion"
)

// createID creates an idempotent ID of rec based on its properties.
func createID(rec *OutputRecord) string {
	str := rec.Type + rec.Path + rec.Datetime + rec.IPAddress + rec.UserID
	sum := sha1.Sum([]byte(str))
	return hex.EncodeToString(sum[:])
}

// Transformer converts a source log object into a destination one
type Transformer struct {
	prevReqs   *PrevReqPool
	numSimilar int
}

// Transform creates a new OutputRecord out of an existing InputRecord
func (t *Transformer) Transform(logRecord *InputRecord, recType string, tzShiftMin int, anonymousUsers []int) (*OutputRecord, error) {
	userID := -1

	r := &OutputRecord{
		Type:        recType,
		time:        logRecord.GetTime(),
		Datetime:    logRecord.GetTime().Add(time.Minute * time.Duration(tzShiftMin)).Format(time.RFC3339),
		IPAddress:   logRecord.Request.RemoteAddr,
		UserAgent:   logRecord.Request.HTTPUserAgent,
		IsAnonymous: userID == -1 || conversion.UserBelongsToList(userID, anonymousUsers),
		IsQuery:     false,
		UserID:      strconv.Itoa(userID),
		Action:      logRecord.Action,
		Path:        logRecord.Path,
		ProcTime:    logRecord.ProcTime,
		Params:      logRecord.Params,
	}
	r.ID = createID(r)
	if t.prevReqs.ContainsSimilar(r) && r.Action == "overlay" ||
		!t.prevReqs.ContainsSimilar(r) && r.Action == "text" {
		r.IsQuery = true
	}
	t.prevReqs.AddItem(r)
	return r, nil
}

// NewTransformer is a default constructor for the Transformer.
// It also loads user ID map from a configured file (if exists).
func NewTransformer() *Transformer {
	return &Transformer{
		prevReqs: NewPrevReqPool(5),
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storage

import (
	"io/ioutil"
	"os"
	"path"
	"reflect"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/control-center/serviced/coordinator/client"
	"github.com/control-center/serviced/domain/host"
	"github.com/zenoss/glog"
)

const monitorSubDir string = "/monitor"

// Monitor monitors the exporting of a file system to clients.
type Monitor struct {
	driver StorageDriver

	monitoredHosts     map[string]bool
	monitoredHostsLock sync.RWMutex

	shouldRestart   bool
	monitorInterval time.Duration
	previousRestart time.Time

	conn               client.Connection
	storageClientsPath string
}

// NewMonitor returns a Monitor object to monitor the exported file system
func NewMonitor(driver StorageDriver, monitorInterval time.Duration) (*Monitor, error) {
	m := &Monitor{
		driver:          driver,
		monitoredHosts:  make(map[string]bool),
		shouldRestart:   getShouldRestartDFSOnFailure(),
		monitorInterval: monitorInterval,
	}

	return m, nil
}

// UpdateRemoteMonitorFile is used by remote clients to write a tiny file to the DFS volume at the given cycle
func UpdateRemoteMonitorFile(localPath string, writeInterval time.Duration, ipAddr string, shutdown <-chan interface{}) {
	remoteFile := path.Join(localPath, monitorSubDir, ipAddr)
	glog.Infof("updating DFS volume monitor file %s at write interval: %s", remoteFile, writeInterval)

	for {
		glog.V(2).Infof("writing DFS file %s", remoteFile)
		if err := ioutil.WriteFile(remoteFile, []byte(ipAddr), 0600); err != nil {
			glog.Warningf("unable to write DFS file %s: %s", remoteFile, err)
		}

		// wait for next cycle or shutdown
		select {
		case <-time.After(writeInterval):

		case <-shutdown:
			glog.Infof("no longer writing remote monitor status for DFS volume %s to %s", localPath, remoteFile)
			return
		}
	}
}

// SetMonitorStorageClients sets the monitored remote IPs that are active
func (m *Monitor) SetMonitorStorageClients(conn client.Connection, storageClientsPath string) {
	m.conn = conn
	m.storageClientsPath = storageClientsPath
	m.SetMonitorRemoteHosts(getActiveRemoteHosts(conn, storageClientsPath, m.monitorInterval)...)
}

// SetMonitorRemoteHosts sets the remote hosts to monitor
func (m *Monitor) SetMonitorRemoteHosts(ipAddrs ...string) {
	if len(ipAddrs) == 0 {
		glog.Warningf("disabled DFS volume monitoring - no remote IPs to monitor")
	} else {
		glog.V(2).Infof("enabled DFS volume monitoring for remote IPs: %+v", ipAddrs)
	}
	m.monitoredHostsLock.Lock()
	defer m.monitoredHostsLock.Unlock()
	for _, ipAddr := range ipAddrs {
		m.monitoredHosts[ipAddr] = true
	}
}

// setMonitorRemoteHost updates the status of the monitored remote
func (m *Monitor) setMonitorRemoteHost(ipAddr string, hasUpdated bool) {
	m.monitoredHostsLock.Lock()
	defer m.monitoredHostsLock.Unlock()
	m.monitoredHosts[ipAddr] = hasUpdated
}

// getMonitorRemoteHosts returns the statuses of the monitored remotes
func (m *Monitor) getMonitorRemoteHosts() map[string]bool {
	m.monitoredHostsLock.RLock()
	defer m.monitoredHostsLock.RUnlock()
	return m.monitoredHosts
}

// ProcessDFSVolumePollUpdateFunc is called by MonitorDFSVolume when updates are detected
type DFSVolumeMonitorPollUpdateFunc func(mountpoint, remoteIP string, isUpdated bool)

// DFSVolumeMonitorPollUpdateFunc restarts nfs based on status of monitored remotes
func (m *Monitor) DFSVolumeMonitorPollUpdateFunc(mountpoint, remoteIP string, hasUpdatedFile bool) {
	// monitor dfs; log warnings each cycle; restart dfs if needed

	if hasUpdatedFile {
		return
	} else if len(m.getMonitorRemoteHosts()) == 0 {
		return
	}

	glog.Warningf("DFS NFS volume %s is not seen by remoteIP:%s - further action may be needed i.e: restart nfs", mountpoint, remoteIP)
	now := time.Now()
	since := now.Sub(m.previousRestart)
	if !m.shouldRestart {
		glog.Warningf("Not restarting DFS NFS service due to configuration setting: SERVICED_MONITOR_DFS_MASTER_RESTART=0")
		return
	} else if since < m.monitorInterval {
		glog.Warningf("Not restarting DFS NFS service - have not surpassed interval: %s since last restart", m.monitorInterval)
		return
	}

	m.previousRestart = now
	if err := m.driver.Restart(); err != nil {
		glog.Errorf("Error restarting DFS NFS service: %s", err)
	}
}

// MonitorVolume monitors the DFS volume - logs on failure and calls pollUpdateFunc
func (m *Monitor) MonitorDFSVolume(mountpoint string, shutdown <-chan interface{}, pollUpdateFunc DFSVolumeMonitorPollUpdateFunc) {
	glog.Infof("monitoring DFS export info for DFS volume %s at polling interval: %s", mountpoint, m.monitorInterval)

	m.previousRestart = time.Now()

	monitorPath := path.Join(mountpoint, monitorSubDir)
	os.RemoveAll(monitorPath)
	if err := os.MkdirAll(monitorPath, 0755); err != nil {
		glog.Errorf("no longer monitoring status for DFS volume %s - unable to mkdir %+v: %s", mountpoint, monitorPath, err)
		return
	}

	for {
		// check all active remotes
		remotes := m.getMonitorRemoteHosts()
		remoteIPs := make([]string, 0, len(remotes))
		for k := range remotes {
			remoteIPs = append(remoteIPs, k)
		}

		activeRemotes := remoteIPs
		if m.conn != nil && len(m.storageClientsPath) != 0 {
			activeRemotes = getActiveRemoteHosts(m.conn, m.storageClientsPath, m.monitorInterval)
			sort.Sort(sort.StringSlice(activeRemotes))
			sort.Sort(sort.StringSlice(remoteIPs))
			if !reflect.DeepEqual(activeRemotes, remoteIPs) {
				glog.Infof("DFS active remotes to be monitored has changed to: %+v", activeRemotes)
			}
			m.SetMonitorRemoteHosts(activeRemotes...)
		}

		for _, remoteIP := range activeRemotes {
			remoteFile := path.Join(monitorPath, remoteIP)
			sinceModified, err := fileSinceModified(remoteFile)
			hasUpdatedFile := false
			if err != nil {
				glog.Warningf("remote agent %s is not synced on DFS volume %s for file %s: %s", remoteIP, mountpoint, remoteFile, err)
			} else if sinceModified > m.monitorInterval {
				glog.Warningf("remote agent %s is not synced DFS volume %s for file %s (since modified: %s)", remoteIP, mountpoint, remoteFile, sinceModified)
			} else {
				glog.V(4).Infof("remote agent %s is synced on DFS volume %s for file %s (since modified: %s)", remoteIP, mountpoint, remoteFile, sinceModified)
				hasUpdatedFile = true
			}

			m.setMonitorRemoteHost(remoteIP, hasUpdatedFile)
			pollUpdateFunc(mountpoint, remoteIP, hasUpdatedFile)
		}

		// wait for next cycle or shutdown
		select {
		case <-time.After(m.monitorInterval):

		case <-shutdown:
			glog.Infof("no longer monitoring status for DFS volume %s", mountpoint)
			return
		}
	}
}

// fileSinceModified returns the duration of the file since last modified
func fileSinceModified(filename string) (time.Duration, error) {
	fileinfo, err := os.Stat(filename)
	if err != nil {
		glog.V(2).Infof("unable to stat file: %s %s", filename, err)
		return 0, err
	}

	timeSince := time.Now().Sub(fileinfo.ModTime())
	glog.V(4).Infof("file %s was modified %s ago", filename, timeSince)
	return timeSince, nil
}

// Get the variable determining if we will restart NFS when we detect a failure
func getShouldRestartDFSOnFailure() bool {
	shouldRestart := os.Getenv("SERVICED_MONITOR_DFS_MASTER_RESTART")
	if len(shouldRestart) == 0 {
		glog.Infof("defaulting SERVICED_MONITOR_DFS_MASTER_RESTART to true")
		return true
	} else if shouldRestart == "1" {
		return true
	} else {
		return false
	}
}

// getDefaultDFSMonitorRemoteInterval returns the duration specified in seconds for the env var SERVICED_DFS_MONITOR_REMOTE_UPDATE_INTERVAL
func getDefaultDFSMonitorRemoteInterval() time.Duration {
	return getEnvMinDuration("SERVICED_MONITOR_DFS_REMOTE_UPDATE_INTERVAL", 60, 60)
}

func getDefaultNFSMonitorMasterInterval() time.Duration {
	min := getDefaultDFSMonitorRemoteInterval()
	return getEnvMinDuration("SERVICED_MONITOR_DFS_MASTER_INTERVAL", 900, int32(min.Seconds())*2)
}

// getEnvMinDuration returns the time.Duration env var meeting minimum and default duration
func getEnvMinDuration(envvar string, def, min int32) time.Duration {
	duration := def
	envval := os.Getenv(envvar)
	if len(strings.TrimSpace(envval)) == 0 {
		// ignore unset envvar
	} else if intVal, intErr := strconv.ParseInt(envval, 0, 32); intErr != nil {
		glog.Warningf("ignoring invalid %s of '%s': %s", envvar, envval, intErr)
		duration = min
	} else if int32(intVal) < min {
		glog.Warningf("ignoring invalid %s of '%s' < minimum:%v seconds", envvar, envval, min)
	} else {
		duration = int32(intVal)
	}

	return time.Duration(duration) * time.Second
}

// StorageClientHostNode is the zookeeper node for a host
type StorageClientHostNode struct {
	host.Host
	version interface{}
}

// Version is an implementation of client.Node
func (v *StorageClientHostNode) Version() interface{} { return v.version }

// SetVersion is an implementation of client.Node
func (v *StorageClientHostNode) SetVersion(version interface{}) { v.version = version }

// getActiveRemoteHosts returns a slice of activeClientIPs
func getActiveRemoteHosts(conn client.Connection, storageClientsPath string, monitorInterval time.Duration) []string {
	// clients is not full list of remotes when called from server.go - retrieve our own list from zookeeper
	var err error
	remoteIPs, err := getAllRemoteHostsFromZookeeper(conn, storageClientsPath)
	if err != nil {
		return []string{}
	}
	glog.V(2).Infof("DFS remote IPs: %+v", remoteIPs)

	// determine active hosts
	var activeClientIPs []string
	now := time.Now()
	for _, clnt := range remoteIPs {
		cp := path.Join(storageClientsPath, clnt)
		glog.V(2).Infof("retrieving info for DFS for remoteIP %s at zookeeper node %s", clnt, cp)

		hnode := StorageClientHostNode{}
		err := conn.Get(cp, &hnode)
		if err != nil && err != client.ErrEmptyNode {
			glog.Errorf("DFS could not get remote host zookeeper node %s: %s", cp, err)
			continue
		}
		if hnode.Host.UpdatedAt.IsZero() {
			glog.Infof("DFS not monitoring non-active host %+v:  HostID:%v  UpdatedAt:%+v", clnt, hnode.ID, hnode.UpdatedAt)
			continue
		}

		elapsed := now.Sub(hnode.Host.UpdatedAt)
		glog.V(2).Infof("retrieved info for DFS for remoteIP %s  UpdatedAt:%s  elapsed:%s  monitorInterval:%s", clnt, hnode.Host.UpdatedAt, elapsed, monitorInterval)
		if elapsed > monitorInterval {
			glog.Infof("DFS not monitoring non-active host %+v:  HostID:%v  UpdatedAt:%+v  lastseen:%s ago", clnt, hnode.ID, hnode.UpdatedAt, elapsed)
			continue
		}

		activeClientIPs = append(activeClientIPs, clnt)
	}

	glog.V(2).Infof("DFS remote active IPs: %+v", activeClientIPs)
	return activeClientIPs
}

// getAllRemoteHostsFromZookeeper retrieves a list of remote storage clients from zookeeper
func getAllRemoteHostsFromZookeeper(conn client.Connection, storageClientsPath string) ([]string, error) {
	clients, err := conn.Children(storageClientsPath)
	if err != nil {
		glog.Errorf("unable to retrieve list of DFS remote hosts: %s", err)
		return []string{}, err
	}

	glog.V(4).Infof("DFS remote IPs: %+v", clients)
	return clients, nil
}
<commit_msg>ensure monitor path in DFS remote agents<commit_after>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storage

import (
	"io/ioutil"
	"os"
	"path"
	"reflect"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/control-center/serviced/coordinator/client"
	"github.com/control-center/serviced/domain/host"
	"github.com/zenoss/glog"
)

const monitorSubDir string = "/monitor"

// Monitor monitors the exporting of a file system to clients.
type Monitor struct {
	driver StorageDriver

	monitoredHosts     map[string]bool
	monitoredHostsLock sync.RWMutex

	shouldRestart   bool
	monitorInterval time.Duration
	previousRestart time.Time

	conn               client.Connection
	storageClientsPath string
}

// NewMonitor returns a Monitor object to monitor the exported file system
func NewMonitor(driver StorageDriver, monitorInterval time.Duration) (*Monitor, error) {
	m := &Monitor{
		driver:          driver,
		monitoredHosts:  make(map[string]bool),
		shouldRestart:   getShouldRestartDFSOnFailure(),
		monitorInterval: monitorInterval,
	}

	return m, nil
}

// UpdateRemoteMonitorFile is used by remote clients to write a tiny file to the DFS volume at the given cycle
func UpdateRemoteMonitorFile(localPath string, writeInterval time.Duration, ipAddr string, shutdown <-chan interface{}) {
	monitorPath := path.Join(localPath, monitorSubDir)
	remoteFile := path.Join(localPath, monitorSubDir, ipAddr)
	glog.Infof("updating DFS volume monitor file %s at write interval: %s", remoteFile, writeInterval)

	for {
		glog.V(2).Infof("checking DFS monitor path %s", monitorPath)
		_, err := os.Stat(monitorPath)
		if err != nil {
			glog.V(2).Infof("unable to stat DFS monitor path: %s %s", monitorPath, err)
			if err := os.MkdirAll(monitorPath, 0755); err != nil {
				glog.Warningf("unable to create DFS volume monitor path %s: %s", monitorPath, err)
			} else {
				glog.Infof("created DFS volume monitor path %s", monitorPath)
			}
		}

		glog.V(2).Infof("writing DFS file %s", remoteFile)
		if err := ioutil.WriteFile(remoteFile, []byte(ipAddr), 0600); err != nil {
			glog.Warningf("unable to write DFS file %s: %s", remoteFile, err)
		}

		// wait for next cycle or shutdown
		select {
		case <-time.After(writeInterval):

		case <-shutdown:
			glog.Infof("no longer writing remote monitor status for DFS volume %s to %s", localPath, remoteFile)
			return
		}
	}
}

// SetMonitorStorageClients sets the monitored remote IPs that are active
func (m *Monitor) SetMonitorStorageClients(conn client.Connection, storageClientsPath string) {
	m.conn = conn
	m.storageClientsPath = storageClientsPath
	m.SetMonitorRemoteHosts(getActiveRemoteHosts(conn, storageClientsPath, m.monitorInterval)...)
}

// SetMonitorRemoteHosts sets the remote hosts to monitor
func (m *Monitor) SetMonitorRemoteHosts(ipAddrs ...string) {
	if len(ipAddrs) == 0 {
		glog.Warningf("disabled DFS volume monitoring - no remote IPs to monitor")
	} else {
		glog.V(2).Infof("enabled DFS volume monitoring for remote IPs: %+v", ipAddrs)
	}
	m.monitoredHostsLock.Lock()
	defer m.monitoredHostsLock.Unlock()
	for _, ipAddr := range ipAddrs {
		m.monitoredHosts[ipAddr] = true
	}
}

// setMonitorRemoteHost updates the status of the monitored remote
func (m *Monitor) setMonitorRemoteHost(ipAddr string, hasUpdated bool) {
	m.monitoredHostsLock.Lock()
	defer m.monitoredHostsLock.Unlock()
	m.monitoredHosts[ipAddr] = hasUpdated
}

// getMonitorRemoteHosts returns the statuses of the monitored remotes
func (m *Monitor) getMonitorRemoteHosts() map[string]bool {
	m.monitoredHostsLock.RLock()
	defer m.monitoredHostsLock.RUnlock()
	return m.monitoredHosts
}

// ProcessDFSVolumePollUpdateFunc is called by MonitorDFSVolume when updates are detected
type DFSVolumeMonitorPollUpdateFunc func(mountpoint, remoteIP string, isUpdated bool)

// DFSVolumeMonitorPollUpdateFunc restarts nfs based on status of monitored remotes
func (m *Monitor) DFSVolumeMonitorPollUpdateFunc(mountpoint, remoteIP string, hasUpdatedFile bool) {
	// monitor dfs; log warnings each cycle; restart dfs if needed

	if hasUpdatedFile {
		return
	} else if len(m.getMonitorRemoteHosts()) == 0 {
		return
	}

	glog.Warningf("DFS NFS volume %s is not seen by remoteIP:%s - further action may be needed i.e: restart nfs", mountpoint, remoteIP)
	now := time.Now()
	since := now.Sub(m.previousRestart)
	if !m.shouldRestart {
		glog.Warningf("Not restarting DFS NFS service due to configuration setting: SERVICED_MONITOR_DFS_MASTER_RESTART=0")
		return
	} else if since < m.monitorInterval {
		glog.Warningf("Not restarting DFS NFS service - have not surpassed interval: %s since last restart", m.monitorInterval)
		return
	}

	m.previousRestart = now
	if err := m.driver.Restart(); err != nil {
		glog.Errorf("Error restarting DFS NFS service: %s", err)
	}
}

// MonitorVolume monitors the DFS volume - logs on failure and calls pollUpdateFunc
func (m *Monitor) MonitorDFSVolume(mountpoint string, shutdown <-chan interface{}, pollUpdateFunc DFSVolumeMonitorPollUpdateFunc) {
	glog.Infof("monitoring DFS export info for DFS volume %s at polling interval: %s", mountpoint, m.monitorInterval)

	m.previousRestart = time.Now()

	monitorPath := path.Join(mountpoint, monitorSubDir)
	os.RemoveAll(monitorPath)
	if err := os.MkdirAll(monitorPath, 0755); err != nil {
		glog.Errorf("no longer monitoring status for DFS volume %s - unable to mkdir %+v: %s", mountpoint, monitorPath, err)
		return
	}

	for {
		// check all active remotes
		remotes := m.getMonitorRemoteHosts()
		remoteIPs := make([]string, 0, len(remotes))
		for k := range remotes {
			remoteIPs = append(remoteIPs, k)
		}

		activeRemotes := remoteIPs
		if m.conn != nil && len(m.storageClientsPath) != 0 {
			activeRemotes = getActiveRemoteHosts(m.conn, m.storageClientsPath, m.monitorInterval)
			sort.Sort(sort.StringSlice(activeRemotes))
			sort.Sort(sort.StringSlice(remoteIPs))
			if !reflect.DeepEqual(activeRemotes, remoteIPs) {
				glog.Infof("DFS active remotes to be monitored has changed to: %+v", activeRemotes)
			}
			m.SetMonitorRemoteHosts(activeRemotes...)
		}

		for _, remoteIP := range activeRemotes {
			remoteFile := path.Join(monitorPath, remoteIP)
			sinceModified, err := fileSinceModified(remoteFile)
			hasUpdatedFile := false
			if err != nil {
				glog.Warningf("remote agent %s is not synced on DFS volume %s for file %s: %s", remoteIP, mountpoint, remoteFile, err)
			} else if sinceModified > m.monitorInterval {
				glog.Warningf("remote agent %s is not synced DFS volume %s for file %s (since modified: %s)", remoteIP, mountpoint, remoteFile, sinceModified)
			} else {
				glog.V(4).Infof("remote agent %s is synced on DFS volume %s for file %s (since modified: %s)", remoteIP, mountpoint, remoteFile, sinceModified)
				hasUpdatedFile = true
			}

			m.setMonitorRemoteHost(remoteIP, hasUpdatedFile)
			pollUpdateFunc(mountpoint, remoteIP, hasUpdatedFile)
		}

		// wait for next cycle or shutdown
		select {
		case <-time.After(m.monitorInterval):

		case <-shutdown:
			glog.Infof("no longer monitoring status for DFS volume %s", mountpoint)
			return
		}
	}
}

// fileSinceModified returns the duration of the file since last modified
func fileSinceModified(filename string) (time.Duration, error) {
	fileinfo, err := os.Stat(filename)
	if err != nil {
		glog.V(2).Infof("unable to stat file: %s %s", filename, err)
		return 0, err
	}

	timeSince := time.Now().Sub(fileinfo.ModTime())
	glog.V(4).Infof("file %s was modified %s ago", filename, timeSince)
	return timeSince, nil
}

// Get the variable determining if we will restart NFS when we detect a failure
func getShouldRestartDFSOnFailure() bool {
	shouldRestart := os.Getenv("SERVICED_MONITOR_DFS_MASTER_RESTART")
	if len(shouldRestart) == 0 {
		glog.Infof("defaulting SERVICED_MONITOR_DFS_MASTER_RESTART to true")
		return true
	} else if shouldRestart == "1" {
		return true
	} else {
		return false
	}
}

// getDefaultDFSMonitorRemoteInterval returns the duration specified in seconds for the env var SERVICED_DFS_MONITOR_REMOTE_UPDATE_INTERVAL
func getDefaultDFSMonitorRemoteInterval() time.Duration {
	return getEnvMinDuration("SERVICED_MONITOR_DFS_REMOTE_UPDATE_INTERVAL", 60, 60)
}

func getDefaultNFSMonitorMasterInterval() time.Duration {
	min := getDefaultDFSMonitorRemoteInterval()
	return getEnvMinDuration("SERVICED_MONITOR_DFS_MASTER_INTERVAL", 900, int32(min.Seconds())*2)
}

// getEnvMinDuration returns the time.Duration env var meeting minimum and default duration
func getEnvMinDuration(envvar string, def, min int32) time.Duration {
	duration := def
	envval := os.Getenv(envvar)
	if len(strings.TrimSpace(envval)) == 0 {
		// ignore unset envvar
	} else if intVal, intErr := strconv.ParseInt(envval, 0, 32); intErr != nil {
		glog.Warningf("ignoring invalid %s of '%s': %s", envvar, envval, intErr)
		duration = min
	} else if int32(intVal) < min {
		glog.Warningf("ignoring invalid %s of '%s' < minimum:%v seconds", envvar, envval, min)
	} else {
		duration = int32(intVal)
	}

	return time.Duration(duration) * time.Second
}

// StorageClientHostNode is the zookeeper node for a host
type StorageClientHostNode struct {
	host.Host
	version interface{}
}

// Version is an implementation of client.Node
func (v *StorageClientHostNode) Version() interface{} { return v.version }

// SetVersion is an implementation of client.Node
func (v *StorageClientHostNode) SetVersion(version interface{}) { v.version = version }

// getActiveRemoteHosts returns a slice of activeClientIPs
func getActiveRemoteHosts(conn client.Connection, storageClientsPath string, monitorInterval time.Duration) []string {
	// clients is not full list of remotes when called from server.go - retrieve our own list from zookeeper
	var err error
	remoteIPs, err := getAllRemoteHostsFromZookeeper(conn, storageClientsPath)
	if err != nil {
		return []string{}
	}
	glog.V(2).Infof("DFS remote IPs: %+v", remoteIPs)

	// determine active hosts
	var activeClientIPs []string
	now := time.Now()
	for _, clnt := range remoteIPs {
		cp := path.Join(storageClientsPath, clnt)
		glog.V(2).Infof("retrieving info for DFS for remoteIP %s at zookeeper node %s", clnt, cp)

		hnode := StorageClientHostNode{}
		err := conn.Get(cp, &hnode)
		if err != nil && err != client.ErrEmptyNode {
			glog.Errorf("DFS could not get remote host zookeeper node %s: %s", cp, err)
			continue
		}
		if hnode.Host.UpdatedAt.IsZero() {
			glog.Infof("DFS not monitoring non-active host %+v:  HostID:%v  UpdatedAt:%+v", clnt, hnode.ID, hnode.UpdatedAt)
			continue
		}

		elapsed := now.Sub(hnode.Host.UpdatedAt)
		glog.V(2).Infof("retrieved info for DFS for remoteIP %s  UpdatedAt:%s  elapsed:%s  monitorInterval:%s", clnt, hnode.Host.UpdatedAt, elapsed, monitorInterval)
		if elapsed > monitorInterval {
			glog.Infof("DFS not monitoring non-active host %+v:  HostID:%v  UpdatedAt:%+v  lastseen:%s ago", clnt, hnode.ID, hnode.UpdatedAt, elapsed)
			continue
		}

		activeClientIPs = append(activeClientIPs, clnt)
	}

	glog.V(2).Infof("DFS remote active IPs: %+v", activeClientIPs)
	return activeClientIPs
}

// getAllRemoteHostsFromZookeeper retrieves a list of remote storage clients from zookeeper
func getAllRemoteHostsFromZookeeper(conn client.Connection, storageClientsPath string) ([]string, error) {
	clients, err := conn.Children(storageClientsPath)
	if err != nil {
		glog.Errorf("unable to retrieve list of DFS remote hosts: %s", err)
		return []string{}, err
	}

	glog.V(4).Infof("DFS remote IPs: %+v", clients)
	return clients, nil
}
<|endoftext|>


<commit_before>// +build e2e

// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific la

package e2etest

import (
	"bytes"
	"context"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"testing"
	"time"

	"github.com/google/exposure-notifications-server/internal/database"
	exportapi "github.com/google/exposure-notifications-server/internal/export"
	"github.com/google/exposure-notifications-server/internal/integration"
	publishdb "github.com/google/exposure-notifications-server/internal/publish/database"
	"github.com/google/exposure-notifications-server/internal/storage"
	"github.com/google/exposure-notifications-server/pkg/secrets"
	"github.com/google/exposure-notifications-server/pkg/util"
	"github.com/sethvargo/go-envconfig"
	"github.com/sethvargo/go-retry"

	publishmodel "github.com/google/exposure-notifications-server/internal/publish/model"
	testutil "github.com/google/exposure-notifications-server/internal/utils"
	verifyapi "github.com/google/exposure-notifications-server/pkg/api/v1"
)

type testConfig struct {
	DbName      string `env:"DB_NAME"`
	ExposureURL string `env:"EXPOSURE_URL"`
	ProjectID   string `env:"PROJECT_ID"`
	DBConfig    *database.Config
}

func initConfig(tb testing.TB, ctx context.Context) *testConfig {
	c := &testConfig{}
	sm, err := secrets.SecretManagerFor(ctx, secrets.SecretManagerTypeGoogleSecretManager)
	if err != nil {
		tb.Fatalf("unable to connect to secret manager: %v", err)
	}
	if err := envconfig.ProcessWith(ctx, c, envconfig.OsLookuper(), secrets.Resolver(sm, &secrets.Config{})); err != nil {
		tb.Fatalf("Unable to process environment: %v", err)
	}
	return c
}

func publishKeys(payload *verifyapi.Publish, publishEndpoint string) (*verifyapi.PublishResponse, error) {
	j, err := json.Marshal(payload)
	if err != nil {
		return nil, fmt.Errorf("failed to marshal json: %w", err)
	}

	resp, err := http.Post(publishEndpoint, "application/json", bytes.NewReader(j))
	if err != nil {
		return nil, fmt.Errorf("failed to POST /publish: %w", err)
	}

	body, err := checkResp(resp)
	if err != nil {
		return nil, fmt.Errorf("failed to POST /publish: %w: %s", err, body)
	}

	var pubResponse verifyapi.PublishResponse
	if err := json.Unmarshal(body, &pubResponse); err != nil {
		return nil, fmt.Errorf("bad publish response")
	}

	return &pubResponse, nil
}

func checkResp(r *http.Response) ([]byte, error) {
	defer r.Body.Close()

	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return nil, fmt.Errorf("failed to read response body: %w", err)
	}

	if r.StatusCode != 200 {
		return nil, fmt.Errorf("response was not 200 OK: %s", body)
	}

	return body, nil
}

func TestPublishEndpoint(t *testing.T) {
	ctx := context.Background()

	tc := initConfig(t, ctx)
	if tc.ExposureURL == "" {
		t.Skip()
	}
	// Increase this so that the db connection won't be canceled while polling for exported files
	tc.DBConfig.PoolMaxConnIdle = 5 * time.Minute

	db, err := database.NewFromEnv(ctx, tc.DBConfig)
	if err != nil {
		t.Fatalf("unable to connect to database: %v", err)
	}
	jwtCfg, bucketName, filenameRoot, appName := integration.Seed(t, ctx, db, 2*time.Minute)

	keys := util.GenerateExposureKeys(3, -1, false)

	// Publish 3 keys
	payload := &verifyapi.Publish{
		Keys:              keys,
		HealthAuthorityID: appName,
	}

	var wantExport []string
	for _, key := range keys {
		wantExport = append(wantExport, key.Key)
	}

	jwtCfg.ExposureKeys = keys
	jwtCfg.JWTWarp = time.Duration(0)
	verification, salt := testutil.IssueJWT(t, jwtCfg)
	payload.VerificationPayload = verification
	payload.HMACKey = salt
	resp, err := publishKeys(payload, tc.ExposureURL+"/v1/publish")
	if err != nil {
		t.Fatalf("Failed publishing keys: \n\tResp: %v\n\t%v", resp, err)
	}

	criteria := publishdb.IterateExposuresCriteria{
		OnlyLocalProvenance: false,
	}
	exposures, err := getExposures(db, criteria)
	if err != nil {
		t.Fatalf("Failed getting exposures: %v", err)
	}

	keysPublished := make(map[string]bool)
	for _, e := range exposures {
		strKey := base64.StdEncoding.EncodeToString(e.ExposureKey)
		keysPublished[strKey] = true
	}

	for _, want := range keys {
		if _, ok := keysPublished[want.Key]; !ok {
			t.Fatalf("Want published key %q not exist in exposures", want.Key)
		}
	}

	blobStore, err := storage.NewGoogleCloudStorage(ctx)
	if err != nil {
		t.Fatal(err)
	}
	gotExported := make(map[string]bool)
	integration.Eventually(t, 30, 10*time.Second, func() error {
		// Attempt to get the index
		index, err := blobStore.GetObject(ctx, bucketName, integration.IndexFilePath(filenameRoot))
		if err != nil {
			if errors.Is(err, storage.ErrNotFound) {
				return retry.RetryableError(fmt.Errorf("Can not find index file: %w", err))
			}
			return err
		}

		// Find the latest file in the index
		lines := strings.Split(string(index), "\n")
		latest := ""
		for _, entry := range lines {
			if strings.HasSuffix(entry, "zip") {
				if entry > latest {
					latest = entry
				}
			}
		}
		if latest == "" {
			return retry.RetryableError(errors.New("failed to find latest export"))
		}

		// Download the latest export file contents
		data, err := blobStore.GetObject(ctx, bucketName, latest)
		if err != nil {
			return fmt.Errorf("failed to open %s/%s: %w", bucketName, latest, err)
		}

		// Process contents as an export
		key, _, err := exportapi.UnmarshalExportFile(data)
		if err != nil {
			return fmt.Errorf("failed to extract export data: %w", err)
		}

		for _, k := range key.Keys {
			gotExported[base64.StdEncoding.EncodeToString(k.KeyData)] = true
		}
		allExported := true
		for _, want := range wantExport {
			if _, ok := gotExported[want]; !ok {
				allExported = false
			}
		}
		if !allExported {
			return retry.RetryableError(errors.New("Not all keys are exported yet, keep waiting"))
		}

		return nil
	})
}

// getExposures finds the exposures that match the given criteria.
func getExposures(db *database.DB, criteria publishdb.IterateExposuresCriteria) ([]*publishmodel.Exposure, error) {
	ctx := context.Background()
	var exposures []*publishmodel.Exposure
	if _, err := publishdb.New(db).IterateExposures(ctx, criteria, func(m *publishmodel.Exposure) error {
		exposures = append(exposures, m)
		return nil
	}); err != nil {
		return nil, err
	}

	return exposures, nil
}
<commit_msg>E2E test: add cleanup scenario (#1071)<commit_after>// +build e2e

// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific la

package e2etest

import (
	"bytes"
	"context"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"testing"
	"time"

	"github.com/google/exposure-notifications-server/internal/database"
	exportapi "github.com/google/exposure-notifications-server/internal/export"
	exportdb "github.com/google/exposure-notifications-server/internal/export/database"
	"github.com/google/exposure-notifications-server/internal/integration"
	publishdb "github.com/google/exposure-notifications-server/internal/publish/database"
	"github.com/google/exposure-notifications-server/internal/storage"
	"github.com/google/exposure-notifications-server/pkg/secrets"
	"github.com/google/exposure-notifications-server/pkg/util"
	"github.com/google/go-cmp/cmp"
	pgx "github.com/jackc/pgx/v4"
	"github.com/sethvargo/go-envconfig"
	"github.com/sethvargo/go-retry"

	publishmodel "github.com/google/exposure-notifications-server/internal/publish/model"
	testutil "github.com/google/exposure-notifications-server/internal/utils"
	verifyapi "github.com/google/exposure-notifications-server/pkg/api/v1"
)

type testConfig struct {
	DbName      string `env:"DB_NAME"`
	ExposureURL string `env:"EXPOSURE_URL"`
	ProjectID   string `env:"PROJECT_ID"`
	DBConfig    *database.Config
}

func initConfig(tb testing.TB, ctx context.Context) *testConfig {
	c := &testConfig{}
	sm, err := secrets.SecretManagerFor(ctx, secrets.SecretManagerTypeGoogleSecretManager)
	if err != nil {
		tb.Fatalf("unable to connect to secret manager: %v", err)
	}
	if err := envconfig.ProcessWith(ctx, c, envconfig.OsLookuper(), secrets.Resolver(sm, &secrets.Config{})); err != nil {
		tb.Fatalf("Unable to process environment: %v", err)
	}
	return c
}

func publishKeys(payload *verifyapi.Publish, publishEndpoint string) (*verifyapi.PublishResponse, error) {
	j, err := json.Marshal(payload)
	if err != nil {
		return nil, fmt.Errorf("failed to marshal json: %w", err)
	}

	resp, err := http.Post(publishEndpoint, "application/json", bytes.NewReader(j))
	if err != nil {
		return nil, fmt.Errorf("failed to POST /publish: %w", err)
	}

	body, err := checkResp(resp)
	if err != nil {
		return nil, fmt.Errorf("failed to POST /publish: %w: %s", err, body)
	}

	var pubResponse verifyapi.PublishResponse
	if err := json.Unmarshal(body, &pubResponse); err != nil {
		return nil, fmt.Errorf("bad publish response")
	}

	return &pubResponse, nil
}

func checkResp(r *http.Response) ([]byte, error) {
	defer r.Body.Close()

	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return nil, fmt.Errorf("failed to read response body: %w", err)
	}

	if r.StatusCode != 200 {
		return nil, fmt.Errorf("response was not 200 OK: %s", body)
	}

	return body, nil
}

func TestPublishEndpoint(t *testing.T) {
	ctx := context.Background()

	tc := initConfig(t, ctx)
	if tc.ExposureURL == "" {
		t.Skip()
	}
	// Increase this so that the db connection won't be canceled while polling for exported files
	tc.DBConfig.PoolMaxConnIdle = 10 * time.Minute

	db, err := database.NewFromEnv(ctx, tc.DBConfig)
	if err != nil {
		t.Fatalf("unable to connect to database: %v", err)
	}
	jwtCfg, bucketName, filenameRoot, appName := integration.Seed(t, ctx, db, 2*time.Minute)

	keys := util.GenerateExposureKeys(3, -1, false)

	// Publish 3 keys
	payload := &verifyapi.Publish{
		Keys:              keys,
		HealthAuthorityID: appName,
	}

	wantExport := make(map[string]bool)
	for _, key := range keys {
		wantExport[key.Key] = true
	}

	jwtCfg.ExposureKeys = keys
	jwtCfg.JWTWarp = time.Duration(0)
	verification, salt := testutil.IssueJWT(t, jwtCfg)
	payload.VerificationPayload = verification
	payload.HMACKey = salt
	resp, err := publishKeys(payload, tc.ExposureURL+"/v1/publish")
	if err != nil {
		t.Fatalf("Failed publishing keys: \n\tResp: %v\n\t%v", resp, err)
	}

	criteria := publishdb.IterateExposuresCriteria{
		OnlyLocalProvenance: false,
	}
	exposures, err := getExposures(db, criteria)
	if err != nil {
		t.Fatalf("Failed getting exposures: %v", err)
	}

	keysPublished := make(map[string]bool)
	for _, e := range exposures {
		strKey := base64.StdEncoding.EncodeToString(e.ExposureKey)
		keysPublished[strKey] = true
	}

	for _, want := range keys {
		if _, ok := keysPublished[want.Key]; !ok {
			t.Fatalf("Want published key %q not exist in exposures", want.Key)
		}
	}

	blobStore, err := storage.NewGoogleCloudStorage(ctx)
	if err != nil {
		t.Fatal(err)
	}
	gotExported := make(map[string]bool)
	var expectedFile string
	integration.Eventually(t, 30, 10*time.Second, func() error {
		// Attempt to get the index
		index, err := blobStore.GetObject(ctx, bucketName, integration.IndexFilePath(filenameRoot))
		if err != nil {
			if errors.Is(err, storage.ErrNotFound) {
				return retry.RetryableError(fmt.Errorf("Can not find index file: %w", err))
			}
			return err
		}

		// Find the latest file in the index
		lines := strings.Split(string(index), "\n")
		latest := ""
		for _, entry := range lines {
			if strings.HasSuffix(entry, "zip") {
				if entry > latest {
					latest = entry
				}
			}
		}
		if latest == "" {
			return retry.RetryableError(errors.New("failed to find latest export"))
		}

		// Download the latest export file contents
		data, err := blobStore.GetObject(ctx, bucketName, latest)
		if err != nil {
			return fmt.Errorf("failed to open %s/%s: %w", bucketName, latest, err)
		}

		// Process contents as an export
		key, _, err := exportapi.UnmarshalExportFile(data)
		if err != nil {
			return fmt.Errorf("failed to extract export data: %w", err)
		}

		for _, k := range key.Keys {
			sk := base64.StdEncoding.EncodeToString(k.KeyData)
			if _, ok := wantExport[sk]; ok {
				expectedFile = latest
				gotExported[sk] = true
			}
		}
		if diff := cmp.Diff(wantExport, gotExported); diff != "" {
			return retry.RetryableError(errors.New("Not all keys are exported yet, keep waiting"))
		}

		return nil
	})

	// Mark the export in the past to force a cleanup
	t.Logf("Exposure was exported: %q. Modify the date to 14 days earlier", expectedFile)
	exportFile, err := exportdb.New(db).LookupExportFile(ctx, expectedFile)
	if err != nil {
		t.Fatal(err)
	}
	if err := db.InTx(ctx, pgx.Serializable, func(tx pgx.Tx) error {
		result, err := tx.Exec(ctx, `
		UPDATE
			ExportBatch
		SET
			start_timestamp = $1,
			end_timestamp = $2
		WHERE
			batch_id = $3
	`,
			time.Now().Add(-30*24*time.Hour),
			time.Now().Add(-29*24*time.Hour),
			exportFile.BatchID,
		)
		if err != nil {
			return err
		}
		if got, want := result.RowsAffected(), int64(1); got != want {
			return fmt.Errorf("expected %v to be %v", got, want)
		}
		return nil
	}); err != nil {
		t.Fatal(err)
	}

	// Ensure the export was deleted
	// Cloud scheduler runs every minute, waiting 2.5 minutes should be sufficient
	t.Logf("Wait until export %q is cleaned up", expectedFile)
	integration.Eventually(t, 30, 10*time.Second, func() error {
		// Attempt to get the index
		index, err := blobStore.GetObject(ctx, bucketName, integration.IndexFilePath(filenameRoot))
		if err != nil {
			if errors.Is(err, storage.ErrNotFound) {
				return retry.RetryableError(err)
			}
			return err
		}

		// Ensure the new export is created
		batchFiles := strings.Split(string(index), "\n")
		for _, f := range batchFiles {
			if f != exportFile.Filename {
				continue
			}

			// Lookup the file, hope it's gone
			if _, err := blobStore.GetObject(ctx, bucketName, f); err != nil {
				if errors.Is(err, storage.ErrNotFound) {
					return nil // expected
				}
				return err
			}

			return retry.RetryableError(fmt.Errorf("export file still exists"))
		}

		return nil
	})
}

// getExposures finds the exposures that match the given criteria.
func getExposures(db *database.DB, criteria publishdb.IterateExposuresCriteria) ([]*publishmodel.Exposure, error) {
	ctx := context.Background()
	var exposures []*publishmodel.Exposure
	if _, err := publishdb.New(db).IterateExposures(ctx, criteria, func(m *publishmodel.Exposure) error {
		exposures = append(exposures, m)
		return nil
	}); err != nil {
		return nil, err
	}

	return exposures, nil
}
<|endoftext|>


<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package frontend

import (
	"context"
	"errors"
	"fmt"
	"html"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"time"

	"github.com/google/safehtml/template"
	"github.com/google/safehtml/template/uncheckedconversions"
	"golang.org/x/pkgsite/internal"
	"golang.org/x/pkgsite/internal/cookie"
	"golang.org/x/pkgsite/internal/derrors"
	"golang.org/x/pkgsite/internal/experiment"
	"golang.org/x/pkgsite/internal/log"
	"golang.org/x/pkgsite/internal/postgres"
	"golang.org/x/pkgsite/internal/stdlib"
	"golang.org/x/pkgsite/internal/version"
)

// errUnitNotFoundWithoutFetch returns a 404 with instructions to the user on
// how to manually fetch the package. No fetch button is provided. This is used
// for very large modules or modules that previously 500ed.
var errUnitNotFoundWithoutFetch = &serverError{
	status: http.StatusNotFound,
	epage: &errorPage{
		messageTemplate: template.MakeTrustedTemplate(`

{{.StatusText}}

Check that you entered the URL correctly or try fetching it following the
                        instructions here.
`),
		MessageData: struct{ StatusText string }{http.StatusText(http.StatusNotFound)},
	},
}

// servePathNotFoundPage serves a 404 page for the requested path, or redirects
// the user to an appropriate location.
func (s *Server) servePathNotFoundPage(w http.ResponseWriter, r *http.Request,
	ds internal.DataSource, fullPath, modulePath, requestedVersion string) (err error) {
	defer derrors.Wrap(&err, "servePathNotFoundPage(w, r, %q, %q)", fullPath, requestedVersion)

	db, ok := ds.(*postgres.DB)
	if !ok {
		return datasourceNotSupportedErr()
	}
	ctx := r.Context()

	if stdlib.Contains(fullPath) {
		var path string
		path, err = stdlibPathForShortcut(ctx, db, fullPath)
		if err != nil {
			// Log the error, but prefer a "path not found" error for a
			// better user experience.
			log.Error(ctx, err)
		}
		if path != "" {
			http.Redirect(w, r, fmt.Sprintf("/%s", path), http.StatusFound)
			return
		}

		if experiment.IsActive(ctx, internal.ExperimentEnableStdFrontendFetch) {
			return &serverError{
				status: http.StatusNotFound,
				epage: &errorPage{
					templateName: "fetch",
					MessageData:  stdlib.ModulePath,
				},
			}
		}
		return &serverError{status: http.StatusNotFound}
	}

	fr, err := previousFetchStatusAndResponse(ctx, db, fullPath, modulePath, requestedVersion)
	if err != nil {
		// If an error occurred, it means that we have never tried to fetch
		// this path before or an error occurred when we tried to
		// gather data about this 404.
		//
		// If the latter, log the error.
		// In either case, give the user the option to fetch that path.
		if !errors.Is(err, derrors.NotFound) {
			log.Error(ctx, err)
		}
		return pathNotFoundError(ctx, fullPath, requestedVersion)
	}

	// If we've reached this point, we know that we've seen this path before.
	// Show a relevant page or redirect the use based on the previous fetch
	// response.
	switch fr.status {
	case http.StatusOK, derrors.ToStatus(derrors.HasIncompletePackages):
		// We will only reach a 2xx status if we found a row in version_map
		// matching exactly the requested path.
		if fr.resolvedVersion != requestedVersion {
			u := constructUnitURL(fullPath, fr.goModPath, fr.resolvedVersion)
			http.Redirect(w, r, u, http.StatusFound)
			return
		}
		// For some reason version_map is telling us that the path@version
		// exists, but earlier in this flow we didn't find it in the units
		// table.
		//
		// Return the fetch page so the user can try requesting again, and log
		// an error.
		log.Errorf(ctx, "version_map reports that %s@%s has status=%d, but this was not found before reaching servePathNotFoundPage", fullPath, requestedVersion)
		return pathNotFoundError(ctx, fullPath, requestedVersion)
	case http.StatusFound, derrors.ToStatus(derrors.AlternativeModule):
		if fr.goModPath == fullPath {
			// The redirectPath and the fullpath are the same. Do not redirect
			// to avoid ending up in a loop.
			return errUnitNotFoundWithoutFetch
		}
		vm, err := db.GetVersionMap(ctx, fr.goModPath, version.Latest)
		if (err != nil && !errors.Is(err, derrors.NotFound)) ||
			(vm != nil && vm.Status != http.StatusOK) {
			// We attempted to fetch the canonical module path before and were
			// not successful. Do not redirect this request.
			return errUnitNotFoundWithoutFetch
		}
		u := constructUnitURL(fr.goModPath, fr.goModPath, version.Latest)
		cookie.Set(w, cookie.AlternativeModuleFlash, fullPath, u)
		http.Redirect(w, r, u, http.StatusFound)
		return nil
	case http.StatusInternalServerError:
		return pathNotFoundError(ctx, fullPath, requestedVersion)
	default:
		if u := githubPathRedirect(fullPath); u != "" {
			http.Redirect(w, r, u, http.StatusFound)
			return
		}

		// If a module has a status of 404, but s.taskIDChangeInterval has
		// passed, allow the module to be refetched.
		if fr.status == http.StatusNotFound && time.Since(fr.updatedAt) > s.taskIDChangeInterval {
			return pathNotFoundError(ctx, fullPath, requestedVersion)
		}

		// Redirect to the search result page for an empty directory that is above nested modules.
		// See https://golang.org/issue/43725 for context.
		nm, err := ds.GetNestedModules(ctx, fullPath)
		if err == nil && len(nm) > 0 {
			http.Redirect(w, r, "/search?q="+url.QueryEscape(fullPath), http.StatusFound)
			return nil
		}
		return &serverError{
			status: fr.status,
			epage: &errorPage{
				messageTemplate: uncheckedconversions.TrustedTemplateFromStringKnownToSatisfyTypeContract(`

{{.StatusText}}

` + html.UnescapeString(fr.responseText) + `
`),
				MessageData: struct{ StatusText string }{http.StatusText(fr.status)},
			},
		}
	}
}

// githubRegexp is regex to match a GitHub URL scheme containing a "/blob" or
// "/tree" element.
var githubRegexp = regexp.MustCompile(`(blob|tree)(/[^/]+)?`)

func githubPathRedirect(fullPath string) string {
	parts := strings.Split(fullPath, "/")
	if len(parts) <= 3 || parts[0] != "github.com" {
		return ""
	}
	m := strings.Split(fullPath, "/"+githubRegexp.FindString(fullPath))
	if len(m) != 2 {
		return ""
	}
	p := m[0]
	if m[1] != "" {
		p = m[0] + m[1]
	}
	return constructUnitURL(p, p, version.Latest)
}

// pathNotFoundError returns a page with an option on how to
// add a package or module to the site.
func pathNotFoundError(ctx context.Context, fullPath, requestedVersion string) error {
	if !isSupportedVersion(fullPath, requestedVersion) {
		return invalidVersionError(fullPath, requestedVersion)
	}
	if stdlib.Contains(fullPath) {
		if experiment.IsActive(ctx, internal.ExperimentEnableStdFrontendFetch) {
			return &serverError{
				status: http.StatusNotFound,
				epage: &errorPage{
					templateName: "fetch",
					MessageData:  stdlib.ModulePath,
				},
			}
		}
		return &serverError{status: http.StatusNotFound}
	}
	path := fullPath
	if requestedVersion != version.Latest {
		path = fmt.Sprintf("%s@%s", fullPath, requestedVersion)
	}
	return &serverError{
		status: http.StatusNotFound,
		epage: &errorPage{
			templateName: "fetch",
			MessageData:  path,
		},
	}
}

// previousFetchStatusAndResponse returns the fetch result from a
// previous fetch of the fullPath and requestedVersion.
func previousFetchStatusAndResponse(ctx context.Context, db *postgres.DB,
	fullPath, modulePath, requestedVersion string) (_ *fetchResult, err error) {
	defer derrors.Wrap(&err, "previousFetchStatusAndResponse(w, r, %q, %q)", fullPath, requestedVersion)

	// Get all candidate module paths for this path.
	paths, err := modulePathsToFetch(ctx, db, fullPath, modulePath)
	if err != nil {
		return nil, err
	}
	// Check if a row exists in the version_map table for the longest candidate
	// path and version.
	//
	// If we have not fetched the path before, a derrors.NotFound will be
	// returned.
	vm, err := db.GetVersionMap(ctx, paths[0], requestedVersion)
	if err != nil {
		return nil, err
	}
	// If the row has been fetched before, and the result was either a 490,
	// 491, or 5xx, return that result, since it is a final state.
	if vm != nil {
		fr := &fetchResult{
			modulePath: vm.ModulePath,
			goModPath:  vm.GoModPath,
			status:     vm.Status,
			err:        errors.New(vm.Error),
		}
		if vm.Status >= 200 && vm.Status < 300 {
			fr.resolvedVersion = vm.ResolvedVersion
			return fr, nil
		}
		if vm.Status >= 500 ||
			vm.Status == derrors.ToStatus(derrors.AlternativeModule) ||
			vm.Status == derrors.ToStatus(derrors.BadModule) {
			return resultFromFetchRequest([]*fetchResult{fr}, fullPath, requestedVersion)
		}
	}

	// Check if the unit path exists at a higher major version.
	// For example, my.module might not exist, but my.module/v3 might.
	// Similarly, my.module/foo might not exist, but my.module/v3/foo might.
	// In either case, the user will be redirected to the highest major version
	// of the path.
	//
	// Do not bother to look for a specific version if this case. If
	// my.module/foo@v2.1.0 was requested, and my.module/foo/v2 exists, just
	// return the latest version of my.module/foo/v2.
	//
	// Only redirect if the majPath returned is different from the fullPath, and
	// the majPath is not at v1. We don't want to redirect my.module/foo/v3 to
	// my.module/foo, or my.module/foo@v1.5.2 to my.module/foo@v1.0.0.
	majPath, maj, err := db.GetLatestMajorPathForV1Path(ctx, fullPath)
	if err != nil && err != derrors.NotFound {
		return nil, err
	}
	if majPath != fullPath && maj != 1 && majPath != "" {
		return &fetchResult{
			modulePath: majPath,
			goModPath:  majPath,
			status:     http.StatusFound,
		}, nil
	}
	vms, err := db.GetVersionMaps(ctx, paths, requestedVersion)
	if err != nil {
		return nil, err
	}
	var fetchResults []*fetchResult
	for _, vm := range vms {
		fr := fetchResultFromVersionMap(vm)
		fetchResults = append(fetchResults, fr)
		if vm.Status == http.StatusOK || vm.Status == 290 {
			fr.err = errPathDoesNotExistInModule
		}
	}
	if len(fetchResults) == 0 {
		return nil, derrors.NotFound
	}
	return resultFromFetchRequest(fetchResults, fullPath, requestedVersion)
}

func fetchResultFromVersionMap(vm *internal.VersionMap) *fetchResult {
	var err error
	if vm.Error != "" {
		err = errors.New(vm.Error)
	}
	return &fetchResult{
		modulePath: vm.ModulePath,
		goModPath:  vm.GoModPath,
		status:     vm.Status,
		updatedAt:  vm.UpdatedAt,
		err:        err,
	}
}
<commit_msg>internal/frontend: add missing arg to log.Errorf<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package frontend

import (
	"context"
	"errors"
	"fmt"
	"html"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"time"

	"github.com/google/safehtml/template"
	"github.com/google/safehtml/template/uncheckedconversions"
	"golang.org/x/pkgsite/internal"
	"golang.org/x/pkgsite/internal/cookie"
	"golang.org/x/pkgsite/internal/derrors"
	"golang.org/x/pkgsite/internal/experiment"
	"golang.org/x/pkgsite/internal/log"
	"golang.org/x/pkgsite/internal/postgres"
	"golang.org/x/pkgsite/internal/stdlib"
	"golang.org/x/pkgsite/internal/version"
)

// errUnitNotFoundWithoutFetch returns a 404 with instructions to the user on
// how to manually fetch the package. No fetch button is provided. This is used
// for very large modules or modules that previously 500ed.
var errUnitNotFoundWithoutFetch = &serverError{
	status: http.StatusNotFound,
	epage: &errorPage{
		messageTemplate: template.MakeTrustedTemplate(`

{{.StatusText}}

Check that you entered the URL correctly or try fetching it following the
                        instructions here.
`),
		MessageData: struct{ StatusText string }{http.StatusText(http.StatusNotFound)},
	},
}

// servePathNotFoundPage serves a 404 page for the requested path, or redirects
// the user to an appropriate location.
func (s *Server) servePathNotFoundPage(w http.ResponseWriter, r *http.Request,
	ds internal.DataSource, fullPath, modulePath, requestedVersion string) (err error) {
	defer derrors.Wrap(&err, "servePathNotFoundPage(w, r, %q, %q)", fullPath, requestedVersion)

	db, ok := ds.(*postgres.DB)
	if !ok {
		return datasourceNotSupportedErr()
	}
	ctx := r.Context()

	if stdlib.Contains(fullPath) {
		var path string
		path, err = stdlibPathForShortcut(ctx, db, fullPath)
		if err != nil {
			// Log the error, but prefer a "path not found" error for a
			// better user experience.
			log.Error(ctx, err)
		}
		if path != "" {
			http.Redirect(w, r, fmt.Sprintf("/%s", path), http.StatusFound)
			return
		}

		if experiment.IsActive(ctx, internal.ExperimentEnableStdFrontendFetch) {
			return &serverError{
				status: http.StatusNotFound,
				epage: &errorPage{
					templateName: "fetch",
					MessageData:  stdlib.ModulePath,
				},
			}
		}
		return &serverError{status: http.StatusNotFound}
	}

	fr, err := previousFetchStatusAndResponse(ctx, db, fullPath, modulePath, requestedVersion)
	if err != nil {
		// If an error occurred, it means that we have never tried to fetch
		// this path before or an error occurred when we tried to
		// gather data about this 404.
		//
		// If the latter, log the error.
		// In either case, give the user the option to fetch that path.
		if !errors.Is(err, derrors.NotFound) {
			log.Error(ctx, err)
		}
		return pathNotFoundError(ctx, fullPath, requestedVersion)
	}

	// If we've reached this point, we know that we've seen this path before.
	// Show a relevant page or redirect the use based on the previous fetch
	// response.
	switch fr.status {
	case http.StatusOK, derrors.ToStatus(derrors.HasIncompletePackages):
		// We will only reach a 2xx status if we found a row in version_map
		// matching exactly the requested path.
		if fr.resolvedVersion != requestedVersion {
			u := constructUnitURL(fullPath, fr.goModPath, fr.resolvedVersion)
			http.Redirect(w, r, u, http.StatusFound)
			return
		}
		// For some reason version_map is telling us that the path@version
		// exists, but earlier in this flow we didn't find it in the units
		// table.
		//
		// Return the fetch page so the user can try requesting again, and log
		// an error.
		log.Errorf(ctx, "version_map reports that %s@%s has status=%d, but this was not found before reaching servePathNotFoundPage",
			fullPath, requestedVersion, fr.status)
		return pathNotFoundError(ctx, fullPath, requestedVersion)
	case http.StatusFound, derrors.ToStatus(derrors.AlternativeModule):
		if fr.goModPath == fullPath {
			// The redirectPath and the fullpath are the same. Do not redirect
			// to avoid ending up in a loop.
			return errUnitNotFoundWithoutFetch
		}
		vm, err := db.GetVersionMap(ctx, fr.goModPath, version.Latest)
		if (err != nil && !errors.Is(err, derrors.NotFound)) ||
			(vm != nil && vm.Status != http.StatusOK) {
			// We attempted to fetch the canonical module path before and were
			// not successful. Do not redirect this request.
			return errUnitNotFoundWithoutFetch
		}
		u := constructUnitURL(fr.goModPath, fr.goModPath, version.Latest)
		cookie.Set(w, cookie.AlternativeModuleFlash, fullPath, u)
		http.Redirect(w, r, u, http.StatusFound)
		return nil
	case http.StatusInternalServerError:
		return pathNotFoundError(ctx, fullPath, requestedVersion)
	default:
		if u := githubPathRedirect(fullPath); u != "" {
			http.Redirect(w, r, u, http.StatusFound)
			return
		}

		// If a module has a status of 404, but s.taskIDChangeInterval has
		// passed, allow the module to be refetched.
		if fr.status == http.StatusNotFound && time.Since(fr.updatedAt) > s.taskIDChangeInterval {
			return pathNotFoundError(ctx, fullPath, requestedVersion)
		}

		// Redirect to the search result page for an empty directory that is above nested modules.
		// See https://golang.org/issue/43725 for context.
		nm, err := ds.GetNestedModules(ctx, fullPath)
		if err == nil && len(nm) > 0 {
			http.Redirect(w, r, "/search?q="+url.QueryEscape(fullPath), http.StatusFound)
			return nil
		}
		return &serverError{
			status: fr.status,
			epage: &errorPage{
				messageTemplate: uncheckedconversions.TrustedTemplateFromStringKnownToSatisfyTypeContract(`

{{.StatusText}}

` + html.UnescapeString(fr.responseText) + `
`),
				MessageData: struct{ StatusText string }{http.StatusText(fr.status)},
			},
		}
	}
}

// githubRegexp is regex to match a GitHub URL scheme containing a "/blob" or
// "/tree" element.
var githubRegexp = regexp.MustCompile(`(blob|tree)(/[^/]+)?`)

func githubPathRedirect(fullPath string) string {
	parts := strings.Split(fullPath, "/")
	if len(parts) <= 3 || parts[0] != "github.com" {
		return ""
	}
	m := strings.Split(fullPath, "/"+githubRegexp.FindString(fullPath))
	if len(m) != 2 {
		return ""
	}
	p := m[0]
	if m[1] != "" {
		p = m[0] + m[1]
	}
	return constructUnitURL(p, p, version.Latest)
}

// pathNotFoundError returns a page with an option on how to
// add a package or module to the site.
func pathNotFoundError(ctx context.Context, fullPath, requestedVersion string) error {
	if !isSupportedVersion(fullPath, requestedVersion) {
		return invalidVersionError(fullPath, requestedVersion)
	}
	if stdlib.Contains(fullPath) {
		if experiment.IsActive(ctx, internal.ExperimentEnableStdFrontendFetch) {
			return &serverError{
				status: http.StatusNotFound,
				epage: &errorPage{
					templateName: "fetch",
					MessageData:  stdlib.ModulePath,
				},
			}
		}
		return &serverError{status: http.StatusNotFound}
	}
	path := fullPath
	if requestedVersion != version.Latest {
		path = fmt.Sprintf("%s@%s", fullPath, requestedVersion)
	}
	return &serverError{
		status: http.StatusNotFound,
		epage: &errorPage{
			templateName: "fetch",
			MessageData:  path,
		},
	}
}

// previousFetchStatusAndResponse returns the fetch result from a
// previous fetch of the fullPath and requestedVersion.
func previousFetchStatusAndResponse(ctx context.Context, db *postgres.DB,
	fullPath, modulePath, requestedVersion string) (_ *fetchResult, err error) {
	defer derrors.Wrap(&err, "previousFetchStatusAndResponse(w, r, %q, %q)", fullPath, requestedVersion)

	// Get all candidate module paths for this path.
	paths, err := modulePathsToFetch(ctx, db, fullPath, modulePath)
	if err != nil {
		return nil, err
	}
	// Check if a row exists in the version_map table for the longest candidate
	// path and version.
	//
	// If we have not fetched the path before, a derrors.NotFound will be
	// returned.
	vm, err := db.GetVersionMap(ctx, paths[0], requestedVersion)
	if err != nil {
		return nil, err
	}
	// If the row has been fetched before, and the result was either a 490,
	// 491, or 5xx, return that result, since it is a final state.
	if vm != nil {
		fr := &fetchResult{
			modulePath: vm.ModulePath,
			goModPath:  vm.GoModPath,
			status:     vm.Status,
			err:        errors.New(vm.Error),
		}
		if vm.Status >= 200 && vm.Status < 300 {
			fr.resolvedVersion = vm.ResolvedVersion
			return fr, nil
		}
		if vm.Status >= 500 ||
			vm.Status == derrors.ToStatus(derrors.AlternativeModule) ||
			vm.Status == derrors.ToStatus(derrors.BadModule) {
			return resultFromFetchRequest([]*fetchResult{fr}, fullPath, requestedVersion)
		}
	}

	// Check if the unit path exists at a higher major version.
	// For example, my.module might not exist, but my.module/v3 might.
	// Similarly, my.module/foo might not exist, but my.module/v3/foo might.
	// In either case, the user will be redirected to the highest major version
	// of the path.
	//
	// Do not bother to look for a specific version if this case. If
	// my.module/foo@v2.1.0 was requested, and my.module/foo/v2 exists, just
	// return the latest version of my.module/foo/v2.
	//
	// Only redirect if the majPath returned is different from the fullPath, and
	// the majPath is not at v1. We don't want to redirect my.module/foo/v3 to
	// my.module/foo, or my.module/foo@v1.5.2 to my.module/foo@v1.0.0.
	majPath, maj, err := db.GetLatestMajorPathForV1Path(ctx, fullPath)
	if err != nil && err != derrors.NotFound {
		return nil, err
	}
	if majPath != fullPath && maj != 1 && majPath != "" {
		return &fetchResult{
			modulePath: majPath,
			goModPath:  majPath,
			status:     http.StatusFound,
		}, nil
	}
	vms, err := db.GetVersionMaps(ctx, paths, requestedVersion)
	if err != nil {
		return nil, err
	}
	var fetchResults []*fetchResult
	for _, vm := range vms {
		fr := fetchResultFromVersionMap(vm)
		fetchResults = append(fetchResults, fr)
		if vm.Status == http.StatusOK || vm.Status == 290 {
			fr.err = errPathDoesNotExistInModule
		}
	}
	if len(fetchResults) == 0 {
		return nil, derrors.NotFound
	}
	return resultFromFetchRequest(fetchResults, fullPath, requestedVersion)
}

func fetchResultFromVersionMap(vm *internal.VersionMap) *fetchResult {
	var err error
	if vm.Error != "" {
		err = errors.New(vm.Error)
	}
	return &fetchResult{
		modulePath: vm.ModulePath,
		goModPath:  vm.GoModPath,
		status:     vm.Status,
		updatedAt:  vm.UpdatedAt,
		err:        err,
	}
}
<|endoftext|>