<commit_before>package beer

import (
	"testing"
)

const targetTestVersion = 1

func TestTestVersion(t *testing.T) {
	if testVersion != targetTestVersion {
		t.Errorf("Found testVersion = %v, want %v.", testVersion, targetTestVersion)
	}
}

const verse8 = "8 bottles of beer on the wall, 8 bottles of beer.\nTake one down and pass it around, 7 bottles of beer on the wall.\n"
const verse3 = "3 bottles of beer on the wall, 3 bottles of beer.\nTake one down and pass it around, 2 bottles of beer on the wall.\n"
const verse2 = "2 bottles of beer on the wall, 2 bottles of beer.\nTake one down and pass it around, 1 bottle of beer on the wall.\n"
const verse1 = "1 bottle of beer on the wall, 1 bottle of beer.\nTake it down and pass it around, no more bottles of beer on the wall.\n"
const verse0 = "No more bottles of beer on the wall, no more bottles of beer.\nGo to the store and buy some more, 99 bottles of beer on the wall.\n"

const verses86 = `8 bottles of beer on the wall, 8 bottles of beer.
Take one down and pass it around, 7 bottles of beer on the wall.

7 bottles of beer on the wall, 7 bottles of beer.
Take one down and pass it around, 6 bottles of beer on the wall.

6 bottles of beer on the wall, 6 bottles of beer.
Take one down and pass it around, 5 bottles of beer on the wall.

`

const verses75 = `7 bottles of beer on the wall, 7 bottles of beer.
Take one down and pass it around, 6 bottles of beer on the wall.

6 bottles of beer on the wall, 6 bottles of beer.
Take one down and pass it around, 5 bottles of beer on the wall.

5 bottles of beer on the wall, 5 bottles of beer.
Take one down and pass it around, 4 bottles of beer on the wall.

`

var verseTestCases = []struct {
	description   string
	verse         int
	expectedVerse string
	expectErr     bool
}{
	{"a typical verse", 8, verse8, false},
	{"another typical verse", 3, verse3, false},
	{"verse 2", 2, verse2, false},
	{"verse 1", 1, verse1, false},
	{"verse 0", 0, verse0, false},
	{"invalid verse", 104, "", true},
}

func TestBottlesVerse(t *testing.T) {
	for _, tt := range verseTestCases {
		actualVerse, err := Verse(tt.verse)
		var _ error = err
		if actualVerse != tt.expectedVerse {
			t.Fatalf("Verse(%d):\nexpected\n%s\nactual\n%s", tt.verse, tt.expectedVerse, actualVerse)
		}

		// if we expect an error and there isn't one
		if tt.expectErr && err == nil {
			t.Errorf("Verse(%d): expected an error, but error is nil", tt.verse)
		}
		// if we don't expect an error and there is one
		if !tt.expectErr && err != nil {
			t.Errorf("Verse(%d): expected no error, but error is: %s", tt.verse, err)
		}
	}
}

var versesTestCases = []struct {
	description   string
	upperBound    int
	lowerBound    int
	expectedVerse string
	expectErr     bool
}{
	{"multiple verses", 8, 6, verses86, false},
	{"a different set of verses", 7, 5, verses75, false},
	{"invalid start", 109, 5, "", true},
	{"invalid stop", 99, -20, "", true},
	{"start less than stop", 8, 14, "", true},
}

func TestSeveralVerses(t *testing.T) {

	for _, tt := range versesTestCases {
		actualVerse, err := Verses(tt.upperBound, tt.lowerBound)
		var _ error = err
		if actualVerse != tt.expectedVerse {
			t.Fatalf("Verses(%d, %d):\nexpected\n%s\nactual\n%s", tt.upperBound, tt.lowerBound, tt.expectedVerse, actualVerse)
		}
		// if we expect an error and there isn't one
		if tt.expectErr && err == nil {
			t.Errorf("Verses(%d, %d): expected an error, but error is nil", tt.upperBound, tt.lowerBound)
		}
		// if we don't expect an error and there is one
		if !tt.expectErr && err != nil {
			t.Errorf("Verses(%d, %d): expected no error, but error is: %s", tt.upperBound, tt.lowerBound, err)
		}
	}
}

func BenchmarkSeveralVerses(b *testing.B) {
	b.StopTimer()
	for _, tt := range versesTestCases {
		b.StartTimer()

		for i := 0; i < b.N; i++ {
			Verses(tt.upperBound, tt.lowerBound)
		}

		b.StopTimer()
	}
}

func TestEntireSong(t *testing.T) {
	expected, err := Verses(99, 0)
	var _ error = err
	if err != nil {
		t.Fatalf("unexpected error calling Verses(99,0)")
	}
	actual := Song()

	if expected != actual {
		msg := `
		  Did not sing the whole song correctly.

			Expected:

			%v

			Actual:

			%v
		`
		t.Fatalf(msg, expected, actual)
	}
}

func BenchmarkEntireSong(b *testing.B) {
	for i := 0; i < b.N; i++ {
		Song()
	}
}
<commit_msg>beer-song: check verse only when no error is expected (#372)<commit_after>package beer

import (
	"testing"
)

const targetTestVersion = 1

func TestTestVersion(t *testing.T) {
	if testVersion != targetTestVersion {
		t.Errorf("Found testVersion = %v, want %v.", testVersion, targetTestVersion)
	}
}

const verse8 = "8 bottles of beer on the wall, 8 bottles of beer.\nTake one down and pass it around, 7 bottles of beer on the wall.\n"
const verse3 = "3 bottles of beer on the wall, 3 bottles of beer.\nTake one down and pass it around, 2 bottles of beer on the wall.\n"
const verse2 = "2 bottles of beer on the wall, 2 bottles of beer.\nTake one down and pass it around, 1 bottle of beer on the wall.\n"
const verse1 = "1 bottle of beer on the wall, 1 bottle of beer.\nTake it down and pass it around, no more bottles of beer on the wall.\n"
const verse0 = "No more bottles of beer on the wall, no more bottles of beer.\nGo to the store and buy some more, 99 bottles of beer on the wall.\n"

const verses86 = `8 bottles of beer on the wall, 8 bottles of beer.
Take one down and pass it around, 7 bottles of beer on the wall.

7 bottles of beer on the wall, 7 bottles of beer.
Take one down and pass it around, 6 bottles of beer on the wall.

6 bottles of beer on the wall, 6 bottles of beer.
Take one down and pass it around, 5 bottles of beer on the wall.

`

const verses75 = `7 bottles of beer on the wall, 7 bottles of beer.
Take one down and pass it around, 6 bottles of beer on the wall.

6 bottles of beer on the wall, 6 bottles of beer.
Take one down and pass it around, 5 bottles of beer on the wall.

5 bottles of beer on the wall, 5 bottles of beer.
Take one down and pass it around, 4 bottles of beer on the wall.

`

var verseTestCases = []struct {
	description   string
	verse         int
	expectedVerse string
	expectErr     bool
}{
	{"a typical verse", 8, verse8, false},
	{"another typical verse", 3, verse3, false},
	{"verse 2", 2, verse2, false},
	{"verse 1", 1, verse1, false},
	{"verse 0", 0, verse0, false},
	{"invalid verse", 104, "", true},
}

func TestBottlesVerse(t *testing.T) {
	for _, tt := range verseTestCases {
		actualVerse, err := Verse(tt.verse)
		if tt.expectErr {
			// check if err is of error type
			var _ error = err

			// if we expect an error and there isn't one
			if err == nil {
				t.Errorf("Verse(%d): expected an error, but error is nil", tt.verse)
			}
		} else {
			if actualVerse != tt.expectedVerse {
				t.Fatalf("Verse(%d):\nexpected\n%s\nactual\n%s", tt.verse, tt.expectedVerse, actualVerse)
			}

			// if we don't expect an error and there is one
			if err != nil {
				t.Errorf("Verse(%d): expected no error, but error is: %s", tt.verse, err)
			}
		}
	}
}

var versesTestCases = []struct {
	description   string
	upperBound    int
	lowerBound    int
	expectedVerse string
	expectErr     bool
}{
	{"multiple verses", 8, 6, verses86, false},
	{"a different set of verses", 7, 5, verses75, false},
	{"invalid start", 109, 5, "", true},
	{"invalid stop", 99, -20, "", true},
	{"start less than stop", 8, 14, "", true},
}

func TestSeveralVerses(t *testing.T) {

	for _, tt := range versesTestCases {
		actualVerse, err := Verses(tt.upperBound, tt.lowerBound)
		if tt.expectErr {
			// check if err is of error type
			var _ error = err

			// if we expect an error and there isn't one
			if err == nil {
				t.Errorf("Verses(%d, %d): expected an error, but error is nil", tt.upperBound, tt.lowerBound)
			}
		} else {
			if actualVerse != tt.expectedVerse {
				t.Fatalf("Verses(%d, %d):\nexpected\n%s\nactual\n%s", tt.upperBound, tt.lowerBound, tt.expectedVerse, actualVerse)
			}

			// if we don't expect an error and there is one
			if err != nil {
				t.Errorf("Verses(%d, %d): expected no error, but error is: %s", tt.upperBound, tt.lowerBound, err)
			}
		}
	}
}

func BenchmarkSeveralVerses(b *testing.B) {
	b.StopTimer()
	for _, tt := range versesTestCases {
		b.StartTimer()

		for i := 0; i < b.N; i++ {
			Verses(tt.upperBound, tt.lowerBound)
		}

		b.StopTimer()
	}
}

func TestEntireSong(t *testing.T) {
	expected, err := Verses(99, 0)
	var _ error = err
	if err != nil {
		t.Fatalf("unexpected error calling Verses(99,0)")
	}
	actual := Song()

	if expected != actual {
		msg := `
		  Did not sing the whole song correctly.

			Expected:

			%v

			Actual:

			%v
		`
		t.Fatalf(msg, expected, actual)
	}
}

func BenchmarkEntireSong(b *testing.B) {
	for i := 0; i < b.N; i++ {
		Song()
	}
}
<|endoftext|>


<commit_before>package mods

import (
	"context"
	"fmt"
	"regexp"
	"strings"
	"time"

	"github.com/Masterminds/semver/v3"
	"github.com/rotisserie/eris"

	"github.com/ngld/knossos/packages/api/common"
	"github.com/ngld/knossos/packages/libknossos/pkg/api"
	"github.com/ngld/knossos/packages/libknossos/pkg/storage"
)

type DependencySnapshot map[string]string

type modConstraint struct {
	constraint *semver.Constraints
	modID      string
}

type resolvePathNode struct {
	versionSnapshot map[string][]string
	modID           string
	version         string
	constraints     []modConstraint
}

func makeVersionSnapshot(versions map[string][]string) map[string][]string {
	result := make(map[string][]string)
	for k, list := range versions {
		result[k] = make([]string, len(list))
		copy(result[k], list)
	}

	return result
}

var noPreRelConstraintPattern = regexp.MustCompile(`[>=~]*\s*[0-9]+\.[0-9]+\.[0-9]+(?:-)?`)

func GetDependencySnapshot(ctx context.Context, mods storage.ModProvider, release *common.Release) (DependencySnapshot, error) {
	startTime := time.Now()

	availableVersions := make(map[string][]string)
	path := make([]resolvePathNode, 0)
	queue := []string{release.Modid}
	conflicts := make(map[string]map[string]string)

	availableVersions[release.Modid] = []string{release.Version}

	for len(queue) > 0 {
		modID := queue[0]
		queue = queue[1:]

	repickVersion:
		if len(availableVersions[modID]) < 1 {
			api.Log(ctx, api.LogDebug, "DEP: No versions left to try for %s, redoing previous mod.", modID)

			if len(path) < 2 {
				api.Log(ctx, api.LogDebug, "DEP: Reached root; no mod left to redo, failing!")

				var messages map[string]string
				for _, msgs := range conflicts {
					if len(msgs) > len(messages) {
						messages = msgs
					}
				}

				if messages == nil {
					return nil, eris.New("unable to satisfy constraints")
				} else {
					msgList := make([]string, 0, len(messages))
					for _, msg := range messages {
						msgList = append(msgList, msg)
					}

					return nil, eris.Errorf("could not resolve conflict: %s which doesn't match what some of the other mods require", strings.Join(msgList, "\n"))
				}
			}

			// Remove the last path mod from the path, put it back in the queue (along with our current mod)
			// restore the snapshot of available versions and remove the previously picked version from the pool.
			lastNode := path[len(path)-1]
			path = path[:len(path)-1]

			queue = append([]string{lastNode.modID, modID}, queue...)
			availableVersions = lastNode.versionSnapshot
			modVersions := availableVersions[lastNode.modID]

			if modVersions[len(modVersions)-1] != lastNode.version {
				panic("consistency error; path version doesn't match expected available version")
			}

			availableVersions[lastNode.modID] = modVersions[:len(modVersions)-1]
			// Let's start processing the mod again...
			continue
		}

		version := availableVersions[modID][len(availableVersions[modID])-1]
		api.Log(ctx, api.LogDebug, "DEP: Trying %s %s", modID, version)

		parsedVersion, err := semver.NewVersion(version)
		if err != nil {
			return nil, eris.Wrapf(err, "failed to parse version %s for mod %s", version, modID)
		}

		// Ensure this version is compatible with previously chosen mods.
		for _, node := range path {
			for _, con := range node.constraints {
				if con.modID == modID {
					ok, err := con.constraint.Validate(parsedVersion)
					if !ok {
						api.Log(ctx, api.LogDebug, "DEP: Conflict with %s %s: %s", node.modID, node.version, err)
						availableVersions[modID] = availableVersions[modID][1:]
						goto repickVersion
					}
				}
			}
		}

		// Collect constraints
		rel, err := mods.GetModRelease(ctx, modID, version)
		if err != nil {
			return nil, eris.Wrapf(err, "failed to retrieve mod %s %s", modID, version)
		}

		cons := make([]modConstraint, 0)
		for _, pkg := range FilterUnsupportedPackages(ctx, rel.Packages) {
			for _, dep := range pkg.Dependencies {
				rawConstraint := dep.Constraint
				if rawConstraint == "" || rawConstraint == "*" {
					rawConstraint = ">= 0.0.0-0"
				}

				// Make sure all constraints that don't require exact versions allow prerelease versions
				rawConstraint = noPreRelConstraintPattern.ReplaceAllStringFunc(rawConstraint, func(s string) string {
					if !strings.HasSuffix(s, "-") && strings.ContainsAny(s, ">~") {
						return s + "-0"
					}
					return s
				})

				constraint, err := semver.NewConstraint(rawConstraint)
				if err != nil {
					return nil, eris.Wrapf(err, "failed to parse constraint %s for mod %s %s", dep.Constraint, modID, version)
				}

				cons = append(cons, modConstraint{
					modID:      dep.Modid,
					constraint: constraint,
				})
			}
		}

		// Remove all conflicting versions from availableVersions
		conflictSnapshot := makeVersionSnapshot(availableVersions)
		for _, con := range cons {
			versions, ok := availableVersions[con.modID]
			if !ok {
				versions, err = mods.GetVersionsForMod(ctx, con.modID)
				if err != nil {
					return nil, eris.Wrapf(err, "failed to fetch versions for mod %s during constraint check", modID)
				}

				availableVersions[con.modID] = versions
				queue = append(queue, con.modID)
			}

			for idx := len(versions) - 1; idx >= 0; idx-- {
				parsedVersion, err := semver.NewVersion(versions[idx])
				if err != nil {
					return nil, eris.Wrapf(err, "failed to parse version %s for mod %s during constraint check", versions[idx], con.modID)
				}

				ok, errs := con.constraint.Validate(parsedVersion)
				if !ok {
					api.Log(ctx, api.LogDebug, "DEP: Removed %s %s due to %s from %s (%s)", con.modID, versions[idx], con.constraint.String(), modID, errs)
					versions = append(versions[:idx], versions[idx+1:]...)
				}
			}

			if len(versions) < 1 {
				api.Log(ctx, api.LogDebug, "DEP: Conflict: no versions left for %s after processing constraints for %s, picking next version.", con.modID, modID)

				// Log conflict
				msgs, ok := conflicts[con.modID]
				if !ok {
					msgs = make(map[string]string)
					conflicts[con.modID] = msgs
				}
				msgs[modID] = fmt.Sprintf("%s requires %s %s which couldn't be fulfilled", modID, con.modID, con.constraint)

				availableVersions = conflictSnapshot
				availableVersions[modID] = availableVersions[modID][1:]
				goto repickVersion
			}

			availableVersions[con.modID] = versions
		}

		path = append(path, resolvePathNode{
			modID:           modID,
			version:         version,
			constraints:     cons,
			versionSnapshot: makeVersionSnapshot(availableVersions),
		})
	}

	api.Log(ctx, api.LogDebug, "DEP: Resolve done in %.3fms; building snapshot", float32(time.Since(startTime).Microseconds())/1000)
	snapshot := make(DependencySnapshot)

	for _, node := range path[1:] {
		snapshot[node.modID] = node.version
	}

	return snapshot, nil
}

func GetModDependents(ctx context.Context, mods storage.ModProvider, modID, version string) ([][2]string, error) {
	result := make([][2]string, 0)
	modList, err := mods.GetAllReleases(ctx)
	if err != nil {
		return nil, eris.Wrap(err, "failed to read mod list")
	}

	for _, rel := range modList {
		if rel.Modid == modID {
			continue
		}

		if rel.DependencySnapshot == nil {
			api.Log(ctx, api.LogWarn, "Mod %s %s doesn't have a dependency snapshot, skipping it!", rel.Modid, rel.Version)
			continue
		}

		if rel.DependencySnapshot[modID] == version {
			result = append(result, [2]string{rel.Modid, rel.Version})
		}
	}

	return result, nil
}
<commit_msg>fix(libknossos): remove the right versions during dep resolve<commit_after>package mods

import (
	"context"
	"fmt"
	"regexp"
	"strings"
	"time"

	"github.com/Masterminds/semver/v3"
	"github.com/rotisserie/eris"

	"github.com/ngld/knossos/packages/api/common"
	"github.com/ngld/knossos/packages/libknossos/pkg/api"
	"github.com/ngld/knossos/packages/libknossos/pkg/storage"
)

type DependencySnapshot map[string]string

type modConstraint struct {
	constraint *semver.Constraints
	modID      string
}

type resolvePathNode struct {
	versionSnapshot map[string][]string
	modID           string
	version         string
	constraints     []modConstraint
}

func makeVersionSnapshot(versions map[string][]string) map[string][]string {
	result := make(map[string][]string)
	for k, list := range versions {
		result[k] = make([]string, len(list))
		copy(result[k], list)
	}

	return result
}

var noPreRelConstraintPattern = regexp.MustCompile(`[>=~]*\s*[0-9]+\.[0-9]+\.[0-9]+(?:-)?`)

func GetDependencySnapshot(ctx context.Context, mods storage.ModProvider, release *common.Release) (DependencySnapshot, error) {
	startTime := time.Now()

	availableVersions := make(map[string][]string)
	path := make([]resolvePathNode, 0)
	queue := []string{release.Modid}
	conflicts := make(map[string]map[string]string)

	availableVersions[release.Modid] = []string{release.Version}

	for len(queue) > 0 {
		modID := queue[0]
		queue = queue[1:]

	repickVersion:
		if len(availableVersions[modID]) < 1 {
			api.Log(ctx, api.LogDebug, "DEP: No versions left to try for %s, redoing previous mod.", modID)

			if len(path) < 2 {
				api.Log(ctx, api.LogDebug, "DEP: Reached root; no mod left to redo, failing!")

				var messages map[string]string
				for _, msgs := range conflicts {
					if len(msgs) > len(messages) {
						messages = msgs
					}
				}

				if messages == nil {
					return nil, eris.New("unable to satisfy constraints")
				} else {
					msgList := make([]string, 0, len(messages))
					for _, msg := range messages {
						msgList = append(msgList, msg)
					}

					return nil, eris.Errorf("could not resolve conflict: %s which doesn't match what some of the other mods require", strings.Join(msgList, "\n"))
				}
			}

			// Remove the last path mod from the path, put it back in the queue (along with our current mod)
			// restore the snapshot of available versions and remove the previously picked version from the pool.
			lastNode := path[len(path)-1]
			path = path[:len(path)-1]

			queue = append([]string{lastNode.modID, modID}, queue...)
			availableVersions = lastNode.versionSnapshot
			modVersions := availableVersions[lastNode.modID]

			if modVersions[len(modVersions)-1] != lastNode.version {
				panic("consistency error; path version doesn't match expected available version")
			}

			availableVersions[lastNode.modID] = modVersions[:len(modVersions)-1]
			// Let's start processing the mod again...
			continue
		}

		version := availableVersions[modID][len(availableVersions[modID])-1]
		api.Log(ctx, api.LogDebug, "DEP: Trying %s %s", modID, version)

		parsedVersion, err := semver.NewVersion(version)
		if err != nil {
			return nil, eris.Wrapf(err, "failed to parse version %s for mod %s", version, modID)
		}

		// Ensure this version is compatible with previously chosen mods.
		for _, node := range path {
			for _, con := range node.constraints {
				if con.modID == modID {
					ok, err := con.constraint.Validate(parsedVersion)
					if !ok {
						api.Log(ctx, api.LogDebug, "DEP: Conflict with %s %s: %s", node.modID, node.version, err)
						availableVersions[modID] = availableVersions[modID][:len(availableVersions[modID])-1]
						goto repickVersion
					}
				}
			}
		}

		// Collect constraints
		rel, err := mods.GetModRelease(ctx, modID, version)
		if err != nil {
			return nil, eris.Wrapf(err, "failed to retrieve mod %s %s", modID, version)
		}

		pkgs := FilterUnsupportedPackages(ctx, rel.Packages)
		if len(pkgs) < 1 {
			api.Log(ctx, api.LogDebug, "DEP: Conflict: %s %s not supported on current platform, picking next version.", modID, version)
			availableVersions[modID] = availableVersions[modID][:len(availableVersions[modID])-1]
			goto repickVersion
		}

		cons := make([]modConstraint, 0)
		for _, pkg := range pkgs {
			for _, dep := range pkg.Dependencies {
				rawConstraint := dep.Constraint
				if rawConstraint == "" || rawConstraint == "*" {
					rawConstraint = ">= 0.0.0-0"
				}

				// Make sure all constraints that don't require exact versions allow prerelease versions
				rawConstraint = noPreRelConstraintPattern.ReplaceAllStringFunc(rawConstraint, func(s string) string {
					if !strings.HasSuffix(s, "-") && strings.ContainsAny(s, ">~") {
						return s + "-0"
					}
					return s
				})

				constraint, err := semver.NewConstraint(rawConstraint)
				if err != nil {
					return nil, eris.Wrapf(err, "failed to parse constraint %s for mod %s %s", dep.Constraint, modID, version)
				}

				cons = append(cons, modConstraint{
					modID:      dep.Modid,
					constraint: constraint,
				})
			}
		}

		conflictSnapshot := makeVersionSnapshot(availableVersions)
		queueSnapshot := make([]string, len(queue))
		copy(queueSnapshot, queue)

		// Remove all conflicting versions from availableVersions
		for _, con := range cons {
			versions, ok := availableVersions[con.modID]
			if !ok {
				versions, err = mods.GetVersionsForMod(ctx, con.modID)
				if err != nil {
					return nil, eris.Wrapf(err, "failed to fetch versions for mod %s during constraint check", modID)
				}

				availableVersions[con.modID] = versions
				queue = append(queue, con.modID)
			}

			for idx := len(versions) - 1; idx >= 0; idx-- {
				parsedVersion, err := semver.NewVersion(versions[idx])
				if err != nil {
					return nil, eris.Wrapf(err, "failed to parse version %s for mod %s during constraint check", versions[idx], con.modID)
				}

				ok, errs := con.constraint.Validate(parsedVersion)
				if !ok {
					api.Log(ctx, api.LogDebug, "DEP: Removed %s %s due to %s from %s (%s)", con.modID, versions[idx], con.constraint.String(), modID, errs)
					versions = append(versions[:idx], versions[idx+1:]...)
				}
			}

			if len(versions) < 1 {
				api.Log(ctx, api.LogDebug, "DEP: Conflict: no versions left for %s after processing constraints for %s, picking next version.", con.modID, modID)

				// Log conflict
				msgs, ok := conflicts[con.modID]
				if !ok {
					msgs = make(map[string]string)
					conflicts[con.modID] = msgs
				}
				msgs[modID] = fmt.Sprintf("%s requires %s %s which couldn't be fulfilled", modID, con.modID, con.constraint)

				availableVersions = conflictSnapshot
				availableVersions[modID] = availableVersions[modID][:len(availableVersions[modID])-1]
				queue = queueSnapshot
				goto repickVersion
			}

			availableVersions[con.modID] = versions
		}

		path = append(path, resolvePathNode{
			modID:           modID,
			version:         version,
			constraints:     cons,
			versionSnapshot: makeVersionSnapshot(availableVersions),
		})
	}

	api.Log(ctx, api.LogDebug, "DEP: Resolve done in %.3fms; building snapshot", float32(time.Since(startTime).Microseconds())/1000)
	snapshot := make(DependencySnapshot)

	for _, node := range path[1:] {
		snapshot[node.modID] = node.version
	}

	return snapshot, nil
}

func GetModDependents(ctx context.Context, mods storage.ModProvider, modID, version string) ([][2]string, error) {
	result := make([][2]string, 0)
	modList, err := mods.GetAllReleases(ctx)
	if err != nil {
		return nil, eris.Wrap(err, "failed to read mod list")
	}

	for _, rel := range modList {
		if rel.Modid == modID {
			continue
		}

		if rel.DependencySnapshot == nil {
			api.Log(ctx, api.LogWarn, "Mod %s %s doesn't have a dependency snapshot, skipping it!", rel.Modid, rel.Version)
			continue
		}

		if rel.DependencySnapshot[modID] == version {
			result = append(result, [2]string{rel.Modid, rel.Version})
		}
	}

	return result, nil
}
<|endoftext|>


<commit_before>package api

import (
	"fmt"
	"net/http"
	"strings"
	"time"

	jwt "github.com/appleboy/gin-jwt/v2"
	contracts "github.com/estafette/estafette-ci-contracts"
	"github.com/gin-gonic/gin"
	"github.com/rs/zerolog/log"
)

// Middleware handles authentication for routes requiring authentication
type Middleware interface {
	GoogleJWTMiddlewareFunc() gin.HandlerFunc
	GinJWTMiddleware(authenticator func(c *gin.Context) (interface{}, error)) (middleware *jwt.GinJWTMiddleware, err error)
	GinJWTMiddlewareForClientLogin(authenticator func(c *gin.Context) (interface{}, error)) (middleware *jwt.GinJWTMiddleware, err error)
}

// NewAuthMiddleware returns a new api.AuthMiddleware
func NewAuthMiddleware(config *APIConfig) (authMiddleware Middleware) {
	authMiddleware = &authMiddlewareImpl{
		config: config,
	}

	return
}

type authMiddlewareImpl struct {
	config *APIConfig
}

func (m *authMiddlewareImpl) GoogleJWTMiddlewareFunc() gin.HandlerFunc {
	return func(c *gin.Context) {

		authorizationHeader := c.Request.Header.Get("Authorization")

		if !strings.HasPrefix(authorizationHeader, "Bearer ") {
			return
		}

		bearerTokenString := strings.TrimPrefix(authorizationHeader, "Bearer ")
		valid, err := isValidGoogleJWT(bearerTokenString)

		if err != nil {
			log.Warn().Err(err).Str("bearer", bearerTokenString).Msgf("Error when validating Google JWT")
			return
		}

		if !valid {
			log.Warn().Err(err).Str("bearer", bearerTokenString).Msgf("Google JWT is not valid")
			return
		}

		// set 'user' to enforce a handler method to require api key auth with `user := c.MustGet(gin.AuthUserKey).(string)` and ensuring the user equals 'google-jwt'
		c.Set(gin.AuthUserKey, "google-jwt")
	}
}

func (m *authMiddlewareImpl) coreGinJWTMiddleware(authenticator func(c *gin.Context) (interface{}, error)) (middleware *jwt.GinJWTMiddleware, err error) {
	return jwt.New(&jwt.GinJWTMiddleware{
		Realm:         m.config.Auth.JWT.Domain,
		Key:           []byte(m.config.Auth.JWT.Key),
		TokenLookup:   "header:Authorization, cookie:jwt",
		Authenticator: authenticator,
		TimeFunc:      time.Now,
	})
}

func (m *authMiddlewareImpl) GinJWTMiddleware(authenticator func(c *gin.Context) (interface{}, error)) (middleware *jwt.GinJWTMiddleware, err error) {
	middleware, err = m.coreGinJWTMiddleware(authenticator)
	if err != nil {
		return nil, err
	}

	// send cookie
	middleware.SendCookie = true
	middleware.SecureCookie = true
	middleware.CookieHTTPOnly = true
	middleware.CookieDomain = m.config.Auth.JWT.Domain

	// redirect after login
	middleware.LoginResponse = func(c *gin.Context, code int, token string, expire time.Time) {
		// see if gin context has a return url
		returnURL, exists := c.Get("returnURL")
		if exists {
			c.Redirect(http.StatusFound, returnURL.(string))
			return
		}

		// cookie is used, so token does not need to be returned via response
		c.Redirect(http.StatusFound, "/")
	}

	// redirect after logout
	middleware.LogoutResponse = func(c *gin.Context, code int) {
		c.Redirect(http.StatusFound, "/login")
	}

	// set some user properties as claims
	middleware.PayloadFunc = func(data interface{}) jwt.MapClaims {
		// add user properties as claims
		if user, ok := data.(*contracts.User); ok {

			organizations := []string{}
			for _, o := range user.Organizations {
				organizations = append(organizations, o.Name)
			}

			groups := []string{}
			for _, g := range user.Groups {
				groups = append(groups, g.Name)
			}

			return jwt.MapClaims{
				jwt.IdentityKey: user.ID,
				"email":         user.GetEmail(),
				"roles":         user.Roles,
				"groups":        groups,
				"organizations": organizations,
			}
		}
		return jwt.MapClaims{}
	}

	return middleware, nil
}

func (m *authMiddlewareImpl) GinJWTMiddlewareForClientLogin(authenticator func(c *gin.Context) (interface{}, error)) (middleware *jwt.GinJWTMiddleware, err error) {
	middleware, err = m.coreGinJWTMiddleware(authenticator)
	if err != nil {
		return nil, err
	}

	// set some client properties as claims
	middleware.PayloadFunc = func(data interface{}) jwt.MapClaims {

		// add client properties as claims
		if client, ok := data.(*contracts.Client); ok {

			organizations := []string{}
			for _, o := range client.Organizations {
				organizations = append(organizations, o.Name)
			}

			return jwt.MapClaims{
				jwt.IdentityKey: client.ID,
				"email":         fmt.Sprintf("%v@client.estafette.io", client.Name),
				"clientID":      client.ClientID,
				"roles":         client.Roles,
				"organizations": organizations,
			}
		}
		return jwt.MapClaims{}
	}

	return middleware, nil
}
<commit_msg>increase jwt timeout to 3 hours from default 1 hour<commit_after>package api

import (
	"fmt"
	"net/http"
	"strings"
	"time"

	jwt "github.com/appleboy/gin-jwt/v2"
	contracts "github.com/estafette/estafette-ci-contracts"
	"github.com/gin-gonic/gin"
	"github.com/rs/zerolog/log"
)

// Middleware handles authentication for routes requiring authentication
type Middleware interface {
	GoogleJWTMiddlewareFunc() gin.HandlerFunc
	GinJWTMiddleware(authenticator func(c *gin.Context) (interface{}, error)) (middleware *jwt.GinJWTMiddleware, err error)
	GinJWTMiddlewareForClientLogin(authenticator func(c *gin.Context) (interface{}, error)) (middleware *jwt.GinJWTMiddleware, err error)
}

// NewAuthMiddleware returns a new api.AuthMiddleware
func NewAuthMiddleware(config *APIConfig) (authMiddleware Middleware) {
	authMiddleware = &authMiddlewareImpl{
		config: config,
	}

	return
}

type authMiddlewareImpl struct {
	config *APIConfig
}

func (m *authMiddlewareImpl) GoogleJWTMiddlewareFunc() gin.HandlerFunc {
	return func(c *gin.Context) {

		authorizationHeader := c.Request.Header.Get("Authorization")

		if !strings.HasPrefix(authorizationHeader, "Bearer ") {
			return
		}

		bearerTokenString := strings.TrimPrefix(authorizationHeader, "Bearer ")
		valid, err := isValidGoogleJWT(bearerTokenString)

		if err != nil {
			log.Warn().Err(err).Str("bearer", bearerTokenString).Msgf("Error when validating Google JWT")
			return
		}

		if !valid {
			log.Warn().Err(err).Str("bearer", bearerTokenString).Msgf("Google JWT is not valid")
			return
		}

		// set 'user' to enforce a handler method to require api key auth with `user := c.MustGet(gin.AuthUserKey).(string)` and ensuring the user equals 'google-jwt'
		c.Set(gin.AuthUserKey, "google-jwt")
	}
}

func (m *authMiddlewareImpl) coreGinJWTMiddleware(authenticator func(c *gin.Context) (interface{}, error)) (middleware *jwt.GinJWTMiddleware, err error) {
	return jwt.New(&jwt.GinJWTMiddleware{
		Realm:         m.config.Auth.JWT.Domain,
		Key:           []byte(m.config.Auth.JWT.Key),
		TokenLookup:   "header:Authorization, cookie:jwt",
		Authenticator: authenticator,
		Timeout:       time.Duration(3) * time.Hour,
		TimeFunc:      time.Now().UTC,
	})
}

func (m *authMiddlewareImpl) GinJWTMiddleware(authenticator func(c *gin.Context) (interface{}, error)) (middleware *jwt.GinJWTMiddleware, err error) {
	middleware, err = m.coreGinJWTMiddleware(authenticator)
	if err != nil {
		return nil, err
	}

	// send cookie
	middleware.SendCookie = true
	middleware.SecureCookie = true
	middleware.CookieHTTPOnly = true
	middleware.CookieDomain = m.config.Auth.JWT.Domain

	// redirect after login
	middleware.LoginResponse = func(c *gin.Context, code int, token string, expire time.Time) {
		// see if gin context has a return url
		returnURL, exists := c.Get("returnURL")
		if exists {
			c.Redirect(http.StatusFound, returnURL.(string))
			return
		}

		// cookie is used, so token does not need to be returned via response
		c.Redirect(http.StatusFound, "/")
	}

	// redirect after logout
	middleware.LogoutResponse = func(c *gin.Context, code int) {
		c.Redirect(http.StatusFound, "/login")
	}

	// set some user properties as claims
	middleware.PayloadFunc = func(data interface{}) jwt.MapClaims {
		// add user properties as claims
		if user, ok := data.(*contracts.User); ok {

			organizations := []string{}
			for _, o := range user.Organizations {
				organizations = append(organizations, o.Name)
			}

			groups := []string{}
			for _, g := range user.Groups {
				groups = append(groups, g.Name)
			}

			return jwt.MapClaims{
				jwt.IdentityKey: user.ID,
				"email":         user.GetEmail(),
				"roles":         user.Roles,
				"groups":        groups,
				"organizations": organizations,
			}
		}
		return jwt.MapClaims{}
	}

	return middleware, nil
}

func (m *authMiddlewareImpl) GinJWTMiddlewareForClientLogin(authenticator func(c *gin.Context) (interface{}, error)) (middleware *jwt.GinJWTMiddleware, err error) {
	middleware, err = m.coreGinJWTMiddleware(authenticator)
	if err != nil {
		return nil, err
	}

	// set some client properties as claims
	middleware.PayloadFunc = func(data interface{}) jwt.MapClaims {

		// add client properties as claims
		if client, ok := data.(*contracts.Client); ok {

			organizations := []string{}
			for _, o := range client.Organizations {
				organizations = append(organizations, o.Name)
			}

			return jwt.MapClaims{
				jwt.IdentityKey: client.ID,
				"email":         fmt.Sprintf("%v@client.estafette.io", client.Name),
				"clientID":      client.ClientID,
				"roles":         client.Roles,
				"organizations": organizations,
			}
		}
		return jwt.MapClaims{}
	}

	return middleware, nil
}
<|endoftext|>


<commit_before>package api

import (
	"bytes"
	"context"
	"io"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/dgrijalva/jwt-go"
	"github.com/netlify/gotrue/models"
)

const (
	jwsSignatureHeaderName = "x-nf-sign"
)

type NetlifyMicroserviceClaims struct {
	SiteURL    string `json:"site_url"`
	Env        string `json:"env"`
	InstanceID string `json:"id"`
	NetlifyID  string `json:"netlify_id"`
	jwt.StandardClaims
}

func addGetBody(w http.ResponseWriter, req *http.Request) (context.Context, error) {
	if req.Method == http.MethodGet {
		return req.Context(), nil
	}

	if req.Body == nil || req.Body == http.NoBody {
		return nil, badRequestError("request must provide a body")
	}

	buf, err := ioutil.ReadAll(req.Body)
	if err != nil {
		return nil, internalServerError("Error reading body").WithInternalError(err)
	}
	req.GetBody = func() (io.ReadCloser, error) {
		return ioutil.NopCloser(bytes.NewReader(buf)), nil
	}
	req.Body, _ = req.GetBody()
	return req.Context(), nil
}

func (api *API) loadInstanceConfig(w http.ResponseWriter, r *http.Request) (context.Context, error) {
	ctx := r.Context()

	signature := r.Header.Get(jwsSignatureHeaderName)
	if signature == "" {
		return nil, badRequestError("Netlify microservice headers missing")
	}

	claims := NetlifyMicroserviceClaims{}
	p := jwt.Parser{ValidMethods: []string{jwt.SigningMethodHS256.Name}}
	_, err := p.ParseWithClaims(signature, &claims, func(token *jwt.Token) (interface{}, error) {
		return []byte(api.config.NetlifySecret), nil
	})
	if err != nil {
		return nil, badRequestError("Netlify microservice headers are invalid: %v", err)
	}

	instanceID := claims.InstanceID
	if instanceID == "" {
		return nil, badRequestError("Netlify microservice headers missing")
	}

	logEntrySetField(r, "instance_id", instanceID)
	instance, err := api.db.GetInstance(instanceID)
	if err != nil {
		if models.IsNotFoundError(err) {
			return nil, notFoundError("Unable to locate site configuration")
		}
		return nil, internalServerError("Database error loading instance").WithInternalError(err)
	}

	env := claims.Env
	if env == "" {
		return nil, badRequestError("No environment specified")
	}
	logEntrySetField(r, "env", env)
	config, err := instance.ConfigForEnvironment(env)
	if err != nil {
		return nil, internalServerError("Error loading environment config").WithInternalError(err)
	}

	if claims.SiteURL != "" {
		config.SiteURL = claims.SiteURL
	}
	logEntrySetField(r, "site_url", config.SiteURL)

	ctx, err = WithInstanceConfig(ctx, config, instanceID)
	if err != nil {
		return nil, internalServerError("Error loading instance config").WithInternalError(err)
	}

	return ctx, nil
}

func (api *API) verifyNetlifyRequest(w http.ResponseWriter, req *http.Request) (context.Context, error) {
	token := strings.TrimPrefix(req.Header.Get("authorization"), "Bearer ")
	if token != api.config.NetlifySecret {
		return nil, unauthorizedError("Request did not originate from Netlify")
	}
	return req.Context(), nil
}
<commit_msg>Clarify error message<commit_after>package api

import (
	"bytes"
	"context"
	"io"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/dgrijalva/jwt-go"
	"github.com/netlify/gotrue/models"
)

const (
	jwsSignatureHeaderName = "x-nf-sign"
)

type NetlifyMicroserviceClaims struct {
	SiteURL    string `json:"site_url"`
	Env        string `json:"env"`
	InstanceID string `json:"id"`
	NetlifyID  string `json:"netlify_id"`
	jwt.StandardClaims
}

func addGetBody(w http.ResponseWriter, req *http.Request) (context.Context, error) {
	if req.Method == http.MethodGet {
		return req.Context(), nil
	}

	if req.Body == nil || req.Body == http.NoBody {
		return nil, badRequestError("request must provide a body")
	}

	buf, err := ioutil.ReadAll(req.Body)
	if err != nil {
		return nil, internalServerError("Error reading body").WithInternalError(err)
	}
	req.GetBody = func() (io.ReadCloser, error) {
		return ioutil.NopCloser(bytes.NewReader(buf)), nil
	}
	req.Body, _ = req.GetBody()
	return req.Context(), nil
}

func (api *API) loadInstanceConfig(w http.ResponseWriter, r *http.Request) (context.Context, error) {
	ctx := r.Context()

	signature := r.Header.Get(jwsSignatureHeaderName)
	if signature == "" {
		return nil, badRequestError("Netlify microservice headers missing")
	}

	claims := NetlifyMicroserviceClaims{}
	p := jwt.Parser{ValidMethods: []string{jwt.SigningMethodHS256.Name}}
	_, err := p.ParseWithClaims(signature, &claims, func(token *jwt.Token) (interface{}, error) {
		return []byte(api.config.NetlifySecret), nil
	})
	if err != nil {
		return nil, badRequestError("Netlify microservice headers are invalid: %v", err)
	}

	instanceID := claims.InstanceID
	if instanceID == "" {
		return nil, badRequestError("Instance ID is missing")
	}

	logEntrySetField(r, "instance_id", instanceID)
	instance, err := api.db.GetInstance(instanceID)
	if err != nil {
		if models.IsNotFoundError(err) {
			return nil, notFoundError("Unable to locate site configuration")
		}
		return nil, internalServerError("Database error loading instance").WithInternalError(err)
	}

	env := claims.Env
	if env == "" {
		return nil, badRequestError("No environment specified")
	}
	logEntrySetField(r, "env", env)
	config, err := instance.ConfigForEnvironment(env)
	if err != nil {
		return nil, internalServerError("Error loading environment config").WithInternalError(err)
	}

	if claims.SiteURL != "" {
		config.SiteURL = claims.SiteURL
	}
	logEntrySetField(r, "site_url", config.SiteURL)

	ctx, err = WithInstanceConfig(ctx, config, instanceID)
	if err != nil {
		return nil, internalServerError("Error loading instance config").WithInternalError(err)
	}

	return ctx, nil
}

func (api *API) verifyNetlifyRequest(w http.ResponseWriter, req *http.Request) (context.Context, error) {
	token := strings.TrimPrefix(req.Header.Get("authorization"), "Bearer ")
	if token != api.config.NetlifySecret {
		return nil, unauthorizedError("Request did not originate from Netlify")
	}
	return req.Context(), nil
}
<|endoftext|>


<commit_before>package twitch

import (
	"fmt"
	"log"
	"sync"
	"time"
)

const (
	heartBeatRate  = time.Minute
	heartBeatLimit = 100
	heartDumpEvery = time.Minute * 30
)

// HeartbeatData - Single Beat of the Heart
type HeartbeatData struct {
	Status    string    `json:"status"`
	Game      string    `json:"game"`
	Time      time.Time `json:"time"`
	IsLive    bool      `json:"live"`
	ViewCount int       `json:"view"`
}

// Heartbeat - A beat which captures some data
// * Live Status
// * Viewer Count
// * host notifications
type Heartbeat struct {
	beats     []HeartbeatData
	hosts     map[ID]time.Time
	heartLock sync.RWMutex

	prevFollowCount int
	followers       []ChannelFollow

	internalBeat *time.Ticker
	client       *Client
}

// StartBeat - Blocking Loop Which
func (heart *Heartbeat) StartBeat() {
	heart.prevFollowCount = -1
	heart.hosts = make(map[ID]time.Time)

	// First Beat
	heart.beat(time.Now())

	timeSinceDump := time.Duration(0)
	heart.internalBeat = time.NewTicker(heartBeatRate)

	// Beat every X minutes
	for ts := range heart.internalBeat.C {
		heart.beat(ts)

		// Dumping to File
		timeSinceDump += heartBeatRate
		if timeSinceDump > heartDumpEvery {
			timeSinceDump = 0
			err := heart.client.DumpViewers()
			if err != nil {
				fmt.Printf("DUMP ERROR: %s", err)
			}

		}
	}
}

// GetBeat - Get Sepecific Beat
func (heart *Heartbeat) GetBeat(beatNum int) HeartbeatData {
	heart.heartLock.RLock()
	defer heart.heartLock.RUnlock()
	return heart.beats[beatNum]
}

// GetAllBeats - Get All the Beats
func (heart *Heartbeat) GetAllBeats() []HeartbeatData {
	heart.heartLock.RLock()
	defer heart.heartLock.RUnlock()

	log.Printf("Beats Source %d", len(heart.beats))
	return heart.beats[0:]
}

// GetAllHosts - Get All Hosts
func (heart *Heartbeat) GetAllHosts() []ID {
	keys := make([]ID, len(heart.hosts), len(heart.hosts))
	i := 0
	for k := range heart.hosts {
		keys[i] = k
		i++
	}
	return keys
}

func (heart *Heartbeat) beat(t time.Time) {
	heart.heartLock.Lock()
	defer heart.heartLock.Unlock()

	fmt.Println("-- BEAT --")

	var prevDataPoint *HeartbeatData
	if len(heart.beats) > 0 {
		prevDataPoint = &heart.beats[len(heart.beats)-1]
	}

	// Stream Viewer Count
	sb, err := heart.client.Stream.GetStreamByUser(heart.client.RoomID)
	fmt.Println(sb)

	if err != nil || sb == nil || (sb.AverageFPS == 0) {
		if prevDataPoint == nil || prevDataPoint.IsLive {
			heart.beats = append(heart.beats, HeartbeatData{Time: t})
			prevDataPoint = &heart.beats[len(heart.beats)-1]
		}
		return
	}
	heart.client.RoomStream = sb

	hbd := HeartbeatData{
		Status:    sb.Channel.Status,
		Game:      sb.Game,
		Time:      t,
		IsLive:    true,
		ViewCount: sb.Viewers,
	}
	if prevDataPoint == nil {
		prevDataPoint = &hbd
	}

	// Get Channel Followers
	// If there are more then 30 follows in a SECOND who cares
	fList, followNum, err := heart.client.Channel.GetFollowers(heart.client.RoomID, 30, true)
	heart.client.Viewers.UpdateFollowers(fList)

	// Check for new followers
	for _, f := range fList {
		t, err := time.Parse(time.RFC3339, f.CreatedAtString)
		if err != nil {
			panic(err)
		}

		if t.After(prevDataPoint.Time) {
			// New Follow
			heart.client.Alerts.Post(f.User.Name, AlertFollow, t)
		} else {
			// Avoid 99% of the work
			break
		}
	}

	heart.prevFollowCount = followNum

	// List of Hosts
	hostList, err := heart.client.Stream.GetHostsByUser(heart.client.RoomID)
	if err != nil {
		fmt.Printf("Host Check failed: %s\n", err.Error())
		return
	}

	hostDiff := len(heart.hosts) - len(hostList)
	for _, h := range hostList {
		srcID := IDFromInt(h.HostID)
		_, ok := heart.hosts[srcID]
		if !ok {
			// Trigger Alert
			heart.hosts[srcID] = time.Now()
			heart.client.Alerts.Post(IrcNick(h.HostLogin), AlertHost, h)
			hostDiff++
		}
	}

	// We lost a host
	if hostDiff > 0 {
		for k := range heart.hosts {
			isFound := false
			for k2 := 0; !isFound && k2 < len(hostList); k2++ {
				srcID := IDFromInt(hostList[k2].HostID)
				isFound = (k == srcID)
			}

			if isFound == false {
				delete(heart.hosts, k)
			}
		}
	}

	// Update Data Points
	prevDataPoint = &hbd
	if len(heart.beats) < heartBeatLimit {
		heart.beats = append(heart.beats, hbd)
	} else {
		heart.beats = append(heart.beats[1:], hbd)
	}

	heart.client.Alerts.Post(heart.client.RoomName, AlertNone, hbd)
}
<commit_msg>Error log<commit_after>package twitch

import (
	"fmt"
	"log"
	"sync"
	"time"
)

const (
	heartBeatRate  = time.Minute
	heartBeatLimit = 100
	heartDumpEvery = time.Minute * 30
)

// HeartbeatData - Single Beat of the Heart
type HeartbeatData struct {
	Status    string    `json:"status"`
	Game      string    `json:"game"`
	Time      time.Time `json:"time"`
	IsLive    bool      `json:"live"`
	ViewCount int       `json:"view"`
}

// Heartbeat - A beat which captures some data
// * Live Status
// * Viewer Count
// * host notifications
type Heartbeat struct {
	beats     []HeartbeatData
	hosts     map[ID]time.Time
	heartLock sync.RWMutex

	prevFollowCount int
	followers       []ChannelFollow

	internalBeat *time.Ticker
	client       *Client
}

// StartBeat - Blocking Loop Which
func (heart *Heartbeat) StartBeat() {
	heart.prevFollowCount = -1
	heart.hosts = make(map[ID]time.Time)

	// First Beat
	heart.beat(time.Now())

	timeSinceDump := time.Duration(0)
	heart.internalBeat = time.NewTicker(heartBeatRate)

	// Beat every X minutes
	for ts := range heart.internalBeat.C {
		heart.beat(ts)

		// Dumping to File
		timeSinceDump += heartBeatRate
		if timeSinceDump > heartDumpEvery {
			timeSinceDump = 0
			err := heart.client.DumpViewers()
			if err != nil {
				fmt.Printf("DUMP ERROR: %s", err)
			}

		}
	}
}

// GetBeat - Get Sepecific Beat
func (heart *Heartbeat) GetBeat(beatNum int) HeartbeatData {
	heart.heartLock.RLock()
	defer heart.heartLock.RUnlock()
	return heart.beats[beatNum]
}

// GetAllBeats - Get All the Beats
func (heart *Heartbeat) GetAllBeats() []HeartbeatData {
	heart.heartLock.RLock()
	defer heart.heartLock.RUnlock()

	log.Printf("Beats Source %d", len(heart.beats))
	return heart.beats[0:]
}

// GetAllHosts - Get All Hosts
func (heart *Heartbeat) GetAllHosts() []ID {
	keys := make([]ID, len(heart.hosts), len(heart.hosts))
	i := 0
	for k := range heart.hosts {
		keys[i] = k
		i++
	}
	return keys
}

func (heart *Heartbeat) beat(t time.Time) {
	heart.heartLock.Lock()
	defer heart.heartLock.Unlock()

	fmt.Println("-- BEAT --")

	var prevDataPoint *HeartbeatData
	if len(heart.beats) > 0 {
		prevDataPoint = &heart.beats[len(heart.beats)-1]
	}

	// Stream Viewer Count
	sb, err := heart.client.Stream.GetStreamByUser(heart.client.RoomID)
	fmt.Println(sb)
	fmt.Println(err)

	if err != nil || sb == nil || (sb.AverageFPS == 0) {
		if prevDataPoint == nil || prevDataPoint.IsLive {
			heart.beats = append(heart.beats, HeartbeatData{Time: t})
			prevDataPoint = &heart.beats[len(heart.beats)-1]
		}
		return
	}
	heart.client.RoomStream = sb

	hbd := HeartbeatData{
		Status:    sb.Channel.Status,
		Game:      sb.Game,
		Time:      t,
		IsLive:    true,
		ViewCount: sb.Viewers,
	}
	if prevDataPoint == nil {
		prevDataPoint = &hbd
	}

	// Get Channel Followers
	// If there are more then 30 follows in a SECOND who cares
	fList, followNum, err := heart.client.Channel.GetFollowers(heart.client.RoomID, 30, true)
	heart.client.Viewers.UpdateFollowers(fList)

	// Check for new followers
	for _, f := range fList {
		t, err := time.Parse(time.RFC3339, f.CreatedAtString)
		if err != nil {
			panic(err)
		}

		if t.After(prevDataPoint.Time) {
			// New Follow
			heart.client.Alerts.Post(f.User.Name, AlertFollow, t)
		} else {
			// Avoid 99% of the work
			break
		}
	}

	heart.prevFollowCount = followNum

	// List of Hosts
	hostList, err := heart.client.Stream.GetHostsByUser(heart.client.RoomID)
	if err != nil {
		fmt.Printf("Host Check failed: %s\n", err.Error())
		return
	}

	hostDiff := len(heart.hosts) - len(hostList)
	for _, h := range hostList {
		srcID := IDFromInt(h.HostID)
		_, ok := heart.hosts[srcID]
		if !ok {
			// Trigger Alert
			heart.hosts[srcID] = time.Now()
			heart.client.Alerts.Post(IrcNick(h.HostLogin), AlertHost, h)
			hostDiff++
		}
	}

	// We lost a host
	if hostDiff > 0 {
		for k := range heart.hosts {
			isFound := false
			for k2 := 0; !isFound && k2 < len(hostList); k2++ {
				srcID := IDFromInt(hostList[k2].HostID)
				isFound = (k == srcID)
			}

			if isFound == false {
				delete(heart.hosts, k)
			}
		}
	}

	// Update Data Points
	prevDataPoint = &hbd
	if len(heart.beats) < heartBeatLimit {
		heart.beats = append(heart.beats, hbd)
	} else {
		heart.beats = append(heart.beats[1:], hbd)
	}

	heart.client.Alerts.Post(heart.client.RoomName, AlertNone, hbd)
}
<|endoftext|>


<commit_before>package model

import (
	"time"
)

type User struct {
	Id          string `gorethink:"id,omitempty"`
	Dateofbirth time.Time
	Email       string
	Phone       string
  Password    string
	Created     time.Time
	Updated     time.Time
}

/*
func (t *User) Completed() bool {
    return t.Status == "complete"
}
*/

func NewUser(email string, phone string, dob time.Time) *User {
	return &User{
		Email:       email,
		Phone:       phone,
		Dateofbirth: dob,
	}
}
<commit_msg>map lowercase name, remove password from json<commit_after>package model

import (
	"time"
)

type User struct {
	Id          string `gorethink:"id,omitempty" json:"id,"`
	Dateofbirth time.Time `json:"dob,omitempty"`
	Email       string `json:"email,omitempty"`
	Phone       string `json:"phone,omitempty"`
  Password    string `json:"-"`
	Created     time.Time `json:"created,omitempty"`
	Updated     time.Time `json:"updated,omitempty"`
}

/*
func (t *User) Completed() bool {
    return t.Status == "complete"
}
*/

func NewUser(email string, phone string, dob time.Time) *User {
	return &User{
		Email:       email,
		Phone:       phone,
		Dateofbirth: dob,
	}
}
<|endoftext|>


<commit_before>// Package api provides rest-like server
package api

import (
	"context"
	"encoding/xml"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/didip/tollbooth/v6"
	"github.com/didip/tollbooth_chi"
	"github.com/go-chi/chi/v5"
	"github.com/go-chi/chi/v5/middleware"
	"github.com/go-chi/render"
	"github.com/go-pkgz/lcw"
	log "github.com/go-pkgz/lgr"
	"github.com/go-pkgz/rest"
	"github.com/go-pkgz/rest/logger"

	"github.com/umputun/feed-master/app/config"
	"github.com/umputun/feed-master/app/feed"
	"github.com/umputun/feed-master/app/proc"
	"github.com/umputun/feed-master/app/youtube"
)

// Server provides HTTP API
type Server struct {
	Version    string
	Conf       config.Conf
	Store      *proc.BoltDB
	YoutubeSvc YoutubeSvc
	httpServer *http.Server
	cache      lcw.LoadingCache
}

// YoutubeSvc provides access to youtube's audio rss
type YoutubeSvc interface {
	RSSFeed(cinfo youtube.FeedInfo) (string, error)
}

// Run starts http server for API with all routes
func (s *Server) Run(ctx context.Context, port int) {
	var err error
	if s.cache, err = lcw.NewExpirableCache(lcw.TTL(time.Minute*5), lcw.MaxCacheSize(10*1024*1024)); err != nil {
		log.Printf("[PANIC] failed to make loading cache, %v", err)
		return
	}

	go func() {
		<-ctx.Done()
		if s.httpServer != nil {
			if clsErr := s.httpServer.Close(); clsErr != nil {
				log.Printf("[ERROR] failed to close proxy http server, %v", clsErr)
			}
		}
	}()

	s.httpServer = &http.Server{
		Addr:              fmt.Sprintf(":%d", port),
		Handler:           s.router(),
		ReadHeaderTimeout: 5 * time.Second,
		WriteTimeout:      30 * time.Second,
		IdleTimeout:       30 * time.Second,
	}
	err = s.httpServer.ListenAndServe()
	log.Printf("[WARN] http server terminated, %s", err)
}

func (s *Server) router() *chi.Mux {
	router := chi.NewRouter()
	router.Use(middleware.RealIP, rest.Recoverer(log.Default()))
	router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second))
	router.Use(rest.AppInfo("feed-master", "umputun", s.Version), rest.Ping)
	router.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)))

	router.Group(func(rimg chi.Router) {
		l := logger.New(logger.Log(log.Default()), logger.Prefix("[DEBUG]"))
		rimg.Use(l.Handler)
		rimg.Get("/images/{name}", s.getImageCtrl)
		rimg.Get("/image/{name}", s.getImageCtrl)
		rimg.Head("/image/{name}", s.getImageHeadCtrl)
		rimg.Head("/images/{name}", s.getImageHeadCtrl)
	})

	router.Group(func(rrss chi.Router) {
		l := logger.New(logger.Log(log.Default()), logger.Prefix("[INFO]"))
		rrss.Use(l.Handler)
		rrss.Get("/rss/{name}", s.getFeedCtrl)
		rrss.Get("/list", s.getListCtrl)
		rrss.Get("/feed/{name}", s.getFeedPageCtrl)
		rrss.Get("/feed/{name}/sources", s.getSourcesPageCtrl)
		rrss.Get("/feeds", s.getFeedsPageCtrl)
	})

	router.Route("/yt", func(r chi.Router) {
		l := logger.New(logger.Log(log.Default()), logger.Prefix("[INFO]"))
		r.Use(l.Handler)
		r.Get("/rss/{channel}", s.getYoutubeFeedCtrl)
	})

	if s.Conf.YouTube.BaseURL != "" {
		baseYtURL, parseErr := url.Parse(s.Conf.YouTube.BaseURL)
		if parseErr != nil {
			log.Printf("[ERROR] failed to parse base url %s, %v", s.Conf.YouTube.BaseURL, parseErr)
		}
		ytfs, fsErr := rest.NewFileServer(baseYtURL.Path, s.Conf.YouTube.FilesLocation)
		if fsErr == nil {
			router.Mount(baseYtURL.Path, ytfs)
		} else {
			log.Printf("[WARN] can't start static file server for yt, %v", fsErr)
		}
	}

	fs, err := rest.NewFileServer("/static", filepath.Join("webapp", "static"))
	if err == nil {
		router.Mount("/static", fs)
	} else {
		log.Printf("[WARN] can't start static file server, %v", err)
	}
	return router
}

// GET /rss/{name} - returns rss for given feeds set
func (s *Server) getFeedCtrl(w http.ResponseWriter, r *http.Request) {
	feedName := chi.URLParam(r, "name")
	items, err := s.Store.Load(feedName, s.Conf.System.MaxTotal, true)
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest, err, "failed to get feed")
		return
	}

	for i, itm := range items {
		// add ts suffix to titles
		switch s.Conf.Feeds[feedName].ExtendDateTitle {
		case "yyyyddmm":
			items[i].Title = fmt.Sprintf("%s (%s)", itm.Title, itm.DT.Format("2006-02-01"))
		case "yyyymmdd":
			items[i].Title = fmt.Sprintf("%s (%s)", itm.Title, itm.DT.Format("2006-01-02"))
		}
	}

	rss := feed.Rss2{
		Version:       "2.0",
		ItemList:      items,
		Title:         s.Conf.Feeds[feedName].Title,
		Description:   s.Conf.Feeds[feedName].Description,
		Language:      s.Conf.Feeds[feedName].Language,
		Link:          s.Conf.Feeds[feedName].Link,
		PubDate:       items[0].PubDate,
		LastBuildDate: time.Now().Format(time.RFC822Z),
		NsItunes:      "http://www.itunes.com/dtds/podcast-1.0.dtd",
		NsMedia:       "http://search.yahoo.com/mrss/",
	}

	// replace link to UI page
	if s.Conf.System.BaseURL != "" {
		baseURL := strings.TrimSuffix(s.Conf.System.BaseURL, "/")
		rss.Link = baseURL + "/feed/" + feedName
		imagesURL := baseURL + "/images/" + feedName
		rss.ItunesImage = feed.ItunesImg{URL: imagesURL}
		rss.MediaThumbnail = feed.MediaThumbnail{URL: imagesURL}
	}

	b, err := xml.MarshalIndent(&rss, "", "  ")
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to marshal rss")
		return
	}

	w.Header().Set("Content-Type", "application/xml; charset=UTF-8")
	res := `<?xml version="1.0" encoding="UTF-8"?>` + "\n" + string(b)
	// this hack to avoid having different items for marshal and unmarshal due to "itunes" namespace
	res = strings.Replace(res, "<duration>", "<itunes:duration>", -1)
	res = strings.Replace(res, "</duration>", "</itunes:duration>", -1)

	_, _ = fmt.Fprintf(w, "%s", res)
}

// GET /image/{name}
func (s *Server) getImageCtrl(w http.ResponseWriter, r *http.Request) {
	fm := chi.URLParam(r, "name")
	fm = strings.TrimSuffix(fm, ".png")
	feedConf, found := s.Conf.Feeds[fm]
	if !found {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest,
			fmt.Errorf("image %s not found", fm), "failed to load image")
		return
	}

	b, err := os.ReadFile(feedConf.Image)
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest,
			errors.New("can't read  "+chi.URLParam(r, "name")), "failed to read image")
		return
	}
	w.Header().Set("Content-Type", "image/png")
	if _, err := w.Write(b); err != nil {
		log.Printf("[WARN] failed to send image, %s", err)
	}
}

// HEAD /image/{name}
func (s *Server) getImageHeadCtrl(w http.ResponseWriter, r *http.Request) {
	fm := chi.URLParam(r, "name")
	fm = strings.TrimSuffix(fm, ".png")
	feedConf, found := s.Conf.Feeds[fm]
	if !found {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	info, err := os.Stat(feedConf.Image)
	if err != nil {
		w.WriteHeader(http.StatusNotFound)
		return
	}
	w.Header().Set("Content-Type", "image/png")
	w.Header().Set("Content-Length", strconv.Itoa(int(info.Size())))
	w.WriteHeader(http.StatusOK)
}

// GET /list - returns feed's image
func (s *Server) getListCtrl(w http.ResponseWriter, r *http.Request) {
	feeds := s.feeds()
	render.JSON(w, r, feeds)
}

// GET /yt/rss/{channel} - returns rss for given youtube channel
func (s *Server) getYoutubeFeedCtrl(w http.ResponseWriter, r *http.Request) {
	channel := chi.URLParam(r, "channel")

	fi := youtube.FeedInfo{ID: channel}
	for _, f := range s.Conf.YouTube.Channels {
		if f.ID == channel {
			fi = f
			break
		}
	}

	res, err := s.YoutubeSvc.RSSFeed(fi)
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to read yt list")
		return
	}

	w.Header().Set("Content-Type", "application/xml; charset=UTF-8")
	res = `<?xml version="1.0" encoding="UTF-8"?>` + "\n" + res
	_, _ = fmt.Fprintf(w, "%s", res)
}

func (s *Server) feeds() []string {
	feeds := make([]string, 0, len(s.Conf.Feeds))
	for k := range s.Conf.Feeds {
		feeds = append(feeds, k)
	}
	return feeds
}
<commit_msg>support HEAD requests<commit_after>// Package api provides rest-like server
package api

import (
	"context"
	"encoding/xml"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/didip/tollbooth/v6"
	"github.com/didip/tollbooth_chi"
	"github.com/go-chi/chi/v5"
	"github.com/go-chi/chi/v5/middleware"
	"github.com/go-chi/render"
	"github.com/go-pkgz/lcw"
	log "github.com/go-pkgz/lgr"
	"github.com/go-pkgz/rest"
	"github.com/go-pkgz/rest/logger"

	"github.com/umputun/feed-master/app/config"
	"github.com/umputun/feed-master/app/feed"
	"github.com/umputun/feed-master/app/proc"
	"github.com/umputun/feed-master/app/youtube"
)

// Server provides HTTP API
type Server struct {
	Version    string
	Conf       config.Conf
	Store      *proc.BoltDB
	YoutubeSvc YoutubeSvc
	httpServer *http.Server
	cache      lcw.LoadingCache
}

// YoutubeSvc provides access to youtube's audio rss
type YoutubeSvc interface {
	RSSFeed(cinfo youtube.FeedInfo) (string, error)
}

// Run starts http server for API with all routes
func (s *Server) Run(ctx context.Context, port int) {
	var err error
	if s.cache, err = lcw.NewExpirableCache(lcw.TTL(time.Minute*5), lcw.MaxCacheSize(10*1024*1024)); err != nil {
		log.Printf("[PANIC] failed to make loading cache, %v", err)
		return
	}

	go func() {
		<-ctx.Done()
		if s.httpServer != nil {
			if clsErr := s.httpServer.Close(); clsErr != nil {
				log.Printf("[ERROR] failed to close proxy http server, %v", clsErr)
			}
		}
	}()

	s.httpServer = &http.Server{
		Addr:              fmt.Sprintf(":%d", port),
		Handler:           s.router(),
		ReadHeaderTimeout: 5 * time.Second,
		WriteTimeout:      30 * time.Second,
		IdleTimeout:       30 * time.Second,
	}
	err = s.httpServer.ListenAndServe()
	log.Printf("[WARN] http server terminated, %s", err)
}

func (s *Server) router() *chi.Mux {
	router := chi.NewRouter()
	router.Use(middleware.RealIP, rest.Recoverer(log.Default()))
	router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second))
	router.Use(rest.AppInfo("feed-master", "umputun", s.Version), rest.Ping)
	router.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)))
	router.Group(func(rimg chi.Router) {
		l := logger.New(logger.Log(log.Default()), logger.Prefix("[DEBUG]"))
		rimg.Use(l.Handler, middleware.GetHead)
		rimg.Get("/images/{name}", s.getImageCtrl)
		rimg.Get("/image/{name}", s.getImageCtrl)
	})

	router.Group(func(rrss chi.Router) {
		l := logger.New(logger.Log(log.Default()), logger.Prefix("[INFO]"))
		rrss.Use(l.Handler, middleware.GetHead)
		rrss.Get("/rss/{name}", s.getFeedCtrl)
		rrss.Head("/rss/{name}", s.getFeedCtrl)
		rrss.Get("/list", s.getListCtrl)
		rrss.Get("/feed/{name}", s.getFeedPageCtrl)
		rrss.Get("/feed/{name}/sources", s.getSourcesPageCtrl)
		rrss.Get("/feeds", s.getFeedsPageCtrl)
	})

	router.Route("/yt", func(r chi.Router) {
		l := logger.New(logger.Log(log.Default()), logger.Prefix("[INFO]"))
		r.Use(l.Handler, middleware.GetHead)
		r.Get("/rss/{channel}", s.getYoutubeFeedCtrl)
	})

	if s.Conf.YouTube.BaseURL != "" {
		baseYtURL, parseErr := url.Parse(s.Conf.YouTube.BaseURL)
		if parseErr != nil {
			log.Printf("[ERROR] failed to parse base url %s, %v", s.Conf.YouTube.BaseURL, parseErr)
		}
		ytfs, fsErr := rest.NewFileServer(baseYtURL.Path, s.Conf.YouTube.FilesLocation)
		if fsErr == nil {
			router.Mount(baseYtURL.Path, ytfs)
		} else {
			log.Printf("[WARN] can't start static file server for yt, %v", fsErr)
		}
	}

	fs, err := rest.NewFileServer("/static", filepath.Join("webapp", "static"))
	if err == nil {
		router.Mount("/static", fs)
	} else {
		log.Printf("[WARN] can't start static file server, %v", err)
	}
	return router
}

// GET /rss/{name} - returns rss for given feeds set
func (s *Server) getFeedCtrl(w http.ResponseWriter, r *http.Request) {
	feedName := chi.URLParam(r, "name")
	items, err := s.Store.Load(feedName, s.Conf.System.MaxTotal, true)
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest, err, "failed to get feed")
		return
	}

	for i, itm := range items {
		// add ts suffix to titles
		switch s.Conf.Feeds[feedName].ExtendDateTitle {
		case "yyyyddmm":
			items[i].Title = fmt.Sprintf("%s (%s)", itm.Title, itm.DT.Format("2006-02-01"))
		case "yyyymmdd":
			items[i].Title = fmt.Sprintf("%s (%s)", itm.Title, itm.DT.Format("2006-01-02"))
		}
	}

	rss := feed.Rss2{
		Version:       "2.0",
		ItemList:      items,
		Title:         s.Conf.Feeds[feedName].Title,
		Description:   s.Conf.Feeds[feedName].Description,
		Language:      s.Conf.Feeds[feedName].Language,
		Link:          s.Conf.Feeds[feedName].Link,
		PubDate:       items[0].PubDate,
		LastBuildDate: time.Now().Format(time.RFC822Z),
		NsItunes:      "http://www.itunes.com/dtds/podcast-1.0.dtd",
		NsMedia:       "http://search.yahoo.com/mrss/",
	}

	// replace link to UI page
	if s.Conf.System.BaseURL != "" {
		baseURL := strings.TrimSuffix(s.Conf.System.BaseURL, "/")
		rss.Link = baseURL + "/feed/" + feedName
		imagesURL := baseURL + "/images/" + feedName
		rss.ItunesImage = feed.ItunesImg{URL: imagesURL}
		rss.MediaThumbnail = feed.MediaThumbnail{URL: imagesURL}
	}

	b, err := xml.MarshalIndent(&rss, "", "  ")
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to marshal rss")
		return
	}

	w.Header().Set("Content-Type", "application/xml; charset=UTF-8")
	res := `<?xml version="1.0" encoding="UTF-8"?>` + "\n" + string(b)
	// this hack to avoid having different items for marshal and unmarshal due to "itunes" namespace
	res = strings.Replace(res, "<duration>", "<itunes:duration>", -1)
	res = strings.Replace(res, "</duration>", "</itunes:duration>", -1)

	_, _ = fmt.Fprintf(w, "%s", res)
}

// GET /image/{name}
func (s *Server) getImageCtrl(w http.ResponseWriter, r *http.Request) {
	fm := chi.URLParam(r, "name")
	fm = strings.TrimSuffix(fm, ".png")
	feedConf, found := s.Conf.Feeds[fm]
	if !found {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest,
			fmt.Errorf("image %s not found", fm), "failed to load image")
		return
	}

	b, err := os.ReadFile(feedConf.Image)
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest,
			errors.New("can't read  "+chi.URLParam(r, "name")), "failed to read image")
		return
	}
	w.Header().Set("Content-Type", "image/png")
	if _, err := w.Write(b); err != nil {
		log.Printf("[WARN] failed to send image, %s", err)
	}
}

// GET /list - returns feed's image
func (s *Server) getListCtrl(w http.ResponseWriter, r *http.Request) {
	feeds := s.feeds()
	render.JSON(w, r, feeds)
}

// GET /yt/rss/{channel} - returns rss for given youtube channel
func (s *Server) getYoutubeFeedCtrl(w http.ResponseWriter, r *http.Request) {
	channel := chi.URLParam(r, "channel")

	fi := youtube.FeedInfo{ID: channel}
	for _, f := range s.Conf.YouTube.Channels {
		if f.ID == channel {
			fi = f
			break
		}
	}

	res, err := s.YoutubeSvc.RSSFeed(fi)
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to read yt list")
		return
	}

	w.Header().Set("Content-Type", "application/xml; charset=UTF-8")
	res = `<?xml version="1.0" encoding="UTF-8"?>` + "\n" + res
	_, _ = fmt.Fprintf(w, "%s", res)
}

func (s *Server) feeds() []string {
	feeds := make([]string, 0, len(s.Conf.Feeds))
	for k := range s.Conf.Feeds {
		feeds = append(feeds, k)
	}
	return feeds
}
<|endoftext|>


<commit_before>// Copyright 2019, OpenTelemetry Authors
// Portions of this file Copyright 2018-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package awsxrayexporter

import (
	"crypto/tls"
	"net/http"
	"net/url"
	"os"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
	"github.com/aws/aws-sdk-go/aws/ec2metadata"
	"github.com/aws/aws-sdk-go/aws/endpoints"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/sts"
	"go.uber.org/zap"
	"golang.org/x/net/http2"
)

type connAttr interface {
	newAWSSession(logger *zap.Logger, roleArn string, region string) (*session.Session, error)
	getEC2Region(s *session.Session) (string, error)
}

// Conn implements connAttr interface.
type Conn struct{}

func (c *Conn) getEC2Region(s *session.Session) (string, error) {
	return ec2metadata.New(s).Region()
}

// AWS STS endpoint constants
const (
	STSEndpointPrefix         = "https://sts."
	STSEndpointSuffix         = ".amazonaws.com"
	STSAwsCnPartitionIDSuffix = ".amazonaws.com.cn" // AWS China partition.
)

// newHTTPClient returns new HTTP client instance with provided configuration.
func newHTTPClient(logger *zap.Logger, maxIdle int, requestTimeout int, noVerify bool,
	proxyAddress string) (*http.Client, error) {
	logger.Debug("Using proxy address: ",
		zap.String("proxyAddr", proxyAddress),
	)
	tls := &tls.Config{
		InsecureSkipVerify: false,
	}

	finalProxyAddress := getProxyAddress(proxyAddress)
	proxyURL, err := getProxyURL(finalProxyAddress)
	if err != nil {
		logger.Error("unable to obtain proxy URL", zap.Error(err))
		return nil, err
	}
	transport := &http.Transport{
		MaxIdleConnsPerHost: maxIdle,
		TLSClientConfig:     tls,
		Proxy:               http.ProxyURL(proxyURL),
	}

	// is not enabled by default as we configure TLSClientConfig for supporting SSL to data plane.
	// http2.ConfigureTransport will setup transport layer to use HTTP2
	http2.ConfigureTransport(transport)
	http := &http.Client{
		Transport: transport,
		Timeout:   time.Second * time.Duration(requestTimeout),
	}
	return http, err
}

func getProxyAddress(proxyAddress string) string {
	var finalProxyAddress string
	if proxyAddress != "" {
		finalProxyAddress = proxyAddress
	} else if proxyAddress == "" && os.Getenv("HTTPS_PROXY") != "" {
		finalProxyAddress = os.Getenv("HTTPS_PROXY")
	} else {
		finalProxyAddress = ""
	}
	return finalProxyAddress
}

func getProxyURL(finalProxyAddress string) (*url.URL, error) {
	var proxyURL *url.URL
	var err error
	if finalProxyAddress != "" {
		proxyURL, err = url.Parse(finalProxyAddress)
	} else {
		proxyURL = nil
		err = nil
	}
	return proxyURL, err
}

// GetAWSConfigSession returns AWS config and session instances.
func GetAWSConfigSession(logger *zap.Logger, cn connAttr, cfg *Config) (*aws.Config, *session.Session, error) {
	var s *session.Session
	var err error
	var awsRegion string
	http, err := newHTTPClient(logger, cfg.NumberOfWorkers, cfg.RequestTimeoutSeconds, cfg.NoVerifySSL, cfg.ProxyAddress)
	if err != nil {
		logger.Error("unable to obtain proxy URL", zap.Error(err))
		return nil, nil, err
	}
	regionEnv := os.Getenv("AWS_REGION")
	if cfg.Region == "" && regionEnv != "" {
		awsRegion = regionEnv
		logger.Debug("Fetch region from environment variables", zap.String("region", awsRegion))
	} else if cfg.Region != "" {
		awsRegion = cfg.Region
		logger.Debug("Fetch region from commandline/config file", zap.String("region", awsRegion))
	} else if !cfg.NoVerifySSL {
		var es *session.Session
		es, err = getDefaultSession(logger)
		if err != nil {
			logger.Error("Unable to retrieve default session", zap.Error(err))
		} else {
			awsRegion, err = cn.getEC2Region(es)
			if err != nil {
				logger.Error("Unable to retrieve the region from the EC2 instance", zap.Error(err))
			} else {
				logger.Debug("Fetch region from ec2 metadata", zap.String("region", awsRegion))
			}
		}
	}
	if awsRegion == "" {
		msg := "Cannot fetch region variable from config file, environment variables and ec2 metadata."
		logger.Error(msg)
		return nil, nil, awserr.New("NoAwsRegion", msg, nil)
	}
	s, err = cn.newAWSSession(logger, cfg.RoleARN, awsRegion)
	if err != nil {
		return nil, nil, err
	}

	config := &aws.Config{
		Region:                 aws.String(awsRegion),
		DisableParamValidation: aws.Bool(true),
		MaxRetries:             aws.Int(cfg.MaxRetries),
		Endpoint:               aws.String(cfg.Endpoint),
		HTTPClient:             http,
	}
	return config, s, nil
}

// ProxyServerTransport configures HTTP transport for TCP Proxy Server.
func ProxyServerTransport(logger *zap.Logger, config *Config) (*http.Transport, error) {
	tls := &tls.Config{
		InsecureSkipVerify: config.NoVerifySSL,
	}

	proxyAddr := getProxyAddress(config.ProxyAddress)
	proxyURL, err := getProxyURL(proxyAddr)
	if err != nil {
		logger.Error("unable to obtain proxy URL", zap.Error(err))
		return nil, err
	}

	// Connection timeout in seconds
	idleConnTimeout := time.Duration(config.RequestTimeoutSeconds) * time.Second

	transport := &http.Transport{
		MaxIdleConns:        config.NumberOfWorkers,
		MaxIdleConnsPerHost: config.NumberOfWorkers,
		IdleConnTimeout:     idleConnTimeout,
		Proxy:               http.ProxyURL(proxyURL),
		TLSClientConfig:     tls,

		// If not disabled the transport will add a gzip encoding header
		// to requests with no `accept-encoding` header value. The header
		// is added after we sign the request which invalidates the
		// signature.
		DisableCompression: true,
	}

	return transport, nil
}

func (c *Conn) newAWSSession(logger *zap.Logger, roleArn string, region string) (*session.Session, error) {
	var s *session.Session
	var err error
	if roleArn == "" {
		s, err = getDefaultSession(logger)
		if err != nil {
			return s, err
		}
	} else {
		stsCreds, _ := getSTSCreds(logger, region, roleArn)

		s, err = session.NewSession(&aws.Config{
			Credentials: stsCreds,
		})

		if err != nil {
			logger.Error("Error in creating session object : ", zap.Error(err))
			return s, err
		}
	}
	return s, nil
}

// getSTSCreds gets STS credentials from regional endpoint. ErrCodeRegionDisabledException is received if the
// STS regional endpoint is disabled. In this case STS credentials are fetched from STS primary regional endpoint
// in the respective AWS partition.
func getSTSCreds(logger *zap.Logger, region string, roleArn string) (*credentials.Credentials, error) {
	t, err := getDefaultSession(logger)
	if err != nil {
		return nil, err
	}

	stsCred := getSTSCredsFromRegionEndpoint(logger, t, region, roleArn)
	// Make explicit call to fetch credentials.
	_, err = stsCred.Get()
	if err != nil {
		if aerr, ok := err.(awserr.Error); ok {
			err = nil
			switch aerr.Code() {
			case sts.ErrCodeRegionDisabledException:
				logger.Error("Region ", zap.String("region", region), zap.String("error", aerr.Error()))
				stsCred = getSTSCredsFromPrimaryRegionEndpoint(logger, t, roleArn, region)
			}
		}
	}
	return stsCred, err
}

// getSTSCredsFromRegionEndpoint fetches STS credentials for provided roleARN from regional endpoint.
// AWS STS recommends that you provide both the Region and endpoint when you make calls to a Regional endpoint.
// Reference: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html#id_credentials_temp_enable-regions_writing_code
func getSTSCredsFromRegionEndpoint(logger *zap.Logger, sess *session.Session, region string,
	roleArn string) *credentials.Credentials {
	regionalEndpoint := getSTSRegionalEndpoint(region)
	// if regionalEndpoint is "", the STS endpoint is Global endpoint for classic regions except ap-east-1 - (HKG)
	// for other opt-in regions, region value will create STS regional endpoint.
	// This will be only in the case, if provided region is not present in aws_regions.go
	c := &aws.Config{Region: aws.String(region), Endpoint: &regionalEndpoint}
	st := sts.New(sess, c)
	logger.Info("STS Endpoint ", zap.String("endpoint", st.Endpoint))
	return stscreds.NewCredentialsWithClient(st, roleArn)
}

// getSTSCredsFromPrimaryRegionEndpoint fetches STS credentials for provided roleARN from primary region endpoint in
// the respective partition.
func getSTSCredsFromPrimaryRegionEndpoint(logger *zap.Logger, t *session.Session, roleArn string,
	region string) *credentials.Credentials {
	logger.Info("Credentials for provided RoleARN being fetched from STS primary region endpoint.")
	partitionID := getPartition(region)
	if partitionID == endpoints.AwsPartitionID {
		return getSTSCredsFromRegionEndpoint(logger, t, endpoints.UsEast1RegionID, roleArn)
	} else if partitionID == endpoints.AwsCnPartitionID {
		return getSTSCredsFromRegionEndpoint(logger, t, endpoints.CnNorth1RegionID, roleArn)
	} else if partitionID == endpoints.AwsUsGovPartitionID {
		return getSTSCredsFromRegionEndpoint(logger, t, endpoints.UsGovWest1RegionID, roleArn)
	}

	return nil
}

func getSTSRegionalEndpoint(r string) string {
	p := getPartition(r)

	var e string
	if p == endpoints.AwsPartitionID || p == endpoints.AwsUsGovPartitionID {
		e = STSEndpointPrefix + r + STSEndpointSuffix
	} else if p == endpoints.AwsCnPartitionID {
		e = STSEndpointPrefix + r + STSAwsCnPartitionIDSuffix
	}
	return e
}

func getDefaultSession(logger *zap.Logger) (*session.Session, error) {
	result, serr := session.NewSession()
	if serr != nil {
		logger.Error("Error in creating session object ", zap.Error(serr))
		return result, serr
	}
	return result, nil
}

// getPartition return AWS Partition for the provided region.
func getPartition(region string) string {
	p, _ := endpoints.PartitionForRegion(endpoints.DefaultPartitions(), region)
	return p.ID()
}
<commit_msg>Setting the tlsconfig InsecureSkipVerify using NoVerifySSL (#1350)<commit_after>// Copyright 2019, OpenTelemetry Authors
// Portions of this file Copyright 2018-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package awsxrayexporter

import (
	"crypto/tls"
	"net/http"
	"net/url"
	"os"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
	"github.com/aws/aws-sdk-go/aws/ec2metadata"
	"github.com/aws/aws-sdk-go/aws/endpoints"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/sts"
	"go.uber.org/zap"
	"golang.org/x/net/http2"
)

type connAttr interface {
	newAWSSession(logger *zap.Logger, roleArn string, region string) (*session.Session, error)
	getEC2Region(s *session.Session) (string, error)
}

// Conn implements connAttr interface.
type Conn struct{}

func (c *Conn) getEC2Region(s *session.Session) (string, error) {
	return ec2metadata.New(s).Region()
}

// AWS STS endpoint constants
const (
	STSEndpointPrefix         = "https://sts."
	STSEndpointSuffix         = ".amazonaws.com"
	STSAwsCnPartitionIDSuffix = ".amazonaws.com.cn" // AWS China partition.
)

// newHTTPClient returns new HTTP client instance with provided configuration.
func newHTTPClient(logger *zap.Logger, maxIdle int, requestTimeout int, noVerify bool,
	proxyAddress string) (*http.Client, error) {
	logger.Debug("Using proxy address: ",
		zap.String("proxyAddr", proxyAddress),
	)
	tls := &tls.Config{
		InsecureSkipVerify: noVerify,
	}

	finalProxyAddress := getProxyAddress(proxyAddress)
	proxyURL, err := getProxyURL(finalProxyAddress)
	if err != nil {
		logger.Error("unable to obtain proxy URL", zap.Error(err))
		return nil, err
	}
	transport := &http.Transport{
		MaxIdleConnsPerHost: maxIdle,
		TLSClientConfig:     tls,
		Proxy:               http.ProxyURL(proxyURL),
	}

	// is not enabled by default as we configure TLSClientConfig for supporting SSL to data plane.
	// http2.ConfigureTransport will setup transport layer to use HTTP2
	http2.ConfigureTransport(transport)
	http := &http.Client{
		Transport: transport,
		Timeout:   time.Second * time.Duration(requestTimeout),
	}
	return http, err
}

func getProxyAddress(proxyAddress string) string {
	var finalProxyAddress string
	if proxyAddress != "" {
		finalProxyAddress = proxyAddress
	} else if proxyAddress == "" && os.Getenv("HTTPS_PROXY") != "" {
		finalProxyAddress = os.Getenv("HTTPS_PROXY")
	} else {
		finalProxyAddress = ""
	}
	return finalProxyAddress
}

func getProxyURL(finalProxyAddress string) (*url.URL, error) {
	var proxyURL *url.URL
	var err error
	if finalProxyAddress != "" {
		proxyURL, err = url.Parse(finalProxyAddress)
	} else {
		proxyURL = nil
		err = nil
	}
	return proxyURL, err
}

// GetAWSConfigSession returns AWS config and session instances.
func GetAWSConfigSession(logger *zap.Logger, cn connAttr, cfg *Config) (*aws.Config, *session.Session, error) {
	var s *session.Session
	var err error
	var awsRegion string
	http, err := newHTTPClient(logger, cfg.NumberOfWorkers, cfg.RequestTimeoutSeconds, cfg.NoVerifySSL, cfg.ProxyAddress)
	if err != nil {
		logger.Error("unable to obtain proxy URL", zap.Error(err))
		return nil, nil, err
	}
	regionEnv := os.Getenv("AWS_REGION")
	if cfg.Region == "" && regionEnv != "" {
		awsRegion = regionEnv
		logger.Debug("Fetch region from environment variables", zap.String("region", awsRegion))
	} else if cfg.Region != "" {
		awsRegion = cfg.Region
		logger.Debug("Fetch region from commandline/config file", zap.String("region", awsRegion))
	} else if !cfg.NoVerifySSL {
		var es *session.Session
		es, err = getDefaultSession(logger)
		if err != nil {
			logger.Error("Unable to retrieve default session", zap.Error(err))
		} else {
			awsRegion, err = cn.getEC2Region(es)
			if err != nil {
				logger.Error("Unable to retrieve the region from the EC2 instance", zap.Error(err))
			} else {
				logger.Debug("Fetch region from ec2 metadata", zap.String("region", awsRegion))
			}
		}
	}
	if awsRegion == "" {
		msg := "Cannot fetch region variable from config file, environment variables and ec2 metadata."
		logger.Error(msg)
		return nil, nil, awserr.New("NoAwsRegion", msg, nil)
	}
	s, err = cn.newAWSSession(logger, cfg.RoleARN, awsRegion)
	if err != nil {
		return nil, nil, err
	}

	config := &aws.Config{
		Region:                 aws.String(awsRegion),
		DisableParamValidation: aws.Bool(true),
		MaxRetries:             aws.Int(cfg.MaxRetries),
		Endpoint:               aws.String(cfg.Endpoint),
		HTTPClient:             http,
	}
	return config, s, nil
}

// ProxyServerTransport configures HTTP transport for TCP Proxy Server.
func ProxyServerTransport(logger *zap.Logger, config *Config) (*http.Transport, error) {
	tls := &tls.Config{
		InsecureSkipVerify: config.NoVerifySSL,
	}

	proxyAddr := getProxyAddress(config.ProxyAddress)
	proxyURL, err := getProxyURL(proxyAddr)
	if err != nil {
		logger.Error("unable to obtain proxy URL", zap.Error(err))
		return nil, err
	}

	// Connection timeout in seconds
	idleConnTimeout := time.Duration(config.RequestTimeoutSeconds) * time.Second

	transport := &http.Transport{
		MaxIdleConns:        config.NumberOfWorkers,
		MaxIdleConnsPerHost: config.NumberOfWorkers,
		IdleConnTimeout:     idleConnTimeout,
		Proxy:               http.ProxyURL(proxyURL),
		TLSClientConfig:     tls,

		// If not disabled the transport will add a gzip encoding header
		// to requests with no `accept-encoding` header value. The header
		// is added after we sign the request which invalidates the
		// signature.
		DisableCompression: true,
	}

	return transport, nil
}

func (c *Conn) newAWSSession(logger *zap.Logger, roleArn string, region string) (*session.Session, error) {
	var s *session.Session
	var err error
	if roleArn == "" {
		s, err = getDefaultSession(logger)
		if err != nil {
			return s, err
		}
	} else {
		stsCreds, _ := getSTSCreds(logger, region, roleArn)

		s, err = session.NewSession(&aws.Config{
			Credentials: stsCreds,
		})

		if err != nil {
			logger.Error("Error in creating session object : ", zap.Error(err))
			return s, err
		}
	}
	return s, nil
}

// getSTSCreds gets STS credentials from regional endpoint. ErrCodeRegionDisabledException is received if the
// STS regional endpoint is disabled. In this case STS credentials are fetched from STS primary regional endpoint
// in the respective AWS partition.
func getSTSCreds(logger *zap.Logger, region string, roleArn string) (*credentials.Credentials, error) {
	t, err := getDefaultSession(logger)
	if err != nil {
		return nil, err
	}

	stsCred := getSTSCredsFromRegionEndpoint(logger, t, region, roleArn)
	// Make explicit call to fetch credentials.
	_, err = stsCred.Get()
	if err != nil {
		if aerr, ok := err.(awserr.Error); ok {
			err = nil
			switch aerr.Code() {
			case sts.ErrCodeRegionDisabledException:
				logger.Error("Region ", zap.String("region", region), zap.String("error", aerr.Error()))
				stsCred = getSTSCredsFromPrimaryRegionEndpoint(logger, t, roleArn, region)
			}
		}
	}
	return stsCred, err
}

// getSTSCredsFromRegionEndpoint fetches STS credentials for provided roleARN from regional endpoint.
// AWS STS recommends that you provide both the Region and endpoint when you make calls to a Regional endpoint.
// Reference: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html#id_credentials_temp_enable-regions_writing_code
func getSTSCredsFromRegionEndpoint(logger *zap.Logger, sess *session.Session, region string,
	roleArn string) *credentials.Credentials {
	regionalEndpoint := getSTSRegionalEndpoint(region)
	// if regionalEndpoint is "", the STS endpoint is Global endpoint for classic regions except ap-east-1 - (HKG)
	// for other opt-in regions, region value will create STS regional endpoint.
	// This will be only in the case, if provided region is not present in aws_regions.go
	c := &aws.Config{Region: aws.String(region), Endpoint: &regionalEndpoint}
	st := sts.New(sess, c)
	logger.Info("STS Endpoint ", zap.String("endpoint", st.Endpoint))
	return stscreds.NewCredentialsWithClient(st, roleArn)
}

// getSTSCredsFromPrimaryRegionEndpoint fetches STS credentials for provided roleARN from primary region endpoint in
// the respective partition.
func getSTSCredsFromPrimaryRegionEndpoint(logger *zap.Logger, t *session.Session, roleArn string,
	region string) *credentials.Credentials {
	logger.Info("Credentials for provided RoleARN being fetched from STS primary region endpoint.")
	partitionID := getPartition(region)
	if partitionID == endpoints.AwsPartitionID {
		return getSTSCredsFromRegionEndpoint(logger, t, endpoints.UsEast1RegionID, roleArn)
	} else if partitionID == endpoints.AwsCnPartitionID {
		return getSTSCredsFromRegionEndpoint(logger, t, endpoints.CnNorth1RegionID, roleArn)
	} else if partitionID == endpoints.AwsUsGovPartitionID {
		return getSTSCredsFromRegionEndpoint(logger, t, endpoints.UsGovWest1RegionID, roleArn)
	}

	return nil
}

func getSTSRegionalEndpoint(r string) string {
	p := getPartition(r)

	var e string
	if p == endpoints.AwsPartitionID || p == endpoints.AwsUsGovPartitionID {
		e = STSEndpointPrefix + r + STSEndpointSuffix
	} else if p == endpoints.AwsCnPartitionID {
		e = STSEndpointPrefix + r + STSAwsCnPartitionIDSuffix
	}
	return e
}

func getDefaultSession(logger *zap.Logger) (*session.Session, error) {
	result, serr := session.NewSession()
	if serr != nil {
		logger.Error("Error in creating session object ", zap.Error(serr))
		return result, serr
	}
	return result, nil
}

// getPartition return AWS Partition for the provided region.
func getPartition(region string) string {
	p, _ := endpoints.PartitionForRegion(endpoints.DefaultPartitions(), region)
	return p.ID()
}
<|endoftext|>


<commit_before>package fuseklient

import (
	"errors"
	"io"
	"koding/fuseklient/transport"
)

const (
	DefaultBlockSize = 4096 * 10
)

var (
	ErrCreateOnNotEmpty = errors.New("Create called on file sized > 0.")
)

type remote interface {
	ReadFileAt(string, int64, int64) (*transport.ReadFileRes, error)
	WriteFile(string, []byte) error
}

type ContentReadWriter struct {
	remote

	Path string
	Size int64

	BlockSize int64

	// isDirty indicates if File has been written to, but not yet been synced
	// with remote.
	isDirty bool
	content []byte
}

func NewContentReadWriter(t remote, path string, size int64) *ContentReadWriter {
	return &ContentReadWriter{
		remote:    t,
		Path:      path,
		Size:      size,
		BlockSize: DefaultBlockSize,
		isDirty:   false,
		content:   nil,
	}
}

func (c *ContentReadWriter) Create() error {
	if c.Size > 0 {
		return ErrCreateOnNotEmpty
	}

	return c.writeContentToRemote(c.content)
}

func (c *ContentReadWriter) ReadAt(p []byte, offset int64) (int, error) {
	if offset >= c.Size {
		return 0, io.EOF
	}

	// everything is fetched from remote already, so return from it
	if int64(len(c.content)) == c.Size {
		copied := copy(p, c.content[offset:])
		return copied, nil
	}

	resp, err := c.remote.ReadFileAt(c.Path, offset, c.BlockSize)
	if err != nil {
		return 0, err
	}

	// TODO: save resp.Content to c.content if curPos is at offset

	copied := copy(p, resp.Content)

	return copied, nil
}

func (c *ContentReadWriter) ResetAndRead() error {
	c.Reset()
	return c.ReadAll()
}

func (c *ContentReadWriter) ReadAll() error {
	// everything is fetched from remote already, so return
	if int64(len(c.content)) == c.Size {
		return nil
	}

	var offset int64 = 0
	for offset < c.Size {
		resp, err := c.remote.ReadFileAt(c.Path, offset, c.BlockSize)
		if err != nil {
			return err
		}

		n := c.writeAt(resp.Content, offset)
		offset += int64(n)
	}

	// TODO: what is this doing here?
	//if offset == 0 {
	//  c.content = c.content[0:len(content)]
	//}

	return nil
}

func (c *ContentReadWriter) WriteAt(content []byte, offset int64) error {
	if c.needSync() {
		if err := c.ReadAll(); err != nil {
			return err
		}
	}

	c.isDirty = true
	c.writeAt(content, offset)
	c.Size = int64(len(c.content))

	return nil
}

// TruncateTo reduces file contents to specified length.
func (c *ContentReadWriter) TruncateTo(size uint64) error {
	if c.needSync() {
		if err := c.ReadAll(); err != nil {
			return err
		}
	}

	s := int(size)
	switch {
	case s < len(c.content):
		// specified size less than size of file, so remove all content over size
		c.content = c.content[:size]
	case s > len(c.content):
		// specified size greater same as size of file, so add empty padding to
		// existing content
		c.content = append(c.content, make([]byte, s-len(c.content))...)
	case s == len(c.content):
		// specified size is same as size of file, so nothing to be done
	}

	c.Size = int64(len(c.content))

	return c.Save(true)
}

func (c *ContentReadWriter) Save(force bool) error {
	if !c.needSave() && !force {
		return nil
	}

	return c.writeContentToRemote(c.content)
}

func (c *ContentReadWriter) Reset() {
	c.isDirty = false
	c.content = nil
}

///// private

func (c *ContentReadWriter) needSync() bool {
	return c.Size != int64(len(c.content))
}

func (c *ContentReadWriter) needSave() bool {
	return c.isDirty
}

func (c *ContentReadWriter) writeAt(content []byte, offset int64) int {
	var newLen = len(content) + int(offset)
	if len(c.content) < newLen {
		padding := make([]byte, newLen-len(c.content))
		c.content = append(c.content, padding...)
	}

	return copy(c.content[offset:], content)
}

func (c *ContentReadWriter) writeContentToRemote(content []byte) error {
	c.isDirty = false
	return c.remote.WriteFile(c.Path, content)
}
<commit_msg>fuseklient: add docstring to Content<commit_after>package fuseklient

import (
	"errors"
	"io"
	"koding/fuseklient/transport"
)

const (
	// DefaultBlockSize specifies how much data to ask from remote when reading a
	// file. This is a tradeoff between time waiting for data vs. amount of remote
	// calls.
	DefaultBlockSize = 4096 * 10
)

var (
	// ErrCreateOnNotEmpty is returned when ContentReadWriter#Create is called
	// file with size greater than 0. This is a precaution to prevent overwriting
	// files that have already been created on remote. Use ContentReadWriter#Save
	// instead.
	ErrCreateOnNotEmpty = errors.New(
		"File#Create called on file sized > 0. Use File#Save instead.",
	)
)

// remote is requried interface to talk to remote machine.
type remote interface {
	ReadFileAt(string, int64, int64) (*transport.ReadFileRes, error)
	WriteFile(string, []byte) error
}

// ContentReadWriter represents remote file content related operations. It's
// upto the caller to lock appropriately to allow for thread safe access.
type ContentReadWriter struct {
	remote

	// Path is the full path on locally mounted folder. Note it does not contain.
	// the remote path prefix, that's in transport.
	Path string

	// Size is the size of file on local. Note this can differ from remote if
	// file was written to, but not yet synced to remote.
	Size int64

	// BlockSize is the size of data to request from remote on each run.
	BlockSize int64

	// isDirty indicates if File has been written to, but not yet been synced
	// with remote.
	isDirty bool

	// content is byte slice of actual contents of file. Note this is lazily
	// populated. Use Size to determine actual size of content, not len(content).
	content []byte
}

// NewContentReadWriter is the required initializer for ContentReadWriter.
func NewContentReadWriter(t remote, path string, size int64) *ContentReadWriter {
	return &ContentReadWriter{
		remote:    t,
		Path:      path,
		Size:      size,
		BlockSize: DefaultBlockSize,
		isDirty:   false,
		content:   nil,
	}
}

// Create saves file on remote. It returns an error if file size is > 0.
func (c *ContentReadWriter) Create() error {
	if c.Size > 0 {
		return ErrCreateOnNotEmpty
	}

	return c.writeContentToRemote(c.content)
}

// ReadAt reads contents of file from offset. Note it uses
// ContentReadWriter#BlockSize to specify how much to read rather than len(p).
// If content has been already fetched, it returns from memory, instead of
// fetching from remote. It returns the number of bytes read and and error, if
// any.
func (c *ContentReadWriter) ReadAt(p []byte, offset int64) (int, error) {
	if offset >= c.Size {
		return 0, io.EOF
	}

	// everything is fetched from remote already, so return from it
	if int64(len(c.content)) == c.Size {
		copied := copy(p, c.content[offset:])
		return copied, nil
	}

	resp, err := c.remote.ReadFileAt(c.Path, offset, c.BlockSize)
	if err != nil {
		return 0, err
	}

	copied := copy(p, resp.Content)
	return copied, nil
}

// ResetAndRead is a helper method that zeroses out ContentReadWriter#content
// and reads it again from remote.
func (c *ContentReadWriter) ResetAndRead() error {
	c.Reset()
	return c.ReadAll()
}

// ReadAll fetches and saves content from remote by making x calls to
// ContentReadWriter#ReadAt. If len of content in memory is same as
// ContentReadWriter#Size, then it does nothing.
func (c *ContentReadWriter) ReadAll() error {
	// everything is fetched from remote already, so return
	if int64(len(c.content)) == c.Size {
		return nil
	}

	var offset int64 = 0
	for offset < c.Size {
		resp, err := c.remote.ReadFileAt(c.Path, offset, c.BlockSize)
		if err != nil {
			return err
		}

		n := c.writeAt(resp.Content, offset)
		offset += int64(n)
	}

	return nil
}

// WriteAt writes specified byte slice to memory at specified offset to locally
// saved content. If necessary, it fetches all content from remote first. This
// done to prevent unsynced file from overwriting itself.
func (c *ContentReadWriter) WriteAt(content []byte, offset int64) error {
	if c.needSync() {
		if err := c.ReadAll(); err != nil {
			return err
		}
	}

	c.isDirty = true
	c.writeAt(content, offset)
	c.Size = int64(len(c.content))

	return nil
}

// TruncateTo reduces content size to specified length. If necessary,
// it fetches all content from remote first.
func (c *ContentReadWriter) TruncateTo(size uint64) error {
	if c.needSync() {
		if err := c.ReadAll(); err != nil {
			return err
		}
	}

	s := int(size)
	switch {
	case s < len(c.content):
		// specified size less than size of file, so remove all content over size
		c.content = c.content[:size]
	case s > len(c.content):
		// specified size greater same as size of file, so add empty padding to
		// existing content
		c.content = append(c.content, make([]byte, s-len(c.content))...)
	case s == len(c.content):
		// specified size is same as size of file, so nothing to be done
	}

	c.Size = int64(len(c.content))

	return c.Save(true)
}

// Save writes locally saved content to remote.
func (c *ContentReadWriter) Save(force bool) error {
	if !c.needSave() && !force {
		return nil
	}

	return c.writeContentToRemote(c.content)
}

// Reset removes locally saved content.
func (c *ContentReadWriter) Reset() {
	c.isDirty = false
	c.content = nil
}

///// private

// needSync returns if locally saved content is not the same size of file on
// remote.
func (c *ContentReadWriter) needSync() bool {
	return c.Size != int64(len(c.content))
}

// needSave returns if new changes have been added to local content, but hasn't
// been saved to remote yet.
func (c *ContentReadWriter) needSave() bool {
	return c.isDirty
}

// writeAt writes specified byte slice to locally saved content slice at specified
// offset. If local content slice doesn't have enough space, it addes the
// required space first before calling copy().
func (c *ContentReadWriter) writeAt(content []byte, offset int64) int {
	var newLen = len(content) + int(offset)
	if len(c.content) < newLen {
		padding := make([]byte, newLen-len(c.content))
		c.content = append(c.content, padding...)
	}

	return copy(c.content[offset:], content)
}

// writeContentToRemote saves specified byte slice to remote.
func (c *ContentReadWriter) writeContentToRemote(content []byte) error {
	c.isDirty = false
	return c.remote.WriteFile(c.Path, content)
}
<|endoftext|>


<commit_before>// khan
// https://github.com/topfreegames/khan
//
// Licensed under the MIT license:
// http://www.opensource.org/licenses/mit-license
// Copyright © 2016 Top Free Games <backend@tfgco.com>

package api

//VERSION identifies Khan's current version
var VERSION string = "0.2.1"
<commit_msg>redundant type<commit_after>// khan
// https://github.com/topfreegames/khan
//
// Licensed under the MIT license:
// http://www.opensource.org/licenses/mit-license
// Copyright © 2016 Top Free Games <backend@tfgco.com>

package api

//VERSION identifies Khan's current version
var VERSION = "0.2.1"
<|endoftext|>