<commit_before>package cmd

import (
	"os"
	"path/filepath"
	"strings"
)

// Project contains name, license and paths to projects.
type Project struct {
	absPath string
	cmdPath string
	srcPath string
	license License
	name    string
}

// NewProject returns Project with specified project name.
// If projectName is blank string, it returns nil.
func NewProject(projectName string) *Project {
	if projectName == "" {
		return nil
	}

	p := new(Project)
	p.name = projectName

	// 1. Find already created protect.
	p.absPath = findPackage(projectName)

	// 2. If there are no created project with this path, and user is in GOPATH,
	// then use GOPATH/src/projectName.
	if p.absPath == "" {
		wd, err := os.Getwd()
		if err != nil {
			er(err)
		}
		for _, srcPath := range srcPaths {
			goPath := filepath.Dir(srcPath)
			if strings.HasPrefix(wd, goPath) {
				p.absPath = filepath.Join(srcPath, projectName)
				break
			}
		}
	}

	// 3. If user is not in GOPATH, then use (first GOPATH)/src/projectName.
	if p.absPath == "" {
		p.absPath = filepath.Join(srcPaths[0], projectName)
	}

	return p
}

// findPackage returns full path to existing go package in GOPATHs.
// findPackage returns "", if it can't find path.
// If packageName is "", findPackage returns "".
func findPackage(packageName string) string {
	if packageName == "" {
		return ""
	}

	for _, srcPath := range srcPaths {
		packagePath := filepath.Join(srcPath, packageName)
		if exists(packagePath) {
			return packagePath
		}
	}

	return ""
}

// NewProjectFromPath returns Project with specified absolute path to
// package.
// If absPath is blank string or if absPath is not actually absolute,
// it returns nil.
func NewProjectFromPath(absPath string) *Project {
	if absPath == "" || !filepath.IsAbs(absPath) {
		return nil
	}

	p := new(Project)
	p.absPath = absPath
	p.absPath = strings.TrimSuffix(p.absPath, findCmdDir(p.absPath))
	p.name = filepath.ToSlash(trimSrcPath(p.absPath, p.SrcPath()))
	return p
}

// trimSrcPath trims at the beginning of absPath the srcPath.
func trimSrcPath(absPath, srcPath string) string {
	relPath, err := filepath.Rel(srcPath, absPath)
	if err != nil {
		er("Cobra supports project only within $GOPATH")
	}
	return relPath
}

// License returns the License object of project.
func (p *Project) License() License {
	if p.license.Text == "" && p.license.Name != "None" {
		p.license = getLicense()
	}

	return p.license
}

// Name returns the name of project, e.g. "github.com/spf13/cobra"
func (p Project) Name() string {
	return p.name
}

// CmdPath returns absolute path to directory, where all commands are located.
//
// CmdPath returns blank string, only if p.AbsPath() is a blank string.
func (p *Project) CmdPath() string {
	if p.absPath == "" {
		return ""
	}
	if p.cmdPath == "" {
		p.cmdPath = filepath.Join(p.absPath, findCmdDir(p.absPath))
	}
	return p.cmdPath
}

// findCmdDir checks if base of absPath is cmd dir and returns it or
// looks for existing cmd dir in absPath.
// If the cmd dir doesn't exist, empty, or cannot be found,
// it returns "cmd".
func findCmdDir(absPath string) string {
	if !exists(absPath) || isEmpty(absPath) {
		return "cmd"
	}

	if isCmdDir(absPath) {
		return filepath.Base(absPath)
	}

	files, _ := filepath.Glob(filepath.Join(absPath, "c*"))
	for _, file := range files {
		if isCmdDir(file) {
			return filepath.Base(file)
		}
	}

	return "cmd"
}

// isCmdDir checks if base of name is one of cmdDir.
func isCmdDir(name string) bool {
	name = filepath.Base(name)
	for _, cmdDir := range cmdDirs {
		if name == cmdDir {
			return true
		}
	}
	return false
}

// AbsPath returns absolute path of project.
func (p Project) AbsPath() string {
	return p.absPath
}

// SrcPath returns absolute path to $GOPATH/src where project is located.
func (p *Project) SrcPath() string {
	if p.srcPath != "" {
		return p.srcPath
	}
	if p.absPath == "" {
		p.srcPath = srcPaths[0]
		return p.srcPath
	}

	for _, srcPath := range srcPaths {
		if strings.HasPrefix(p.absPath, srcPath) {
			p.srcPath = srcPath
			break
		}
	}

	return p.srcPath
}
<commit_msg>cmd: Make detailed error when project not in $GOPATH<commit_after>package cmd

import (
	"os"
	"path/filepath"
	"strings"
)

// Project contains name, license and paths to projects.
type Project struct {
	absPath string
	cmdPath string
	srcPath string
	license License
	name    string
}

// NewProject returns Project with specified project name.
// If projectName is blank string, it returns nil.
func NewProject(projectName string) *Project {
	if projectName == "" {
		return nil
	}

	p := new(Project)
	p.name = projectName

	// 1. Find already created protect.
	p.absPath = findPackage(projectName)

	// 2. If there are no created project with this path, and user is in GOPATH,
	// then use GOPATH/src/projectName.
	if p.absPath == "" {
		wd, err := os.Getwd()
		if err != nil {
			er(err)
		}
		for _, srcPath := range srcPaths {
			goPath := filepath.Dir(srcPath)
			if strings.HasPrefix(wd, goPath) {
				p.absPath = filepath.Join(srcPath, projectName)
				break
			}
		}
	}

	// 3. If user is not in GOPATH, then use (first GOPATH)/src/projectName.
	if p.absPath == "" {
		p.absPath = filepath.Join(srcPaths[0], projectName)
	}

	return p
}

// findPackage returns full path to existing go package in GOPATHs.
// findPackage returns "", if it can't find path.
// If packageName is "", findPackage returns "".
func findPackage(packageName string) string {
	if packageName == "" {
		return ""
	}

	for _, srcPath := range srcPaths {
		packagePath := filepath.Join(srcPath, packageName)
		if exists(packagePath) {
			return packagePath
		}
	}

	return ""
}

// NewProjectFromPath returns Project with specified absolute path to
// package.
// If absPath is blank string or if absPath is not actually absolute,
// it returns nil.
func NewProjectFromPath(absPath string) *Project {
	if absPath == "" || !filepath.IsAbs(absPath) {
		return nil
	}

	p := new(Project)
	p.absPath = absPath
	p.absPath = strings.TrimSuffix(p.absPath, findCmdDir(p.absPath))
	p.name = filepath.ToSlash(trimSrcPath(p.absPath, p.SrcPath()))
	return p
}

// trimSrcPath trims at the beginning of absPath the srcPath.
func trimSrcPath(absPath, srcPath string) string {
	relPath, err := filepath.Rel(srcPath, absPath)
	if err != nil {
		er("Cobra supports project only within $GOPATH: " + err.Error())
	}
	return relPath
}

// License returns the License object of project.
func (p *Project) License() License {
	if p.license.Text == "" && p.license.Name != "None" {
		p.license = getLicense()
	}

	return p.license
}

// Name returns the name of project, e.g. "github.com/spf13/cobra"
func (p Project) Name() string {
	return p.name
}

// CmdPath returns absolute path to directory, where all commands are located.
//
// CmdPath returns blank string, only if p.AbsPath() is a blank string.
func (p *Project) CmdPath() string {
	if p.absPath == "" {
		return ""
	}
	if p.cmdPath == "" {
		p.cmdPath = filepath.Join(p.absPath, findCmdDir(p.absPath))
	}
	return p.cmdPath
}

// findCmdDir checks if base of absPath is cmd dir and returns it or
// looks for existing cmd dir in absPath.
// If the cmd dir doesn't exist, empty, or cannot be found,
// it returns "cmd".
func findCmdDir(absPath string) string {
	if !exists(absPath) || isEmpty(absPath) {
		return "cmd"
	}

	if isCmdDir(absPath) {
		return filepath.Base(absPath)
	}

	files, _ := filepath.Glob(filepath.Join(absPath, "c*"))
	for _, file := range files {
		if isCmdDir(file) {
			return filepath.Base(file)
		}
	}

	return "cmd"
}

// isCmdDir checks if base of name is one of cmdDir.
func isCmdDir(name string) bool {
	name = filepath.Base(name)
	for _, cmdDir := range cmdDirs {
		if name == cmdDir {
			return true
		}
	}
	return false
}

// AbsPath returns absolute path of project.
func (p Project) AbsPath() string {
	return p.absPath
}

// SrcPath returns absolute path to $GOPATH/src where project is located.
func (p *Project) SrcPath() string {
	if p.srcPath != "" {
		return p.srcPath
	}
	if p.absPath == "" {
		p.srcPath = srcPaths[0]
		return p.srcPath
	}

	for _, srcPath := range srcPaths {
		if strings.HasPrefix(p.absPath, srcPath) {
			p.srcPath = srcPath
			break
		}
	}

	return p.srcPath
}
<|endoftext|>


<commit_before>package sdees

import (
	"os"
	"os/exec"
	"strings"
	"time"
)

func ListFiles(gitfolder string) []string {

	defer timeTrack(time.Now(), "Listing files")
	cwd, _ := os.Getwd()
	defer os.Chdir(cwd)
	err := os.Chdir(gitfolder)
	if err != nil {
		logger.Error("Cannot chdir into " + gitfolder)
	}

	cmd := exec.Command("git", "ls-tree", "--name-only", "master")
	stdout, err := cmd.Output()
	if err != nil {
		logger.Error("Problem doing ls-tree")
	}
	documents := strings.Split(strings.TrimSpace(string(stdout)), "\n")
	return documents
}
<commit_msg>List files works with gpg files now<commit_after>package sdees

import (
	"os"
	"os/exec"
	"strings"
	"time"
)

func ListFiles(gitfolder string) []string {
	defer timeTrack(time.Now(), "Listing files")
	cwd, _ := os.Getwd()
	defer os.Chdir(cwd)
	err := os.Chdir(gitfolder)
	if err != nil {
		logger.Error("Cannot chdir into " + gitfolder)
	}

	cmd := exec.Command("git", "ls-tree", "--name-only", "master")
	stdout, err := cmd.Output()
	if err != nil {
		logger.Error("Problem doing ls-tree")
	}
	documents := []string{}
	for _, document := range strings.Split(strings.TrimSpace(string(stdout)), "\n") {
		if document[0] == '.' {
			continue
		}
		documents = append(documents, strings.Replace(document, ".gpg", "", -1))
	}
	return documents
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
)

func main() {
	var a int
	var b int8
	var c int16
	var d int32
	var e int64

	var f bool
	var g float32
	var h float64

	var i string

	var j [5] int

	fmt.Println(a, "\t", b, "\t", c, "\t", d, "\t", e, "\t", f, "\t", g, "\t", h, "\t", i, "\t", j)

}
<commit_msg>Update<commit_after>package main

import (
	"fmt"
	"math"
	"time"
)

func main() {
	var a int
	var b int8
	var c int16
	var d int32
	var e int64

	var f bool
	var g float32
	var h float64

	var i string

	var j [5]int

	var k uint
	var l uint8
	var m uint16
	var n uint32
	var o uint64

	fmt.Println(a, "\t", b, "\t", c, "\t", d, "\t", e, "\t", f, "\t", g, "\t", h, "\t", i, "\t", j)

	a = math.MinInt8
	b = math.MinInt8
	c = math.MinInt16
	d = math.MinInt32
	e = math.MinInt64

	fmt.Println(a, "\t", b, "\t", c, "\t", d, "\t", e)

	a = math.MaxInt8
	b = math.MaxInt8
	c = math.MaxInt16
	d = math.MaxInt32
	e = math.MaxInt64

	k = math.MaxUint8
	l = math.MaxUint8
	m = math.MaxUint16
	n = math.MaxUint32
	o = math.MaxUint64

	fmt.Println(a, "\t", b, "\t", c, "\t", d, "\t", e)
	fmt.Println(k, "\t", l, "\t", m, "\t", n, "\t", o)

	fmt.Println("Hello World, ", time.Now())

}
<|endoftext|>


<commit_before>package exhibit

import (
	"flag"
	"fmt"
	"path"
	"regexp"
	"runtime"
	"strings"
)

var (
	fixup      *bool
	whitespace *regexp.Regexp
	testname   *regexp.Regexp
	maxdepth   int
)

func init() {
	fixup = flag.Bool("fixup", false, "Fixup failing tests by overwriting the approved content")

	flag.Parse()

	whitespace = regexp.MustCompile(`\s+`)
	testname = regexp.MustCompile(`^.*\.Test[^a-z].*`)
	maxdepth = 12
}

func makeEvidenceFilename(evidence Evidence, caller *callerInfo, label string) string {
	label = strings.TrimSpace(label)
	if len(label) > 0 {
		label = "." + string(whitespace.ReplaceAll([]byte(label), []byte{'_'}))
	}

	name := fmt.Sprintf("%s.exhibit%s.%s", caller.function, label, evidence.Extension())
	dir := path.Dir(caller.file)
	return path.Join(dir, name)
}

type callerInfo struct {
	file, function string
}

func getCallerInfo() (*callerInfo, error) {
	for i := 2; i < maxdepth; i++ {
		// program counter, filename, line, ok
		pc, file, _, ok := runtime.Caller(i)
		if !ok {
			// return nil, fmt.Errorf("Could not retrieve caller %d", i)
			continue
		}

		caller := runtime.FuncForPC(pc)

		if testname.Match([]byte(caller.Name())) {
			return &callerInfo{
				file:     file,
				function: caller.Name(),
			}, nil
		}
	}
	return nil, fmt.Errorf("Max stack depth (%d) reached, no test method found", maxdepth)
}
<commit_msg>Better file names for exhibits<commit_after>package exhibit

import (
	"flag"
	"fmt"
	"regexp"
	"runtime"
	"strings"
)

var (
	fixup      *bool
	whitespace *regexp.Regexp
	testname   *regexp.Regexp
	prefix     *regexp.Regexp
	maxdepth   int
)

func init() {
	fixup = flag.Bool("fixup", false, "Fixup failing tests by overwriting the approved content")

	flag.Parse()

	whitespace = regexp.MustCompile(`\s+`)
	testname = regexp.MustCompile(`^.*\.Test[^a-z].*`)
	prefix = regexp.MustCompile(`(^.*\.)?`)
	maxdepth = 12
}

func makeEvidenceFilename(evidence Evidence, caller *callerInfo, label string) string {
	label = strings.TrimSpace(label)
	if len(label) > 0 {
		label = "-" + string(whitespace.ReplaceAllString(label, "_"))
	}

	name := fmt.Sprintf("%s.exhibit%s.%s", prefix.ReplaceAllString(caller.function, ""), label, evidence.Extension())
	return fmt.Sprintf("%s.%s", caller.file, name)
}

type callerInfo struct {
	file, function string
}

func getCallerInfo() (*callerInfo, error) {
	for i := 2; i < maxdepth; i++ {
		// program counter, filename, line, ok
		pc, file, _, ok := runtime.Caller(i)
		if !ok {
			// return nil, fmt.Errorf("Could not retrieve caller %d", i)
			continue
		}

		caller := runtime.FuncForPC(pc)

		if testname.Match([]byte(caller.Name())) {
			return &callerInfo{
				file:     file,
				function: caller.Name(),
			}, nil
		}
	}
	return nil, fmt.Errorf("Max stack depth (%d) reached, no test method found", maxdepth)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path"
	"strconv"
	"strings"

	"github.com/Zamiell/isaac-racing-server/src/models"
	"github.com/gin-contrib/sessions"
	"github.com/gin-gonic/gin"
)

/*
	The user will login with their Steam account according to this authentication flow:
	https://partner.steamgames.com/documentation/auth#client_to_backend_webapi
	(you have to be logged in for the link to work)
*/

func httpLogin(c *gin.Context) {
	// Local variables
	r := c.Request
	w := c.Writer
	ip, _, _ := net.SplitHostPort(r.RemoteAddr)

	/*
		Validation
	*/

	// Check to see if their IP is banned
	if userIsBanned, err := db.BannedIPs.Check(ip); err != nil {
		logger.Error("Database error when checking to see if IP \""+ip+"\" was banned:", err)
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	} else if userIsBanned {
		logger.Info("IP \"" + ip + "\" tried to log in, but they are banned.")
		http.Error(w, "Your IP address has been banned. Please contact an administrator if you think this is a mistake.", http.StatusUnauthorized)
		return
	}

	// Check to see if they are already logged in
	// (which should probably never happen since the cookie lasts 5 seconds)
	session := sessions.Default(c)
	if v := session.Get("userID"); v != nil {
		logger.Info("User from IP \"" + ip + "\" tried to get a session cookie, but they are already logged in.")
		http.Error(w, "You are already logged in. Please wait 5 seconds, then try again.", http.StatusUnauthorized)
		return
	}

	// Validate that the user sent the Steam ID, the ticket, and the version number of the client
	steamID := c.PostForm("steamID")
	if steamID == "" {
		logger.Error("User from IP \"" + ip + "\" tried to log in, but they did not provide the \"steamID\" parameter.")
		http.Error(w, "You must provide the \"steamID\" parameter to log in.", http.StatusUnauthorized)
		return
	}
	ticket := c.PostForm("ticket")
	if ticket == "" {
		logger.Error("User from IP \"" + ip + "\" tried to log in, but they did not provide the \"ticket\" parameter.")
		http.Error(w, "You must provide the \"ticket\" parameter to log in.", http.StatusUnauthorized)
		return
	}
	version := c.PostForm("version")
	if version == "" {
		logger.Error("User from IP \"" + ip + "\" tried to log in, but they did not provide the \"version\" parameter.")
		http.Error(w, "You must provide the \"version\" parameter to log in.", http.StatusUnauthorized)
		return
	}

	// Validate that the provided Steam ID is sane
	var steamIDint int
	if v, err := strconv.Atoi(steamID); err != nil {
		logger.Error("Failed to convert the steam ID to an integer.")
		http.Error(w, "You provided an invalid \"steamID\".", http.StatusUnauthorized)
		return
	} else {
		steamIDint = v
	}

	// Validate that the Racing+ client version is the latest version
	if steamIDint > 0 {
		if !validateLatestVersion(version, w) {
			return
		}
	}

	// Validate the ticket with the Steam API
	if !validateSteamTicket(steamID, ticket, ip, w) {
		return
	}

	// Check to see if this Steam ID exists in the database
	var sessionValues *models.SessionValues
	if v, err := db.Users.Login(steamID); err != nil {
		logger.Error("Database error when checking to see if steam ID "+steamID+" exists:", err)
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	} else if v == nil {
		// This is a new user, so return a success, but don't give them a WebSocket cookie
		// (the client is expected to now make a POST request to "/register")
		http.Error(w, http.StatusText(http.StatusAccepted), http.StatusAccepted)
		return
	} else {
		sessionValues = v
	}

	// Check to see if this user is banned
	if sessionValues.Banned {
		logger.Info("User \"" + sessionValues.Username + "\" tried to log in, but they are banned.")
		http.Error(w, "Your user account has been banned. Please contact an administrator if you think this is a mistake.", http.StatusUnauthorized)
		return
	}

	/*
		Login
	*/

	// Update the database with datetime_last_login and last_ip
	if err := db.Users.SetLogin(sessionValues.UserID, ip); err != nil {
		logger.Error("Database error when setting the login values:", err)
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	}

	// Save the information to the session
	session.Set("userID", sessionValues.UserID)
	session.Set("username", sessionValues.Username)
	session.Set("admin", sessionValues.Admin)
	session.Set("muted", sessionValues.Muted)
	session.Set("streamURL", sessionValues.StreamURL)
	session.Set("twitchBotEnabled", sessionValues.TwitchBotEnabled)
	session.Set("twitchBotDelay", sessionValues.TwitchBotDelay)
	if err := session.Save(); err != nil {
		logger.Error("Failed to save the session:", err)
	}

	// Log the login request
	logger.Info("User \""+sessionValues.Username+"\" logged in from:", ip)
}

/*
	We need to create some structures that emulate the JSON that the Steam API returns
*/

type SteamAPIReply struct {
	Response SteamAPIResponse `json:"response"`
}
type SteamAPIResponse struct {
	Params SteamAPIParams `json:"params"`
	Error  SteamAPIError  `json:"error"`
}
type SteamAPIParams struct {
	Result          string `json:"result"`
	SteamID         string `json:"steamid"`
	OwnerSteamID    string `json:"ownersteamid"`
	VACBanned       bool   `json:"vacbanned"`
	PublisherBanned bool   `json:"publisherbanned"`
}
type SteamAPIError struct {
	Code int    `json:"errorcode"`
	Desc string `json:"errordesc"`
}

/*
	Validate that the ticket is valid using the Steam web API
	E.g. https://api.steampowered.com/ISteamUserAuth/AuthenticateUserTicket/v1?key=secret&appid=250900&ticket=longhex
*/

func validateSteamTicket(steamID string, ticket string, ip string, w http.ResponseWriter) bool {
	// Automatically validate test accounts
	if ticket == "debug" &&
		steamID == "-1" || // These 10 fake steam IDs allow for 10 test accounts
		steamID == "-2" ||
		steamID == "-3" ||
		steamID == "-4" ||
		steamID == "-5" ||
		steamID == "-6" ||
		steamID == "-7" ||
		steamID == "-8" ||
		steamID == "-9" ||
		steamID == "-10" {

		IPWhitelist := os.Getenv("DEV_IP_WHITELIST")
		IPs := strings.Split(IPWhitelist, ",")
		for _, validIP := range IPs {
			if ip == validIP {
				return true
			}
		}

		logger.Warning("IP \"" + ip + "\" tried to use a debug ticket, but they are not on the whitelist.")
		http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
		return false
	}

	// Make the request
	apiKey := os.Getenv("STEAM_WEB_API_KEY")
	if len(apiKey) == 0 {
		logger.Error("The \"STEAM_WEB_API_KEY\" environment variable is blank; aborting the login request.")
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return false
	}
	appID := "250900" // This is the app ID on Steam for The Binding of Isaac: Rebirth
	url := "https://api.steampowered.com/ISteamUserAuth/AuthenticateUserTicket/v1"
	args := "?key=" + apiKey + "&appid=" + appID + "&ticket=" + ticket
	resp, err := HTTPClientWithTimeout.Get(url + args)
	if err != nil {
		logger.Error("Failed to query the Steam web API for IP \""+ip+"\": ", err)
		http.Error(w, "An error occurred while verifying your Steam account. Please try again later.", http.StatusUnauthorized)
		return false
	}
	defer resp.Body.Close()

	// Read the body
	raw, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		logger.Error("Failed to read the body of the response from the Steam web API for IP \""+ip+"\": ", err)
		http.Error(w, "An error occurred while verifying your Steam account. Please try again later.", http.StatusUnauthorized)
		return false
	}

	// Unmarshall the JSON of the body from the response
	var steamAPIReply SteamAPIReply
	if err := json.Unmarshal(raw, &steamAPIReply); err != nil {
		logger.Error("Failed to unmarshall the body of the response from the Steam web API for IP \""+ip+":", err)
		logger.Error("The response was as follows:", raw)
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return false
	}

	invalidMessage := "Your Steam account appears to be invalid. Please make sure you have the latest version of Steam installed and are correctly logged in."

	// Check to see if we got an error
	steamError := steamAPIReply.Response.Error
	if steamError.Code != 0 {
		logger.Error("The Steam web API returned error code " + strconv.Itoa(steamError.Code) + " for IP " + ip + " and Steam ID \"" + steamID + "\" and ticket \"" + ticket + "\": " + steamError.Desc)
		http.Error(w, invalidMessage, http.StatusUnauthorized)
		return false
	}

	// Check to see if the ticket is valid
	result := steamAPIReply.Response.Params.Result
	if result == "" {
		logger.Error("The Steam web API response does not have a \"result\" property.")
		http.Error(w, "An error occurred while verifying your Steam account. Please try again later.", http.StatusUnauthorized)
		return false
	} else if result != "OK" {
		logger.Warning("A user from IP \"" + ip + "\" tried to log in, but their Steam ticket was invalid.")
		http.Error(w, invalidMessage, http.StatusUnauthorized)
		return false
	}

	// Check to see if the Steam ID matches who they claim to be
	ticketSteamID := steamAPIReply.Response.Params.SteamID
	if ticketSteamID == "" {
		logger.Error("The Steam web API response does not have a \"steamID\" property.")
		http.Error(w, "An error occurred while verifying your Steam account. Please try again later.", http.StatusUnauthorized)
		return false
	} else if ticketSteamID != steamID {
		logger.Warning("A user from IP \"" + ip + "\" submitted a Steam ticket that does not match their submitted Steam ID.")
		http.Error(w, invalidMessage, http.StatusUnauthorized)
		return false
	}

	return true
}

func validateLatestVersion(version string, w http.ResponseWriter) bool {
	// Make an exception for users on macOS
	if version == "macOS" {
		return true
	}

	latestVersionRaw, err := ioutil.ReadFile(path.Join(projectPath, "latest_client_version.txt"))
	if err != nil {
		logger.Error("Failed to read the \"latest_client_version.txt\" file:", err)
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return false
	}
	latestVersion := string(latestVersionRaw)
	latestVersion = strings.TrimSpace(latestVersion)
	if len(latestVersion) == 0 {
		logger.Error("The \"latest_client_version.txt\" file is empty, so users will not be able to login to the WebSocket server.")
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return false
	}
	if version != latestVersion {
		errorMsg := "Your client version is " + version + " and the latest version is " + latestVersion + ".

Please restart the Racing+ program and it should automatically update to the latest version. If that does not work, you can try manually downloading the latest version from the Racing+ website."
		http.Error(w, errorMsg, http.StatusUnauthorized)
		return false
	}

	return true
}
<commit_msg>better logging<commit_after>package main

import (
	"encoding/json"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path"
	"strconv"
	"strings"

	"github.com/Zamiell/isaac-racing-server/src/models"
	"github.com/gin-contrib/sessions"
	"github.com/gin-gonic/gin"
)

/*
	The user will login with their Steam account according to this authentication flow:
	https://partner.steamgames.com/documentation/auth#client_to_backend_webapi
	(you have to be logged in for the link to work)
*/

func httpLogin(c *gin.Context) {
	// Local variables
	r := c.Request
	w := c.Writer
	ip, _, _ := net.SplitHostPort(r.RemoteAddr)

	/*
		Validation
	*/

	// Check to see if their IP is banned
	if userIsBanned, err := db.BannedIPs.Check(ip); err != nil {
		logger.Error("Database error when checking to see if IP \""+ip+"\" was banned:", err)
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	} else if userIsBanned {
		logger.Info("IP \"" + ip + "\" tried to log in, but they are banned.")
		http.Error(w, "Your IP address has been banned. Please contact an administrator if you think this is a mistake.", http.StatusUnauthorized)
		return
	}

	// Check to see if they are already logged in
	// (which should probably never happen since the cookie lasts 5 seconds)
	session := sessions.Default(c)
	if v := session.Get("userID"); v != nil {
		logger.Info("User from IP \"" + ip + "\" tried to get a session cookie, but they are already logged in.")
		http.Error(w, "You are already logged in. Please wait 5 seconds, then try again.", http.StatusUnauthorized)
		return
	}

	// Validate that the user sent the Steam ID, the ticket, and the version number of the client
	steamID := c.PostForm("steamID")
	if steamID == "" {
		logger.Error("User from IP \"" + ip + "\" tried to log in, but they did not provide the \"steamID\" parameter.")
		http.Error(w, "You must provide the \"steamID\" parameter to log in.", http.StatusUnauthorized)
		return
	}
	ticket := c.PostForm("ticket")
	if ticket == "" {
		logger.Error("User from IP \"" + ip + "\" tried to log in, but they did not provide the \"ticket\" parameter.")
		http.Error(w, "You must provide the \"ticket\" parameter to log in.", http.StatusUnauthorized)
		return
	}
	version := c.PostForm("version")
	if version == "" {
		logger.Error("User from IP \"" + ip + "\" tried to log in, but they did not provide the \"version\" parameter.")
		http.Error(w, "You must provide the \"version\" parameter to log in.", http.StatusUnauthorized)
		return
	}

	// Validate that the provided Steam ID is sane
	var steamIDint int
	if v, err := strconv.Atoi(steamID); err != nil {
		logger.Error("Failed to convert the steam ID to an integer.")
		http.Error(w, "You provided an invalid \"steamID\".", http.StatusUnauthorized)
		return
	} else {
		steamIDint = v
	}

	// Validate that the Racing+ client version is the latest version
	if steamIDint > 0 {
		if !validateLatestVersion(version, w) {
			return
		}
	}

	// Validate the ticket with the Steam API
	if !validateSteamTicket(steamID, ticket, ip, w) {
		return
	}

	// Check to see if this Steam ID exists in the database
	var sessionValues *models.SessionValues
	if v, err := db.Users.Login(steamID); err != nil {
		logger.Error("Database error when checking to see if steam ID "+steamID+" exists:", err)
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	} else if v == nil {
		// This is a new user, so return a success, but don't give them a WebSocket cookie
		// (the client is expected to now make a POST request to "/register")
		http.Error(w, http.StatusText(http.StatusAccepted), http.StatusAccepted)
		return
	} else {
		sessionValues = v
	}

	// Check to see if this user is banned
	if sessionValues.Banned {
		logger.Info("User \"" + sessionValues.Username + "\" tried to log in, but they are banned.")
		http.Error(w, "Your user account has been banned. Please contact an administrator if you think this is a mistake.", http.StatusUnauthorized)
		return
	}

	/*
		Login
	*/

	// Update the database with datetime_last_login and last_ip
	if err := db.Users.SetLogin(sessionValues.UserID, ip); err != nil {
		logger.Error("Database error when setting the login values:", err)
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	}

	// Save the information to the session
	session.Set("userID", sessionValues.UserID)
	session.Set("username", sessionValues.Username)
	session.Set("admin", sessionValues.Admin)
	session.Set("muted", sessionValues.Muted)
	session.Set("streamURL", sessionValues.StreamURL)
	session.Set("twitchBotEnabled", sessionValues.TwitchBotEnabled)
	session.Set("twitchBotDelay", sessionValues.TwitchBotDelay)
	if err := session.Save(); err != nil {
		logger.Error("Failed to save the session:", err)
	}

	// Log the login request
	logger.Info("User \""+sessionValues.Username+"\" logged in from:", ip)
}

/*
	We need to create some structures that emulate the JSON that the Steam API returns
*/

type SteamAPIReply struct {
	Response SteamAPIResponse `json:"response"`
}
type SteamAPIResponse struct {
	Params SteamAPIParams `json:"params"`
	Error  SteamAPIError  `json:"error"`
}
type SteamAPIParams struct {
	Result          string `json:"result"`
	SteamID         string `json:"steamid"`
	OwnerSteamID    string `json:"ownersteamid"`
	VACBanned       bool   `json:"vacbanned"`
	PublisherBanned bool   `json:"publisherbanned"`
}
type SteamAPIError struct {
	Code int    `json:"errorcode"`
	Desc string `json:"errordesc"`
}

/*
	Validate that the ticket is valid using the Steam web API
	E.g. https://api.steampowered.com/ISteamUserAuth/AuthenticateUserTicket/v1?key=secret&appid=250900&ticket=longhex
*/

func validateSteamTicket(steamID string, ticket string, ip string, w http.ResponseWriter) bool {
	// Automatically validate test accounts
	if ticket == "debug" &&
		steamID == "-1" || // These 10 fake steam IDs allow for 10 test accounts
		steamID == "-2" ||
		steamID == "-3" ||
		steamID == "-4" ||
		steamID == "-5" ||
		steamID == "-6" ||
		steamID == "-7" ||
		steamID == "-8" ||
		steamID == "-9" ||
		steamID == "-10" {

		IPWhitelist := os.Getenv("DEV_IP_WHITELIST")
		IPs := strings.Split(IPWhitelist, ",")
		for _, validIP := range IPs {
			if ip == validIP {
				return true
			}
		}

		logger.Warning("IP \"" + ip + "\" tried to use a debug ticket, but they are not on the whitelist.")
		http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
		return false
	}

	// Make the request
	apiKey := os.Getenv("STEAM_WEB_API_KEY")
	if len(apiKey) == 0 {
		logger.Error("The \"STEAM_WEB_API_KEY\" environment variable is blank; aborting the login request.")
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return false
	}
	appID := "250900" // This is the app ID on Steam for The Binding of Isaac: Rebirth
	url := "https://api.steampowered.com/ISteamUserAuth/AuthenticateUserTicket/v1"
	args := "?key=" + apiKey + "&appid=" + appID + "&ticket=" + ticket
	resp, err := HTTPClientWithTimeout.Get(url + args)
	if err != nil {
		logger.Error("Failed to query the Steam web API for IP \""+ip+"\": ", err)
		http.Error(w, "An error occurred while verifying your Steam account. Please try again later.", http.StatusUnauthorized)
		return false
	}
	defer resp.Body.Close()

	// Read the body
	raw, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		logger.Error("Failed to read the body of the response from the Steam web API for IP \""+ip+"\": ", err)
		http.Error(w, "An error occurred while verifying your Steam account. Please try again later.", http.StatusUnauthorized)
		return false
	}

	// Unmarshall the JSON of the body from the response
	var steamAPIReply SteamAPIReply
	if err := json.Unmarshal(raw, &steamAPIReply); err != nil {
		logger.Error("Failed to unmarshall the body of the response from the Steam web API for IP \""+ip+":", err)
		logger.Error("The response was as follows:", string(raw))
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return false
	}

	invalidMessage := "Your Steam account appears to be invalid. Please make sure you have the latest version of Steam installed and are correctly logged in."

	// Check to see if we got an error
	steamError := steamAPIReply.Response.Error
	if steamError.Code != 0 {
		logger.Error("The Steam web API returned error code " + strconv.Itoa(steamError.Code) + " for IP " + ip + " and Steam ID \"" + steamID + "\" and ticket \"" + ticket + "\": " + steamError.Desc)
		http.Error(w, invalidMessage, http.StatusUnauthorized)
		return false
	}

	// Check to see if the ticket is valid
	result := steamAPIReply.Response.Params.Result
	if result == "" {
		logger.Error("The Steam web API response does not have a \"result\" property.")
		http.Error(w, "An error occurred while verifying your Steam account. Please try again later.", http.StatusUnauthorized)
		return false
	} else if result != "OK" {
		logger.Warning("A user from IP \"" + ip + "\" tried to log in, but their Steam ticket was invalid.")
		http.Error(w, invalidMessage, http.StatusUnauthorized)
		return false
	}

	// Check to see if the Steam ID matches who they claim to be
	ticketSteamID := steamAPIReply.Response.Params.SteamID
	if ticketSteamID == "" {
		logger.Error("The Steam web API response does not have a \"steamID\" property.")
		http.Error(w, "An error occurred while verifying your Steam account. Please try again later.", http.StatusUnauthorized)
		return false
	} else if ticketSteamID != steamID {
		logger.Warning("A user from IP \"" + ip + "\" submitted a Steam ticket that does not match their submitted Steam ID.")
		http.Error(w, invalidMessage, http.StatusUnauthorized)
		return false
	}

	return true
}

func validateLatestVersion(version string, w http.ResponseWriter) bool {
	// Make an exception for users on macOS
	if version == "macOS" {
		return true
	}

	latestVersionRaw, err := ioutil.ReadFile(path.Join(projectPath, "latest_client_version.txt"))
	if err != nil {
		logger.Error("Failed to read the \"latest_client_version.txt\" file:", err)
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return false
	}
	latestVersion := string(latestVersionRaw)
	latestVersion = strings.TrimSpace(latestVersion)
	if len(latestVersion) == 0 {
		logger.Error("The \"latest_client_version.txt\" file is empty, so users will not be able to login to the WebSocket server.")
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return false
	}
	if version != latestVersion {
		errorMsg := "Your client version is " + version + " and the latest version is " + latestVersion + ".

Please restart the Racing+ program and it should automatically update to the latest version. If that does not work, you can try manually downloading the latest version from the Racing+ website."
		http.Error(w, errorMsg, http.StatusUnauthorized)
		return false
	}

	return true
}
<|endoftext|>


<commit_before>package inputs

import (
	"bufio"
	"encoding/csv"
	"fmt"
	"io"
	"strings"

	"github.com/subparlabs/bonjourno/log"
)

// MessageBuilder - Turns raw data into a list of messages
type MessageBuilder <-chan []string

func CSVField(fieldIndex int, source DataSource) MessageBuilder {
	c := make(chan []string)

	go func() {
		for {
			reader := csv.NewReader(strings.NewReader(<-source))
			reader.TrimLeadingSpace = true
			reader.FieldsPerRecord = -1

			var values []string

			// Ignore first line - comment
			record, err := reader.Read()
			for err == nil {
				record, err = reader.Read()
				if err == nil && len(record) > fieldIndex {
					values = append(values, record[fieldIndex])
				}
			}
			if err != nil && err != io.EOF {
				log.Error("Failed to read CSV", "err", err)
			} else if len(values) > 0 {
				c <- values
			}
		}
	}()

	return c
}

func Lines(source DataSource) MessageBuilder {
	c := make(chan []string)

	go func() {
		for {
			var lines []string

			scanner := bufio.NewScanner(strings.NewReader(<-source))
			for scanner.Scan() {
				line := scanner.Text()
				if line != "" {
					lines = append(lines, line)
				}
			}
			if scanner.Err() == nil && len(lines) > 0 {
				c <- lines
			}
		}
	}()

	return c
}

func WordGroups(source DataSource) MessageBuilder {
	c := make(chan []string)

	go func() {
		for {
			var words []string

			scanner := bufio.NewScanner(strings.NewReader(<-source))
			scanner.Split(bufio.ScanWords)
			for scanner.Scan() {
				words = append(words, scanner.Text())
			}
			if scanner.Err() == nil && len(words) > 0 {
				// Combine into groups so that they're as big as they can
				// be without going over a limit.
				var groups []string
				group, words := words[0], words[1:]

				for _, word := range words {
					if len(group)+1+len(word) <= 20 {
						group = fmt.Sprintf("%s %s", group, word)
					} else {
						groups = append(groups, group)
						group = word
					}
				}
				if group != "" {
					groups = append(groups, group)
				}

				c <- groups
			}
		}
	}()

	return c
}
<commit_msg>Log and ignore CSV errors<commit_after>package inputs

import (
	"bufio"
	"encoding/csv"
	"fmt"
	"io"
	"strings"

	"github.com/subparlabs/bonjourno/log"
)

// MessageBuilder - Turns raw data into a list of messages
type MessageBuilder <-chan []string

func CSVField(fieldIndex int, source DataSource) MessageBuilder {
	c := make(chan []string)

	go func() {
		for {
			reader := csv.NewReader(strings.NewReader(<-source))
			reader.TrimLeadingSpace = true
			reader.FieldsPerRecord = -1

			var values []string

			// Ignore first line - comment
			reader.Read()
			for {
				if record, err := reader.Read(); err == io.EOF {
					break
				} else if err != nil {
					log.Error("Error reading CSV data", "err", err)
				} else if len(record) <= fieldIndex {
					log.Error("Error reading CSV data: index out of bounds", "# fields in record", record, "index", fieldIndex)
				} else {
					values = append(values, record[fieldIndex])
				}
			}
			if len(values) == 0 {
				log.Error("Didn't get any values from CSV")
			} else {
				c <- values
			}
		}
	}()

	return c
}

func Lines(source DataSource) MessageBuilder {
	c := make(chan []string)

	go func() {
		for {
			var lines []string

			scanner := bufio.NewScanner(strings.NewReader(<-source))
			for scanner.Scan() {
				line := scanner.Text()
				if line != "" {
					lines = append(lines, line)
				}
			}
			if scanner.Err() == nil && len(lines) > 0 {
				c <- lines
			}
		}
	}()

	return c
}

func WordGroups(source DataSource) MessageBuilder {
	c := make(chan []string)

	go func() {
		for {
			var words []string

			scanner := bufio.NewScanner(strings.NewReader(<-source))
			scanner.Split(bufio.ScanWords)
			for scanner.Scan() {
				words = append(words, scanner.Text())
			}
			if scanner.Err() == nil && len(words) > 0 {
				// Combine into groups so that they're as big as they can
				// be without going over a limit.
				var groups []string
				group, words := words[0], words[1:]

				for _, word := range words {
					if len(group)+1+len(word) <= 20 {
						group = fmt.Sprintf("%s %s", group, word)
					} else {
						groups = append(groups, group)
						group = word
					}
				}
				if group != "" {
					groups = append(groups, group)
				}

				c <- groups
			}
		}
	}()

	return c
}
<|endoftext|>


<commit_before>// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package crypto

import (
	"context"
	"encoding/base64"
	"errors"
	"net/http"
	"path"
	"strings"

	jsoniter "github.com/json-iterator/go"
	xhttp "github.com/minio/minio/internal/http"
	"github.com/minio/minio/internal/kms"
	"github.com/minio/minio/internal/logger"
)

type ssekms struct{}

var (
	// S3KMS represents AWS SSE-KMS. It provides functionality to
	// handle SSE-KMS requests.
	S3KMS = ssekms{}

	_ Type = S3KMS
)

// String returns the SSE domain as string. For SSE-KMS the
// domain is "SSE-KMS".
func (ssekms) String() string { return "SSE-KMS" }

// IsRequested returns true if the HTTP headers contains
// at least one SSE-KMS header.
func (ssekms) IsRequested(h http.Header) bool {
	if _, ok := h[xhttp.AmzServerSideEncryptionKmsID]; ok {
		return true
	}
	if _, ok := h[xhttp.AmzServerSideEncryptionKmsContext]; ok {
		return true
	}
	if _, ok := h[xhttp.AmzServerSideEncryption]; ok {
		return strings.ToUpper(h.Get(xhttp.AmzServerSideEncryption)) != xhttp.AmzEncryptionAES // Return only true if the SSE header is specified and does not contain the SSE-S3 value
	}
	return false
}

// ParseHTTP parses the SSE-KMS headers and returns the SSE-KMS key ID
// and the KMS context on success.
func (ssekms) ParseHTTP(h http.Header) (string, kms.Context, error) {
	algorithm := h.Get(xhttp.AmzServerSideEncryption)
	if algorithm != xhttp.AmzEncryptionKMS {
		return "", nil, ErrInvalidEncryptionMethod
	}

	var ctx kms.Context
	if context, ok := h[xhttp.AmzServerSideEncryptionKmsContext]; ok {
		b, err := base64.StdEncoding.DecodeString(context[0])
		if err != nil {
			return "", nil, err
		}

		var json = jsoniter.ConfigCompatibleWithStandardLibrary
		if err := json.Unmarshal(b, &ctx); err != nil {
			return "", nil, err
		}
	}
	return h.Get(xhttp.AmzServerSideEncryptionKmsID), ctx, nil
}

// IsEncrypted returns true if the object metadata indicates
// that the object was uploaded using SSE-KMS.
func (ssekms) IsEncrypted(metadata map[string]string) bool {
	if _, ok := metadata[MetaSealedKeyKMS]; ok {
		return true
	}
	return false
}

// UnsealObjectKey extracts and decrypts the sealed object key
// from the metadata using KMS and returns the decrypted object
// key.
func (s3 ssekms) UnsealObjectKey(KMS kms.KMS, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
	keyID, kmsKey, sealedKey, ctx, err := s3.ParseMetadata(metadata)
	if err != nil {
		return key, err
	}
	if ctx == nil {
		ctx = kms.Context{bucket: path.Join(bucket, object)}
	} else if _, ok := ctx[bucket]; !ok {
		ctx[bucket] = path.Join(bucket, object)
	}
	unsealKey, err := KMS.DecryptKey(keyID, kmsKey, ctx)
	if err != nil {
		return key, err
	}
	err = key.Unseal(unsealKey[:], sealedKey, s3.String(), bucket, object)
	return key, err
}

// CreateMetadata encodes the sealed object key into the metadata and returns
// the modified metadata. If the keyID and the kmsKey is not empty it encodes
// both into the metadata as well. It allocates a new metadata map if metadata
// is nil.
func (ssekms) CreateMetadata(metadata map[string]string, keyID string, kmsKey []byte, sealedKey SealedKey, ctx kms.Context) map[string]string {
	if sealedKey.Algorithm != SealAlgorithm {
		logger.CriticalIf(context.Background(), Errorf("The seal algorithm '%s' is invalid for SSE-S3", sealedKey.Algorithm))
	}

	// There are two possibilites:
	// - We use a KMS -> There must be non-empty key ID and a KMS data key.
	// - We use a K/V -> There must be no key ID and no KMS data key.
	// Otherwise, the caller has passed an invalid argument combination.
	if keyID == "" && len(kmsKey) != 0 {
		logger.CriticalIf(context.Background(), errors.New("The key ID must not be empty if a KMS data key is present"))
	}
	if keyID != "" && len(kmsKey) == 0 {
		logger.CriticalIf(context.Background(), errors.New("The KMS data key must not be empty if a key ID is present"))
	}

	if metadata == nil {
		metadata = make(map[string]string, 5)
	}

	metadata[MetaAlgorithm] = sealedKey.Algorithm
	metadata[MetaIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])
	metadata[MetaSealedKeyKMS] = base64.StdEncoding.EncodeToString(sealedKey.Key[:])
	if len(ctx) > 0 {
		b, _ := ctx.MarshalText()
		metadata[MetaContext] = base64.StdEncoding.EncodeToString(b)
	}
	if len(kmsKey) > 0 && keyID != "" { // We use a KMS -> Store key ID and sealed KMS data key.
		metadata[MetaKeyID] = keyID
		metadata[MetaDataEncryptionKey] = base64.StdEncoding.EncodeToString(kmsKey)
	}
	return metadata
}

// ParseMetadata extracts all SSE-KMS related values from the object metadata
// and checks whether they are well-formed. It returns the sealed object key
// on success. If the metadata contains both, a KMS master key ID and a sealed
// KMS data key it returns both. If the metadata does not contain neither a
// KMS master key ID nor a sealed KMS data key it returns an empty keyID and
// KMS data key. Otherwise, it returns an error.
func (ssekms) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte, sealedKey SealedKey, ctx kms.Context, err error) {
	// Extract all required values from object metadata
	b64IV, ok := metadata[MetaIV]
	if !ok {
		return keyID, kmsKey, sealedKey, ctx, errMissingInternalIV
	}
	algorithm, ok := metadata[MetaAlgorithm]
	if !ok {
		return keyID, kmsKey, sealedKey, ctx, errMissingInternalSealAlgorithm
	}
	b64SealedKey, ok := metadata[MetaSealedKeyKMS]
	if !ok {
		return keyID, kmsKey, sealedKey, ctx, Errorf("The object metadata is missing the internal sealed key for SSE-S3")
	}

	// There are two possibilites:
	// - We use a KMS -> There must be a key ID and a KMS data key.
	// - We use a K/V -> There must be no key ID and no KMS data key.
	// Otherwise, the metadata is corrupted.
	keyID, idPresent := metadata[MetaKeyID]
	b64KMSSealedKey, kmsKeyPresent := metadata[MetaDataEncryptionKey]
	if !idPresent && kmsKeyPresent {
		return keyID, kmsKey, sealedKey, ctx, Errorf("The object metadata is missing the internal KMS key-ID for SSE-S3")
	}
	if idPresent && !kmsKeyPresent {
		return keyID, kmsKey, sealedKey, ctx, Errorf("The object metadata is missing the internal sealed KMS data key for SSE-S3")
	}

	// Check whether all extracted values are well-formed
	iv, err := base64.StdEncoding.DecodeString(b64IV)
	if err != nil || len(iv) != 32 {
		return keyID, kmsKey, sealedKey, ctx, errInvalidInternalIV
	}
	if algorithm != SealAlgorithm {
		return keyID, kmsKey, sealedKey, ctx, errInvalidInternalSealAlgorithm
	}
	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
	if err != nil || len(encryptedKey) != 64 {
		return keyID, kmsKey, sealedKey, ctx, Errorf("The internal sealed key for SSE-KMS is invalid")
	}
	if idPresent && kmsKeyPresent { // We are using a KMS -> parse the sealed KMS data key.
		kmsKey, err = base64.StdEncoding.DecodeString(b64KMSSealedKey)
		if err != nil {
			return keyID, kmsKey, sealedKey, ctx, Errorf("The internal sealed KMS data key for SSE-KMS is invalid")
		}
	}
	b64Ctx, ok := metadata[MetaContext]
	if ok {
		b, err := base64.StdEncoding.DecodeString(b64Ctx)
		if err != nil {
			return keyID, kmsKey, sealedKey, ctx, Errorf("The internal KMS context is not base64-encoded")
		}
		var json = jsoniter.ConfigCompatibleWithStandardLibrary
		if err = json.Unmarshal(b, ctx); err != nil {
			return keyID, kmsKey, sealedKey, ctx, Errorf("The internal sealed KMS context is invalid")
		}
	}

	sealedKey.Algorithm = algorithm
	copy(sealedKey.IV[:], iv)
	copy(sealedKey.Key[:], encryptedKey)
	return keyID, kmsKey, sealedKey, ctx, nil
}
<commit_msg>fix sse-kms context unmarshal failure (#13206)<commit_after>// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package crypto

import (
	"context"
	"encoding/base64"
	"errors"
	"net/http"
	"path"
	"strings"

	jsoniter "github.com/json-iterator/go"
	xhttp "github.com/minio/minio/internal/http"
	"github.com/minio/minio/internal/kms"
	"github.com/minio/minio/internal/logger"
)

type ssekms struct{}

var (
	// S3KMS represents AWS SSE-KMS. It provides functionality to
	// handle SSE-KMS requests.
	S3KMS = ssekms{}

	_ Type = S3KMS
)

// String returns the SSE domain as string. For SSE-KMS the
// domain is "SSE-KMS".
func (ssekms) String() string { return "SSE-KMS" }

// IsRequested returns true if the HTTP headers contains
// at least one SSE-KMS header.
func (ssekms) IsRequested(h http.Header) bool {
	if _, ok := h[xhttp.AmzServerSideEncryptionKmsID]; ok {
		return true
	}
	if _, ok := h[xhttp.AmzServerSideEncryptionKmsContext]; ok {
		return true
	}
	if _, ok := h[xhttp.AmzServerSideEncryption]; ok {
		return strings.ToUpper(h.Get(xhttp.AmzServerSideEncryption)) != xhttp.AmzEncryptionAES // Return only true if the SSE header is specified and does not contain the SSE-S3 value
	}
	return false
}

// ParseHTTP parses the SSE-KMS headers and returns the SSE-KMS key ID
// and the KMS context on success.
func (ssekms) ParseHTTP(h http.Header) (string, kms.Context, error) {
	algorithm := h.Get(xhttp.AmzServerSideEncryption)
	if algorithm != xhttp.AmzEncryptionKMS {
		return "", nil, ErrInvalidEncryptionMethod
	}

	var ctx kms.Context
	if context, ok := h[xhttp.AmzServerSideEncryptionKmsContext]; ok {
		b, err := base64.StdEncoding.DecodeString(context[0])
		if err != nil {
			return "", nil, err
		}

		var json = jsoniter.ConfigCompatibleWithStandardLibrary
		if err := json.Unmarshal(b, &ctx); err != nil {
			return "", nil, err
		}
	}
	return h.Get(xhttp.AmzServerSideEncryptionKmsID), ctx, nil
}

// IsEncrypted returns true if the object metadata indicates
// that the object was uploaded using SSE-KMS.
func (ssekms) IsEncrypted(metadata map[string]string) bool {
	if _, ok := metadata[MetaSealedKeyKMS]; ok {
		return true
	}
	return false
}

// UnsealObjectKey extracts and decrypts the sealed object key
// from the metadata using KMS and returns the decrypted object
// key.
func (s3 ssekms) UnsealObjectKey(KMS kms.KMS, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
	keyID, kmsKey, sealedKey, ctx, err := s3.ParseMetadata(metadata)
	if err != nil {
		return key, err
	}
	if ctx == nil {
		ctx = kms.Context{bucket: path.Join(bucket, object)}
	} else if _, ok := ctx[bucket]; !ok {
		ctx[bucket] = path.Join(bucket, object)
	}
	unsealKey, err := KMS.DecryptKey(keyID, kmsKey, ctx)
	if err != nil {
		return key, err
	}
	err = key.Unseal(unsealKey[:], sealedKey, s3.String(), bucket, object)
	return key, err
}

// CreateMetadata encodes the sealed object key into the metadata and returns
// the modified metadata. If the keyID and the kmsKey is not empty it encodes
// both into the metadata as well. It allocates a new metadata map if metadata
// is nil.
func (ssekms) CreateMetadata(metadata map[string]string, keyID string, kmsKey []byte, sealedKey SealedKey, ctx kms.Context) map[string]string {
	if sealedKey.Algorithm != SealAlgorithm {
		logger.CriticalIf(context.Background(), Errorf("The seal algorithm '%s' is invalid for SSE-S3", sealedKey.Algorithm))
	}

	// There are two possibilites:
	// - We use a KMS -> There must be non-empty key ID and a KMS data key.
	// - We use a K/V -> There must be no key ID and no KMS data key.
	// Otherwise, the caller has passed an invalid argument combination.
	if keyID == "" && len(kmsKey) != 0 {
		logger.CriticalIf(context.Background(), errors.New("The key ID must not be empty if a KMS data key is present"))
	}
	if keyID != "" && len(kmsKey) == 0 {
		logger.CriticalIf(context.Background(), errors.New("The KMS data key must not be empty if a key ID is present"))
	}

	if metadata == nil {
		metadata = make(map[string]string, 5)
	}

	metadata[MetaAlgorithm] = sealedKey.Algorithm
	metadata[MetaIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])
	metadata[MetaSealedKeyKMS] = base64.StdEncoding.EncodeToString(sealedKey.Key[:])
	if len(ctx) > 0 {
		b, _ := ctx.MarshalText()
		metadata[MetaContext] = base64.StdEncoding.EncodeToString(b)
	}
	if len(kmsKey) > 0 && keyID != "" { // We use a KMS -> Store key ID and sealed KMS data key.
		metadata[MetaKeyID] = keyID
		metadata[MetaDataEncryptionKey] = base64.StdEncoding.EncodeToString(kmsKey)
	}
	return metadata
}

// ParseMetadata extracts all SSE-KMS related values from the object metadata
// and checks whether they are well-formed. It returns the sealed object key
// on success. If the metadata contains both, a KMS master key ID and a sealed
// KMS data key it returns both. If the metadata does not contain neither a
// KMS master key ID nor a sealed KMS data key it returns an empty keyID and
// KMS data key. Otherwise, it returns an error.
func (ssekms) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte, sealedKey SealedKey, ctx kms.Context, err error) {
	// Extract all required values from object metadata
	b64IV, ok := metadata[MetaIV]
	if !ok {
		return keyID, kmsKey, sealedKey, ctx, errMissingInternalIV
	}
	algorithm, ok := metadata[MetaAlgorithm]
	if !ok {
		return keyID, kmsKey, sealedKey, ctx, errMissingInternalSealAlgorithm
	}
	b64SealedKey, ok := metadata[MetaSealedKeyKMS]
	if !ok {
		return keyID, kmsKey, sealedKey, ctx, Errorf("The object metadata is missing the internal sealed key for SSE-S3")
	}

	// There are two possibilites:
	// - We use a KMS -> There must be a key ID and a KMS data key.
	// - We use a K/V -> There must be no key ID and no KMS data key.
	// Otherwise, the metadata is corrupted.
	keyID, idPresent := metadata[MetaKeyID]
	b64KMSSealedKey, kmsKeyPresent := metadata[MetaDataEncryptionKey]
	if !idPresent && kmsKeyPresent {
		return keyID, kmsKey, sealedKey, ctx, Errorf("The object metadata is missing the internal KMS key-ID for SSE-S3")
	}
	if idPresent && !kmsKeyPresent {
		return keyID, kmsKey, sealedKey, ctx, Errorf("The object metadata is missing the internal sealed KMS data key for SSE-S3")
	}

	// Check whether all extracted values are well-formed
	iv, err := base64.StdEncoding.DecodeString(b64IV)
	if err != nil || len(iv) != 32 {
		return keyID, kmsKey, sealedKey, ctx, errInvalidInternalIV
	}
	if algorithm != SealAlgorithm {
		return keyID, kmsKey, sealedKey, ctx, errInvalidInternalSealAlgorithm
	}
	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
	if err != nil || len(encryptedKey) != 64 {
		return keyID, kmsKey, sealedKey, ctx, Errorf("The internal sealed key for SSE-KMS is invalid")
	}
	if idPresent && kmsKeyPresent { // We are using a KMS -> parse the sealed KMS data key.
		kmsKey, err = base64.StdEncoding.DecodeString(b64KMSSealedKey)
		if err != nil {
			return keyID, kmsKey, sealedKey, ctx, Errorf("The internal sealed KMS data key for SSE-KMS is invalid")
		}
	}
	b64Ctx, ok := metadata[MetaContext]
	if ok {
		b, err := base64.StdEncoding.DecodeString(b64Ctx)
		if err != nil {
			return keyID, kmsKey, sealedKey, ctx, Errorf("The internal KMS context is not base64-encoded")
		}
		var json = jsoniter.ConfigCompatibleWithStandardLibrary
		if err = json.Unmarshal(b, &ctx); err != nil {
			return keyID, kmsKey, sealedKey, ctx, Errorf("The internal sealed KMS context is invalid %w", err)
		}
	}

	sealedKey.Algorithm = algorithm
	copy(sealedKey.IV[:], iv)
	copy(sealedKey.Key[:], encryptedKey)
	return keyID, kmsKey, sealedKey, ctx, nil
}
<|endoftext|>


<commit_before>// +build darwin freebsd linux

package fuse

import (
	"bytes"
	"math/rand"
	"os"
	"testing"
	"time"

	"golang.org/x/net/context"

	"github.com/restic/restic/internal/repository"
	"github.com/restic/restic/internal/restic"

	"bazil.org/fuse"
	"bazil.org/fuse/fs"

	rtest "github.com/restic/restic/internal/test"
)

func TestCache(t *testing.T) {
	var id1, id2, id3 restic.ID
	id1[0] = 1
	id2[0] = 2
	id3[0] = 3

	const (
		kiB       = 1 << 10
		cacheSize = 64*kiB + 3*cacheOverhead
	)

	c := newBlobCache(cacheSize)

	addAndCheck := func(id restic.ID, exp []byte) {
		c.add(id, exp)
		blob, ok := c.get(id)
		rtest.Assert(t, ok, "blob %v added but not found in cache", id)
		rtest.Equals(t, &exp[0], &blob[0])
		rtest.Equals(t, exp, blob)
	}

	addAndCheck(id1, make([]byte, 32*kiB))
	addAndCheck(id2, make([]byte, 30*kiB))
	addAndCheck(id3, make([]byte, 10*kiB))

	_, ok := c.get(id2)
	rtest.Assert(t, ok, "blob %v not present", id2)
	_, ok = c.get(id1)
	rtest.Assert(t, !ok, "blob %v present, but should have been evicted", id1)

	c.add(id1, make([]byte, 1+c.size))
	_, ok = c.get(id1)
	rtest.Assert(t, !ok, "blob %v too large but still added to cache")

	c.c.Remove(id1)
	c.c.Remove(id3)
	c.c.Remove(id2)

	rtest.Equals(t, cacheSize, c.size)
	rtest.Equals(t, cacheSize, c.free)
}

func testRead(t testing.TB, f *file, offset, length int, data []byte) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	req := &fuse.ReadRequest{
		Offset: int64(offset),
		Size:   length,
	}
	resp := &fuse.ReadResponse{
		Data: data,
	}
	rtest.OK(t, f.Read(ctx, req, resp))
}

func firstSnapshotID(t testing.TB, repo restic.Repository) (first restic.ID) {
	err := repo.List(context.TODO(), restic.SnapshotFile, func(id restic.ID, size int64) error {
		if first.IsNull() {
			first = id
		}
		return nil
	})

	if err != nil {
		t.Fatal(err)
	}

	return first
}

func loadFirstSnapshot(t testing.TB, repo restic.Repository) *restic.Snapshot {
	id := firstSnapshotID(t, repo)
	sn, err := restic.LoadSnapshot(context.TODO(), repo, id)
	rtest.OK(t, err)
	return sn
}

func loadTree(t testing.TB, repo restic.Repository, id restic.ID) *restic.Tree {
	tree, err := repo.LoadTree(context.TODO(), id)
	rtest.OK(t, err)
	return tree
}

func TestFuseFile(t *testing.T) {
	repo, cleanup := repository.TestRepository(t)
	defer cleanup()

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	timestamp, err := time.Parse(time.RFC3339, "2017-01-24T10:42:56+01:00")
	rtest.OK(t, err)
	restic.TestCreateSnapshot(t, repo, timestamp, 2, 0.1)

	sn := loadFirstSnapshot(t, repo)
	tree := loadTree(t, repo, *sn.Tree)

	var content restic.IDs
	for _, node := range tree.Nodes {
		content = append(content, node.Content...)
	}
	t.Logf("tree loaded, content: %v", content)

	var (
		filesize uint64
		memfile  []byte
	)
	for _, id := range content {
		size, found := repo.LookupBlobSize(id, restic.DataBlob)
		rtest.Assert(t, found, "Expected to find blob id %v", id)
		filesize += uint64(size)

		buf, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, nil)
		rtest.OK(t, err)

		if len(buf) != int(size) {
			t.Fatalf("not enough bytes read for id %v: want %v, got %v", id.Str(), size, len(buf))
		}

		if uint(len(buf)) != size {
			t.Fatalf("buffer has wrong length for id %v: want %v, got %v", id.Str(), size, len(buf))
		}

		memfile = append(memfile, buf...)
	}

	t.Logf("filesize is %v, memfile has size %v", filesize, len(memfile))

	node := &restic.Node{
		Name:    "foo",
		Inode:   23,
		Mode:    0742,
		Size:    filesize,
		Content: content,
	}
	root := NewRoot(context.TODO(), repo, Config{})

	t.Logf("blob cache has %d entries", len(root.blobSizeCache.m))

	inode := fs.GenerateDynamicInode(1, "foo")
	f, err := newFile(context.TODO(), root, inode, node)
	rtest.OK(t, err)

	attr := fuse.Attr{}
	rtest.OK(t, f.Attr(ctx, &attr))

	rtest.Equals(t, inode, attr.Inode)
	rtest.Equals(t, node.Mode, attr.Mode)
	rtest.Equals(t, node.Size, attr.Size)
	rtest.Equals(t, (node.Size/uint64(attr.BlockSize))+1, attr.Blocks)

	for i := 0; i < 200; i++ {
		offset := rand.Intn(int(filesize))
		length := rand.Intn(int(filesize)-offset) + 100

		b := memfile[offset : offset+length]

		buf := make([]byte, length)

		testRead(t, f, offset, length, buf)
		if !bytes.Equal(b, buf) {
			t.Errorf("test %d failed, wrong data returned (offset %v, length %v)", i, offset, length)
		}
	}
}

// Test top-level directories for their UID and GID.
func TestTopUidGid(t *testing.T) {
	repo, cleanup := repository.TestRepository(t)
	defer cleanup()

	restic.TestCreateSnapshot(t, repo, time.Unix(1460289341, 207401672), 0, 0)

	testTopUidGid(t, Config{}, repo, uint32(os.Getuid()), uint32(os.Getgid()))
	testTopUidGid(t, Config{OwnerIsRoot: true}, repo, 0, 0)
}

func testTopUidGid(t *testing.T, cfg Config, repo restic.Repository, uid, gid uint32) {
	t.Helper()

	ctx := context.Background()
	root := NewRoot(ctx, repo, cfg)

	var attr fuse.Attr
	err := root.Attr(ctx, &attr)
	rtest.OK(t, err)
	rtest.Equals(t, uid, attr.Uid)
	rtest.Equals(t, gid, attr.Gid)

	idsdir, err := root.Lookup(ctx, "ids")
	rtest.OK(t, err)

	err = idsdir.Attr(ctx, &attr)
	rtest.OK(t, err)
	rtest.Equals(t, uid, attr.Uid)
	rtest.Equals(t, gid, attr.Gid)

	snapID := loadFirstSnapshot(t, repo).ID().Str()
	snapshotdir, err := idsdir.(fs.NodeStringLookuper).Lookup(ctx, snapID)

	err = snapshotdir.Attr(ctx, &attr)
	rtest.OK(t, err)
	rtest.Equals(t, uid, attr.Uid)
	rtest.Equals(t, gid, attr.Gid)
}
<commit_msg>internal/fuse: fix dropped test error<commit_after>// +build darwin freebsd linux

package fuse

import (
	"bytes"
	"math/rand"
	"os"
	"testing"
	"time"

	"golang.org/x/net/context"

	"github.com/restic/restic/internal/repository"
	"github.com/restic/restic/internal/restic"

	"bazil.org/fuse"
	"bazil.org/fuse/fs"

	rtest "github.com/restic/restic/internal/test"
)

func TestCache(t *testing.T) {
	var id1, id2, id3 restic.ID
	id1[0] = 1
	id2[0] = 2
	id3[0] = 3

	const (
		kiB       = 1 << 10
		cacheSize = 64*kiB + 3*cacheOverhead
	)

	c := newBlobCache(cacheSize)

	addAndCheck := func(id restic.ID, exp []byte) {
		c.add(id, exp)
		blob, ok := c.get(id)
		rtest.Assert(t, ok, "blob %v added but not found in cache", id)
		rtest.Equals(t, &exp[0], &blob[0])
		rtest.Equals(t, exp, blob)
	}

	addAndCheck(id1, make([]byte, 32*kiB))
	addAndCheck(id2, make([]byte, 30*kiB))
	addAndCheck(id3, make([]byte, 10*kiB))

	_, ok := c.get(id2)
	rtest.Assert(t, ok, "blob %v not present", id2)
	_, ok = c.get(id1)
	rtest.Assert(t, !ok, "blob %v present, but should have been evicted", id1)

	c.add(id1, make([]byte, 1+c.size))
	_, ok = c.get(id1)
	rtest.Assert(t, !ok, "blob %v too large but still added to cache")

	c.c.Remove(id1)
	c.c.Remove(id3)
	c.c.Remove(id2)

	rtest.Equals(t, cacheSize, c.size)
	rtest.Equals(t, cacheSize, c.free)
}

func testRead(t testing.TB, f *file, offset, length int, data []byte) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	req := &fuse.ReadRequest{
		Offset: int64(offset),
		Size:   length,
	}
	resp := &fuse.ReadResponse{
		Data: data,
	}
	rtest.OK(t, f.Read(ctx, req, resp))
}

func firstSnapshotID(t testing.TB, repo restic.Repository) (first restic.ID) {
	err := repo.List(context.TODO(), restic.SnapshotFile, func(id restic.ID, size int64) error {
		if first.IsNull() {
			first = id
		}
		return nil
	})

	if err != nil {
		t.Fatal(err)
	}

	return first
}

func loadFirstSnapshot(t testing.TB, repo restic.Repository) *restic.Snapshot {
	id := firstSnapshotID(t, repo)
	sn, err := restic.LoadSnapshot(context.TODO(), repo, id)
	rtest.OK(t, err)
	return sn
}

func loadTree(t testing.TB, repo restic.Repository, id restic.ID) *restic.Tree {
	tree, err := repo.LoadTree(context.TODO(), id)
	rtest.OK(t, err)
	return tree
}

func TestFuseFile(t *testing.T) {
	repo, cleanup := repository.TestRepository(t)
	defer cleanup()

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	timestamp, err := time.Parse(time.RFC3339, "2017-01-24T10:42:56+01:00")
	rtest.OK(t, err)
	restic.TestCreateSnapshot(t, repo, timestamp, 2, 0.1)

	sn := loadFirstSnapshot(t, repo)
	tree := loadTree(t, repo, *sn.Tree)

	var content restic.IDs
	for _, node := range tree.Nodes {
		content = append(content, node.Content...)
	}
	t.Logf("tree loaded, content: %v", content)

	var (
		filesize uint64
		memfile  []byte
	)
	for _, id := range content {
		size, found := repo.LookupBlobSize(id, restic.DataBlob)
		rtest.Assert(t, found, "Expected to find blob id %v", id)
		filesize += uint64(size)

		buf, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, nil)
		rtest.OK(t, err)

		if len(buf) != int(size) {
			t.Fatalf("not enough bytes read for id %v: want %v, got %v", id.Str(), size, len(buf))
		}

		if uint(len(buf)) != size {
			t.Fatalf("buffer has wrong length for id %v: want %v, got %v", id.Str(), size, len(buf))
		}

		memfile = append(memfile, buf...)
	}

	t.Logf("filesize is %v, memfile has size %v", filesize, len(memfile))

	node := &restic.Node{
		Name:    "foo",
		Inode:   23,
		Mode:    0742,
		Size:    filesize,
		Content: content,
	}
	root := NewRoot(context.TODO(), repo, Config{})

	t.Logf("blob cache has %d entries", len(root.blobSizeCache.m))

	inode := fs.GenerateDynamicInode(1, "foo")
	f, err := newFile(context.TODO(), root, inode, node)
	rtest.OK(t, err)

	attr := fuse.Attr{}
	rtest.OK(t, f.Attr(ctx, &attr))

	rtest.Equals(t, inode, attr.Inode)
	rtest.Equals(t, node.Mode, attr.Mode)
	rtest.Equals(t, node.Size, attr.Size)
	rtest.Equals(t, (node.Size/uint64(attr.BlockSize))+1, attr.Blocks)

	for i := 0; i < 200; i++ {
		offset := rand.Intn(int(filesize))
		length := rand.Intn(int(filesize)-offset) + 100

		b := memfile[offset : offset+length]

		buf := make([]byte, length)

		testRead(t, f, offset, length, buf)
		if !bytes.Equal(b, buf) {
			t.Errorf("test %d failed, wrong data returned (offset %v, length %v)", i, offset, length)
		}
	}
}

// Test top-level directories for their UID and GID.
func TestTopUidGid(t *testing.T) {
	repo, cleanup := repository.TestRepository(t)
	defer cleanup()

	restic.TestCreateSnapshot(t, repo, time.Unix(1460289341, 207401672), 0, 0)

	testTopUidGid(t, Config{}, repo, uint32(os.Getuid()), uint32(os.Getgid()))
	testTopUidGid(t, Config{OwnerIsRoot: true}, repo, 0, 0)
}

func testTopUidGid(t *testing.T, cfg Config, repo restic.Repository, uid, gid uint32) {
	t.Helper()

	ctx := context.Background()
	root := NewRoot(ctx, repo, cfg)

	var attr fuse.Attr
	err := root.Attr(ctx, &attr)
	rtest.OK(t, err)
	rtest.Equals(t, uid, attr.Uid)
	rtest.Equals(t, gid, attr.Gid)

	idsdir, err := root.Lookup(ctx, "ids")
	rtest.OK(t, err)

	err = idsdir.Attr(ctx, &attr)
	rtest.OK(t, err)
	rtest.Equals(t, uid, attr.Uid)
	rtest.Equals(t, gid, attr.Gid)

	snapID := loadFirstSnapshot(t, repo).ID().Str()
	snapshotdir, err := idsdir.(fs.NodeStringLookuper).Lookup(ctx, snapID)
	rtest.OK(t, err)

	err = snapshotdir.Attr(ctx, &attr)
	rtest.OK(t, err)
	rtest.Equals(t, uid, attr.Uid)
	rtest.Equals(t, gid, attr.Gid)
}
<|endoftext|>


<commit_before>package hmac

import (
	"crypto/sha1"
	"fmt"
	"testing"
)

func BenchmarkHMACSHA1_512(b *testing.B) {
	key := make([]byte, 32)
	buf := make([]byte, 512)
	b.ReportAllocs()
	h := AcquireSHA1(key)
	b.SetBytes(int64(len(buf)))
	for i := 0; i < b.N; i++ {
		h.Write(buf)
		h.Reset()
		mac := h.Sum(nil)
		buf[0] = mac[0]
	}
}

func BenchmarkHMACSHA1_512_Pool(b *testing.B) {
	key := make([]byte, 32)
	buf := make([]byte, 512)
	tBuf := make([]byte, 0, 512)
	b.ReportAllocs()
	b.SetBytes(int64(len(buf)))
	for i := 0; i < b.N; i++ {
		h := AcquireSHA1(key)
		h.Write(buf)
		h.Reset()
		mac := h.Sum(tBuf)
		buf[0] = mac[0]
		PutSHA1(h)
	}
}

func TestHMACReset(t *testing.T) {
	for i, tt := range hmacTests {
		h := New(tt.hash, tt.key)
		h.(*hmac).resetTo(tt.key)
		if s := h.Size(); s != tt.size {
			t.Errorf("Size: got %v, want %v", s, tt.size)
		}
		if b := h.BlockSize(); b != tt.blocksize {
			t.Errorf("BlockSize: got %v, want %v", b, tt.blocksize)
		}
		for j := 0; j < 2; j++ {
			n, err := h.Write(tt.in)
			if n != len(tt.in) || err != nil {
				t.Errorf("test %d.%d: Write(%d) = %d, %v", i, j, len(tt.in), n, err)
				continue
			}

			// Repetitive Sum() calls should return the same value
			for k := 0; k < 2; k++ {
				sum := fmt.Sprintf("%x", h.Sum(nil))
				if sum != tt.out {
					t.Errorf("test %d.%d.%d: have %s want %s\n", i, j, k, sum, tt.out)
				}
			}

			// Second iteration: make sure reset works.
			h.Reset()
		}
	}
}

func TestHMACPool(t *testing.T) {
	for i, tt := range hmacTests {
		if tt.blocksize != sha1.BlockSize {
			continue
		}
		h := AcquireSHA1(tt.key)
		if s := h.Size(); s != tt.size {
			t.Errorf("Size: got %v, want %v", s, tt.size)
		}
		if b := h.BlockSize(); b != tt.blocksize {
			t.Errorf("BlockSize: got %v, want %v", b, tt.blocksize)
		}
		for j := 0; j < 2; j++ {
			n, err := h.Write(tt.in)
			if n != len(tt.in) || err != nil {
				t.Errorf("test %d.%d: Write(%d) = %d, %v", i, j, len(tt.in), n, err)
				continue
			}

			// Repetitive Sum() calls should return the same value
			for k := 0; k < 2; k++ {
				sum := fmt.Sprintf("%x", h.Sum(nil))
				if sum != tt.out {
					t.Errorf("test %d.%d.%d: have %s want %s\n", i, j, k, sum, tt.out)
				}
			}

			// Second iteration: make sure reset works.
			h.Reset()
		}
		PutSHA1(h)
	}
}
<commit_msg>hmac: fix TestHMACPool<commit_after>package hmac

import (
	"crypto/sha1"
	"fmt"
	"testing"
)

func BenchmarkHMACSHA1_512(b *testing.B) {
	key := make([]byte, 32)
	buf := make([]byte, 512)
	b.ReportAllocs()
	h := AcquireSHA1(key)
	b.SetBytes(int64(len(buf)))
	for i := 0; i < b.N; i++ {
		h.Write(buf)
		h.Reset()
		mac := h.Sum(nil)
		buf[0] = mac[0]
	}
}

func BenchmarkHMACSHA1_512_Pool(b *testing.B) {
	key := make([]byte, 32)
	buf := make([]byte, 512)
	tBuf := make([]byte, 0, 512)
	b.ReportAllocs()
	b.SetBytes(int64(len(buf)))
	for i := 0; i < b.N; i++ {
		h := AcquireSHA1(key)
		h.Write(buf)
		h.Reset()
		mac := h.Sum(tBuf)
		buf[0] = mac[0]
		PutSHA1(h)
	}
}

func TestHMACReset(t *testing.T) {
	for i, tt := range hmacTests {
		h := New(tt.hash, tt.key)
		h.(*hmac).resetTo(tt.key)
		if s := h.Size(); s != tt.size {
			t.Errorf("Size: got %v, want %v", s, tt.size)
		}
		if b := h.BlockSize(); b != tt.blocksize {
			t.Errorf("BlockSize: got %v, want %v", b, tt.blocksize)
		}
		for j := 0; j < 2; j++ {
			n, err := h.Write(tt.in)
			if n != len(tt.in) || err != nil {
				t.Errorf("test %d.%d: Write(%d) = %d, %v", i, j, len(tt.in), n, err)
				continue
			}

			// Repetitive Sum() calls should return the same value
			for k := 0; k < 2; k++ {
				sum := fmt.Sprintf("%x", h.Sum(nil))
				if sum != tt.out {
					t.Errorf("test %d.%d.%d: have %s want %s\n", i, j, k, sum, tt.out)
				}
			}

			// Second iteration: make sure reset works.
			h.Reset()
		}
	}
}

func TestHMACPool(t *testing.T) {
	for i, tt := range hmacTests {
		if tt.blocksize != sha1.BlockSize || tt.size != sha1.Size {
			continue
		}
		h := AcquireSHA1(tt.key)
		if s := h.Size(); s != tt.size {
			t.Errorf("Size: got %v, want %v", s, tt.size)
		}
		if b := h.BlockSize(); b != tt.blocksize {
			t.Errorf("BlockSize: got %v, want %v", b, tt.blocksize)
		}
		for j := 0; j < 2; j++ {
			n, err := h.Write(tt.in)
			if n != len(tt.in) || err != nil {
				t.Errorf("test %d.%d: Write(%d) = %d, %v", i, j, len(tt.in), n, err)
				continue
			}

			// Repetitive Sum() calls should return the same value
			for k := 0; k < 2; k++ {
				sum := fmt.Sprintf("%x", h.Sum(nil))
				if sum != tt.out {
					t.Errorf("test %d.%d.%d: have %s want %s\n", i, j, k, sum, tt.out)
				}
			}

			// Second iteration: make sure reset works.
			h.Reset()
		}
		PutSHA1(h)
	}
}
<|endoftext|>


<commit_before>// +build darwin

package nat

import (
	"fmt"
	"log"
	"net"
	"os/exec"
	"strings"
	ppf "github.com/datawire/pf"
	"github.com/google/shlex"
)

type Translator struct {
	commonTranslator
	dev *ppf.Handle
}

func pf(argline, stdin string) (err error) {
	args, err := shlex.Split(argline)
	if err != nil { panic(err) }
	args = append([]string{}, args...)
	cmd := exec.Command("pfctl", args...)
	cmd.Stdin = strings.NewReader(stdin)
	log.Printf("pfctl %s < %s\n", argline, stdin)
	out, err := cmd.CombinedOutput()
	if len(out) > 0 {
		log.Printf("%s", out)
	}
	if err != nil {
		log.Println(err)
		return fmt.Errorf("IN:%s\nOUT:%s\nERR:%s\n", strings.TrimSpace(stdin), strings.TrimSpace(string(out)),
			err)
	}
	return
}

func (t *Translator) rules() string {
	if t.dev == nil { return "" }

	entries := t.sorted()

	result := ""
	for _, entry := range entries {
		dst := entry.Destination
		result += ("rdr pass on lo0 inet proto " + dst.Proto + " to " + dst.Ip + " -> 127.0.0.1 port " +
			entry.Port + "\n")
	}

	result += "pass out quick inet proto tcp to 127.0.0.1/32\n"

	for _, entry := range entries {
		dst := entry.Destination
		result += "pass out route-to lo0 inet proto " + dst.Proto + " to " + dst.Ip + " keep state\n"
	}

	return result
}

var actions = []ppf.Action{ppf.ActionPass, ppf.ActionRDR}

func (t *Translator) Enable() {
	var err error
	t.dev, err = ppf.Open()
	if err != nil { panic(err) }

	for _, action := range actions {
		var rule ppf.Rule
		err = rule.SetAnchorCall(t.Name)
		if err != nil { panic(err) }
		rule.SetAction(action)
		err = t.dev.PrependRule(rule)
		if err != nil { panic(err) }
	}

	pf("-a " + t.Name + " -F all", "")
	pf("-a " + t.Name + " -f /dev/stdin", t.rules())

	t.dev.Start()
}

func (t *Translator) Disable() {
	if t.dev != nil {
		t.dev.Stop()

		for _, action := range actions {
			OUTER: for {
				rules, err := t.dev.Rules(action)
				if err != nil { panic(err) }

				for _, rule := range rules {
					if rule.AnchorCall() == t.Name {
						log.Printf("Removing rule: %v\n", rule)
						err = t.dev.RemoveRule(rule)
						if err != nil { panic(err) }
						continue OUTER
					}
				}
				break
			}
		}
	}

	pf("-a " + t.Name + " -F all", "")
}

func (t *Translator) ForwardTCP(ip, toPort string) {
	t.forward("tcp", ip, toPort)
}

func (t *Translator) ForwardUDP(ip, toPort string) {
	t.forward("udp", ip, toPort)
}

func (t *Translator) forward(protocol, ip, toPort string) {
	t.clear(protocol, ip)
	t.Mappings[Address{protocol, ip}] = toPort
	pf("-a " + t.Name + " -f /dev/stdin", t.rules())
}

func (t *Translator) ClearTCP(ip string) {
	t.clear("tcp", ip)
	pf("-a " + t.Name + " -f /dev/stdin", t.rules())
}

func (t *Translator) clear(protocol, ip string) {
	if _, exists := t.Mappings[Address{protocol, ip}]; exists {
		delete(t.Mappings, Address{protocol, ip})
	}
}

func (t *Translator) GetOriginalDst(conn *net.TCPConn) (rawaddr []byte, host string, err error) {
	remote := conn.RemoteAddr().(*net.TCPAddr)
	local := conn.LocalAddr().(*net.TCPAddr)
	addr, port, err := t.dev.NatLook(remote.IP.String(), remote.Port, local.IP.String(), local.Port)
	if err != nil {
		return
	}

	return nil, fmt.Sprintf("%s:%d", addr, port), nil
}
<commit_msg>Ensure pf is active on loopback. Ensure we stop processing rules if our change matches.<commit_after>// +build darwin

package nat

import (
	"fmt"
	"log"
	"net"
	"os/exec"
	"strings"
	ppf "github.com/datawire/pf"
	"github.com/google/shlex"
)

type Translator struct {
	commonTranslator
	dev *ppf.Handle
}

func pf(argline, stdin string) (err error) {
	args, err := shlex.Split(argline)
	if err != nil { panic(err) }
	args = append([]string{}, args...)
	cmd := exec.Command("pfctl", args...)
	cmd.Stdin = strings.NewReader(stdin)
	log.Printf("pfctl %s < %s\n", argline, stdin)
	out, err := cmd.CombinedOutput()
	if len(out) > 0 {
		log.Printf("%s", out)
	}
	if err != nil {
		log.Println(err)
		return fmt.Errorf("IN:%s\nOUT:%s\nERR:%s\n", strings.TrimSpace(stdin), strings.TrimSpace(string(out)),
			err)
	}
	return
}

func (t *Translator) rules() string {
	if t.dev == nil { return "" }

	entries := t.sorted()

	result := ""
	for _, entry := range entries {
		dst := entry.Destination
		result += ("rdr pass on lo0 inet proto " + dst.Proto + " to " + dst.Ip + " -> 127.0.0.1 port " +
			entry.Port + "\n")
	}

	result += "pass out quick inet proto tcp to 127.0.0.1/32\n"

	for _, entry := range entries {
		dst := entry.Destination
		result += "pass out route-to lo0 inet proto " + dst.Proto + " to " + dst.Ip + " keep state\n"
	}

	return result
}

var actions = []ppf.Action{ppf.ActionPass, ppf.ActionRDR}

func (t *Translator) Enable() {
	var err error
	t.dev, err = ppf.Open()
	if err != nil { panic(err) }

	for _, action := range actions {
		var rule ppf.Rule
		err = rule.SetAnchorCall(t.Name)
		if err != nil { panic(err) }
		rule.SetAction(action)
		rule.SetQuick(true)
		err = t.dev.PrependRule(rule)
		if err != nil { panic(err) }
	}

	pf("-a " + t.Name + " -F all", "")

	pf("-f /dev/stdin", "pass on lo0")
	pf("-a " + t.Name + " -f /dev/stdin", t.rules())

	t.dev.Start()
}

func (t *Translator) Disable() {
	if t.dev != nil {
		t.dev.Stop()

		for _, action := range actions {
			OUTER: for {
				rules, err := t.dev.Rules(action)
				if err != nil { panic(err) }

				for _, rule := range rules {
					if rule.AnchorCall() == t.Name {
						log.Printf("Removing rule: %v\n", rule)
						err = t.dev.RemoveRule(rule)
						if err != nil { panic(err) }
						continue OUTER
					}
				}
				break
			}
		}
	}

	pf("-a " + t.Name + " -F all", "")
}

func (t *Translator) ForwardTCP(ip, toPort string) {
	t.forward("tcp", ip, toPort)
}

func (t *Translator) ForwardUDP(ip, toPort string) {
	t.forward("udp", ip, toPort)
}

func (t *Translator) forward(protocol, ip, toPort string) {
	t.clear(protocol, ip)
	t.Mappings[Address{protocol, ip}] = toPort
	pf("-a " + t.Name + " -f /dev/stdin", t.rules())
}

func (t *Translator) ClearTCP(ip string) {
	t.clear("tcp", ip)
	pf("-a " + t.Name + " -f /dev/stdin", t.rules())
}

func (t *Translator) clear(protocol, ip string) {
	if _, exists := t.Mappings[Address{protocol, ip}]; exists {
		delete(t.Mappings, Address{protocol, ip})
	}
}

func (t *Translator) GetOriginalDst(conn *net.TCPConn) (rawaddr []byte, host string, err error) {
	remote := conn.RemoteAddr().(*net.TCPAddr)
	local := conn.LocalAddr().(*net.TCPAddr)
	addr, port, err := t.dev.NatLook(remote.IP.String(), remote.Port, local.IP.String(), local.Port)
	if err != nil {
		return
	}

	return nil, fmt.Sprintf("%s:%d", addr, port), nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"

	"github.com/cloudfoundry/bosh-bootloader/storage"
)

var (
	backendURL string
	stateFile  *string
)

func main() {
	if checkFastFail() {
		log.Fatal("failed to terraform")
	}

	if os.Args[1] == "apply" {
		flagSet := flag.NewFlagSet("apply", flag.PanicOnError)
		stateFile = flagSet.String("state", "fake-terraform.tfstate", "output tfvars")
		flagSet.Parse(os.Args[2:])
	} else {
		stateFile = flag.String("state", "fake-terraform.tfstate", "output tfvars")
		flag.Parse()
	}

	if contains(os.Args, "region=fail-to-terraform") {
		err := ioutil.WriteFile(*stateFile, []byte(`{"key":"partial-apply"}`), storage.StateMode)
		if err != nil {
			panic(err)
		}

		log.Fatal("failed to terraform")
	}

	if os.Args[1] == "apply" {
		postArgs, err := json.Marshal(os.Args[1:])
		if err != nil {
			panic(err)
		}

		_, err = http.Post(fmt.Sprintf("%s/args", backendURL), "application/json", strings.NewReader(string(postArgs)))
		if err != nil {
			panic(err)
		}

		err = ioutil.WriteFile(*stateFile, []byte(`{"key":"value"}`), storage.StateMode)
		if err != nil {
			panic(err)
		}

		dir, err := os.Getwd()
		if err != nil {
			panic(err)
		}

		fmt.Printf("working directory: %s\n", dir)
		fmt.Printf("data directory: %s\n", os.Getenv("TF_DATA_DIR"))
		fmt.Printf("terraform %s/n", removeBrackets(fmt.Sprintf("%+v", os.Args)))
	}
}

func removeBrackets(contents string) string {
	contents = strings.Replace(contents, "[", "", -1)
	contents = strings.Replace(contents, "]", "", -1)
	return contents
}

func checkFastFail() bool {
	resp, err := http.Get(fmt.Sprintf("%s/fastfail", backendURL))
	if err != nil {
		panic(err)
	}

	return resp.StatusCode == http.StatusInternalServerError
}

func contains(slice []string, word string) bool {
	for _, item := range slice {
		if item == word {
			return true
		}
	}
	return false
}
<commit_msg>Fake terraform logs args which trigger failure<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"

	"github.com/cloudfoundry/bosh-bootloader/storage"
)

var (
	backendURL string
	stateFile  *string
)

func main() {
	if checkFastFail() {
		log.Fatal("failed to terraform")
	}

	if os.Args[1] == "apply" {
		flagSet := flag.NewFlagSet("apply", flag.PanicOnError)
		stateFile = flagSet.String("state", "fake-terraform.tfstate", "output tfvars")
		flagSet.Parse(os.Args[2:])
	} else {
		stateFile = flag.String("state", "fake-terraform.tfstate", "output tfvars")
		flag.Parse()
	}

	if contains(os.Args, "region=fail-to-terraform") {
		fmt.Printf("received args: %+v\n", os.Args)
		err := ioutil.WriteFile(*stateFile, []byte(`{"key":"partial-apply"}`), storage.StateMode)
		if err != nil {
			panic(err)
		}

		log.Fatal("failed to terraform")
	}

	if os.Args[1] == "apply" {
		postArgs, err := json.Marshal(os.Args[1:])
		if err != nil {
			panic(err)
		}

		_, err = http.Post(fmt.Sprintf("%s/args", backendURL), "application/json", strings.NewReader(string(postArgs)))
		if err != nil {
			panic(err)
		}

		err = ioutil.WriteFile(*stateFile, []byte(`{"key":"value"}`), storage.StateMode)
		if err != nil {
			panic(err)
		}

		dir, err := os.Getwd()
		if err != nil {
			panic(err)
		}

		fmt.Printf("working directory: %s\n", dir)
		fmt.Printf("data directory: %s\n", os.Getenv("TF_DATA_DIR"))
		fmt.Printf("terraform %s/n", removeBrackets(fmt.Sprintf("%+v", os.Args)))
	}
}

func removeBrackets(contents string) string {
	contents = strings.Replace(contents, "[", "", -1)
	contents = strings.Replace(contents, "]", "", -1)
	return contents
}

func checkFastFail() bool {
	resp, err := http.Get(fmt.Sprintf("%s/fastfail", backendURL))
	if err != nil {
		panic(err)
	}

	return resp.StatusCode == http.StatusInternalServerError
}

func contains(slice []string, word string) bool {
	for _, item := range slice {
		if item == word {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright (C) 2017-2018 Pilosa Corp. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package roaring

type sliceContainers struct {
	keys          []uint64
	containers    []*Container
	lastKey       uint64
	lastContainer *Container
}

func newSliceContainers() *sliceContainers {
	return &sliceContainers{}
}

func (sc *sliceContainers) Get(key uint64) *Container {
	i := search64(sc.keys, key)
	if i < 0 {
		return nil
	}
	return sc.containers[i]
}

func (sc *sliceContainers) Put(key uint64, c *Container) {
	i := search64(sc.keys, key)

	// If index is negative then there's not an exact match
	// and a container needs to be added.
	if i < 0 {
		sc.insertAt(key, c, -i-1)
	} else {
		sc.containers[i] = c
	}

}

func (sc *sliceContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) {
	i := search64(sc.keys, key)
	if i < 0 {
		c := NewContainer()
		c.containerType = containerType
		c.n = n
		c.mapped = mapped
		sc.insertAt(key, c, -i-1)
	} else {
		c := sc.containers[i]
		c.containerType = containerType
		c.n = n
		c.mapped = mapped
	}

}

func (sc *sliceContainers) Remove(key uint64) {
	i := search64(sc.keys, key)
	if i < 0 {
		return
	}
	sc.keys = append(sc.keys[:i], sc.keys[i+1:]...)
	sc.containers = append(sc.containers[:i], sc.containers[i+1:]...)

}
func (sc *sliceContainers) insertAt(key uint64, c *Container, i int) {
	sc.keys = append(sc.keys, 0)
	copy(sc.keys[i+1:], sc.keys[i:])
	sc.keys[i] = key

	sc.containers = append(sc.containers, nil)
	copy(sc.containers[i+1:], sc.containers[i:])
	sc.containers[i] = c
}

func (sc *sliceContainers) GetOrCreate(key uint64) *Container {
	// Check the last* cache for same container.
	if key == sc.lastKey && sc.lastContainer != nil {
		return sc.lastContainer
	}

	sc.lastKey = key
	i := search64(sc.keys, key)
	if i < 0 {
		c := NewContainer()
		sc.insertAt(key, c, -i-1)
		sc.lastContainer = c
		return c
	}

	sc.lastContainer = sc.containers[i]
	return sc.lastContainer
}

func (sc *sliceContainers) Clone() Containers {
	other := newSliceContainers()
	other.keys = make([]uint64, len(sc.keys))
	other.containers = make([]*Container, len(sc.containers))
	copy(other.keys, sc.keys)
	for i, c := range sc.containers {
		other.containers[i] = c.Clone()
	}
	return other
}

func (sc *sliceContainers) Last() (key uint64, c *Container) {
	if len(sc.keys) == 0 {
		return 0, nil
	}
	return sc.keys[len(sc.keys)-1], sc.containers[len(sc.keys)-1]
}

func (sc *sliceContainers) Size() int {
	return len(sc.keys)

}

func (sc *sliceContainers) Count() uint64 {
	n := uint64(0)
	for i := range sc.containers {
		n += uint64(sc.containers[i].n)
	}
	return n
}

func (sc *sliceContainers) Reset() {
	sc.keys = sc.keys[:0]
	sc.containers = sc.containers[:0]
	sc.lastContainer = nil
	sc.lastKey = 0
}

func (sc *sliceContainers) seek(key uint64) (int, bool) {
	i := search64(sc.keys, key)
	found := true
	if i < 0 {
		found = false
		i = -i - 1
	}
	return i, found
}

func (sc *sliceContainers) Iterator(key uint64) (citer ContainerIterator, found bool) {
	i, found := sc.seek(key)
	return &SliceIterator{e: sc, i: i}, found
}

type SliceIterator struct {
	e     *sliceContainers
	i     int
	key   uint64
	value *Container
}

func (si *SliceIterator) Next() bool {
	if si.e == nil || si.i > len(si.e.keys)-1 {
		return false
	}
	si.key = si.e.keys[si.i]
	si.value = si.e.containers[si.i]
	si.i++

	return true
}

func (si *SliceIterator) Value() (uint64, *Container) {
	return si.key, si.value
}
<commit_msg>Unexport roaring.SliceIterator<commit_after>// Copyright (C) 2017-2018 Pilosa Corp. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package roaring

type sliceContainers struct {
	keys          []uint64
	containers    []*Container
	lastKey       uint64
	lastContainer *Container
}

func newSliceContainers() *sliceContainers {
	return &sliceContainers{}
}

func (sc *sliceContainers) Get(key uint64) *Container {
	i := search64(sc.keys, key)
	if i < 0 {
		return nil
	}
	return sc.containers[i]
}

func (sc *sliceContainers) Put(key uint64, c *Container) {
	i := search64(sc.keys, key)

	// If index is negative then there's not an exact match
	// and a container needs to be added.
	if i < 0 {
		sc.insertAt(key, c, -i-1)
	} else {
		sc.containers[i] = c
	}

}

func (sc *sliceContainers) PutContainerValues(key uint64, containerType byte, n int, mapped bool) {
	i := search64(sc.keys, key)
	if i < 0 {
		c := NewContainer()
		c.containerType = containerType
		c.n = n
		c.mapped = mapped
		sc.insertAt(key, c, -i-1)
	} else {
		c := sc.containers[i]
		c.containerType = containerType
		c.n = n
		c.mapped = mapped
	}

}

func (sc *sliceContainers) Remove(key uint64) {
	i := search64(sc.keys, key)
	if i < 0 {
		return
	}
	sc.keys = append(sc.keys[:i], sc.keys[i+1:]...)
	sc.containers = append(sc.containers[:i], sc.containers[i+1:]...)

}
func (sc *sliceContainers) insertAt(key uint64, c *Container, i int) {
	sc.keys = append(sc.keys, 0)
	copy(sc.keys[i+1:], sc.keys[i:])
	sc.keys[i] = key

	sc.containers = append(sc.containers, nil)
	copy(sc.containers[i+1:], sc.containers[i:])
	sc.containers[i] = c
}

func (sc *sliceContainers) GetOrCreate(key uint64) *Container {
	// Check the last* cache for same container.
	if key == sc.lastKey && sc.lastContainer != nil {
		return sc.lastContainer
	}

	sc.lastKey = key
	i := search64(sc.keys, key)
	if i < 0 {
		c := NewContainer()
		sc.insertAt(key, c, -i-1)
		sc.lastContainer = c
		return c
	}

	sc.lastContainer = sc.containers[i]
	return sc.lastContainer
}

func (sc *sliceContainers) Clone() Containers {
	other := newSliceContainers()
	other.keys = make([]uint64, len(sc.keys))
	other.containers = make([]*Container, len(sc.containers))
	copy(other.keys, sc.keys)
	for i, c := range sc.containers {
		other.containers[i] = c.Clone()
	}
	return other
}

func (sc *sliceContainers) Last() (key uint64, c *Container) {
	if len(sc.keys) == 0 {
		return 0, nil
	}
	return sc.keys[len(sc.keys)-1], sc.containers[len(sc.keys)-1]
}

func (sc *sliceContainers) Size() int {
	return len(sc.keys)

}

func (sc *sliceContainers) Count() uint64 {
	n := uint64(0)
	for i := range sc.containers {
		n += uint64(sc.containers[i].n)
	}
	return n
}

func (sc *sliceContainers) Reset() {
	sc.keys = sc.keys[:0]
	sc.containers = sc.containers[:0]
	sc.lastContainer = nil
	sc.lastKey = 0
}

func (sc *sliceContainers) seek(key uint64) (int, bool) {
	i := search64(sc.keys, key)
	found := true
	if i < 0 {
		found = false
		i = -i - 1
	}
	return i, found
}

func (sc *sliceContainers) Iterator(key uint64) (citer ContainerIterator, found bool) {
	i, found := sc.seek(key)
	return &sliceIterator{e: sc, i: i}, found
}

type sliceIterator struct {
	e     *sliceContainers
	i     int
	key   uint64
	value *Container
}

func (si *sliceIterator) Next() bool {
	if si.e == nil || si.i > len(si.e.keys)-1 {
		return false
	}
	si.key = si.e.keys[si.i]
	si.value = si.e.containers[si.i]
	si.i++

	return true
}

func (si *sliceIterator) Value() (uint64, *Container) {
	return si.key, si.value
}
<|endoftext|>