<commit_before>// Useful utility functions
package common

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"os"
	"time"
)

// The main function which handles database upload processing for both the webUI and DB4S end points
func AddDatabase(r *http.Request, loggedInUser string, dbOwner string, dbFolder string, dbName string,
	branchName string, public bool, licenceName string, commitMsg string, sourceURL string, newDB io.Reader,
	serverSw string) (numBytes int64, err error) {

	// Write the temporary file locally, so we can try opening it with SQLite to verify it's ok
	var buf bytes.Buffer
	numBytes, err = io.Copy(&buf, newDB)
	if err != nil {
		log.Printf("Error: %v\n", err)
		return numBytes, err
	}
	if numBytes == 0 {
		log.Printf("Database seems to be 0 bytes in length. Username: %s, Database: %s\n", loggedInUser, dbName)
		return numBytes, err
	}
	tempDB, err := ioutil.TempFile("", "dbhub-upload-")
	if err != nil {
		log.Printf("Error creating temporary file. User: '%s', Database: '%s%s%s', Filename: '%s', Error: %v\n",
			loggedInUser, dbOwner, dbFolder, dbName, tempDB.Name(), err)
		return numBytes, err
	}
	_, err = tempDB.Write(buf.Bytes())
	if err != nil {
		log.Printf("Error when writing the uploaded db to a temp file. User: '%s', Database: '%s%s%s' "+
			"Error: %v\n", loggedInUser, dbOwner, dbFolder, dbName, err)
		return numBytes, err
	}
	tempDBName := tempDB.Name()

	// Delete the temporary file when this function finishes
	defer os.Remove(tempDBName)

	// Sanity check the uploaded database
	err = SanityCheck(tempDBName)
	if err != nil {
		return numBytes, err
	}

	// Generate sha256 of the uploaded file
	s := sha256.Sum256(buf.Bytes())
	sha := hex.EncodeToString(s[:])

	// Check if the database already exists in the system
	needDefaultBranchCreated := false
	var branches map[string]BranchEntry
	exists, err := CheckDBExists(loggedInUser, loggedInUser, dbFolder, dbName)
	if err != err {
		return numBytes, err
	}
	if exists {
		// Load the existing branchHeads for the database
		branches, err = GetBranches(loggedInUser, dbFolder, dbName)
		if err != nil {
			return numBytes, err
		}

		// If no branch name was given, use the default for the database
		if branchName == "" {
			branchName, err = GetDefaultBranchName(loggedInUser, dbFolder, dbName)
			if err != nil {
				return numBytes, err
			}
		}
	} else {
		// No existing branches, so this will be the first
		branches = make(map[string]BranchEntry)

		// Set the default branch name for the database
		if branchName == "" {
			branchName = "master"
		}
		needDefaultBranchCreated = true
	}

	// Create a dbTree entry for the individual database file
	var e DBTreeEntry
	e.EntryType = DATABASE
	e.Name = dbName
	e.Sha256 = sha
	e.Last_Modified = time.Now()
	// TODO: Check if there's a way to pass the last modified timestamp through a standard file upload control.  If
	// TODO  not, then it might only be possible through db4s, dio cli and similar
	//e.Last_Modified, err = time.Parse(time.RFC3339, modTime)
	//if err != nil {
	//	log.Println(err.Error())
	//	w.WriteHeader(http.StatusInternalServerError)
	//	return
	//}
	e.Size = buf.Len()
	if licenceName == "" || licenceName == "Not specified" {
		// No licence was specified by the client, so check if the database is already in the system and
		// already has one.  If so, we use that.
		if exists {
			headBranch, ok := branches[branchName]
			if !ok {
				return numBytes, errors.New("Error retrieving branch details")
			}
			commits, err := GetCommitList(loggedInUser, dbFolder, dbName)
			if err != nil {
				return numBytes, errors.New("Error retrieving commit list")
			}
			headCommit, ok := commits[headBranch.Commit]
			if !ok {
				return numBytes, fmt.Errorf("Err when looking up commit '%s' in commit list", headBranch.Commit)

			}
			if headCommit.Tree.Entries[0].LicenceSHA != "" {
				// The previous commit for the database had a licence, so we use that for this commit too
				e.LicenceSHA = headCommit.Tree.Entries[0].LicenceSHA
			}
		} else {
			// It's a new database, and the licence hasn't been specified
			e.LicenceSHA, err = GetLicenceSha256FromName(loggedInUser, licenceName)
			if err != nil {
				return numBytes, err
			}

			// If no commit message was given, use a default one and include the info of no licence being specified
			if commitMsg == "" {
				commitMsg = "Initial database upload, licence not specified."
			}
		}
	} else {
		// A licence was specified by the client, so use that
		e.LicenceSHA, err = GetLicenceSha256FromName(loggedInUser, licenceName)
		if err != nil {
			return numBytes, err
		}

		// Generate a reasonable commit message if none was given
		if !exists && commitMsg == "" {
			commitMsg = fmt.Sprintf("Initial database upload, using licence %s.", licenceName)
		}
	}

	// Create a dbTree structure for the database entry
	var t DBTree
	t.Entries = append(t.Entries, e)
	t.ID = CreateDBTreeID(t.Entries)

	// Retrieve the display name and email address for the user
	dn, em, err := GetUserDetails(loggedInUser)
	if err != nil {
		return numBytes, err
	}

	// If either the display name or email address is empty, tell the user we need them first
	if dn == "" || em == "" {
		return numBytes, errors.New("You need to set your full name and email address in Preferences first")
	}

	// Construct a commit structure pointing to the tree
	var c CommitEntry
	c.AuthorName = dn
	c.AuthorEmail = em
	c.Message = commitMsg
	c.Timestamp = time.Now()
	c.Tree = t

	// If the database already exists, use the head commit for the appropriate branch as the parent for our new
	// uploads' commit
	if exists {
		b, ok := branches[branchName]
		if !ok {
			return numBytes, errors.New("Error when looking up branch details")
		}
		c.Parent = b.Commit
	}

	// Create the commit ID for the new upload
	c.ID = CreateCommitID(c)

	// If the database already exists, count the number of commits in the new branch
	commitCount := 1
	if exists {
		commitList, err := GetCommitList(loggedInUser, dbFolder, dbName)
		if err != nil {
			return numBytes, err
		}
		var ok bool
		var c2 CommitEntry
		c2.Parent = c.Parent
		for c2.Parent != "" {
			commitCount++
			c2, ok = commitList[c2.Parent]
			if !ok {
				m := fmt.Sprintf("Error when counting commits in branch '%s' of database '%s%s%s'\n", branchName,
					loggedInUser, dbFolder, dbName)
				log.Print(m)
				return numBytes, errors.New(m)
			}
		}
	}

	// Update the branch with the commit for this new database upload & the updated commit count for the branch
	b := branches[branchName]
	b.Commit = c.ID
	b.CommitCount = commitCount
	branches[branchName] = b
	err = StoreDatabase(loggedInUser, dbFolder, dbName, branches, c, public, buf.Bytes(), sha, "",
		"", needDefaultBranchCreated, branchName, sourceURL)
	if err != nil {
		return numBytes, err
	}

	// If the database already existed, update it's contributor count
	if exists {
		err = UpdateContributorsCount(loggedInUser, dbFolder, dbName)
		if err != nil {
			return numBytes, err
		}
	}

	// Was a user agent part of the request?
	var userAgent string
	ua, ok := r.Header["User-Agent"]
	if ok {
		userAgent = ua[0]
	}

	// Make a record of the upload
	err = LogUpload(loggedInUser, dbFolder, dbName, loggedInUser, r.RemoteAddr, serverSw, userAgent, time.Now(), sha)
	if err != nil {
		return 0, err
	}

	// Invalidate the memcached entry for the database (only really useful if we're updating an existing database)
	err = InvalidateCacheEntry(loggedInUser, loggedInUser, "/", dbName, "") // Empty string indicates "for all versions"
	if err != nil {
		// Something went wrong when invalidating memcached entries for the database
		log.Printf("Error when invalidating memcache entries: %s\n", err.Error())
		return
	}

	// Invalidate any memcached entries for the previous highest version # of the database
	err = InvalidateCacheEntry(loggedInUser, loggedInUser, dbFolder, dbName, c.ID) // And empty string indicates "for all commits"
	if err != nil {
		// Something went wrong when invalidating memcached entries for any previous database
		log.Printf("Error when invalidating memcache entries: %s\n", err.Error())
		return
	}

	// Database successfully uploaded
	return numBytes, nil
}

// Generate a stable SHA256 for a commit.
func CreateCommitID(c CommitEntry) string {
	var b bytes.Buffer
	b.WriteString(fmt.Sprintf("tree %s\n", c.Tree.ID))
	if c.Parent != "" {
		b.WriteString(fmt.Sprintf("parent %s\n", c.Parent))
	}
	b.WriteString(fmt.Sprintf("author %s <%s> %v\n", c.AuthorName, c.AuthorEmail,
		c.Timestamp.Format(time.UnixDate)))
	if c.CommitterEmail != "" {
		b.WriteString(fmt.Sprintf("committer %s <%s> %v\n", c.CommitterName, c.CommitterEmail,
			c.Timestamp.Format(time.UnixDate)))
	}
	b.WriteString("\n" + c.Message)
	b.WriteByte(0)
	s := sha256.Sum256(b.Bytes())
	return hex.EncodeToString(s[:])
}

// Generate the SHA256 for a tree.
// Tree entry structure is:
// * [ entry type ] [ licence sha256] [ file sha256 ] [ file name ] [ last modified (timestamp) ] [ file size (bytes) ]
func CreateDBTreeID(entries []DBTreeEntry) string {
	var b bytes.Buffer
	for _, j := range entries {
		b.WriteString(string(j.EntryType))
		b.WriteByte(0)
		b.WriteString(string(j.LicenceSHA))
		b.WriteByte(0)
		b.WriteString(j.Sha256)
		b.WriteByte(0)
		b.WriteString(j.Name)
		b.WriteByte(0)
		b.WriteString(j.Last_Modified.Format(time.RFC3339))
		b.WriteByte(0)
		b.WriteString(fmt.Sprintf("%d\n", j.Size))
	}
	s := sha256.Sum256(b.Bytes())
	return hex.EncodeToString(s[:])
}

// Look for the next child fork in a fork tree
func nextChild(loggedInUser string, rawListPtr *[]ForkEntry, outputListPtr *[]ForkEntry, forkTrailPtr *[]int, iconDepth int) ([]ForkEntry, []int, bool) {
	// TODO: This approach feels half arsed.  Maybe redo it as a recursive function instead?

	// Resolve the pointers
	rawList := *rawListPtr
	outputList := *outputListPtr
	forkTrail := *forkTrailPtr

	// Grab the last database ID from the fork trail
	parentID := forkTrail[len(forkTrail)-1:][0]

	// Scan unprocessed rows for the first child of parentID
	numResults := len(rawList)
	for j := 1; j < numResults; j++ {
		// Skip already processed entries
		if rawList[j].Processed == false {
			if rawList[j].ForkedFrom == parentID {
				// * Found a fork of the parent *

				// Set the icon list for display in the browser
				for k := 0; k < iconDepth; k++ {
					rawList[j].IconList = append(rawList[j].IconList, SPACE)
				}
				rawList[j].IconList = append(rawList[j].IconList, END)

				// If the database is no longer public, then use placeholder details instead
				if !rawList[j].Public && (rawList[j].Owner != loggedInUser) {
					rawList[j].DBName = "private database"
				}

				// If the database is deleted, use a placeholder indicating that instead
				if rawList[j].Deleted {
					rawList[j].DBName = "deleted database"
				}

				// Add this database to the output list
				outputList = append(outputList, rawList[j])

				// Append this database ID to the fork trail
				forkTrail = append(forkTrail, rawList[j].ID)

				// Mark this database entry as processed
				rawList[j].Processed = true

				// Indicate a child fork was found
				return outputList, forkTrail, true
			}
		}
	}

	// Indicate no child fork was found
	return outputList, forkTrail, false
}

// Generate a random string
func RandomString(length int) string {
	rand.Seed(time.Now().UnixNano())
	const alphaNum = "abcdefghijklmnopqrstuvwxyz0123456789"
	randomString := make([]byte, length)
	for i := range randomString {
		randomString[i] = alphaNum[rand.Intn(len(alphaNum))]
	}

	return string(randomString)
}
<commit_msg>Add util function to return the name of the calling function<commit_after>// Useful utility functions
package common

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"os"
	"runtime"
	"time"
)

// The main function which handles database upload processing for both the webUI and DB4S end points
func AddDatabase(r *http.Request, loggedInUser string, dbOwner string, dbFolder string, dbName string,
	branchName string, public bool, licenceName string, commitMsg string, sourceURL string, newDB io.Reader,
	serverSw string) (numBytes int64, err error) {

	// Write the temporary file locally, so we can try opening it with SQLite to verify it's ok
	var buf bytes.Buffer
	numBytes, err = io.Copy(&buf, newDB)
	if err != nil {
		log.Printf("Error: %v\n", err)
		return numBytes, err
	}
	if numBytes == 0 {
		log.Printf("Database seems to be 0 bytes in length. Username: %s, Database: %s\n", loggedInUser, dbName)
		return numBytes, err
	}
	tempDB, err := ioutil.TempFile("", "dbhub-upload-")
	if err != nil {
		log.Printf("Error creating temporary file. User: '%s', Database: '%s%s%s', Filename: '%s', Error: %v\n",
			loggedInUser, dbOwner, dbFolder, dbName, tempDB.Name(), err)
		return numBytes, err
	}
	_, err = tempDB.Write(buf.Bytes())
	if err != nil {
		log.Printf("Error when writing the uploaded db to a temp file. User: '%s', Database: '%s%s%s' "+
			"Error: %v\n", loggedInUser, dbOwner, dbFolder, dbName, err)
		return numBytes, err
	}
	tempDBName := tempDB.Name()

	// Delete the temporary file when this function finishes
	defer os.Remove(tempDBName)

	// Sanity check the uploaded database
	err = SanityCheck(tempDBName)
	if err != nil {
		return numBytes, err
	}

	// Generate sha256 of the uploaded file
	s := sha256.Sum256(buf.Bytes())
	sha := hex.EncodeToString(s[:])

	// Check if the database already exists in the system
	needDefaultBranchCreated := false
	var branches map[string]BranchEntry
	exists, err := CheckDBExists(loggedInUser, loggedInUser, dbFolder, dbName)
	if err != err {
		return numBytes, err
	}
	if exists {
		// Load the existing branchHeads for the database
		branches, err = GetBranches(loggedInUser, dbFolder, dbName)
		if err != nil {
			return numBytes, err
		}

		// If no branch name was given, use the default for the database
		if branchName == "" {
			branchName, err = GetDefaultBranchName(loggedInUser, dbFolder, dbName)
			if err != nil {
				return numBytes, err
			}
		}
	} else {
		// No existing branches, so this will be the first
		branches = make(map[string]BranchEntry)

		// Set the default branch name for the database
		if branchName == "" {
			branchName = "master"
		}
		needDefaultBranchCreated = true
	}

	// Create a dbTree entry for the individual database file
	var e DBTreeEntry
	e.EntryType = DATABASE
	e.Name = dbName
	e.Sha256 = sha
	e.Last_Modified = time.Now()
	// TODO: Check if there's a way to pass the last modified timestamp through a standard file upload control.  If
	// TODO  not, then it might only be possible through db4s, dio cli and similar
	//e.Last_Modified, err = time.Parse(time.RFC3339, modTime)
	//if err != nil {
	//	log.Println(err.Error())
	//	w.WriteHeader(http.StatusInternalServerError)
	//	return
	//}
	e.Size = buf.Len()
	if licenceName == "" || licenceName == "Not specified" {
		// No licence was specified by the client, so check if the database is already in the system and
		// already has one.  If so, we use that.
		if exists {
			headBranch, ok := branches[branchName]
			if !ok {
				return numBytes, errors.New("Error retrieving branch details")
			}
			commits, err := GetCommitList(loggedInUser, dbFolder, dbName)
			if err != nil {
				return numBytes, errors.New("Error retrieving commit list")
			}
			headCommit, ok := commits[headBranch.Commit]
			if !ok {
				return numBytes, fmt.Errorf("Err when looking up commit '%s' in commit list", headBranch.Commit)

			}
			if headCommit.Tree.Entries[0].LicenceSHA != "" {
				// The previous commit for the database had a licence, so we use that for this commit too
				e.LicenceSHA = headCommit.Tree.Entries[0].LicenceSHA
			}
		} else {
			// It's a new database, and the licence hasn't been specified
			e.LicenceSHA, err = GetLicenceSha256FromName(loggedInUser, licenceName)
			if err != nil {
				return numBytes, err
			}

			// If no commit message was given, use a default one and include the info of no licence being specified
			if commitMsg == "" {
				commitMsg = "Initial database upload, licence not specified."
			}
		}
	} else {
		// A licence was specified by the client, so use that
		e.LicenceSHA, err = GetLicenceSha256FromName(loggedInUser, licenceName)
		if err != nil {
			return numBytes, err
		}

		// Generate a reasonable commit message if none was given
		if !exists && commitMsg == "" {
			commitMsg = fmt.Sprintf("Initial database upload, using licence %s.", licenceName)
		}
	}

	// Create a dbTree structure for the database entry
	var t DBTree
	t.Entries = append(t.Entries, e)
	t.ID = CreateDBTreeID(t.Entries)

	// Retrieve the display name and email address for the user
	dn, em, err := GetUserDetails(loggedInUser)
	if err != nil {
		return numBytes, err
	}

	// If either the display name or email address is empty, tell the user we need them first
	if dn == "" || em == "" {
		return numBytes, errors.New("You need to set your full name and email address in Preferences first")
	}

	// Construct a commit structure pointing to the tree
	var c CommitEntry
	c.AuthorName = dn
	c.AuthorEmail = em
	c.Message = commitMsg
	c.Timestamp = time.Now()
	c.Tree = t

	// If the database already exists, use the head commit for the appropriate branch as the parent for our new
	// uploads' commit
	if exists {
		b, ok := branches[branchName]
		if !ok {
			return numBytes, errors.New("Error when looking up branch details")
		}
		c.Parent = b.Commit
	}

	// Create the commit ID for the new upload
	c.ID = CreateCommitID(c)

	// If the database already exists, count the number of commits in the new branch
	commitCount := 1
	if exists {
		commitList, err := GetCommitList(loggedInUser, dbFolder, dbName)
		if err != nil {
			return numBytes, err
		}
		var ok bool
		var c2 CommitEntry
		c2.Parent = c.Parent
		for c2.Parent != "" {
			commitCount++
			c2, ok = commitList[c2.Parent]
			if !ok {
				m := fmt.Sprintf("Error when counting commits in branch '%s' of database '%s%s%s'\n", branchName,
					loggedInUser, dbFolder, dbName)
				log.Print(m)
				return numBytes, errors.New(m)
			}
		}
	}

	// Update the branch with the commit for this new database upload & the updated commit count for the branch
	b := branches[branchName]
	b.Commit = c.ID
	b.CommitCount = commitCount
	branches[branchName] = b
	err = StoreDatabase(loggedInUser, dbFolder, dbName, branches, c, public, buf.Bytes(), sha, "",
		"", needDefaultBranchCreated, branchName, sourceURL)
	if err != nil {
		return numBytes, err
	}

	// If the database already existed, update it's contributor count
	if exists {
		err = UpdateContributorsCount(loggedInUser, dbFolder, dbName)
		if err != nil {
			return numBytes, err
		}
	}

	// Was a user agent part of the request?
	var userAgent string
	ua, ok := r.Header["User-Agent"]
	if ok {
		userAgent = ua[0]
	}

	// Make a record of the upload
	err = LogUpload(loggedInUser, dbFolder, dbName, loggedInUser, r.RemoteAddr, serverSw, userAgent, time.Now(), sha)
	if err != nil {
		return 0, err
	}

	// Invalidate the memcached entry for the database (only really useful if we're updating an existing database)
	err = InvalidateCacheEntry(loggedInUser, loggedInUser, "/", dbName, "") // Empty string indicates "for all versions"
	if err != nil {
		// Something went wrong when invalidating memcached entries for the database
		log.Printf("Error when invalidating memcache entries: %s\n", err.Error())
		return
	}

	// Invalidate any memcached entries for the previous highest version # of the database
	err = InvalidateCacheEntry(loggedInUser, loggedInUser, dbFolder, dbName, c.ID) // And empty string indicates "for all commits"
	if err != nil {
		// Something went wrong when invalidating memcached entries for any previous database
		log.Printf("Error when invalidating memcache entries: %s\n", err.Error())
		return
	}

	// Database successfully uploaded
	return numBytes, nil
}

// Generate a stable SHA256 for a commit.
func CreateCommitID(c CommitEntry) string {
	var b bytes.Buffer
	b.WriteString(fmt.Sprintf("tree %s\n", c.Tree.ID))
	if c.Parent != "" {
		b.WriteString(fmt.Sprintf("parent %s\n", c.Parent))
	}
	b.WriteString(fmt.Sprintf("author %s <%s> %v\n", c.AuthorName, c.AuthorEmail,
		c.Timestamp.Format(time.UnixDate)))
	if c.CommitterEmail != "" {
		b.WriteString(fmt.Sprintf("committer %s <%s> %v\n", c.CommitterName, c.CommitterEmail,
			c.Timestamp.Format(time.UnixDate)))
	}
	b.WriteString("\n" + c.Message)
	b.WriteByte(0)
	s := sha256.Sum256(b.Bytes())
	return hex.EncodeToString(s[:])
}

// Generate the SHA256 for a tree.
// Tree entry structure is:
// * [ entry type ] [ licence sha256] [ file sha256 ] [ file name ] [ last modified (timestamp) ] [ file size (bytes) ]
func CreateDBTreeID(entries []DBTreeEntry) string {
	var b bytes.Buffer
	for _, j := range entries {
		b.WriteString(string(j.EntryType))
		b.WriteByte(0)
		b.WriteString(string(j.LicenceSHA))
		b.WriteByte(0)
		b.WriteString(j.Sha256)
		b.WriteByte(0)
		b.WriteString(j.Name)
		b.WriteByte(0)
		b.WriteString(j.Last_Modified.Format(time.RFC3339))
		b.WriteByte(0)
		b.WriteString(fmt.Sprintf("%d\n", j.Size))
	}
	s := sha256.Sum256(b.Bytes())
	return hex.EncodeToString(s[:])
}

// Returns the name of the function this was called from
func GetCurrentFunctionName() (FuncName string) {
	stk := make([]uintptr, 1)
	runtime.Callers(2, stk[:])
	FuncName = runtime.FuncForPC(stk[0]).Name() + "()"
	return
}

// Look for the next child fork in a fork tree
func nextChild(loggedInUser string, rawListPtr *[]ForkEntry, outputListPtr *[]ForkEntry, forkTrailPtr *[]int, iconDepth int) ([]ForkEntry, []int, bool) {
	// TODO: This approach feels half arsed.  Maybe redo it as a recursive function instead?

	// Resolve the pointers
	rawList := *rawListPtr
	outputList := *outputListPtr
	forkTrail := *forkTrailPtr

	// Grab the last database ID from the fork trail
	parentID := forkTrail[len(forkTrail)-1:][0]

	// Scan unprocessed rows for the first child of parentID
	numResults := len(rawList)
	for j := 1; j < numResults; j++ {
		// Skip already processed entries
		if rawList[j].Processed == false {
			if rawList[j].ForkedFrom == parentID {
				// * Found a fork of the parent *

				// Set the icon list for display in the browser
				for k := 0; k < iconDepth; k++ {
					rawList[j].IconList = append(rawList[j].IconList, SPACE)
				}
				rawList[j].IconList = append(rawList[j].IconList, END)

				// If the database is no longer public, then use placeholder details instead
				if !rawList[j].Public && (rawList[j].Owner != loggedInUser) {
					rawList[j].DBName = "private database"
				}

				// If the database is deleted, use a placeholder indicating that instead
				if rawList[j].Deleted {
					rawList[j].DBName = "deleted database"
				}

				// Add this database to the output list
				outputList = append(outputList, rawList[j])

				// Append this database ID to the fork trail
				forkTrail = append(forkTrail, rawList[j].ID)

				// Mark this database entry as processed
				rawList[j].Processed = true

				// Indicate a child fork was found
				return outputList, forkTrail, true
			}
		}
	}

	// Indicate no child fork was found
	return outputList, forkTrail, false
}

// Generate a random string
func RandomString(length int) string {
	rand.Seed(time.Now().UnixNano())
	const alphaNum = "abcdefghijklmnopqrstuvwxyz0123456789"
	randomString := make([]byte, length)
	for i := range randomString {
		randomString[i] = alphaNum[rand.Intn(len(alphaNum))]
	}

	return string(randomString)
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Arista Networks, Inc.
// Use of this source code is governed by the Apache License 2.0
// that can be found in the COPYING file.

package test

import (
	"io"
	"os"
	"testing"
)

// CopyFile copies a file
func CopyFile(t *testing.T, srcPath, dstPath string) {
	src, err := os.Open(srcPath)
	if err != nil {
		t.Fatal(err)
	}
	defer src.Close()
	dst, err := os.Create(dstPath)
	if err != nil {
		t.Fatal(err)
	}
	defer dst.Close()
	_, err = io.Copy(dst, src)
	if err != nil {
		t.Fatal(err)
	}
}
<commit_msg>test/fileutil: Add t.Helper() to CopyFile()<commit_after>// Copyright (c) 2015 Arista Networks, Inc.
// Use of this source code is governed by the Apache License 2.0
// that can be found in the COPYING file.

package test

import (
	"io"
	"os"
	"testing"
)

// CopyFile copies a file
func CopyFile(t *testing.T, srcPath, dstPath string) {
	t.Helper()
	src, err := os.Open(srcPath)
	if err != nil {
		t.Fatal(err)
	}
	defer src.Close()
	dst, err := os.Create(dstPath)
	if err != nil {
		t.Fatal(err)
	}
	defer dst.Close()
	_, err = io.Copy(dst, src)
	if err != nil {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package structmapper_test

import (
	"testing"

	"net"

	"github.com/anexia-it/go-structmapper"
	"github.com/stretchr/testify/require"
)

type mapperTestStructSimple struct {
	A string `mapper:"eff,omitempty"`
}

type mapperTestStructNested struct {
	// Even though a tag is set, this should be ignored
	privateTest string                  `mapper:"private"`
	A           string                  `mapper:"a"`
	B           int                     `mapper:"b"`
	C           float64                 `mapper:"c"`
	D           uint64                  `mapper:"dee,omitempty"`
	E           *mapperTestStructSimple `mapper:"e"`
}

type mapperTestStructArraySlice struct {
	A []string                  `mapper:"a"`
	B []*mapperTestStructSimple `mapper:"b,omitempty"`
	C [2]string                 `mapper:"c"`
}

type mapperTestStructTextMarshaler struct {
	IP net.IP
}

type mapperTestStructMap struct {
	A map[int]string  `mapper:"a"`
	B map[int]float32 `mapper:"bee"`
	C map[string]int  `mapper:"z"`
}

type mapperTestStructBool struct {
	A bool
}

type mapperTestStructAnonymousInner struct {
	A string `mapper:"a_inner"`
}

type mapperTestStructAnonymousOuter struct {
	mapperTestStructAnonymousInner
	A string `mapper:"a_outer"`
}

func TestMapper_Roundtrip_Map(t *testing.T) {
	// Initialize Mapper without options
	sm, err := structmapper.NewMapper()
	require.NoError(t, err)
	require.NotNil(t, sm)

	source := &mapperTestStructMap{
		A: map[int]string{
			10:   "a",
			1024: "b",
			30:   "c",
		},
		B: map[int]float32{
			1: 1.1,
			2: 2.2,
		},
		C: map[string]int{
			"a": 1,
			"b": 2,
			"c": 3,
		},
	}

	target := &mapperTestStructMap{}

	// Convert struct to map
	m, err := sm.ToMap(source)

	require.NoError(t, err)
	require.NotNil(t, m)

	// Convert back to struct
	require.NoError(t, sm.ToStruct(m, target))

	// Check if source and target are equal
	require.EqualValues(t, source, target)
}

func TestMapper_Roundtrip_Simple(t *testing.T) {
	// Initialize Mapper without options
	sm, err := structmapper.NewMapper()
	require.NoError(t, err)
	require.NotNil(t, sm)

	source := &mapperTestStructSimple{
		A: "test value",
	}

	target := &mapperTestStructSimple{}

	// Convert struct to map
	m, err := sm.ToMap(source)

	require.NoError(t, err)
	require.NotNil(t, m)

	// Convert back to struct
	require.NoError(t, sm.ToStruct(m, target))

	// Check if source and target are equal
	require.EqualValues(t, source, target)
}

func TestMapper_Roundtrip_Nested(t *testing.T) {
	// Initialize Mapper without options
	sm, err := structmapper.NewMapper()
	require.NoError(t, err)
	require.NotNil(t, sm)

	source := &mapperTestStructNested{
		A: "0",
		B: 1,
		C: 2.1,
		D: 3,
		E: &mapperTestStructSimple{
			A: "4",
		},
	}

	target := &mapperTestStructNested{}

	// Convert struct to map
	m, err := sm.ToMap(source)
	require.NoError(t, err)
	require.NotNil(t, m)

	// Convert back to struct
	require.NoError(t, sm.ToStruct(m, target))

	// Check if source and target are equal
	require.EqualValues(t, source, target)

	// Define second source
	source2 := &mapperTestStructArraySlice{
		A: []string{"0.0", "0.1"},
		B: []*mapperTestStructSimple{
			{
				A: "1.0",
			},
			{
				A: "",
			},
		},
		C: [2]string{"2.0", ""},
	}

	target2 := &mapperTestStructArraySlice{}

	// Convert struct to map
	m, err = sm.ToMap(source2)
	require.NoError(t, err)
	require.NotNil(t, m)

	// Convert back to struct
	require.NoError(t, sm.ToStruct(m, target2))

	// Check if source and target are equal
	require.EqualValues(t, source2, target2)

}

func TestMapper_Roundtrip_ArraySlice(t *testing.T) {
	// Initialize Mapper without options
	sm, err := structmapper.NewMapper()
	require.NoError(t, err)
	require.NotNil(t, sm)

	source := &mapperTestStructArraySlice{
		A: []string{"test value", "test value 1"},
		B: []*mapperTestStructSimple{
			{
				A: "test0",
			},
			{
				A: "test1",
			},
		},
		C: [2]string{"a", "b"},
	}

	target := &mapperTestStructArraySlice{}

	// Convert struct to map
	m, err := sm.ToMap(source)

	require.NoError(t, err)
	require.NotNil(t, m)

	// Convert back to struct
	require.NoError(t, sm.ToStruct(m, target))

	// Check if source and target are equal
	require.EqualValues(t, source, target)
}

func TestMapper_Roundtrip_Bool(t *testing.T) {
	// Initialize Mapper without options
	sm, err := structmapper.NewMapper()
	require.NoError(t, err)
	require.NotNil(t, sm)

	source := &mapperTestStructBool{
		A: true,
	}

	target := &mapperTestStructBool{}

	// Convert struct to map
	m, err := sm.ToMap(source)

	require.NoError(t, err)
	require.NotNil(t, m)

	// Convert back to struct
	require.NoError(t, sm.ToStruct(m, target))

	// Check if source and target are equal
	require.EqualValues(t, source, target)

	source = &mapperTestStructBool{
		A: false,
	}

	target = &mapperTestStructBool{}

	// Convert struct to map
	m, err = sm.ToMap(source)

	require.NoError(t, err)
	require.NotNil(t, m)

	// Convert back to struct
	require.NoError(t, sm.ToStruct(m, target))

	// Check if source and target are equal
	require.EqualValues(t, source, target)
}

func TestMapper_Roundtrip_Anonymous(t *testing.T) {
	// Initialize Mapper without options
	sm, err := structmapper.NewMapper()
	require.NoError(t, err)
	require.NotNil(t, sm)

	source := &mapperTestStructAnonymousOuter{
		mapperTestStructAnonymousInner: mapperTestStructAnonymousInner{
			A: "inner",
		},
		A: "outer",
	}

	target := &mapperTestStructAnonymousOuter{}

	m, err := sm.ToMap(source)
	require.NoError(t, err)
	require.NotNil(t, m)

	require.NoError(t, sm.ToStruct(m, target))

	require.EqualValues(t, source, target)
}
<commit_msg>Add additional testcases<commit_after>package structmapper_test

import (
	"testing"

	"net"

	"github.com/anexia-it/go-structmapper"
	"github.com/stretchr/testify/require"
)

type mapperTestStructSimple struct {
	A string `mapper:"eff,omitempty"`
}

type mapperTestStructNested struct {
	// Even though a tag is set, this should be ignored
	privateTest string                  `mapper:"private"`
	A           string                  `mapper:"a"`
	B           int                     `mapper:"b"`
	C           float64                 `mapper:"c"`
	D           uint64                  `mapper:"dee,omitempty"`
	E           *mapperTestStructSimple `mapper:"e"`
}

type mapperTestStructArraySlice struct {
	A []string                  `mapper:"a"`
	B []*mapperTestStructSimple `mapper:"b,omitempty"`
	C [2]string                 `mapper:"c"`
}

type mapperTestStructTextMarshaler struct {
	IP net.IP
}

type mapperTestStructMap struct {
	A map[int]string  `mapper:"a"`
	B map[int]float32 `mapper:"bee"`
	C map[string]int  `mapper:"z"`
}

type mapperTestStructBool struct {
	A bool
}

type mapperTestStructAnonymousInner struct {
	A string `mapper:"a_inner"`
}

type mapperTestStructAnonymousOuter struct {
	mapperTestStructAnonymousInner
	A string `mapper:"a_outer"`
}

type mapperTestStructMapStringString struct {
	A string
	B map[string]string
}

type mapperTestStructNestedStructSlice struct {
	A string
	B []mapperTestStructSimple
}

type mapperTestStructNestedNestedStructSlice struct {
	A string
	B mapperTestStructNestedStructSlice
}

func TestMapper_Roundtrip_Map(t *testing.T) {
	// Initialize Mapper without options
	sm, err := structmapper.NewMapper()
	require.NoError(t, err)
	require.NotNil(t, sm)

	source := &mapperTestStructMap{
		A: map[int]string{
			10:   "a",
			1024: "b",
			30:   "c",
		},
		B: map[int]float32{
			1: 1.1,
			2: 2.2,
		},
		C: map[string]int{
			"a": 1,
			"b": 2,
			"c": 3,
		},
	}

	target := &mapperTestStructMap{}

	// Convert struct to map
	m, err := sm.ToMap(source)

	require.NoError(t, err)
	require.NotNil(t, m)

	// Convert back to struct
	require.NoError(t, sm.ToStruct(m, target))

	// Check if source and target are equal
	require.EqualValues(t, source, target)
}

func TestMapper_Roundtrip_Simple(t *testing.T) {
	// Initialize Mapper without options
	sm, err := structmapper.NewMapper()
	require.NoError(t, err)
	require.NotNil(t, sm)

	source := &mapperTestStructSimple{
		A: "test value",
	}

	target := &mapperTestStructSimple{}

	// Convert struct to map
	m, err := sm.ToMap(source)

	require.NoError(t, err)
	require.NotNil(t, m)

	// Convert back to struct
	require.NoError(t, sm.ToStruct(m, target))

	// Check if source and target are equal
	require.EqualValues(t, source, target)
}

func TestMapper_Roundtrip_Nested(t *testing.T) {
	// Initialize Mapper without options
	sm, err := structmapper.NewMapper()
	require.NoError(t, err)
	require.NotNil(t, sm)

	source := &mapperTestStructNested{
		A: "0",
		B: 1,
		C: 2.1,
		D: 3,
		E: &mapperTestStructSimple{
			A: "4",
		},
	}

	target := &mapperTestStructNested{}

	// Convert struct to map
	m, err := sm.ToMap(source)
	require.NoError(t, err)
	require.NotNil(t, m)

	// Convert back to struct
	require.NoError(t, sm.ToStruct(m, target))

	// Check if source and target are equal
	require.EqualValues(t, source, target)

	// Define second source
	source2 := &mapperTestStructArraySlice{
		A: []string{"0.0", "0.1"},
		B: []*mapperTestStructSimple{
			{
				A: "1.0",
			},
			{
				A: "",
			},
		},
		C: [2]string{"2.0", ""},
	}

	target2 := &mapperTestStructArraySlice{}

	// Convert struct to map
	m, err = sm.ToMap(source2)
	require.NoError(t, err)
	require.NotNil(t, m)

	// Convert back to struct
	require.NoError(t, sm.ToStruct(m, target2))

	// Check if source and target are equal
	require.EqualValues(t, source2, target2)

}

func TestMapper_Roundtrip_ArraySlice(t *testing.T) {
	// Initialize Mapper without options
	sm, err := structmapper.NewMapper()
	require.NoError(t, err)
	require.NotNil(t, sm)

	source := &mapperTestStructArraySlice{
		A: []string{"test value", "test value 1"},
		B: []*mapperTestStructSimple{
			{
				A: "test0",
			},
			{
				A: "test1",
			},
		},
		C: [2]string{"a", "b"},
	}

	target := &mapperTestStructArraySlice{}

	// Convert struct to map
	m, err := sm.ToMap(source)

	require.NoError(t, err)
	require.NotNil(t, m)

	// Convert back to struct
	require.NoError(t, sm.ToStruct(m, target))

	// Check if source and target are equal
	require.EqualValues(t, source, target)
}

func TestMapper_Roundtrip_Bool(t *testing.T) {
	// Initialize Mapper without options
	sm, err := structmapper.NewMapper()
	require.NoError(t, err)
	require.NotNil(t, sm)

	source := &mapperTestStructBool{
		A: true,
	}

	target := &mapperTestStructBool{}

	// Convert struct to map
	m, err := sm.ToMap(source)

	require.NoError(t, err)
	require.NotNil(t, m)

	// Convert back to struct
	require.NoError(t, sm.ToStruct(m, target))

	// Check if source and target are equal
	require.EqualValues(t, source, target)

	source = &mapperTestStructBool{
		A: false,
	}

	target = &mapperTestStructBool{}

	// Convert struct to map
	m, err = sm.ToMap(source)

	require.NoError(t, err)
	require.NotNil(t, m)

	// Convert back to struct
	require.NoError(t, sm.ToStruct(m, target))

	// Check if source and target are equal
	require.EqualValues(t, source, target)
}

func TestMapper_Roundtrip_Anonymous(t *testing.T) {
	// Initialize Mapper without options
	sm, err := structmapper.NewMapper()
	require.NoError(t, err)
	require.NotNil(t, sm)

	source := &mapperTestStructAnonymousOuter{
		mapperTestStructAnonymousInner: mapperTestStructAnonymousInner{
			A: "inner",
		},
		A: "outer",
	}

	target := &mapperTestStructAnonymousOuter{}

	m, err := sm.ToMap(source)
	require.NoError(t, err)
	require.NotNil(t, m)

	require.NoError(t, sm.ToStruct(m, target))

	require.EqualValues(t, source, target)
}

func TestMapper_Roundtrip_MapStringString(t *testing.T) {
	// Initialize Mapper without options
	sm, err := structmapper.NewMapper()
	require.NoError(t, err)
	require.NotNil(t, sm)

	source := &mapperTestStructMapStringString{
		A: "test0",
		B: map[string]string{
			"b0": "1",
			"b1": "2",
		},
	}

	target := &mapperTestStructMapStringString{}

	m, err := sm.ToMap(source)
	require.NoError(t, err)
	require.NotNil(t, m)

	require.NoError(t, sm.ToStruct(m, target))

	require.EqualValues(t, source, target)
}

func TestMapper_Roundtrip_NestedStructSlice(t *testing.T) {
	// Initialize Mapper without options
	sm, err := structmapper.NewMapper()
	require.NoError(t, err)
	require.NotNil(t, sm)

	source := &mapperTestStructNestedStructSlice{
		A: "test0",
		B: []mapperTestStructSimple{
			{
				A: "test1",
			},
			{
				A: "test2",
			},
		},
	}

	target := &mapperTestStructNestedStructSlice{}

	m, err := sm.ToMap(source)
	require.NoError(t, err)
	require.NotNil(t, m)

	require.NoError(t, sm.ToStruct(m, target))

	require.EqualValues(t, source, target)
}

// mapperTestStructNestedNestedStructSlice

func TestMapper_Roundtrip_NestedNestedStructSlice(t *testing.T) {
	// Initialize Mapper without options
	sm, err := structmapper.NewMapper()
	require.NoError(t, err)
	require.NotNil(t, sm)

	source := &mapperTestStructNestedNestedStructSlice{
		A: "test0",
		B: mapperTestStructNestedStructSlice{
			A: "test1",
			B: []mapperTestStructSimple{
				{
					A: "test1",
				},
				{
					A: "test2",
				},
			},
		},
	}

	target := &mapperTestStructNestedNestedStructSlice{}

	m, err := sm.ToMap(source)
	require.NoError(t, err)
	require.NotNil(t, m)

	require.NoError(t, sm.ToStruct(m, target))

	require.EqualValues(t, source, target)
}
<|endoftext|>


<commit_before>// Copyright 2012 Arne Roomann-Kurrik
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package json

import (
	"reflect"
	"fmt"
	"testing"
)

type TestCase struct {
	Raw    string
	Result interface{}
}

var errors = map[string]TestCase{
	"HTML": TestCase{
		Raw: "<!DOCTYPE html>Foo",
		Result: "Unrecognized type in ' --><<-- !DOCTYPE '",
	},
	"Blank": TestCase{
		Raw: "",
		Result: "Unrecognized type in ' --><-- '",
	},
	"Empty": TestCase{
		Raw: "    ",
		Result: "Unrecognized type in '    --><-- '",
	},
}

var cases = map[string]TestCase{
	"Number": TestCase{
		Raw: "1234",
		Result: int64(1234),
	},
	"Number - negative": TestCase{
		Raw: "-1234",
		Result: int64(-1234),
	},
	"Number - float": TestCase{
		Raw: "1234.5678",
		Result: float64(1234.5678),
	},
	"Number - negative float": TestCase{
		Raw: "-1234.5678",
		Result: float64(-1234.5678),
	},
	"String": TestCase{
		Raw: "\"foobar\"",
		Result: "foobar",
	},
	"String with encoded UTF-8": TestCase{
		Raw: "\"\\u6211\\u7231\\u4f60\"",
		Result: "我爱你",
	},
	"String with unencoded UTF-8": TestCase{
		Raw: "\"我爱你\"",
		Result: "我爱你",
	},
	"String with big-U encoded multibyte UTF-8": TestCase{
		Raw: "\"\\U0001D11E\"",
		Result: "𝄞",
	},
	"String with octal encoded multibyte UTF-8": TestCase{
		Raw: "\"\\360\\235\\204\\236\"",
		Result: "𝄞",
	},
	"String with hex encoded multibyte UTF-8": TestCase{
		Raw: "\"\\xF0\\x9D\\x84\\x9E\"",
		Result: "𝄞",
	},
	"String with encoded UTF-8 and backslash": TestCase{
		Raw: "\"10\\\\10 ~ \\u2764\"",
		Result: "10\\10 ~ ❤",
	},
	"String with backslash": TestCase{
		Raw: "\"10\\\\10\"",
		Result: "10\\10",
	},
	"String with backslash and tab": TestCase{
		Raw: "\"10\\\\\t10\"",
		Result: "10\\	10",
	},
	"String with backslash and backspace": TestCase{
		Raw: "\"10\\\\\b10\"",
		Result: "10\\\b10",
	},
	"String with escaped forward slash": TestCase{
		Raw: "\"\\\\\\/\"",
		Result: "\\/",
	},
	"String with just backslash": TestCase{
		Raw: "\"\\\\\"",
		Result: "\\",
	},
	"Object": TestCase{
		Raw: "{\"foo\":\"bar\"}",
		Result: map[string]interface{}{
			"foo": "bar",
		},
	},
	"Object with spaces": TestCase{
		Raw: "{ \"foo\" : \"bar\" }",
		Result: map[string]interface{}{
			"foo": "bar",
		},
	},
	"Object with UTF-8 value": TestCase{
		Raw: "{ \"foo\" : \"\\u6211\" }",
		Result: map[string]interface{}{
			"foo": "我",
		},
	},
	"Object with tabs": TestCase{
		Raw: "{	\"foo\"	:	\"bar\"	}",
		Result: map[string]interface{}{
			"foo": "bar",
		},
	},
	"Object with empty nested object": TestCase{
		Raw: "{ \"foo\": {}}",
		Result: map[string]interface{}{
			"foo": map[string]interface{}{},
		},
	},
	"Object with empty nested array": TestCase{
		Raw: "{\"foo\": []}",
		Result: map[string]interface{}{
			"foo": []interface{}{},
		},
	},
	"Array": TestCase{
		Raw: "[1234,\"foobar\"]",
		Result: []interface{}{
			int64(1234),
			"foobar",
		},
	},
	"Array with spaces": TestCase{
		Raw: "[ 1234 , \"foobar\" ]",
		Result: []interface{}{
			int64(1234),
			"foobar",
		},
	},
	"Array with tabs": TestCase{
		Raw: "[	1234	,	\"foobar\"	]",
		Result: []interface{}{
			int64(1234),
			"foobar",
		},
	},
	"Array with multiple tabs": TestCase{
		Raw: "[				1234,\"foobar\"]",
		Result: []interface{}{
			int64(1234),
			"foobar",
		},
	},
	"Array with no contents": TestCase{
		Raw: "[]",
		Result: []interface{}{},
	},
	"Array with empty object": TestCase{
		Raw: "[{}]",
		Result: []interface{}{
			map[string]interface{}{},
		},
	},
}

func TestCases(t *testing.T) {
	var (
		err    error
		decode interface{}
	)
	for desc, testcase := range cases {
		if err = Unmarshal([]byte(testcase.Raw), &decode); err != nil {
			t.Fatalf("Error decoding '%v': %v", desc, err)
		}
		if !reflect.DeepEqual(decode, testcase.Result) {
			t.Logf("%v\n", reflect.TypeOf(decode))
			t.Logf("%v\n", reflect.TypeOf(testcase.Result))
			if reflect.TypeOf(decode) == reflect.TypeOf("") {
				t.Logf("Decode: %v\n", []byte(decode.(string)))
				t.Logf("Expected: %v\n",
					[]byte(testcase.Result.(string)))
			}
			t.Fatalf("Problem decoding '%v' Expected: %v, Got %v",
				desc, testcase.Result, decode)
		}
	}
}

func TestErrors(t *testing.T) {
	var (
		err error
		str string
		res string
		decode interface{}
	)
	for desc, tcase := range errors {
		if err = Unmarshal([]byte(tcase.Raw), &decode); err == nil {
			t.Fatalf("Expected error for '%v': %v", desc, tcase.Raw)
		}
		str = fmt.Sprintf("%v", err)
		res = tcase.Result.(string)
		if str != res {
			t.Fatalf("Invalid error '%v' expected '%v'", str, res)
		}
	}
}
<commit_msg>Add a test to make sure invalid UTF-8 is caught.<commit_after>// Copyright 2012 Arne Roomann-Kurrik
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package json

import (
	"fmt"
	"reflect"
	"testing"
)

type TestCase struct {
	Raw    string
	Result interface{}
}

var errors = map[string]TestCase{
	"HTML": TestCase{
		Raw:    "<!DOCTYPE html>Foo",
		Result: "Unrecognized type in ' --><<-- !DOCTYPE '",
	},
	"Blank": TestCase{
		Raw:    "",
		Result: "Unrecognized type in ' --><-- '",
	},
	"Empty": TestCase{
		Raw:    "    ",
		Result: "Unrecognized type in '    --><-- '",
	},
}

var cases = map[string]TestCase{
	"Number": TestCase{
		Raw:    "1234",
		Result: int64(1234),
	},
	"Number - negative": TestCase{
		Raw:    "-1234",
		Result: int64(-1234),
	},
	"Number - float": TestCase{
		Raw:    "1234.5678",
		Result: float64(1234.5678),
	},
	"Number - negative float": TestCase{
		Raw:    "-1234.5678",
		Result: float64(-1234.5678),
	},
	"String": TestCase{
		Raw:    "\"foobar\"",
		Result: "foobar",
	},
	"String with encoded UTF-8": TestCase{
		Raw:    "\"\\u6211\\u7231\\u4f60\"",
		Result: "我爱你",
	},
	"String with unencoded UTF-8": TestCase{
		Raw:    "\"我爱你\"",
		Result: "我爱你",
	},
	"String with big-U encoded multibyte UTF-8": TestCase{
		Raw:    "\"\\U0001D11E\"",
		Result: "𝄞",
	},
	"String with octal encoded multibyte UTF-8": TestCase{
		Raw:    "\"\\360\\235\\204\\236\"",
		Result: "𝄞",
	},
	"String with hex encoded multibyte UTF-8": TestCase{
		Raw:    "\"\\xF0\\x9D\\x84\\x9E\"",
		Result: "𝄞",
	},
	"String with encoded UTF-8 and backslash": TestCase{
		Raw:    "\"10\\\\10 ~ \\u2764\"",
		Result: "10\\10 ~ ❤",
	},
	"Invalid string with small-U encoded multibyte UTF-8": TestCase{
		Raw:    "\"\\uD834\\uDD1E\"",
		Result: "��",
		// This is pretty dependent on implementation
		// but I'd like to get a heads up if it changes.
	},
	"String with backslash": TestCase{
		Raw:    "\"10\\\\10\"",
		Result: "10\\10",
	},
	"String with backslash and tab": TestCase{
		Raw: "\"10\\\\\t10\"",
		Result: "10\\	10",
	},
	"String with backslash and backspace": TestCase{
		Raw:    "\"10\\\\\b10\"",
		Result: "10\\\b10",
	},
	"String with escaped forward slash": TestCase{
		Raw:    "\"\\\\\\/\"",
		Result: "\\/",
	},
	"String with just backslash": TestCase{
		Raw:    "\"\\\\\"",
		Result: "\\",
	},
	"Object": TestCase{
		Raw: "{\"foo\":\"bar\"}",
		Result: map[string]interface{}{
			"foo": "bar",
		},
	},
	"Object with spaces": TestCase{
		Raw: "{ \"foo\" : \"bar\" }",
		Result: map[string]interface{}{
			"foo": "bar",
		},
	},
	"Object with UTF-8 value": TestCase{
		Raw: "{ \"foo\" : \"\\u6211\" }",
		Result: map[string]interface{}{
			"foo": "我",
		},
	},
	"Object with tabs": TestCase{
		Raw: "{	\"foo\"	:	\"bar\"	}",
		Result: map[string]interface{}{
			"foo": "bar",
		},
	},
	"Object with empty nested object": TestCase{
		Raw: "{ \"foo\": {}}",
		Result: map[string]interface{}{
			"foo": map[string]interface{}{},
		},
	},
	"Object with empty nested array": TestCase{
		Raw: "{\"foo\": []}",
		Result: map[string]interface{}{
			"foo": []interface{}{},
		},
	},
	"Array": TestCase{
		Raw: "[1234,\"foobar\"]",
		Result: []interface{}{
			int64(1234),
			"foobar",
		},
	},
	"Array with spaces": TestCase{
		Raw: "[ 1234 , \"foobar\" ]",
		Result: []interface{}{
			int64(1234),
			"foobar",
		},
	},
	"Array with tabs": TestCase{
		Raw: "[	1234	,	\"foobar\"	]",
		Result: []interface{}{
			int64(1234),
			"foobar",
		},
	},
	"Array with multiple tabs": TestCase{
		Raw: "[				1234,\"foobar\"]",
		Result: []interface{}{
			int64(1234),
			"foobar",
		},
	},
	"Array with no contents": TestCase{
		Raw:    "[]",
		Result: []interface{}{},
	},
	"Array with empty object": TestCase{
		Raw: "[{}]",
		Result: []interface{}{
			map[string]interface{}{},
		},
	},
}

func TestCases(t *testing.T) {
	var (
		err    error
		decode interface{}
	)
	for desc, testcase := range cases {
		if err = Unmarshal([]byte(testcase.Raw), &decode); err != nil {
			t.Fatalf("Error decoding '%v': %v", desc, err)
		}
		if !reflect.DeepEqual(decode, testcase.Result) {
			t.Logf("%v\n", reflect.TypeOf(decode))
			t.Logf("%v\n", reflect.TypeOf(testcase.Result))
			if reflect.TypeOf(decode) == reflect.TypeOf("") {
				t.Logf("Decode: %v\n", []byte(decode.(string)))
				t.Logf("Expected: %v\n",
					[]byte(testcase.Result.(string)))
			}
			t.Fatalf("Problem decoding '%v' Expected: %v, Got %v",
				desc, testcase.Result, decode)
		}
	}
}

func TestErrors(t *testing.T) {
	var (
		err    error
		str    string
		res    string
		decode interface{}
	)
	for desc, tcase := range errors {
		if err = Unmarshal([]byte(tcase.Raw), &decode); err == nil {
			t.Fatalf("Expected error for '%v': %v", desc, tcase.Raw)
		}
		str = fmt.Sprintf("%v", err)
		res = tcase.Result.(string)
		if str != res {
			t.Fatalf("Invalid error '%v' expected '%v'", str, res)
		}
	}
}
<|endoftext|>


<commit_before>package device

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
)

const (
	PatternReg        = ":regid"
	PatternRes        = ":resname"
	PatternUuid       = ":uuid"
	PatternFType      = ":type"
	PatternFPath      = ":path"
	PatternFOp        = ":op"
	PatternFValue     = ":value"
	FTypeDevice       = "device"
	FTypeDevices      = "devices"
	FTypeResource     = "resource"
	FTypeResources    = "resources"
	GetParamPage      = "page"
	GetParamPerPage   = "per_page"
	CollectionType    = "DeviceCatalog"
	CurrentApiVersion = "0.2.1"
)

type Collection struct {
	Context   string                 `json:"@context,omitempty"`
	Id        string                 `json:"id"`
	Type      string                 `json:"type"`
	Devices   map[string]EmptyDevice `json:"devices"`
	Resources []Resource             `json:"resources"`
	Page      int                    `json:"page"`
	PerPage   int                    `json:"per_page"`
	Total     int                    `json:"total"`
}

// Device object with empty resources
type EmptyDevice struct {
	*Device
	Resources []Resource `json:"resources,omitempty"`
}

// Device object with paginated resources
type PaginatedDevice struct {
	*Device
	Resources []Resource `json:"resources"`
	Page      int        `json:"page"`
	PerPage   int        `json:"per_page"`
	Total     int        `json:"total"`
}

// Read-only catalog api
type ReadableCatalogAPI struct {
	catalogStorage CatalogStorage
	contextUrl     string
}

// Writable catalog api
type WritableCatalogAPI struct {
	*ReadableCatalogAPI
}

func NewReadableCatalogAPI(storage CatalogStorage, contextUrl string) *ReadableCatalogAPI {
	return &ReadableCatalogAPI{
		catalogStorage: storage,
		contextUrl:     contextUrl,
	}
}

func NewWritableCatalogAPI(storage CatalogStorage, contextUrl string) *WritableCatalogAPI {
	return &WritableCatalogAPI{
		&ReadableCatalogAPI{
			catalogStorage: storage,
			contextUrl:     contextUrl,
		}}
}

func (self *Device) ldify() Device {
	rc := self.copy()
	for i, res := range rc.Resources {
		rc.Resources[i] = res.ldify()
	}
	rc.Id = fmt.Sprintf("%v/%v", CatalogBaseUrl, self.Id)
	return rc
}

func (self *Resource) ldify() Resource {
	resc := self.copy()
	resc.Id = fmt.Sprintf("%v/%v", CatalogBaseUrl, self.Id)
	resc.Device = fmt.Sprintf("%v/%v", CatalogBaseUrl, self.Device)
	return resc
}

func (self *Device) unLdify() Device {
	rc := self.copy()
	for i, res := range rc.Resources {
		rc.Resources[i] = res.unLdify()
	}
	rc.Id = strings.TrimPrefix(self.Id, CatalogBaseUrl+"/")
	return rc
}

func (self *Resource) unLdify() Resource {
	resc := self.copy()
	resc.Id = strings.TrimPrefix(self.Id, CatalogBaseUrl+"/")
	resc.Device = strings.TrimPrefix(self.Device, CatalogBaseUrl+"/")
	return resc
}

func (self ReadableCatalogAPI) collectionFromDevices(devices []Device, page, perPage, total int) *Collection {
	respDevices := make(map[string]EmptyDevice)
	respResources := make([]Resource, 0, self.catalogStorage.getResourcesCount())

	for _, d := range devices {
		dld := d.ldify()
		for _, res := range dld.Resources {
			respResources = append(respResources, res)
		}

		respDevices[d.Id] = EmptyDevice{
			&dld,
			nil,
		}
	}

	return &Collection{
		Context:   self.contextUrl,
		Id:        CatalogBaseUrl,
		Type:      CollectionType,
		Devices:   respDevices,
		Resources: respResources,
		Page:      page,
		PerPage:   perPage,
		Total:     total,
	}
}

func (self ReadableCatalogAPI) paginatedDeviceFromDevice(d Device, page, perPage int) *PaginatedDevice {
	// Never return more than the defined maximum
	if perPage > MaxPerPage || perPage == 0 {
		perPage = MaxPerPage
	}

	pd := &PaginatedDevice{
		&d,
		make([]Resource, 0, len(d.Resources)),
		page,
		perPage,
		len(d.Resources),
	}

	// if 1, not specified or negative - return the first page
	if page < 2 {
		// first page
		if perPage > pd.Total {
			pd.Resources = d.Resources
		} else {
			pd.Resources = d.Resources[:perPage]
		}
	} else if page == int(pd.Total/perPage)+1 {
		// last page
		pd.Resources = d.Resources[perPage*(page-1):]

	} else if page <= pd.Total/perPage && page*perPage <= pd.Total {
		// slice
		r := page * perPage
		l := r - perPage
		pd.Resources = d.Resources[l:r]
	}
	return pd
}

func (self ReadableCatalogAPI) List(w http.ResponseWriter, req *http.Request) {
	req.ParseForm()
	page, _ := strconv.Atoi(req.Form.Get(GetParamPage))
	perPage, _ := strconv.Atoi(req.Form.Get(GetParamPerPage))

	// use defaults if not specified
	if page == 0 {
		page = 1
	}
	if perPage == 0 {
		perPage = MaxPerPage
	}

	devices, total, _ := self.catalogStorage.getMany(page, perPage)
	coll := self.collectionFromDevices(devices, page, perPage, total)

	b, _ := json.Marshal(coll)
	w.Header().Set("Content-Type", "application/ld+json;version="+CurrentApiVersion)
	w.Write(b)

	return
}

func (self ReadableCatalogAPI) Filter(w http.ResponseWriter, req *http.Request) {
	ftype := req.URL.Query().Get(PatternFType)
	fpath := req.URL.Query().Get(PatternFPath)
	fop := req.URL.Query().Get(PatternFOp)
	fvalue := req.URL.Query().Get(PatternFValue)

	var data interface{}
	var err error
	matched := false

	switch ftype {
	case FTypeDevice:
		data, err = self.catalogStorage.pathFilterDevice(fpath, fop, fvalue)
		if data.(Device).Id != "" {
			d := data.(Device)
			data = d.ldify()
			matched = true
		}

	case FTypeDevices:
		data, err = self.catalogStorage.pathFilterDevices(fpath, fop, fvalue)
		if len(data.([]Device)) > 0 {
			data = self.collectionFromDevices(data.([]Device), 0, 0, 0) //FIXME
			matched = true
		}

	case FTypeResource:
		data, err = self.catalogStorage.pathFilterResource(fpath, fop, fvalue)
		if data.(Resource).Id != "" {
			res := data.(Resource)
			data = res.ldify()
			matched = true
		}

	case FTypeResources:
		data, err = self.catalogStorage.pathFilterResources(fpath, fop, fvalue)
		if len(data.([]Resource)) > 0 {
			devs := self.catalogStorage.devicesFromResources(data.([]Resource))
			data = self.collectionFromDevices(devs, 0, 0, 0) //FIXME
			matched = true
		}
	}

	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "Error processing the request: %s\n", err.Error())
	}

	if matched == false {
		w.WriteHeader(http.StatusNotFound)
		fmt.Fprintf(w, "Not found\n")
		return
	}

	b, _ := json.Marshal(data)
	w.Header().Set("Content-Type", "application/ld+json;version="+CurrentApiVersion)
	w.Write(b)
}

func (self ReadableCatalogAPI) Get(w http.ResponseWriter, req *http.Request) {
	req.ParseForm()
	page, _ := strconv.Atoi(req.Form.Get(GetParamPage))
	perPage, _ := strconv.Atoi(req.Form.Get(GetParamPerPage))
	id := fmt.Sprintf("%v/%v", req.URL.Query().Get(PatternUuid), req.URL.Query().Get(PatternReg))

	d, err := self.catalogStorage.get(id)
	if err != nil || d.Id == "" {
		w.WriteHeader(http.StatusNotFound)
		fmt.Fprintf(w, "Registration not found\n")
		return
	}

	// use defaults if not specified
	if page == 0 {
		page = 1
	}
	if perPage == 0 {
		perPage = MaxPerPage
	}

	pd := self.paginatedDeviceFromDevice(d, page, perPage)
	b, _ := json.Marshal(pd)

	w.Header().Set("Content-Type", "application/ld+json;version="+CurrentApiVersion)
	w.Write(b)
	return
}

func (self ReadableCatalogAPI) GetResource(w http.ResponseWriter, req *http.Request) {
	devid := fmt.Sprintf("%v/%v", req.URL.Query().Get(PatternUuid), req.URL.Query().Get(PatternReg))
	resid := fmt.Sprintf("%v/%v", devid, req.URL.Query().Get(PatternRes))

	// check if device devid exists
	_, err := self.catalogStorage.get(devid)
	if err != nil {
		w.WriteHeader(http.StatusNotFound)
		fmt.Fprintf(w, "Device not found\n")
		return
	}

	// check if it has a resource resid
	res, err := self.catalogStorage.getResourceById(resid)
	if err != nil {
		w.WriteHeader(http.StatusNotFound)
		fmt.Fprintf(w, "Resource not found\n")
		return
	}

	b, _ := json.Marshal(res.ldify())
	w.Header().Set("Content-Type", "application/ld+json;version="+CurrentApiVersion)
	w.Write(b)
	return
}

func (self WritableCatalogAPI) Add(w http.ResponseWriter, req *http.Request) {
	body, err := ioutil.ReadAll(req.Body)
	req.Body.Close()

	var d Device
	err = json.Unmarshal(body, &d)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "Error processing the request: %s\n", err.Error())
		return
	}

	da, err := self.catalogStorage.add(d)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintf(w, "Error creating the registration: %s\n", err.Error())
		return
	}

	b, _ := json.Marshal(da.ldify())
	w.Header().Set("Content-Type", "application/ld+json;version="+CurrentApiVersion)
	w.WriteHeader(http.StatusCreated)
	w.Write(b)
	return
}

func (self WritableCatalogAPI) Update(w http.ResponseWriter, req *http.Request) {
	id := fmt.Sprintf("%v/%v", req.URL.Query().Get(PatternUuid), req.URL.Query().Get(PatternReg))

	body, err := ioutil.ReadAll(req.Body)
	req.Body.Close()

	var d Device
	err = json.Unmarshal(body, &d)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "Error processing the request:: %s\n", err.Error())
		return
	}

	du, err := self.catalogStorage.update(id, d)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintf(w, "Error updating the device: %s\n", err.Error())
		return
	}

	if du.Id == "" {
		w.WriteHeader(http.StatusNotFound)
		fmt.Fprintf(w, "Not found\n")
		return
	}

	b, _ := json.Marshal(du.ldify())
	w.Header().Set("Content-Type", "application/ld+json;version="+CurrentApiVersion)
	w.WriteHeader(http.StatusOK)
	w.Write(b)

	return
}

func (self WritableCatalogAPI) Delete(w http.ResponseWriter, req *http.Request) {
	id := fmt.Sprintf("%v/%v", req.URL.Query().Get(PatternUuid), req.URL.Query().Get(PatternReg))

	dd, err := self.catalogStorage.delete(id)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintf(w, "Error deleting the device: %s\n", err.Error())
		return
	}

	if dd.Id == "" {
		w.WriteHeader(http.StatusNotFound)
		fmt.Fprintf(w, "Not found\n")
		return
	}

	b, _ := json.Marshal(dd.ldify())
	w.Header().Set("Content-Type", "application/ld+json;version="+CurrentApiVersion)
	w.WriteHeader(http.StatusOK)
	w.Write(b)
	return
}
<commit_msg>Device Catalog: fixed wrong ids of resources in /dc/devid<commit_after>package device

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
)

const (
	PatternReg        = ":regid"
	PatternRes        = ":resname"
	PatternUuid       = ":uuid"
	PatternFType      = ":type"
	PatternFPath      = ":path"
	PatternFOp        = ":op"
	PatternFValue     = ":value"
	FTypeDevice       = "device"
	FTypeDevices      = "devices"
	FTypeResource     = "resource"
	FTypeResources    = "resources"
	GetParamPage      = "page"
	GetParamPerPage   = "per_page"
	CollectionType    = "DeviceCatalog"
	CurrentApiVersion = "0.2.1"
)

type Collection struct {
	Context   string                 `json:"@context,omitempty"`
	Id        string                 `json:"id"`
	Type      string                 `json:"type"`
	Devices   map[string]EmptyDevice `json:"devices"`
	Resources []Resource             `json:"resources"`
	Page      int                    `json:"page"`
	PerPage   int                    `json:"per_page"`
	Total     int                    `json:"total"`
}

// Device object with empty resources
type EmptyDevice struct {
	*Device
	Resources []Resource `json:"resources,omitempty"`
}

// Device object with paginated resources
type PaginatedDevice struct {
	*Device
	Resources []Resource `json:"resources"`
	Page      int        `json:"page"`
	PerPage   int        `json:"per_page"`
	Total     int        `json:"total"`
}

// Read-only catalog api
type ReadableCatalogAPI struct {
	catalogStorage CatalogStorage
	contextUrl     string
}

// Writable catalog api
type WritableCatalogAPI struct {
	*ReadableCatalogAPI
}

func NewReadableCatalogAPI(storage CatalogStorage, contextUrl string) *ReadableCatalogAPI {
	return &ReadableCatalogAPI{
		catalogStorage: storage,
		contextUrl:     contextUrl,
	}
}

func NewWritableCatalogAPI(storage CatalogStorage, contextUrl string) *WritableCatalogAPI {
	return &WritableCatalogAPI{
		&ReadableCatalogAPI{
			catalogStorage: storage,
			contextUrl:     contextUrl,
		}}
}

func (self *Device) ldify() Device {
	rc := self.copy()
	for i, res := range rc.Resources {
		rc.Resources[i] = res.ldify()
	}
	rc.Id = fmt.Sprintf("%v/%v", CatalogBaseUrl, self.Id)
	return rc
}

func (self *Resource) ldify() Resource {
	resc := self.copy()
	resc.Id = fmt.Sprintf("%v/%v", CatalogBaseUrl, self.Id)
	resc.Device = fmt.Sprintf("%v/%v", CatalogBaseUrl, self.Device)
	return resc
}

func (self *Device) unLdify() Device {
	rc := self.copy()
	for i, res := range rc.Resources {
		rc.Resources[i] = res.unLdify()
	}
	rc.Id = strings.TrimPrefix(self.Id, CatalogBaseUrl+"/")
	return rc
}

func (self *Resource) unLdify() Resource {
	resc := self.copy()
	resc.Id = strings.TrimPrefix(self.Id, CatalogBaseUrl+"/")
	resc.Device = strings.TrimPrefix(self.Device, CatalogBaseUrl+"/")
	return resc
}

func (self ReadableCatalogAPI) collectionFromDevices(devices []Device, page, perPage, total int) *Collection {
	respDevices := make(map[string]EmptyDevice)
	respResources := make([]Resource, 0, self.catalogStorage.getResourcesCount())

	for _, d := range devices {
		dld := d.ldify()
		for _, res := range dld.Resources {
			respResources = append(respResources, res)
		}

		respDevices[d.Id] = EmptyDevice{
			&dld,
			nil,
		}
	}

	return &Collection{
		Context:   self.contextUrl,
		Id:        CatalogBaseUrl,
		Type:      CollectionType,
		Devices:   respDevices,
		Resources: respResources,
		Page:      page,
		PerPage:   perPage,
		Total:     total,
	}
}

func (self ReadableCatalogAPI) paginatedDeviceFromDevice(d Device, page, perPage int) *PaginatedDevice {
	// Never return more than the defined maximum
	if perPage > MaxPerPage || perPage == 0 {
		perPage = MaxPerPage
	}

	pd := &PaginatedDevice{
		&d,
		make([]Resource, 0, len(d.Resources)),
		page,
		perPage,
		len(d.Resources),
	}

	// if 1, not specified or negative - return the first page
	if page < 2 {
		// first page
		if perPage > pd.Total {
			pd.Resources = d.Resources
		} else {
			pd.Resources = d.Resources[:perPage]
		}
	} else if page == int(pd.Total/perPage)+1 {
		// last page
		pd.Resources = d.Resources[perPage*(page-1):]

	} else if page <= pd.Total/perPage && page*perPage <= pd.Total {
		// slice
		r := page * perPage
		l := r - perPage
		pd.Resources = d.Resources[l:r]
	}

	for i, r := range pd.Resources {
		rld := r.ldify()
		pd.Resources[i] = rld
	}

	return pd
}

func (self ReadableCatalogAPI) List(w http.ResponseWriter, req *http.Request) {
	req.ParseForm()
	page, _ := strconv.Atoi(req.Form.Get(GetParamPage))
	perPage, _ := strconv.Atoi(req.Form.Get(GetParamPerPage))

	// use defaults if not specified
	if page == 0 {
		page = 1
	}
	if perPage == 0 {
		perPage = MaxPerPage
	}

	devices, total, _ := self.catalogStorage.getMany(page, perPage)
	coll := self.collectionFromDevices(devices, page, perPage, total)

	b, _ := json.Marshal(coll)
	w.Header().Set("Content-Type", "application/ld+json;version="+CurrentApiVersion)
	w.Write(b)

	return
}

func (self ReadableCatalogAPI) Filter(w http.ResponseWriter, req *http.Request) {
	ftype := req.URL.Query().Get(PatternFType)
	fpath := req.URL.Query().Get(PatternFPath)
	fop := req.URL.Query().Get(PatternFOp)
	fvalue := req.URL.Query().Get(PatternFValue)

	var data interface{}
	var err error
	matched := false

	switch ftype {
	case FTypeDevice:
		data, err = self.catalogStorage.pathFilterDevice(fpath, fop, fvalue)
		if data.(Device).Id != "" {
			d := data.(Device)
			data = d.ldify()
			matched = true
		}

	case FTypeDevices:
		data, err = self.catalogStorage.pathFilterDevices(fpath, fop, fvalue)
		if len(data.([]Device)) > 0 {
			data = self.collectionFromDevices(data.([]Device), 0, 0, 0) //FIXME
			matched = true
		}

	case FTypeResource:
		data, err = self.catalogStorage.pathFilterResource(fpath, fop, fvalue)
		if data.(Resource).Id != "" {
			res := data.(Resource)
			data = res.ldify()
			matched = true
		}

	case FTypeResources:
		data, err = self.catalogStorage.pathFilterResources(fpath, fop, fvalue)
		if len(data.([]Resource)) > 0 {
			devs := self.catalogStorage.devicesFromResources(data.([]Resource))
			data = self.collectionFromDevices(devs, 0, 0, 0) //FIXME
			matched = true
		}
	}

	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "Error processing the request: %s\n", err.Error())
	}

	if matched == false {
		w.WriteHeader(http.StatusNotFound)
		fmt.Fprintf(w, "Not found\n")
		return
	}

	b, _ := json.Marshal(data)
	w.Header().Set("Content-Type", "application/ld+json;version="+CurrentApiVersion)
	w.Write(b)
}

func (self ReadableCatalogAPI) Get(w http.ResponseWriter, req *http.Request) {
	req.ParseForm()
	page, _ := strconv.Atoi(req.Form.Get(GetParamPage))
	perPage, _ := strconv.Atoi(req.Form.Get(GetParamPerPage))
	id := fmt.Sprintf("%v/%v", req.URL.Query().Get(PatternUuid), req.URL.Query().Get(PatternReg))

	d, err := self.catalogStorage.get(id)
	if err != nil || d.Id == "" {
		w.WriteHeader(http.StatusNotFound)
		fmt.Fprintf(w, "Registration not found\n")
		return
	}

	// use defaults if not specified
	if page == 0 {
		page = 1
	}
	if perPage == 0 {
		perPage = MaxPerPage
	}

	pd := self.paginatedDeviceFromDevice(d, page, perPage)
	b, _ := json.Marshal(pd)

	w.Header().Set("Content-Type", "application/ld+json;version="+CurrentApiVersion)
	w.Write(b)
	return
}

func (self ReadableCatalogAPI) GetResource(w http.ResponseWriter, req *http.Request) {
	devid := fmt.Sprintf("%v/%v", req.URL.Query().Get(PatternUuid), req.URL.Query().Get(PatternReg))
	resid := fmt.Sprintf("%v/%v", devid, req.URL.Query().Get(PatternRes))

	// check if device devid exists
	_, err := self.catalogStorage.get(devid)
	if err != nil {
		w.WriteHeader(http.StatusNotFound)
		fmt.Fprintf(w, "Device not found\n")
		return
	}

	// check if it has a resource resid
	res, err := self.catalogStorage.getResourceById(resid)
	if err != nil {
		w.WriteHeader(http.StatusNotFound)
		fmt.Fprintf(w, "Resource not found\n")
		return
	}

	b, _ := json.Marshal(res.ldify())
	w.Header().Set("Content-Type", "application/ld+json;version="+CurrentApiVersion)
	w.Write(b)
	return
}

func (self WritableCatalogAPI) Add(w http.ResponseWriter, req *http.Request) {
	body, err := ioutil.ReadAll(req.Body)
	req.Body.Close()

	var d Device
	err = json.Unmarshal(body, &d)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "Error processing the request: %s\n", err.Error())
		return
	}

	da, err := self.catalogStorage.add(d)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintf(w, "Error creating the registration: %s\n", err.Error())
		return
	}

	b, _ := json.Marshal(da.ldify())
	w.Header().Set("Content-Type", "application/ld+json;version="+CurrentApiVersion)
	w.WriteHeader(http.StatusCreated)
	w.Write(b)
	return
}

func (self WritableCatalogAPI) Update(w http.ResponseWriter, req *http.Request) {
	id := fmt.Sprintf("%v/%v", req.URL.Query().Get(PatternUuid), req.URL.Query().Get(PatternReg))

	body, err := ioutil.ReadAll(req.Body)
	req.Body.Close()

	var d Device
	err = json.Unmarshal(body, &d)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "Error processing the request:: %s\n", err.Error())
		return
	}

	du, err := self.catalogStorage.update(id, d)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintf(w, "Error updating the device: %s\n", err.Error())
		return
	}

	if du.Id == "" {
		w.WriteHeader(http.StatusNotFound)
		fmt.Fprintf(w, "Not found\n")
		return
	}

	b, _ := json.Marshal(du.ldify())
	w.Header().Set("Content-Type", "application/ld+json;version="+CurrentApiVersion)
	w.WriteHeader(http.StatusOK)
	w.Write(b)

	return
}

func (self WritableCatalogAPI) Delete(w http.ResponseWriter, req *http.Request) {
	id := fmt.Sprintf("%v/%v", req.URL.Query().Get(PatternUuid), req.URL.Query().Get(PatternReg))

	dd, err := self.catalogStorage.delete(id)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintf(w, "Error deleting the device: %s\n", err.Error())
		return
	}

	if dd.Id == "" {
		w.WriteHeader(http.StatusNotFound)
		fmt.Fprintf(w, "Not found\n")
		return
	}

	b, _ := json.Marshal(dd.ldify())
	w.Header().Set("Content-Type", "application/ld+json;version="+CurrentApiVersion)
	w.WriteHeader(http.StatusOK)
	w.Write(b)
	return
}
<|endoftext|>


<commit_before>package sessions

import (
	"github.com/apratheek/schemamagic"
	pgx "gopkg.in/jackc/pgx.v2"
)

// TableSessions creates the Sessions table
func TableSessions(tx *pgx.Tx, defaultSchema string, database string) *schemamagic.Table {
	/*
		CREATE TABLE sessions (
			id bigserial UNIQUE PRIMARY KEY,
			key text NOT NULL, -- stores the session key
			token text UNIQUE NOT NULL, -- stores the generated session token, in response to the session key
			expires_at timestamp with timezone NOT NULL, -- stores the time when this session will expire
			ip text NOT NULL, -- stores the ip address of the client
			active bool DEFAULT true,
			timestamp bigint DEFAULT EXTRACT(EPOCH FROM NOW())::bigint
		)
	*/
	table := schemamagic.NewTable(schemamagic.Table{Name: "sessions", DefaultSchema: defaultSchema, Database: database, Tx: tx})
	table.Append(schemamagic.NewColumn(schemamagic.Column{Name: "id", Datatype: "bigserial", IsPrimary: true, IsUnique: true}))
	table.Append(schemamagic.NewColumn(schemamagic.Column{Name: "key", Datatype: "text", IsNotNull: true}))
	table.Append(schemamagic.NewColumn(schemamagic.Column{Name: "token", Datatype: "text", IsNotNull: true, IsUnique: true}))
	table.Append(schemamagic.NewColumn(schemamagic.Column{Name: "expires_at", Datatype: "timestamp with time zone", IsNotNull: true, PseudoDatatype: "timestamp with time zone"}))
	table.Append(schemamagic.NewColumn(schemamagic.Column{Name: "ip", Datatype: "text", IsNotNull: true}))
	table.Append(schemamagic.NewColumn(schemamagic.Column{Name: "active", Datatype: "boolean", DefaultExists: true, DefaultValue: "true"}))
	table.Append(schemamagic.NewColumn(schemamagic.Column{Name: "timestamp", Datatype: "bigint", DefaultExists: true, DefaultValue: "date_part('epoch'::text, now())::bigint"}))
	return table
}
<commit_msg>tableSessions has been converted to a private function. This will be called internally.<commit_after>package sessions

import (
	"github.com/apratheek/schemamagic"
	pgx "gopkg.in/jackc/pgx.v2"
)

// tableSessions creates the Sessions table
func tableSessions(tx *pgx.Tx, defaultSchema string, database string) *schemamagic.Table {
	/*
		CREATE TABLE sessions (
			id bigserial UNIQUE PRIMARY KEY,
			key text NOT NULL, -- stores the session key
			token text UNIQUE NOT NULL, -- stores the generated session token, in response to the session key
			expires_at timestamp with timezone NOT NULL, -- stores the time when this session will expire
			ip text NOT NULL, -- stores the ip address of the client
			active bool DEFAULT true,
			timestamp bigint DEFAULT EXTRACT(EPOCH FROM NOW())::bigint
		)
	*/
	table := schemamagic.NewTable(schemamagic.Table{Name: "sessions", DefaultSchema: defaultSchema, Database: database, Tx: tx})
	table.Append(schemamagic.NewColumn(schemamagic.Column{Name: "id", Datatype: "bigserial", IsPrimary: true, IsUnique: true}))
	table.Append(schemamagic.NewColumn(schemamagic.Column{Name: "key", Datatype: "text", IsNotNull: true}))
	table.Append(schemamagic.NewColumn(schemamagic.Column{Name: "token", Datatype: "text", IsNotNull: true, IsUnique: true}))
	table.Append(schemamagic.NewColumn(schemamagic.Column{Name: "expires_at", Datatype: "timestamp with time zone", IsNotNull: true, PseudoDatatype: "timestamp with time zone"}))
	table.Append(schemamagic.NewColumn(schemamagic.Column{Name: "ip", Datatype: "text", IsNotNull: true}))
	table.Append(schemamagic.NewColumn(schemamagic.Column{Name: "active", Datatype: "boolean", DefaultExists: true, DefaultValue: "true"}))
	table.Append(schemamagic.NewColumn(schemamagic.Column{Name: "timestamp", Datatype: "bigint", DefaultExists: true, DefaultValue: "date_part('epoch'::text, now())::bigint"}))
	return table
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"time"

	"github.com/luizbranco/eventsource"

	"golang.org/x/net/context"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/cloudmonitoring/v2beta2"
)

type GoogleCloudMonitoring struct {
	base_url string
	remote   cloudmonitoring.TimeseriesService
}

type Timeseries struct {
	base_url    string
	metric_name string
	start       time.Time
	end         time.Time
	value       float64
}

func createTimeseries(args Timeseries) *cloudmonitoring.TimeseriesPoint {

	var end_string string
	var start_string string

	start_string = args.start.Format(time.RFC3339)

	if &args.end != nil {
		end_string = args.end.Format(time.RFC3339)
	} else {
		end_string = start_string
	}

	description := cloudmonitoring.TimeseriesDescriptor{
		Labels: map[string]string{
			args.base_url + "implementation": "golang",
		},
		Metric:  args.base_url + args.metric_name,
		Project: "replay-gaming",
	}

	point := cloudmonitoring.Point{
		Start:       start_string,
		End:         end_string,
		DoubleValue: &args.value,
	}

	timeseries := cloudmonitoring.TimeseriesPoint{
		Point:          &point,
		TimeseriesDesc: &description,
	}

	return &timeseries
}

func pushMetrics(points []*cloudmonitoring.TimeseriesPoint, remote cloudmonitoring.TimeseriesService) {
	request := cloudmonitoring.WriteTimeseriesRequest{
		CommonLabels: map[string]string{
			"container.googleapis.com/container_name": "eventsource",
		},
		Timeseries: points,
	}

	response, err := remote.Write("replay-gaming", &request).Do()
	if err != nil {
		log.Fatal("pushMetrics - Unable to write timeseries: ", err)
	}
	log.Printf("pushMetrics - Response: %s", response)
}

func createMetricDescriptor(prefix string, name string, description string) *cloudmonitoring.MetricDescriptor {
	metric_type := cloudmonitoring.MetricDescriptorTypeDescriptor{
		MetricType: "gauge",
		ValueType:  "double",
	}

	label := cloudmonitoring.MetricDescriptorLabelDescriptor{
		Description: "Application",
		Key:         "eventsource",
	}

	return &cloudmonitoring.MetricDescriptor{
		Description: description,
		Labels: []*cloudmonitoring.MetricDescriptorLabelDescriptor{
			&label,
		},
		Name:           "custom.cloudmonitoring.googleapis.com/" + prefix + "/" + name,
		Project:        "replay-gaming",
		TypeDescriptor: &metric_type,
	}
}

func createMetric(prefix string, metricDescriptorsService *cloudmonitoring.MetricDescriptorsService, name string, description string) {
	metricDescriptor := createMetricDescriptor(prefix, name, description)
	metricDescriptor, err := metricDescriptorsService.Create("replay-gaming", metricDescriptor).Do()
	if err != nil {
		log.Fatal("Unable to create '"+name+"' metric: ", err)
	}
}

func createMetrics(prefix string, cloudmonitoringService *cloudmonitoring.Service) {
	metricDescriptorsService := cloudmonitoring.NewMetricDescriptorsService(cloudmonitoringService)

	createMetric(prefix, metricDescriptorsService, "clients", "Number of clients that the event was distributed to")
	createMetric(prefix, metricDescriptorsService, "avg_time", "Average time to send an event to all connected clients")
	createMetric(prefix, metricDescriptorsService, "connections", "Number of connected SSE clients (browser sessions)")
}

func NewMetrics(prefix string) (GoogleCloudMonitoring, error) {
	monitor := GoogleCloudMonitoring{base_url: "custom.cloudmonitoring.googleapis.com/" + prefix + "/"}

	client, err := google.DefaultClient(
		context.Background(),
		cloudmonitoring.MonitoringScope,
	)
	if err != nil {
		log.Fatal("Unable to get default client: ", err)
	}

	cloudmonitoringService, err := cloudmonitoring.New(client)
	if err != nil {
		log.Fatal("Unable to create monitoring service: ", err)
	}

	createMetrics(prefix, cloudmonitoringService)

	timeseriesService := cloudmonitoring.NewTimeseriesService(cloudmonitoringService)

	monitor.remote = *timeseriesService
	return monitor, nil
}

func (monitor GoogleCloudMonitoring) ClientCount(count int) {
	log.Printf("[METRIC] %sconnections: %d\n", monitor.base_url, count)

	timeseries := createTimeseries(Timeseries{
		base_url:    monitor.base_url,
		metric_name: "connections",
		start:       time.Now().UTC(),
		value:       float64(count),
	})

	points := []*cloudmonitoring.TimeseriesPoint{
		timeseries,
	}

	pushMetrics(points, monitor.remote)
}

func (monitor GoogleCloudMonitoring) EventDone(event eventsource.Event, duration time.Duration, eventdurations []time.Duration) {
	var sum int64
	var count int64
	var avg float64

	for _, d := range eventdurations {
		if d > 0 {
			sum += d.Nanoseconds()
		}
	}

	count = int64(len(eventdurations))

	if count > 0 {
		avg = float64(sum) / float64(count)
	}

	log.Printf("[METRIC] %s.event_distributed.clients: %d\n", monitor.base_url, count)
	log.Printf("[METRIC] %s.event_distributed.avg_time: %dns\n", monitor.base_url, avg)

	clients_timeseries := createTimeseries(Timeseries{
		base_url:    monitor.base_url,
		metric_name: "clients",
		start:       time.Now().UTC(),
		value:       float64(count),
	})

	avg_time_timeseries := createTimeseries(Timeseries{
		base_url:    monitor.base_url,
		metric_name: "avg_time",
		start:       time.Now().UTC(),
		value:       avg,
	})

	points := []*cloudmonitoring.TimeseriesPoint{
		clients_timeseries,
		avg_time_timeseries,
	}

	pushMetrics(points, monitor.remote)
}
<commit_msg>Metric failures are no longer fatal.<commit_after>package main

import (
	"log"
	"time"

	"github.com/luizbranco/eventsource"

	"golang.org/x/net/context"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/cloudmonitoring/v2beta2"
)

type GoogleCloudMonitoring struct {
	base_url string
	remote   cloudmonitoring.TimeseriesService
}

type Timeseries struct {
	base_url    string
	metric_name string
	start       time.Time
	end         time.Time
	value       float64
}

func createTimeseries(args Timeseries) *cloudmonitoring.TimeseriesPoint {

	var end_string string
	var start_string string

	start_string = args.start.Format(time.RFC3339)

	if &args.end != nil {
		end_string = args.end.Format(time.RFC3339)
	} else {
		end_string = start_string
	}

	description := cloudmonitoring.TimeseriesDescriptor{
		Labels: map[string]string{
			args.base_url + "implementation": "golang",
		},
		Metric:  args.base_url + args.metric_name,
		Project: "replay-gaming",
	}

	point := cloudmonitoring.Point{
		Start:       start_string,
		End:         end_string,
		DoubleValue: &args.value,
	}

	timeseries := cloudmonitoring.TimeseriesPoint{
		Point:          &point,
		TimeseriesDesc: &description,
	}

	return &timeseries
}

func pushMetrics(points []*cloudmonitoring.TimeseriesPoint, remote cloudmonitoring.TimeseriesService) {
	request := cloudmonitoring.WriteTimeseriesRequest{
		CommonLabels: map[string]string{
			"container.googleapis.com/container_name": "eventsource",
		},
		Timeseries: points,
	}

	response, err := remote.Write("replay-gaming", &request).Do()
	if err != nil {
		log.Printf("pushMetrics - Unable to write timeseries: %v", err)
	} else {
		log.Printf("pushMetrics - Response: %s", response)
	}
}

func createMetricDescriptor(prefix string, name string, description string) *cloudmonitoring.MetricDescriptor {
	metric_type := cloudmonitoring.MetricDescriptorTypeDescriptor{
		MetricType: "gauge",
		ValueType:  "double",
	}

	label := cloudmonitoring.MetricDescriptorLabelDescriptor{
		Description: "Application",
		Key:         "eventsource",
	}

	return &cloudmonitoring.MetricDescriptor{
		Description: description,
		Labels: []*cloudmonitoring.MetricDescriptorLabelDescriptor{
			&label,
		},
		Name:           "custom.cloudmonitoring.googleapis.com/" + prefix + "/" + name,
		Project:        "replay-gaming",
		TypeDescriptor: &metric_type,
	}
}

func createMetric(prefix string, metricDescriptorsService *cloudmonitoring.MetricDescriptorsService, name string, description string) {
	metricDescriptor := createMetricDescriptor(prefix, name, description)
	metricDescriptor, err := metricDescriptorsService.Create("replay-gaming", metricDescriptor).Do()
	if err != nil {
		log.Printf("Unable to create '%s' metric: %v", name, err)
	}
}

func createMetrics(prefix string, cloudmonitoringService *cloudmonitoring.Service) {
	metricDescriptorsService := cloudmonitoring.NewMetricDescriptorsService(cloudmonitoringService)

	createMetric(prefix, metricDescriptorsService, "clients", "Number of clients that the event was distributed to")
	createMetric(prefix, metricDescriptorsService, "avg_time", "Average time to send an event to all connected clients")
	createMetric(prefix, metricDescriptorsService, "connections", "Number of connected SSE clients (browser sessions)")
}

func NewMetrics(prefix string) (GoogleCloudMonitoring, error) {
	monitor := GoogleCloudMonitoring{base_url: "custom.cloudmonitoring.googleapis.com/" + prefix + "/"}

	client, err := google.DefaultClient(
		context.Background(),
		cloudmonitoring.MonitoringScope,
	)
	if err != nil {
		log.Printf("Unable to get default client: %v", err)
	}

	cloudmonitoringService, err := cloudmonitoring.New(client)
	if err != nil {
		log.Printf("Unable to create monitoring service: %v", err)
	}

	createMetrics(prefix, cloudmonitoringService)

	timeseriesService := cloudmonitoring.NewTimeseriesService(cloudmonitoringService)

	monitor.remote = *timeseriesService
	return monitor, nil
}

func (monitor GoogleCloudMonitoring) ClientCount(count int) {
	log.Printf("[METRIC] %sconnections: %d\n", monitor.base_url, count)

	timeseries := createTimeseries(Timeseries{
		base_url:    monitor.base_url,
		metric_name: "connections",
		start:       time.Now().UTC(),
		value:       float64(count),
	})

	points := []*cloudmonitoring.TimeseriesPoint{
		timeseries,
	}

	pushMetrics(points, monitor.remote)
}

func (monitor GoogleCloudMonitoring) EventDone(event eventsource.Event, duration time.Duration, eventdurations []time.Duration) {
	var sum int64
	var count int64
	var avg float64

	for _, d := range eventdurations {
		if d > 0 {
			sum += d.Nanoseconds()
		}
	}

	count = int64(len(eventdurations))

	if count > 0 {
		avg = float64(sum) / float64(count)
	}

	log.Printf("[METRIC] %s.event_distributed.clients: %d\n", monitor.base_url, count)
	log.Printf("[METRIC] %s.event_distributed.avg_time: %dns\n", monitor.base_url, avg)

	clients_timeseries := createTimeseries(Timeseries{
		base_url:    monitor.base_url,
		metric_name: "clients",
		start:       time.Now().UTC(),
		value:       float64(count),
	})

	avg_time_timeseries := createTimeseries(Timeseries{
		base_url:    monitor.base_url,
		metric_name: "avg_time",
		start:       time.Now().UTC(),
		value:       avg,
	})

	points := []*cloudmonitoring.TimeseriesPoint{
		clients_timeseries,
		avg_time_timeseries,
	}

	pushMetrics(points, monitor.remote)
}
<|endoftext|>


<commit_before>package worker

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"time"

	"github.com/bitly/go-simplejson"
	"github.com/pkg/errors"
	"github.com/travis-ci/worker/backend"

	gocontext "golang.org/x/net/context"
)

// HTTPJobQueue is a JobQueue that uses http
type HTTPJobQueue struct {
	processorPool *ProcessorPool
	jobBoardURL   *url.URL
	site          string
	queue         string
	workerID      string

	DefaultLanguage, DefaultDist, DefaultGroup, DefaultOS string
}

type httpFetchJobsRequest struct {
	Jobs []string `json:"jobs"`
}

type httpFetchJobsResponse struct {
	Jobs []string `json:"jobs"`
}

// NewHTTPJobQueue creates a new job-board job queue
func NewHTTPJobQueue(pool *ProcessorPool, jobBoardURL *url.URL, site, queue, workerID string) (*HTTPJobQueue, error) {
	return &HTTPJobQueue{
		processorPool: pool,
		jobBoardURL:   jobBoardURL,
		site:          site,
		queue:         queue,
		workerID:      workerID,
	}, nil
}

// Jobs consumes new jobs from job-board
func (q *HTTPJobQueue) Jobs(ctx gocontext.Context) (outChan <-chan Job, err error) {
	buildJobChan := make(chan Job)
	outChan = buildJobChan

	go func() {
		for {
			jobIds, err := q.fetchJobs()
			if err != nil {
				fmt.Fprintf(os.Stderr, "TODO: handle error from httpJobQueue.fetchJobs: %#v", err)
				panic("whoops!")
			}
			for _, id := range jobIds {
				go func(id uint64) {
					buildJob, err := q.fetchJob(id)
					if err != nil {
						fmt.Fprintf(os.Stderr, "TODO: handle error from httpJobQueue.fetchJob: %#v", err)
					}
					buildJobChan <- buildJob
				}(id)
			}

			time.Sleep(time.Second)
		}
	}()

	return outChan, nil
}

func (q *HTTPJobQueue) fetchJobs() ([]uint64, error) {
	// POST /jobs?count=17&queue=flah
	// Content-Type: application/json
	// Travis-Site: ${SITE}
	// Authorization: Basic ${BASE64_BASIC_AUTH}
	// From: ${UNIQUE_ID}

	fetchRequestPayload := &httpFetchJobsRequest{Jobs: []string{}}
	numWaiting := 0
	q.processorPool.Each(func(i int, p *Processor) {
		// CurrentStatus is one of "new", "waiting", "processing" or "done"
		switch p.CurrentStatus {
		case "processing":
			fetchRequestPayload.Jobs = append(fetchRequestPayload.Jobs, fmt.Sprintf("%d", p.LastJobID))
		case "waiting", "new":
			numWaiting++
		}
	})

	jobIdsJSON, err := json.Marshal(fetchRequestPayload)
	if err != nil {
		return nil, errors.Wrap(err, "failed to marshal job board jobs request payload")
	}

	// copy jobBoardURL
	url := *q.jobBoardURL

	query := url.Query()
	query.Add("count", fmt.Sprintf("%d", numWaiting))
	query.Add("queue", q.queue)

	url.Path = "/jobs"
	url.RawQuery = query.Encode()

	client := &http.Client{}

	req, err := http.NewRequest("POST", url.String(), bytes.NewReader(jobIdsJSON))
	if err != nil {
		return nil, errors.Wrap(err, "failed to create job board jobs request")
	}

	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("Travis-Site", q.site)
	req.Header.Add("From", q.workerID)

	resp, err := client.Do(req)
	if err != nil {
		return nil, errors.Wrap(err, "failed to make job board jobs request")
	}

	fetchResponsePayload := &httpFetchJobsResponse{}
	err = json.NewDecoder(resp.Body).Decode(&fetchResponsePayload)
	if err != nil {
		return nil, errors.Wrap(err, "failed to decode job board jobs response")
	}

	var jobIds []uint64
	for _, strID := range fetchResponsePayload.Jobs {
		alreadyRunning := false
		for _, prevStrID := range fetchRequestPayload.Jobs {
			if strID == prevStrID {
				alreadyRunning = true
			}
		}
		if alreadyRunning {
			continue
		}

		id, err := strconv.ParseUint(strID, 10, 64)
		if err != nil {
			return nil, errors.Wrap(err, "failed to parse job ID")
		}
		jobIds = append(jobIds, id)
	}

	return jobIds, nil
}

func (q *HTTPJobQueue) fetchJob(id uint64) (Job, error) {
	// GET /jobs/:id
	// Authorization: Basic ${BASE64_BASIC_AUTH}
	// Travis-Site: ${SITE}
	// From: ${UNIQUE_ID}

	buildJob := &httpJob{
		payload: &httpJobPayload{
			Data: &JobPayload{},
		},
		startAttributes: &backend.StartAttributes{},
	}
	startAttrs := &httpJobPayloadStartAttrs{
		Data: &jobPayloadStartAttrs{
			Config: &backend.StartAttributes{},
		},
	}

	// copy jobBoardURL
	url := *q.jobBoardURL
	url.Path = fmt.Sprintf("/jobs/%d", id)

	client := &http.Client{}

	req, err := http.NewRequest("GET", url.String(), nil)
	if err != nil {
		return nil, errors.Wrap(err, "couldn't make job board job request")
	}

	req.Header.Add("Travis-Site", q.site)
	req.Header.Add("From", q.workerID)

	resp, err := client.Do(req)
	if err != nil {
		return nil, errors.Wrap(err, "error making job board job request")
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, errors.Wrap(err, "error reading body from job board job request")
	}

	err = json.Unmarshal(body, buildJob.payload)
	if err != nil {
		return nil, errors.Wrap(err, "failed to unmarshal job board payload")
	}

	err = json.Unmarshal(body, &startAttrs)
	if err != nil {
		return nil, errors.Wrap(err, "failed to unmarshal start attributes from job board")
	}

	rawPayload, err := simplejson.NewJson(body)
	if err != nil {
		return nil, errors.Wrap(err, "failed to parse raw payload with simplejson")
	}
	buildJob.rawPayload = rawPayload.Get("data")

	buildJob.startAttributes = startAttrs.Data.Config
	buildJob.startAttributes.VMType = buildJob.payload.Data.VMType
	buildJob.startAttributes.SetDefaults(q.DefaultLanguage, q.DefaultDist, q.DefaultGroup, q.DefaultOS, VMTypeDefault)

	return buildJob, nil
}

// Cleanup does not do anything!
func (q *HTTPJobQueue) Cleanup() error {
	return nil
}
<commit_msg>Use strconv instead of fmt.Sprintf("%d", …)<commit_after>package worker

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"time"

	"github.com/bitly/go-simplejson"
	"github.com/pkg/errors"
	"github.com/travis-ci/worker/backend"

	gocontext "golang.org/x/net/context"
)

// HTTPJobQueue is a JobQueue that uses http
type HTTPJobQueue struct {
	processorPool *ProcessorPool
	jobBoardURL   *url.URL
	site          string
	queue         string
	workerID      string

	DefaultLanguage, DefaultDist, DefaultGroup, DefaultOS string
}

type httpFetchJobsRequest struct {
	Jobs []string `json:"jobs"`
}

type httpFetchJobsResponse struct {
	Jobs []string `json:"jobs"`
}

// NewHTTPJobQueue creates a new job-board job queue
func NewHTTPJobQueue(pool *ProcessorPool, jobBoardURL *url.URL, site, queue, workerID string) (*HTTPJobQueue, error) {
	return &HTTPJobQueue{
		processorPool: pool,
		jobBoardURL:   jobBoardURL,
		site:          site,
		queue:         queue,
		workerID:      workerID,
	}, nil
}

// Jobs consumes new jobs from job-board
func (q *HTTPJobQueue) Jobs(ctx gocontext.Context) (outChan <-chan Job, err error) {
	buildJobChan := make(chan Job)
	outChan = buildJobChan

	go func() {
		for {
			jobIds, err := q.fetchJobs()
			if err != nil {
				fmt.Fprintf(os.Stderr, "TODO: handle error from httpJobQueue.fetchJobs: %#v", err)
				panic("whoops!")
			}
			for _, id := range jobIds {
				go func(id uint64) {
					buildJob, err := q.fetchJob(id)
					if err != nil {
						fmt.Fprintf(os.Stderr, "TODO: handle error from httpJobQueue.fetchJob: %#v", err)
					}
					buildJobChan <- buildJob
				}(id)
			}

			time.Sleep(time.Second)
		}
	}()

	return outChan, nil
}

func (q *HTTPJobQueue) fetchJobs() ([]uint64, error) {
	// POST /jobs?count=17&queue=flah
	// Content-Type: application/json
	// Travis-Site: ${SITE}
	// Authorization: Basic ${BASE64_BASIC_AUTH}
	// From: ${UNIQUE_ID}

	fetchRequestPayload := &httpFetchJobsRequest{Jobs: []string{}}
	numWaiting := 0
	q.processorPool.Each(func(i int, p *Processor) {
		// CurrentStatus is one of "new", "waiting", "processing" or "done"
		switch p.CurrentStatus {
		case "processing":
			fetchRequestPayload.Jobs = append(fetchRequestPayload.Jobs, strconv.FormatUint(p.LastJobID, 10))
		case "waiting", "new":
			numWaiting++
		}
	})

	jobIdsJSON, err := json.Marshal(fetchRequestPayload)
	if err != nil {
		return nil, errors.Wrap(err, "failed to marshal job board jobs request payload")
	}

	// copy jobBoardURL
	url := *q.jobBoardURL

	query := url.Query()
	query.Add("count", strconv.Itoa(numWaiting))
	query.Add("queue", q.queue)

	url.Path = "/jobs"
	url.RawQuery = query.Encode()

	client := &http.Client{}

	req, err := http.NewRequest("POST", url.String(), bytes.NewReader(jobIdsJSON))
	if err != nil {
		return nil, errors.Wrap(err, "failed to create job board jobs request")
	}

	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("Travis-Site", q.site)
	req.Header.Add("From", q.workerID)

	resp, err := client.Do(req)
	if err != nil {
		return nil, errors.Wrap(err, "failed to make job board jobs request")
	}

	fetchResponsePayload := &httpFetchJobsResponse{}
	err = json.NewDecoder(resp.Body).Decode(&fetchResponsePayload)
	if err != nil {
		return nil, errors.Wrap(err, "failed to decode job board jobs response")
	}

	var jobIds []uint64
	for _, strID := range fetchResponsePayload.Jobs {
		alreadyRunning := false
		for _, prevStrID := range fetchRequestPayload.Jobs {
			if strID == prevStrID {
				alreadyRunning = true
			}
		}
		if alreadyRunning {
			continue
		}

		id, err := strconv.ParseUint(strID, 10, 64)
		if err != nil {
			return nil, errors.Wrap(err, "failed to parse job ID")
		}
		jobIds = append(jobIds, id)
	}

	return jobIds, nil
}

func (q *HTTPJobQueue) fetchJob(id uint64) (Job, error) {
	// GET /jobs/:id
	// Authorization: Basic ${BASE64_BASIC_AUTH}
	// Travis-Site: ${SITE}
	// From: ${UNIQUE_ID}

	buildJob := &httpJob{
		payload: &httpJobPayload{
			Data: &JobPayload{},
		},
		startAttributes: &backend.StartAttributes{},
	}
	startAttrs := &httpJobPayloadStartAttrs{
		Data: &jobPayloadStartAttrs{
			Config: &backend.StartAttributes{},
		},
	}

	// copy jobBoardURL
	url := *q.jobBoardURL
	url.Path = fmt.Sprintf("/jobs/%d", id)

	client := &http.Client{}

	req, err := http.NewRequest("GET", url.String(), nil)
	if err != nil {
		return nil, errors.Wrap(err, "couldn't make job board job request")
	}

	req.Header.Add("Travis-Site", q.site)
	req.Header.Add("From", q.workerID)

	resp, err := client.Do(req)
	if err != nil {
		return nil, errors.Wrap(err, "error making job board job request")
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, errors.Wrap(err, "error reading body from job board job request")
	}

	err = json.Unmarshal(body, buildJob.payload)
	if err != nil {
		return nil, errors.Wrap(err, "failed to unmarshal job board payload")
	}

	err = json.Unmarshal(body, &startAttrs)
	if err != nil {
		return nil, errors.Wrap(err, "failed to unmarshal start attributes from job board")
	}

	rawPayload, err := simplejson.NewJson(body)
	if err != nil {
		return nil, errors.Wrap(err, "failed to parse raw payload with simplejson")
	}
	buildJob.rawPayload = rawPayload.Get("data")

	buildJob.startAttributes = startAttrs.Data.Config
	buildJob.startAttributes.VMType = buildJob.payload.Data.VMType
	buildJob.startAttributes.SetDefaults(q.DefaultLanguage, q.DefaultDist, q.DefaultGroup, q.DefaultOS, VMTypeDefault)

	return buildJob, nil
}

// Cleanup does not do anything!
func (q *HTTPJobQueue) Cleanup() error {
	return nil
}
<|endoftext|>