<commit_before>// Package upload provides functions for dealing with uploading files in a fast and safe way
package upload

import (
	"bytes"
	"fmt"
	"io"
	"net/http"
	"net/textproto"
	"os"
	"path/filepath"
	"strconv"

	log "github.com/sirupsen/logrus"
	"github.com/xyproto/algernon/utils"
	"github.com/xyproto/gopher-lua"
)

// For dealing with uploaded files in POST method handlers

const (
	// Class is an identifier for the UploadedFile class in Lua
	Class = "UploadedFile"

	// Upload limit, in bytes
	defaultUploadLimit int64 = 32 * utils.MiB

	// Memory usage while uploading
	defaultMemoryLimit int64 = 32 * utils.MiB

	// Chunk size when reading uploaded file
	chunkSize int64 = 4 * utils.KiB
	//chunkSize = defaultMemoryLimit
)

// UploadedFile represents a file that has been uploaded but not yet been
// written to file.
type UploadedFile struct {
	req       *http.Request
	scriptdir string
	header    textproto.MIMEHeader
	filename  string
	buf       *bytes.Buffer
}

// New creates a struct that is used for accepting an uploaded file
//
// The client will send all the data, if the data is over the given size,
// if the Content-Length is wrongly set to a value below the the uploadLimit.
// However, the buffer and memory usage will not grow despite this.
//
// uploadLimit is in bytes.
//
// Note that the client may appear to keep sending the file even when the
// server has stopped receiving it, for files that are too large.
func New(req *http.Request, scriptdir, formID string, uploadLimit int64) (*UploadedFile, error) {

	clientLengthTotal, err := strconv.Atoi(req.Header.Get("Content-Length"))
	if err != nil {
		log.Error("Invalid Content-Length: ", req.Header.Get("Content-Length"))
	}
	// Remove the extra 20 bytes and convert to int64
	clientLength := int64(clientLengthTotal - 20)

	if clientLength > uploadLimit {
		return nil, fmt.Errorf("uploaded file was too large: %s according to Content-Length (current limit is %s)", utils.DescribeBytes(clientLength), utils.DescribeBytes(uploadLimit))
	}

	// For specifying the memory usage when uploading
	if errMem := req.ParseMultipartForm(defaultMemoryLimit); errMem != nil {
		return nil, errMem
	}
	file, handler, err := req.FormFile(formID)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	// Store the data in a buffer, for later usage.
	buf := new(bytes.Buffer)

	// Read the data in chunks
	var totalWritten, writtenBytes, i int64
	for i = 0; i < int64(uploadLimit); i += chunkSize {
		writtenBytes, err = io.CopyN(buf, file, chunkSize)
		totalWritten += writtenBytes
		if totalWritten > uploadLimit {
			// File too large
			return nil, fmt.Errorf("uploaded file was too large: %d bytes (limit is %d bytes)", totalWritten, uploadLimit)
		} else if writtenBytes < chunkSize || err == io.EOF {
			// Done writing
			break
		} else if err != nil {
			// Error when copying data
			return nil, err
		}
	}

	// all ok
	return &UploadedFile{req, scriptdir, handler.Header, handler.Filename, buf}, nil
}

// Get the first argument, "self", and cast it from userdata to
// an UploadedFile, which contains the file data and information.
func checkUploadedFile(L *lua.LState) *UploadedFile {
	ud := L.CheckUserData(1)
	if uploadedfile, ok := ud.Value.(*UploadedFile); ok {
		return uploadedfile
	}
	L.ArgError(1, "UploadedFile expected")
	return nil
}

// Create a new Upload file
func constructUploadedFile(L *lua.LState, req *http.Request, scriptdir, formID string, uploadLimit int64) (*lua.LUserData, error) {
	// Create a new UploadedFile
	uploadedfile, err := New(req, scriptdir, formID, uploadLimit)
	if err != nil {
		return nil, err
	}
	// Create a new userdata struct
	ud := L.NewUserData()
	ud.Value = uploadedfile
	L.SetMetatable(ud, L.GetTypeMetatable(Class))
	return ud, nil
}

// String representation
func uploadedfileToString(L *lua.LState) int {
	L.Push(lua.LString("Uploaded file"))
	return 1 // number of results
}

// File name
func uploadedfileName(L *lua.LState) int {
	ulf := checkUploadedFile(L) // arg 1
	L.Push(lua.LString(ulf.filename))
	return 1 // number of results
}

// File size
func uploadedfileSize(L *lua.LState) int {
	ulf := checkUploadedFile(L) // arg 1
	L.Push(lua.LNumber(ulf.buf.Len()))
	return 1 // number of results
}

// Mime type
func uploadedfileMimeType(L *lua.LState) int {
	ulf := checkUploadedFile(L) // arg 1
	contentType := ""
	if contentTypes, ok := ulf.header["Content-Type"]; ok {
		if len(contentTypes) > 0 {
			contentType = contentTypes[0]
		}
	}
	L.Push(lua.LString(contentType))
	return 1 // number of results
}

// Write the uploaded file to the given full filename.
// Does not overwrite files.
func (ulf *UploadedFile) write(fullFilename string, fperm os.FileMode) error {
	// Check if the file already exists
	if _, err := os.Stat(fullFilename); err == nil { // exists
		log.Error(fullFilename, " already exists")
		return fmt.Errorf("File exists: " + fullFilename)
	}
	// Write the uploaded file
	f, err := os.OpenFile(fullFilename, os.O_WRONLY|os.O_CREATE, fperm)
	if err != nil {
		log.Error("Error when creating ", fullFilename)
		return err
	}
	defer f.Close()
	// Copy the data to a new buffer, to keep the data and the length
	fileDataBuffer := bytes.NewBuffer(ulf.buf.Bytes())
	if _, err := io.Copy(f, fileDataBuffer); err != nil {
		log.Error("Error when writing: " + err.Error())
		return err
	}
	return nil
}

// Save the file locally
func uploadedfileSave(L *lua.LState) int {
	ulf := checkUploadedFile(L) // arg 1
	givenFilename := ""
	if L.GetTop() == 2 {
		givenFilename = L.ToString(2) // optional argument
	}
	// optional argument, file permissions
	var givenPermissions os.FileMode = 0660
	if L.GetTop() == 3 {
		givenPermissions = os.FileMode(L.ToInt(3))
	}

	// Use the given filename instead of the default one, if given
	var filename string
	if givenFilename != "" {
		filename = givenFilename
	} else {
		filename = ulf.filename
	}

	// Get the full path
	writeFilename := filepath.Join(ulf.scriptdir, filename)

	// Write the file and return true if successful
	L.Push(lua.LBool(ulf.write(writeFilename, givenPermissions) == nil))
	return 1 // number of results
}

// Save the file locally, to a given directory
func uploadedfileSaveIn(L *lua.LState) int {
	ulf := checkUploadedFile(L)     // arg 1
	givenDirectory := L.ToString(2) // required argument

	// optional argument, file permissions
	var givenPermissions os.FileMode = 0660
	if L.GetTop() == 3 {
		givenPermissions = os.FileMode(L.ToInt(3))
	}

	// Get the full path
	var writeFilename string
	if filepath.IsAbs(givenDirectory) {
		writeFilename = filepath.Join(givenDirectory, ulf.filename)
	} else {
		writeFilename = filepath.Join(ulf.scriptdir, givenDirectory, ulf.filename)
	}

	// Write the file and return true if successful
	L.Push(lua.LBool(ulf.write(writeFilename, givenPermissions) == nil))
	return 1 // number of results
}

// The hash map methods that are to be registered
var uploadedfileMethods = map[string]lua.LGFunction{
	"__tostring": uploadedfileToString,
	"filename":   uploadedfileName,
	"size":       uploadedfileSize,
	"mimetype":   uploadedfileMimeType,
	"save":       uploadedfileSave,
	"savein":     uploadedfileSaveIn,
}

// Load makes functions related to saving an uploaded file available
func Load(L *lua.LState, w http.ResponseWriter, req *http.Request, scriptdir string) {

	// Register the UploadedFile class and the methods that belongs with it.
	mt := L.NewTypeMetatable(Class)
	mt.RawSetH(lua.LString("__index"), mt)
	L.SetFuncs(mt, uploadedfileMethods)

	// The constructor for the UploadedFile userdata
	// Takes a form ID (string) and an optional file upload limit in MiB
	// (number). Returns the userdata and an empty string on success.
	// Returns nil and an error message on failure.
	L.SetGlobal("UploadedFile", L.NewFunction(func(L *lua.LState) int {
		formID := L.ToString(1)
		if formID == "" {
			L.ArgError(1, "form ID expected")
		}
		uploadLimit := defaultUploadLimit
		if L.GetTop() == 2 {
			uploadLimit = int64(L.ToInt(2)) * utils.MiB // optional upload limit, in MiB
		}
		// Construct a new UploadedFile
		userdata, err := constructUploadedFile(L, req, scriptdir, formID, uploadLimit)
		if err != nil {
			// Log the error
			log.Error(err)

			// Return an invalid UploadedFile object and an error string.
			// It's up to the Lua script to send an error to the client.
			L.Push(lua.LNil)
			L.Push(lua.LString(err.Error()))
			return 2 // Number of returned values
		}

		// Return the Lua UploadedFile object and an empty error string
		L.Push(userdata)
		L.Push(lua.LString(""))
		return 2 // Number of returned values
	}))

}
<commit_msg>Add method to retrieve raw content of an uploaded file.<commit_after>// Package upload provides functions for dealing with uploading files in a fast and safe way
package upload

import (
	"bytes"
	"fmt"
	"io"
	"net/http"
	"net/textproto"
	"os"
	"path/filepath"
	"strconv"

	log "github.com/sirupsen/logrus"
	"github.com/xyproto/algernon/utils"
	"github.com/xyproto/gopher-lua"
)

// For dealing with uploaded files in POST method handlers

const (
	// Class is an identifier for the UploadedFile class in Lua
	Class = "UploadedFile"

	// Upload limit, in bytes
	defaultUploadLimit int64 = 32 * utils.MiB

	// Memory usage while uploading
	defaultMemoryLimit int64 = 32 * utils.MiB

	// Chunk size when reading uploaded file
	chunkSize int64 = 4 * utils.KiB
	//chunkSize = defaultMemoryLimit
)

// UploadedFile represents a file that has been uploaded but not yet been
// written to file.
type UploadedFile struct {
	req       *http.Request
	scriptdir string
	header    textproto.MIMEHeader
	filename  string
	buf       *bytes.Buffer
}

// New creates a struct that is used for accepting an uploaded file
//
// The client will send all the data, if the data is over the given size,
// if the Content-Length is wrongly set to a value below the the uploadLimit.
// However, the buffer and memory usage will not grow despite this.
//
// uploadLimit is in bytes.
//
// Note that the client may appear to keep sending the file even when the
// server has stopped receiving it, for files that are too large.
func New(req *http.Request, scriptdir, formID string, uploadLimit int64) (*UploadedFile, error) {

	clientLengthTotal, err := strconv.Atoi(req.Header.Get("Content-Length"))
	if err != nil {
		log.Error("Invalid Content-Length: ", req.Header.Get("Content-Length"))
	}
	// Remove the extra 20 bytes and convert to int64
	clientLength := int64(clientLengthTotal - 20)

	if clientLength > uploadLimit {
		return nil, fmt.Errorf("uploaded file was too large: %s according to Content-Length (current limit is %s)", utils.DescribeBytes(clientLength), utils.DescribeBytes(uploadLimit))
	}

	// For specifying the memory usage when uploading
	if errMem := req.ParseMultipartForm(defaultMemoryLimit); errMem != nil {
		return nil, errMem
	}
	file, handler, err := req.FormFile(formID)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	// Store the data in a buffer, for later usage.
	buf := new(bytes.Buffer)

	// Read the data in chunks
	var totalWritten, writtenBytes, i int64
	for i = 0; i < int64(uploadLimit); i += chunkSize {
		writtenBytes, err = io.CopyN(buf, file, chunkSize)
		totalWritten += writtenBytes
		if totalWritten > uploadLimit {
			// File too large
			return nil, fmt.Errorf("uploaded file was too large: %d bytes (limit is %d bytes)", totalWritten, uploadLimit)
		} else if writtenBytes < chunkSize || err == io.EOF {
			// Done writing
			break
		} else if err != nil {
			// Error when copying data
			return nil, err
		}
	}

	// all ok
	return &UploadedFile{req, scriptdir, handler.Header, handler.Filename, buf}, nil
}

// Get the first argument, "self", and cast it from userdata to
// an UploadedFile, which contains the file data and information.
func checkUploadedFile(L *lua.LState) *UploadedFile {
	ud := L.CheckUserData(1)
	if uploadedfile, ok := ud.Value.(*UploadedFile); ok {
		return uploadedfile
	}
	L.ArgError(1, "UploadedFile expected")
	return nil
}

// Create a new Upload file
func constructUploadedFile(L *lua.LState, req *http.Request, scriptdir, formID string, uploadLimit int64) (*lua.LUserData, error) {
	// Create a new UploadedFile
	uploadedfile, err := New(req, scriptdir, formID, uploadLimit)
	if err != nil {
		return nil, err
	}
	// Create a new userdata struct
	ud := L.NewUserData()
	ud.Value = uploadedfile
	L.SetMetatable(ud, L.GetTypeMetatable(Class))
	return ud, nil
}

// String representation
func uploadedfileToString(L *lua.LState) int {
	L.Push(lua.LString("Uploaded file"))
	return 1 // number of results
}

// File name
func uploadedfileName(L *lua.LState) int {
	ulf := checkUploadedFile(L) // arg 1
	L.Push(lua.LString(ulf.filename))
	return 1 // number of results
}

// File size
func uploadedfileSize(L *lua.LState) int {
	ulf := checkUploadedFile(L) // arg 1
	L.Push(lua.LNumber(ulf.buf.Len()))
	return 1 // number of results
}

// Mime type
func uploadedfileMimeType(L *lua.LState) int {
	ulf := checkUploadedFile(L) // arg 1
	contentType := ""
	if contentTypes, ok := ulf.header["Content-Type"]; ok {
		if len(contentTypes) > 0 {
			contentType = contentTypes[0]
		}
	}
	L.Push(lua.LString(contentType))
	return 1 // number of results
}

// Write the uploaded file to the given full filename.
// Does not overwrite files.
func (ulf *UploadedFile) write(fullFilename string, fperm os.FileMode) error {
	// Check if the file already exists
	if _, err := os.Stat(fullFilename); err == nil { // exists
		log.Error(fullFilename, " already exists")
		return fmt.Errorf("File exists: " + fullFilename)
	}
	// Write the uploaded file
	f, err := os.OpenFile(fullFilename, os.O_WRONLY|os.O_CREATE, fperm)
	if err != nil {
		log.Error("Error when creating ", fullFilename)
		return err
	}
	defer f.Close()
	// Copy the data to a new buffer, to keep the data and the length
	fileDataBuffer := bytes.NewBuffer(ulf.buf.Bytes())
	if _, err := io.Copy(f, fileDataBuffer); err != nil {
		log.Error("Error when writing: " + err.Error())
		return err
	}
	return nil
}

// Save the file locally
func uploadedfileSave(L *lua.LState) int {
	ulf := checkUploadedFile(L) // arg 1
	givenFilename := ""
	if L.GetTop() == 2 {
		givenFilename = L.ToString(2) // optional argument
	}
	// optional argument, file permissions
	var givenPermissions os.FileMode = 0660
	if L.GetTop() == 3 {
		givenPermissions = os.FileMode(L.ToInt(3))
	}

	// Use the given filename instead of the default one, if given
	var filename string
	if givenFilename != "" {
		filename = givenFilename
	} else {
		filename = ulf.filename
	}

	// Get the full path
	writeFilename := filepath.Join(ulf.scriptdir, filename)

	// Write the file and return true if successful
	L.Push(lua.LBool(ulf.write(writeFilename, givenPermissions) == nil))
	return 1 // number of results
}

// Save the file locally, to a given directory
func uploadedfileSaveIn(L *lua.LState) int {
	ulf := checkUploadedFile(L)     // arg 1
	givenDirectory := L.ToString(2) // required argument

	// optional argument, file permissions
	var givenPermissions os.FileMode = 0660
	if L.GetTop() == 3 {
		givenPermissions = os.FileMode(L.ToInt(3))
	}

	// Get the full path
	var writeFilename string
	if filepath.IsAbs(givenDirectory) {
		writeFilename = filepath.Join(givenDirectory, ulf.filename)
	} else {
		writeFilename = filepath.Join(ulf.scriptdir, givenDirectory, ulf.filename)
	}

	// Write the file and return true if successful
	L.Push(lua.LBool(ulf.write(writeFilename, givenPermissions) == nil))
	return 1 // number of results
}

// Retrieve the file content
func uploadedfileGet(L *lua.LState) int {
	ulf := checkUploadedFile(L) // arg 1
	L.Push(lua.LString(ulf.buf.Bytes()))
	return 1 // number of results
}

// The hash map methods that are to be registered
var uploadedfileMethods = map[string]lua.LGFunction{
	"__tostring": uploadedfileToString,
	"filename":   uploadedfileName,
	"size":       uploadedfileSize,
	"mimetype":   uploadedfileMimeType,
	"save":       uploadedfileSave,
	"savein":     uploadedfileSaveIn,
	"content":    uploadedfileGet,
}

// Load makes functions related to saving an uploaded file available
func Load(L *lua.LState, w http.ResponseWriter, req *http.Request, scriptdir string) {

	// Register the UploadedFile class and the methods that belongs with it.
	mt := L.NewTypeMetatable(Class)
	mt.RawSetH(lua.LString("__index"), mt)
	L.SetFuncs(mt, uploadedfileMethods)

	// The constructor for the UploadedFile userdata
	// Takes a form ID (string) and an optional file upload limit in MiB
	// (number). Returns the userdata and an empty string on success.
	// Returns nil and an error message on failure.
	L.SetGlobal("UploadedFile", L.NewFunction(func(L *lua.LState) int {
		formID := L.ToString(1)
		if formID == "" {
			L.ArgError(1, "form ID expected")
		}
		uploadLimit := defaultUploadLimit
		if L.GetTop() == 2 {
			uploadLimit = int64(L.ToInt(2)) * utils.MiB // optional upload limit, in MiB
		}
		// Construct a new UploadedFile
		userdata, err := constructUploadedFile(L, req, scriptdir, formID, uploadLimit)
		if err != nil {
			// Log the error
			log.Error(err)

			// Return an invalid UploadedFile object and an error string.
			// It's up to the Lua script to send an error to the client.
			L.Push(lua.LNil)
			L.Push(lua.LString(err.Error()))
			return 2 // Number of returned values
		}

		// Return the Lua UploadedFile object and an empty error string
		L.Push(userdata)
		L.Push(lua.LString(""))
		return 2 // Number of returned values
	}))

}
<|endoftext|>


<commit_before>package query

import (
	"context"
	"database/sql"
	"fmt"
	"sort"
	"strconv"
	"strings"
	"time"
)

// Dump returns a SQL text dump of all rows across all tables, similar to
// sqlite3's dump feature
func Dump(ctx context.Context, tx *sql.Tx, schema string, schemaOnly bool) (string, error) {
	schemas := dumpParseSchema(schema)

	// Begin
	dump := `PRAGMA foreign_keys=OFF;
BEGIN TRANSACTION;
`
	// Schema table
	tableDump, err := dumpTable(ctx, tx, "schema", dumpSchemaTable)
	if err != nil {
		return "", fmt.Errorf("Failed to dump table schema: %w", err)
	}
	dump += tableDump

	// All other tables
	tables := make([]string, 0)
	for table := range schemas {
		tables = append(tables, table)
	}
	sort.Strings(tables)
	for _, table := range tables {
		if schemaOnly {
			// Dump only the schema.
			dump += schemas[table] + "\n"
			continue
		}

		tableDump, err := dumpTable(ctx, tx, table, schemas[table])
		if err != nil {
			return "", fmt.Errorf("Failed to dump table %s: %w", table, err)
		}

		dump += tableDump
	}

	// Sequences (unless the schemaOnly flag is true)
	if !schemaOnly {
		tableDump, err = dumpTable(ctx, tx, "sqlite_sequence", "DELETE FROM sqlite_sequence;")
		if err != nil {
			return "", fmt.Errorf("Failed to dump table sqlite_sequence: %w", err)
		}

		dump += tableDump
	}

	// Commit
	dump += "COMMIT;\n"

	return dump, nil
}

// Return a map from table names to their schema definition, taking a full
// schema SQL text generated with schema.Schema.Dump().
func dumpParseSchema(schema string) map[string]string {
	tables := map[string]string{}
	for _, statement := range strings.Split(schema, ";") {
		statement = strings.Trim(statement, " \n") + ";"
		if !strings.HasPrefix(statement, "CREATE TABLE") {
			continue
		}
		table := strings.Split(statement, " ")[2]
		tables[table] = statement
	}
	return tables
}

// Dump a single table, returning a SQL text containing statements for its
// schema and data.
func dumpTable(ctx context.Context, tx *sql.Tx, table, schema string) (string, error) {
	statements := []string{schema}

	// Query all rows.
	rows, err := tx.QueryContext(ctx, fmt.Sprintf("SELECT * FROM %s ORDER BY rowid", table))
	if err != nil {
		return "", fmt.Errorf("failed to fetch rows: %w", err)
	}
	defer rows.Close()

	// Figure column names
	columns, err := rows.Columns()
	if err != nil {
		return "", fmt.Errorf("failed to get columns: %w", err)
	}

	// Generate an INSERT statement for each row.
	for i := 0; rows.Next(); i++ {
		raw := make([]any, len(columns)) // Raw column values
		row := make([]any, len(columns))
		for i := range raw {
			row[i] = &raw[i]
		}
		err := rows.Scan(row...)
		if err != nil {
			return "", fmt.Errorf("failed to scan row %d: %w", i, err)
		}
		values := make([]string, len(columns))
		for j, v := range raw {
			switch v := v.(type) {
			case int64:
				values[j] = strconv.FormatInt(v, 10)
			case string:
				values[j] = fmt.Sprintf("'%s'", v)
			case []byte:
				values[j] = fmt.Sprintf("'%s'", string(v))
			case time.Time:
				values[j] = strconv.FormatInt(v.Unix(), 10)
			default:
				if v != nil {
					return "", fmt.Errorf("bad type in column %s of row %d", columns[j], i)
				}
				values[j] = "NULL"
			}
		}
		statement := fmt.Sprintf("INSERT INTO %s VALUES(%s);", table, strings.Join(values, ","))
		statements = append(statements, statement)
	}
	return strings.Join(statements, "\n") + "\n", nil
}

// Schema of the schema table.
const dumpSchemaTable = `CREATE TABLE schema (
    id         INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    version    INTEGER NOT NULL,
    updated_at DATETIME NOT NULL,
    UNIQUE (version)
);`
<commit_msg>lxd/db/query: Gets both schema and data from sqlite.<commit_after>package query

import (
	"context"
	"database/sql"
	"fmt"
	"strconv"
	"strings"
	"time"
)

// Dump returns a SQL text dump of all rows across all tables, similar to
// sqlite3's dump feature.
func Dump(ctx context.Context, tx *sql.Tx, schemaOnly bool) (string, error) {
	tablesSchemas, tableNames, err := getTablesSchemas(ctx, tx)
	if err != nil {
		return "", err
	}

	// Begin dump string.
	var builder strings.Builder
	builder.WriteString("PRAGMA foreign_keys=OFF;\n")
	builder.WriteString("BEGIN TRANSACTION;\n")

	// For each table, write the schema and optionally write the data.
	for _, tableName := range tableNames {
		builder.WriteString(tablesSchemas[tableName] + "\n")

		if !schemaOnly {
			tableData, err := getTableData(ctx, tx, tableName)
			if err != nil {
				return "", err
			}

			for _, stmt := range tableData {
				builder.WriteString(stmt + "\n")
			}
		}
	}

	// Sequences (unless the schemaOnly flag is true).
	if !schemaOnly {
		builder.WriteString("DELETE FROM sqlite_sequence;\n")

		tableData, err := getTableData(ctx, tx, "sqlite_sequence")
		if err != nil {
			return "", fmt.Errorf("Failed to dump table sqlite_sequence: %w", err)
		}

		for _, stmt := range tableData {
			builder.WriteString(stmt + "\n")
		}
	}

	// Commit.
	builder.WriteString("COMMIT;\n")

	return builder.String(), nil
}

// getTablesSchemas gets all the tables and their schema, as well as a list of table names in their default order from
// the sqlite_master table.
func getTablesSchemas(ctx context.Context, tx *sql.Tx) (map[string]string, []string, error) {
	rows, err := tx.QueryContext(ctx, `SELECT name, sql FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY rowid`)
	if err != nil {
		return nil, nil, fmt.Errorf("Could not get table names and their schema: %w", err)
	}

	defer rows.Close()

	tablesSchemas := make(map[string]string)
	var names []string
	for rows.Next() {
		var name string
		var schema string
		err := rows.Scan(&name, &schema)
		if err != nil {
			return nil, nil, fmt.Errorf("Could not scan table name and schema: %w", err)
		}

		names = append(names, name)

		// Whether a table name is quoted or not can depend on if it was quoted when originally created, or if it
		// collides with a keyword (and maybe more). Regardless, sqlite3 quotes table names in create statements when
		// executing a dump. If the table name is already quoted, add the "IF NOT EXISTS" clause, else quote it and add
		// the same clause.
		isQuoted := strings.Contains(schema, fmt.Sprintf("TABLE %q", name))
		if isQuoted {
			schema = strings.Replace(schema, "TABLE", "TABLE IF NOT EXISTS", 1)
		} else {
			schema = strings.Replace(schema, name, fmt.Sprintf("IF NOT EXISTS %q", name), 1)
		}

		tablesSchemas[name] = schema + ";"
	}

	return tablesSchemas, names, nil
}

// getTableData gets all the data for a single table, returning a string slice where each element is an insert statement
// for the data.
func getTableData(ctx context.Context, tx *sql.Tx, table string) ([]string, error) {
	var statements []string

	// Query all rows.
	rows, err := tx.QueryContext(ctx, fmt.Sprintf("SELECT * FROM %s ORDER BY rowid", table))
	if err != nil {
		return nil, fmt.Errorf("Failed to fetch rows for table %q: %w", table, err)
	}

	defer rows.Close()

	// Get the column names.
	columns, err := rows.Columns()
	if err != nil {
		return nil, fmt.Errorf("Failed to get columns for table %q: %w", table, err)
	}

	// Generate an INSERT statement for each row.
	for i := 0; rows.Next(); i++ {
		raw := make([]any, len(columns)) // Raw column values
		row := make([]any, len(columns))
		for i := range raw {
			row[i] = &raw[i]
		}

		err := rows.Scan(row...)
		if err != nil {
			return nil, fmt.Errorf("Failed to scan row %d in table %q: %w", i, table, err)
		}

		values := make([]string, len(columns))
		for j, v := range raw {
			switch v := v.(type) {
			case int64:
				values[j] = strconv.FormatInt(v, 10)
			case string:
				values[j] = fmt.Sprintf("'%s'", v)
			case []byte:
				values[j] = fmt.Sprintf("'%s'", string(v))
			case time.Time:
				values[j] = strconv.FormatInt(v.Unix(), 10)
			default:
				if v != nil {
					return nil, fmt.Errorf("Bad type in column %q of row %d in table %q", columns[j], i, table)
				}

				values[j] = "NULL"
			}
		}

		statement := fmt.Sprintf("INSERT INTO %s VALUES(%s);", table, strings.Join(values, ","))
		statements = append(statements, statement)
	}

	return statements, nil
}
<|endoftext|>


<commit_before>package events

import (
	"context"
	"sync"
	"time"

	"github.com/gorilla/websocket"
	log "gopkg.in/inconshreveable/log15.v2"

	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/logger"
)

// EventHandler called when the connection receives an event from the client.
type EventHandler func(event api.Event)

// serverCommon represents an instance of a comon event server.
type serverCommon struct {
	debug   bool
	verbose bool
	lock    sync.Mutex
}

// listenerCommon describes a common event listener.
type listenerCommon struct {
	*websocket.Conn

	messageTypes []string
	ctx          context.Context
	ctxCancel    func()
	id           string
	lock         sync.Mutex
	pongsPending uint
	recvFunc     EventHandler

	// If true, this listener won't get events forwarded from other
	// nodes. It only used by listeners created internally by LXD nodes
	// connecting to other LXD nodes to get their local events only.
	localOnly bool
}

func (e *listenerCommon) heartbeat() {
	logger.Debug("Event listener server handler started", log.Ctx{"listener": e.ID(), "local": e.Conn.LocalAddr(), "remote": e.Conn.RemoteAddr(), "localOnly": e.localOnly})

	defer e.Close()

	pingInterval := time.Second * 10
	e.pongsPending = 0

	e.SetPongHandler(func(msg string) error {
		e.lock.Lock()
		e.pongsPending = 0
		e.lock.Unlock()
		return nil
	})

	// Start reader from client.
	go func() {
		defer e.Close()

		if e.recvFunc != nil {
			for {
				var event api.Event
				err := e.Conn.ReadJSON(&event)
				if err != nil {
					return // This detects if client has disconnected or sent invalid data.
				}

				// Pass received event to the handler.
				e.recvFunc(event)
			}
		} else {
			// Run a blocking reader to detect if the client has disconnected. We don't expect to get
			// anything from the remote side, so this should remain blocked until disconnected.
			e.Conn.NextReader()
		}
	}()

	for {
		if e.IsClosed() {
			return
		}

		e.lock.Lock()
		if e.pongsPending > 2 {
			e.lock.Unlock()
			logger.Warn("Hearbeat for event listener handler timed out", log.Ctx{"listener": e.ID(), "local": e.Conn.LocalAddr(), "remote": e.Conn.RemoteAddr(), "localOnly": e.localOnly})
			return
		}
		err := e.WriteControl(websocket.PingMessage, []byte("keepalive"), time.Now().Add(5*time.Second))
		if err != nil {
			e.lock.Unlock()
			return
		}

		e.pongsPending++
		e.lock.Unlock()

		select {
		case <-time.After(pingInterval):
		case <-e.ctx.Done():
			return
		}
	}
}

// IsClosed returns true if the listener is closed.
func (e *listenerCommon) IsClosed() bool {
	return e.ctx.Err() != nil
}

// ID returns the listener ID.
func (e *listenerCommon) ID() string {
	return e.id
}

// Wait waits for a message on its active channel or the context is cancelled, then returns.
func (e *listenerCommon) Wait(ctx context.Context) {
	select {
	case <-ctx.Done():
	case <-e.ctx.Done():
	}
}

// Close Disconnects the listener.
func (e *listenerCommon) Close() {
	e.lock.Lock()
	defer e.lock.Unlock()

	if e.IsClosed() {
		return
	}

	logger.Debug("Event listener server handler stopped", log.Ctx{"listener": e.ID(), "local": e.Conn.LocalAddr(), "remote": e.Conn.RemoteAddr(), "localOnly": e.localOnly})

	e.Conn.Close()
	e.ctxCancel()
}

// WriteJSON message to the connection.
func (e *listenerCommon) WriteJSON(v interface{}) error {
	e.lock.Lock()
	defer e.lock.Unlock()

	return e.Conn.WriteJSON(v)
}

// WriteMessage to the connection.
func (e *listenerCommon) WriteMessage(messageType int, data []byte) error {
	e.lock.Lock()
	defer e.lock.Unlock()

	return e.Conn.WriteMessage(messageType, data)
}
<commit_msg>lxd/events/common: Removes localOnly concept from common listener<commit_after>package events

import (
	"context"
	"sync"
	"time"

	"github.com/gorilla/websocket"
	log "gopkg.in/inconshreveable/log15.v2"

	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/logger"
)

// EventHandler called when the connection receives an event from the client.
type EventHandler func(event api.Event)

// serverCommon represents an instance of a comon event server.
type serverCommon struct {
	debug   bool
	verbose bool
	lock    sync.Mutex
}

// listenerCommon describes a common event listener.
type listenerCommon struct {
	*websocket.Conn

	messageTypes []string
	ctx          context.Context
	ctxCancel    func()
	id           string
	lock         sync.Mutex
	pongsPending uint
	recvFunc     EventHandler
}

func (e *listenerCommon) heartbeat() {
	logger.Debug("Event listener server handler started", log.Ctx{"listener": e.ID(), "local": e.Conn.LocalAddr(), "remote": e.Conn.RemoteAddr()})

	defer e.Close()

	pingInterval := time.Second * 10
	e.pongsPending = 0

	e.SetPongHandler(func(msg string) error {
		e.lock.Lock()
		e.pongsPending = 0
		e.lock.Unlock()
		return nil
	})

	// Start reader from client.
	go func() {
		defer e.Close()

		if e.recvFunc != nil {
			for {
				var event api.Event
				err := e.Conn.ReadJSON(&event)
				if err != nil {
					return // This detects if client has disconnected or sent invalid data.
				}

				// Pass received event to the handler.
				e.recvFunc(event)
			}
		} else {
			// Run a blocking reader to detect if the client has disconnected. We don't expect to get
			// anything from the remote side, so this should remain blocked until disconnected.
			e.Conn.NextReader()
		}
	}()

	for {
		if e.IsClosed() {
			return
		}

		e.lock.Lock()
		if e.pongsPending > 2 {
			e.lock.Unlock()
			logger.Warn("Hearbeat for event listener handler timed out", log.Ctx{"listener": e.ID(), "local": e.Conn.LocalAddr(), "remote": e.Conn.RemoteAddr()})
			return
		}
		err := e.WriteControl(websocket.PingMessage, []byte("keepalive"), time.Now().Add(5*time.Second))
		if err != nil {
			e.lock.Unlock()
			return
		}

		e.pongsPending++
		e.lock.Unlock()

		select {
		case <-time.After(pingInterval):
		case <-e.ctx.Done():
			return
		}
	}
}

// IsClosed returns true if the listener is closed.
func (e *listenerCommon) IsClosed() bool {
	return e.ctx.Err() != nil
}

// ID returns the listener ID.
func (e *listenerCommon) ID() string {
	return e.id
}

// Wait waits for a message on its active channel or the context is cancelled, then returns.
func (e *listenerCommon) Wait(ctx context.Context) {
	select {
	case <-ctx.Done():
	case <-e.ctx.Done():
	}
}

// Close Disconnects the listener.
func (e *listenerCommon) Close() {
	e.lock.Lock()
	defer e.lock.Unlock()

	if e.IsClosed() {
		return
	}

	logger.Debug("Event listener server handler stopped", log.Ctx{"listener": e.ID(), "local": e.Conn.LocalAddr(), "remote": e.Conn.RemoteAddr()})

	e.Conn.Close()
	e.ctxCancel()
}

// WriteJSON message to the connection.
func (e *listenerCommon) WriteJSON(v interface{}) error {
	e.lock.Lock()
	defer e.lock.Unlock()

	return e.Conn.WriteJSON(v)
}

// WriteMessage to the connection.
func (e *listenerCommon) WriteMessage(messageType int, data []byte) error {
	e.lock.Lock()
	defer e.lock.Unlock()

	return e.Conn.WriteMessage(messageType, data)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"sort"
	"strconv"
	"strings"

	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/lxd/state"
	storageDrivers "github.com/lxc/lxd/lxd/storage/drivers"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/units"
)

// For 'btrfs' storage backend.
func btrfsSubVolumeCreate(subvol string) error {
	parentDestPath := filepath.Dir(subvol)
	if !shared.PathExists(parentDestPath) {
		err := os.MkdirAll(parentDestPath, 0711)
		if err != nil {
			return err
		}
	}

	_, err := shared.RunCommand(
		"btrfs",
		"subvolume",
		"create",
		subvol)
	if err != nil {
		return err
	}

	return nil
}

func btrfsSubVolumeQGroup(subvol string) (string, error) {
	output, err := shared.RunCommand(
		"btrfs",
		"qgroup",
		"show",
		"-e",
		"-f",
		subvol)

	if err != nil {
		return "", fmt.Errorf("Quotas disabled on filesystem")
	}

	var qgroup string
	for _, line := range strings.Split(output, "\n") {
		if line == "" || strings.HasPrefix(line, "qgroupid") || strings.HasPrefix(line, "---") {
			continue
		}

		fields := strings.Fields(line)
		if len(fields) != 4 {
			continue
		}

		qgroup = fields[0]
	}

	if qgroup == "" {
		return "", fmt.Errorf("Unable to find quota group")
	}

	return qgroup, nil
}

func btrfsSubVolumeDelete(subvol string) error {
	// Attempt (but don't fail on) to delete any qgroup on the subvolume
	qgroup, err := btrfsSubVolumeQGroup(subvol)
	if err == nil {
		shared.RunCommand(
			"btrfs",
			"qgroup",
			"destroy",
			qgroup,
			subvol)
	}

	// Attempt to make the subvolume writable
	shared.RunCommand("btrfs", "property", "set", subvol, "ro", "false")

	// Delete the subvolume itself
	_, err = shared.RunCommand(
		"btrfs",
		"subvolume",
		"delete",
		subvol)

	return err
}

func btrfsSubVolumesDelete(subvol string) error {
	// Delete subsubvols.
	subsubvols, err := storageDrivers.BTRFSSubVolumesGet(subvol)
	if err != nil {
		return err
	}
	sort.Sort(sort.Reverse(sort.StringSlice(subsubvols)))

	for _, subsubvol := range subsubvols {
		err := btrfsSubVolumeDelete(path.Join(subvol, subsubvol))
		if err != nil {
			return err
		}
	}

	// Delete the subvol itself
	err = btrfsSubVolumeDelete(subvol)
	if err != nil {
		return err
	}

	return nil
}

func btrfsSnapshot(s *state.State, source string, dest string, readonly bool) error {
	var output string
	var err error
	if readonly && !s.OS.RunningInUserNS {
		output, err = shared.RunCommand(
			"btrfs",
			"subvolume",
			"snapshot",
			"-r",
			source,
			dest)
	} else {
		output, err = shared.RunCommand(
			"btrfs",
			"subvolume",
			"snapshot",
			source,
			dest)
	}
	if err != nil {
		return fmt.Errorf(
			"subvolume snapshot failed, source=%s, dest=%s, output=%s",
			source,
			dest,
			output,
		)
	}

	return err
}

// For 'lvm' storage backend.
func lvmLVRename(vgName string, oldName string, newName string) error {
	_, err := shared.TryRunCommand("lvrename", vgName, oldName, newName)
	if err != nil {
		return fmt.Errorf("could not rename volume group from \"%s\" to \"%s\": %v", oldName, newName, err)
	}

	return nil
}

func lvmLVExists(lvName string) (bool, error) {
	_, err := shared.RunCommand("lvs", "--noheadings", "-o", "lv_attr", lvName)
	if err != nil {
		runErr, ok := err.(shared.RunError)
		if ok {
			exitError, ok := runErr.Err.(*exec.ExitError)
			if ok {
				if exitError.ExitCode() == 5 {
					// logical volume not found
					return false, nil
				}
			}
		}

		return false, fmt.Errorf("error checking for logical volume \"%s\"", lvName)
	}

	return true, nil
}

func lvmVGActivate(lvmVolumePath string) error {
	_, err := shared.TryRunCommand("vgchange", "-ay", lvmVolumePath)
	if err != nil {
		return fmt.Errorf("could not activate volume group \"%s\": %v", lvmVolumePath, err)
	}

	return nil
}

func lvmNameToLVName(containerName string) string {
	lvName := strings.Replace(containerName, "-", "--", -1)
	return strings.Replace(lvName, shared.SnapshotDelimiter, "-", -1)
}

func lvmDevPath(projectName, lvmPool string, volumeType string, lvmVolume string) string {
	lvmVolume = project.Instance(projectName, lvmVolume)
	if volumeType == "" {
		return fmt.Sprintf("/dev/%s/%s", lvmPool, lvmVolume)
	}

	return fmt.Sprintf("/dev/%s/%s_%s", lvmPool, volumeType, lvmVolume)
}

func lvmGetLVSize(lvPath string) (string, error) {
	msg, err := shared.TryRunCommand("lvs", "--noheadings", "-o", "size", "--nosuffix", "--units", "b", lvPath)
	if err != nil {
		return "", fmt.Errorf("failed to retrieve size of logical volume: %s: %s", string(msg), err)
	}

	sizeString := string(msg)
	sizeString = strings.TrimSpace(sizeString)
	size, err := strconv.ParseInt(sizeString, 10, 64)
	if err != nil {
		return "", err
	}

	detectedSize := units.GetByteSizeString(size, 0)

	return detectedSize, nil
}

// volumeFillDefault fills default settings into a volume config.
// Deprecated. Please use FillInstanceConfig() on the storage pool.
func volumeFillDefault(config map[string]string, parentPool *api.StoragePool) error {
	if parentPool.Driver == "lvm" || parentPool.Driver == "ceph" {
		if config["block.filesystem"] == "" {
			config["block.filesystem"] = parentPool.Config["volume.block.filesystem"]
		}
		if config["block.filesystem"] == "" {
			// Unchangeable volume property: Set unconditionally.
			config["block.filesystem"] = storageDrivers.DefaultFilesystem
		}

		if config["block.mount_options"] == "" {
			config["block.mount_options"] = parentPool.Config["volume.block.mount_options"]
		}
		if config["block.mount_options"] == "" {
			// Unchangeable volume property: Set unconditionally.
			config["block.mount_options"] = "discard"
		}
	}

	return nil
}
<commit_msg>lxd/patches/utils: Adds containerMountPoint as deprecated legacy function<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"sort"
	"strconv"
	"strings"

	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/lxd/state"
	storageDrivers "github.com/lxc/lxd/lxd/storage/drivers"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/units"
)

// For 'btrfs' storage backend.
func btrfsSubVolumeCreate(subvol string) error {
	parentDestPath := filepath.Dir(subvol)
	if !shared.PathExists(parentDestPath) {
		err := os.MkdirAll(parentDestPath, 0711)
		if err != nil {
			return err
		}
	}

	_, err := shared.RunCommand(
		"btrfs",
		"subvolume",
		"create",
		subvol)
	if err != nil {
		return err
	}

	return nil
}

func btrfsSubVolumeQGroup(subvol string) (string, error) {
	output, err := shared.RunCommand(
		"btrfs",
		"qgroup",
		"show",
		"-e",
		"-f",
		subvol)

	if err != nil {
		return "", fmt.Errorf("Quotas disabled on filesystem")
	}

	var qgroup string
	for _, line := range strings.Split(output, "\n") {
		if line == "" || strings.HasPrefix(line, "qgroupid") || strings.HasPrefix(line, "---") {
			continue
		}

		fields := strings.Fields(line)
		if len(fields) != 4 {
			continue
		}

		qgroup = fields[0]
	}

	if qgroup == "" {
		return "", fmt.Errorf("Unable to find quota group")
	}

	return qgroup, nil
}

func btrfsSubVolumeDelete(subvol string) error {
	// Attempt (but don't fail on) to delete any qgroup on the subvolume
	qgroup, err := btrfsSubVolumeQGroup(subvol)
	if err == nil {
		shared.RunCommand(
			"btrfs",
			"qgroup",
			"destroy",
			qgroup,
			subvol)
	}

	// Attempt to make the subvolume writable
	shared.RunCommand("btrfs", "property", "set", subvol, "ro", "false")

	// Delete the subvolume itself
	_, err = shared.RunCommand(
		"btrfs",
		"subvolume",
		"delete",
		subvol)

	return err
}

func btrfsSubVolumesDelete(subvol string) error {
	// Delete subsubvols.
	subsubvols, err := storageDrivers.BTRFSSubVolumesGet(subvol)
	if err != nil {
		return err
	}
	sort.Sort(sort.Reverse(sort.StringSlice(subsubvols)))

	for _, subsubvol := range subsubvols {
		err := btrfsSubVolumeDelete(path.Join(subvol, subsubvol))
		if err != nil {
			return err
		}
	}

	// Delete the subvol itself
	err = btrfsSubVolumeDelete(subvol)
	if err != nil {
		return err
	}

	return nil
}

func btrfsSnapshot(s *state.State, source string, dest string, readonly bool) error {
	var output string
	var err error
	if readonly && !s.OS.RunningInUserNS {
		output, err = shared.RunCommand(
			"btrfs",
			"subvolume",
			"snapshot",
			"-r",
			source,
			dest)
	} else {
		output, err = shared.RunCommand(
			"btrfs",
			"subvolume",
			"snapshot",
			source,
			dest)
	}
	if err != nil {
		return fmt.Errorf(
			"subvolume snapshot failed, source=%s, dest=%s, output=%s",
			source,
			dest,
			output,
		)
	}

	return err
}

// For 'lvm' storage backend.
func lvmLVRename(vgName string, oldName string, newName string) error {
	_, err := shared.TryRunCommand("lvrename", vgName, oldName, newName)
	if err != nil {
		return fmt.Errorf("could not rename volume group from \"%s\" to \"%s\": %v", oldName, newName, err)
	}

	return nil
}

func lvmLVExists(lvName string) (bool, error) {
	_, err := shared.RunCommand("lvs", "--noheadings", "-o", "lv_attr", lvName)
	if err != nil {
		runErr, ok := err.(shared.RunError)
		if ok {
			exitError, ok := runErr.Err.(*exec.ExitError)
			if ok {
				if exitError.ExitCode() == 5 {
					// logical volume not found
					return false, nil
				}
			}
		}

		return false, fmt.Errorf("error checking for logical volume \"%s\"", lvName)
	}

	return true, nil
}

func lvmVGActivate(lvmVolumePath string) error {
	_, err := shared.TryRunCommand("vgchange", "-ay", lvmVolumePath)
	if err != nil {
		return fmt.Errorf("could not activate volume group \"%s\": %v", lvmVolumePath, err)
	}

	return nil
}

func lvmNameToLVName(containerName string) string {
	lvName := strings.Replace(containerName, "-", "--", -1)
	return strings.Replace(lvName, shared.SnapshotDelimiter, "-", -1)
}

func lvmDevPath(projectName, lvmPool string, volumeType string, lvmVolume string) string {
	lvmVolume = project.Instance(projectName, lvmVolume)
	if volumeType == "" {
		return fmt.Sprintf("/dev/%s/%s", lvmPool, lvmVolume)
	}

	return fmt.Sprintf("/dev/%s/%s_%s", lvmPool, volumeType, lvmVolume)
}

func lvmGetLVSize(lvPath string) (string, error) {
	msg, err := shared.TryRunCommand("lvs", "--noheadings", "-o", "size", "--nosuffix", "--units", "b", lvPath)
	if err != nil {
		return "", fmt.Errorf("failed to retrieve size of logical volume: %s: %s", string(msg), err)
	}

	sizeString := string(msg)
	sizeString = strings.TrimSpace(sizeString)
	size, err := strconv.ParseInt(sizeString, 10, 64)
	if err != nil {
		return "", err
	}

	detectedSize := units.GetByteSizeString(size, 0)

	return detectedSize, nil
}

// volumeFillDefault fills default settings into a volume config.
// Deprecated. Please use FillInstanceConfig() on the storage pool.
func volumeFillDefault(config map[string]string, parentPool *api.StoragePool) error {
	if parentPool.Driver == "lvm" || parentPool.Driver == "ceph" {
		if config["block.filesystem"] == "" {
			config["block.filesystem"] = parentPool.Config["volume.block.filesystem"]
		}
		if config["block.filesystem"] == "" {
			// Unchangeable volume property: Set unconditionally.
			config["block.filesystem"] = storageDrivers.DefaultFilesystem
		}

		if config["block.mount_options"] == "" {
			config["block.mount_options"] = parentPool.Config["volume.block.mount_options"]
		}
		if config["block.mount_options"] == "" {
			// Unchangeable volume property: Set unconditionally.
			config["block.mount_options"] = "discard"
		}
	}

	return nil
}

// containerMountPoint returns the mountpoint of the given container.
// ${LXD_DIR}/storage-pools/<pool>/containers/[<project_name>_]<container_name>
// Deprecated.
func containerMountPoint(projectName string, poolName string, containerName string) string {
	return shared.VarPath("storage-pools", poolName, "containers", project.Instance(projectName, containerName))
}
<|endoftext|>


<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package common_test

import (
	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/state/apiserver/common"
)

type stateAddresserSuite struct {
	addresser *common.StateAddresser
}

type apiAddresserSuite struct {
	addresser *common.APIAddresser
}

var _ = gc.Suite(&stateAddresserSuite{})
var _ = gc.Suite(&apiAddresserSuite{})

func (s *stateAddresserSuite) SetUpTest(c *gc.C) {
	s.addresser = common.NewStateAddresser(fakeAddresses{})
}

// Verify that AddressAndCertGetter is satisfied by *state.State.
var _ common.AddressAndCertGetter = (*state.State)(nil)

func (s *stateAddresserSuite) TestStateAddresses(c *gc.C) {
	result, err := s.addresser.StateAddresses()
	c.Assert(err, gc.IsNil)
	c.Assert(result.Result, gc.DeepEquals, []string{"addresses:1", "addresses:2"})
}

func (s *stateAddresserSuite) TestCACert(c *gc.C) {
	result := s.addresser.CACert()
	c.Assert(string(result.Result), gc.Equals, "a cert")
}

func (s *apiAddresserSuite) SetUpTest(c *gc.C) {
	s.addresser = common.NewAPIAddresser(fakeAddresses{})
}

func (s *apiAddresserSuite) TestAPIAddresses(c *gc.C) {
	result, err := s.addresser.APIAddresses()
	c.Assert(err, gc.IsNil)
	c.Assert(result.Result, gc.DeepEquals, []string{"apiaddresses:1", "apiaddresses:2"})
}

type fakeAddresses struct{}

func (fakeAddresses) Addresses() ([]string, error) {
	return []string{"addresses:1", "addresses:2"}, nil
}

func (fakeAddresses) APIAddressesFromMachines() ([]string, error) {
	return []string{"apiaddresses:1", "apiaddresses:2"}, nil
}

func (fakeAddresses) CACert() []byte {
	return []byte("a cert")
}
<commit_msg>Add stub method to fakeAddress<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package common_test

import (
	gc "launchpad.net/gocheck"

        "launchpad.net/juju-core/instance"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/state/apiserver/common"
)

type stateAddresserSuite struct {
	addresser *common.StateAddresser
}

type apiAddresserSuite struct {
	addresser *common.APIAddresser
}

var _ = gc.Suite(&stateAddresserSuite{})
var _ = gc.Suite(&apiAddresserSuite{})

func (s *stateAddresserSuite) SetUpTest(c *gc.C) {
	s.addresser = common.NewStateAddresser(fakeAddresses{})
}

// Verify that AddressAndCertGetter is satisfied by *state.State.
var _ common.AddressAndCertGetter = (*state.State)(nil)

func (s *stateAddresserSuite) TestStateAddresses(c *gc.C) {
	result, err := s.addresser.StateAddresses()
	c.Assert(err, gc.IsNil)
	c.Assert(result.Result, gc.DeepEquals, []string{"addresses:1", "addresses:2"})
}

func (s *stateAddresserSuite) TestCACert(c *gc.C) {
	result := s.addresser.CACert()
	c.Assert(string(result.Result), gc.Equals, "a cert")
}

func (s *apiAddresserSuite) SetUpTest(c *gc.C) {
	s.addresser = common.NewAPIAddresser(fakeAddresses{})
}

func (s *apiAddresserSuite) TestAPIAddresses(c *gc.C) {
	result, err := s.addresser.APIAddresses()
	c.Assert(err, gc.IsNil)
	c.Assert(result.Result, gc.DeepEquals, []string{"apiaddresses:1", "apiaddresses:2"})
}

type fakeAddresses struct{}

func (fakeAddresses) Addresses() ([]string, error) {
	return []string{"addresses:1", "addresses:2"}, nil
}

func (fakeAddresses) APIAddressesFromMachines() ([]string, error) {
	return []string{"apiaddresses:1", "apiaddresses:2"}, nil
}

func (fakeAddresses) CACert() []byte {
	return []byte("a cert")
}

func (fakeAdresses) ([][]instance.HostPort, error) {
    panic("should never be called")
    return nil, nil
}
<|endoftext|>


<commit_before>package maintain

import (
	"errors"
	"os"
	"syscall"
	"time"

	Bbs "github.com/cloudfoundry-incubator/runtime-schema/bbs"
	steno "github.com/cloudfoundry/gosteno"
)

var ErrFailedToAquireLock = errors.New("Failed to aquire maintain presence lock")

type Maintainer struct {
	id                string
	bbs               Bbs.ExecutorBBS
	logger            *steno.Logger
	heartbeatInterval time.Duration
}

func New(id string, bbs Bbs.ExecutorBBS, logger *steno.Logger, heartbeatInterval time.Duration) *Maintainer {
	return &Maintainer{
		id:                id,
		bbs:               bbs,
		logger:            logger,
		heartbeatInterval: heartbeatInterval,
	}
}

func (m *Maintainer) Run(sigChan <-chan os.Signal, ready chan<- struct{}) error {
	presence, status, err := m.bbs.MaintainExecutorPresence(m.heartbeatInterval, m.id)
	if err != nil {
		m.logger.Errord(map[string]interface{}{
			"error": err.Error(),
		}, "executor.maintain_presence_begin.failed")
	}

	close(ready)

	for {
		select {
		case sig := <-sigChan:
			if sig != syscall.SIGUSR1 {
				go func() {
					presence.Remove()
				}()
			}

		case locked, ok := <-status:
			if !ok {
				return nil
			}

			if !locked {
				m.logger.Error("executor.maintain_presence.failed")
				return ErrFailedToAquireLock
			}
		}
	}
}
<commit_msg>log when shutting down the maintainer<commit_after>package maintain

import (
	"errors"
	"os"
	"syscall"
	"time"

	Bbs "github.com/cloudfoundry-incubator/runtime-schema/bbs"
	steno "github.com/cloudfoundry/gosteno"
)

var ErrFailedToAquireLock = errors.New("Failed to aquire maintain presence lock")

type Maintainer struct {
	id                string
	bbs               Bbs.ExecutorBBS
	logger            *steno.Logger
	heartbeatInterval time.Duration
}

func New(id string, bbs Bbs.ExecutorBBS, logger *steno.Logger, heartbeatInterval time.Duration) *Maintainer {
	return &Maintainer{
		id:                id,
		bbs:               bbs,
		logger:            logger,
		heartbeatInterval: heartbeatInterval,
	}
}

func (m *Maintainer) Run(sigChan <-chan os.Signal, ready chan<- struct{}) error {
	presence, status, err := m.bbs.MaintainExecutorPresence(m.heartbeatInterval, m.id)
	if err != nil {
		m.logger.Errord(map[string]interface{}{
			"error": err.Error(),
		}, "executor.maintain_presence_begin.failed")
	}

	close(ready)

	for {
		select {
		case sig := <-sigChan:
			if sig != syscall.SIGUSR1 {
				go func() {
					m.logger.Info("executor.maintain_presence.will-stop-maintaining-presence")
					presence.Remove()
				}()
			}

		case locked, ok := <-status:
			if !ok {
				m.logger.Info("executor.maintain_presence.shutting-down")
				return nil
			}

			if !locked {
				m.logger.Error("executor.maintain_presence.failed")
				return ErrFailedToAquireLock
			}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package fmt

/*
	C-like printf, but because of reflection knowledge does not need
	to be told about sizes and signedness (no %llud etc. - just %d).
*/

import (
	"fmt";
	"reflect";
	"os";
)

const Runeself = 0x80
const AllocSize = 32

export type P struct {
	n	int;
	buf	*[]byte;
	fmt	*Fmt;
}

export func Printer() *P {
	p := new(P);
	p.fmt = fmt.New();
	return p;
}

func (p *P) ensure(n int) {
	if p.buf == nil || len(p.buf) < n {
		newn := AllocSize;
		if p.buf != nil {
			newn += len(p.buf);
		}
		if newn < n {
			newn = n + AllocSize
		}
		b := new([]byte, newn);
		for i := 0; i < p.n; i++ {
			b[i] = p.buf[i];
		}
		p.buf = b;
	}
}

func (p *P) addstr(s string) {
	n := len(s);
	p.ensure(p.n + n);
	for i := 0; i < n; i++ {
		p.buf[p.n] = s[i];
		p.n++;
	}
}

func (p *P) addbytes(b *[]byte, start, end int) {
	p.ensure(p.n + end-start);
	for i := start; i < end; i++ {
		p.buf[p.n] = b[i];
		p.n++;
	}
}

func (p *P) add(c int) {
	p.ensure(p.n + 1);
	if c < Runeself {
		p.buf[p.n] = byte(c);
		p.n++;
	} else {
		p.addstr(string(c));
	}
}

func (p *P) reset() {
	p.n = 0;
}

export type Writer interface {
	Write(b *[]byte) (ret int, err *os.Error);
}

func (p *P) doprintf(format string, v reflect.StructValue);
func (p *P) doprint(v reflect.StructValue, addspace bool);

// These routines end in 'f' and take a format string.

func (p *P) fprintf(w Writer, format string, a reflect.Empty) (n int, error *os.Error) {
	v := reflect.NewValue(a).(reflect.PtrValue).Sub().(reflect.StructValue);
	p.doprintf(format, v);
	n, error = w.Write(p.buf[0:p.n]);
	p.reset();
	return n, error;
}

func (p *P) printf(format string, v reflect.Empty) (n int, errno *os.Error) {
	n, errno = p.fprintf(os.Stdout, format, v);
	return n, errno;
}

func (p *P) sprintf(format string, v reflect.Empty) string {
	p.doprintf(format, reflect.NewValue(v).(reflect.StructValue));
	s := string(p.buf)[0 : p.n];
	p.reset();
	return s;
}

// These routines do not take a format string and add spaces only
// when the operand on neither side is a string.

func (p *P) fprint(w Writer, a reflect.Empty) (n int, error *os.Error) {
	v := reflect.NewValue(a).(reflect.PtrValue).Sub().(reflect.StructValue);
	p.doprint(v, false);
	n, error = w.Write(p.buf[0:p.n]);
	p.reset();
	return n, error;
}

func (p *P) print(v reflect.Empty) (n int, errno *os.Error) {
	n, errno = p.fprint(os.Stdout, v);
	return n, errno;
}

func (p *P) sprint(v reflect.Empty) string {
	p.doprint(reflect.NewValue(v).(reflect.StructValue), false);
	s := string(p.buf)[0 : p.n];
	p.reset();
	return s;
}

// These routines end in 'ln', do not take a format string,
// always add spaces between operands, and add a newline
// after the last operand.

func (p *P) fprintln(w Writer, a reflect.Empty) (n int, error *os.Error) {
	v := reflect.NewValue(a).(reflect.PtrValue).Sub().(reflect.StructValue);
	p.doprint(v, true);
	n, error = w.Write(p.buf[0:p.n]);
	p.reset();
	return n, error;
}

func (p *P) println(v reflect.Empty) (n int, errno *os.Error) {
	n, errno = p.fprintln(os.Stdout, v);
	return n, errno;
}

func (p *P) sprintln(v reflect.Empty) string {
	p.doprint(reflect.NewValue(v).(reflect.StructValue), true);
	s := string(p.buf)[0 : p.n];
	p.reset();
	return s;
}

// Getters for the fields of the argument structure.

func getInt(v reflect.Value) (val int64, signed, ok bool) {
	switch v.Kind() {
	case reflect.IntKind:
		return int64(v.(reflect.IntValue).Get()), true, true;
	case reflect.Int8Kind:
		return int64(v.(reflect.Int8Value).Get()), true, true;
	case reflect.Int16Kind:
		return int64(v.(reflect.Int16Value).Get()), true, true;
	case reflect.Int32Kind:
		return int64(v.(reflect.Int32Value).Get()), true, true;
	case reflect.Int64Kind:
		return int64(v.(reflect.Int64Value).Get()), true, true;
	case reflect.UintKind:
		return int64(v.(reflect.UintValue).Get()), false, true;
	case reflect.Uint8Kind:
		return int64(v.(reflect.Uint8Value).Get()), false, true;
	case reflect.Uint16Kind:
		return int64(v.(reflect.Uint16Value).Get()), false, true;
	case reflect.Uint32Kind:
		return int64(v.(reflect.Uint32Value).Get()), false, true;
	case reflect.Uint64Kind:
		return int64(v.(reflect.Uint64Value).Get()), false, true;
	}
	return 0, false, false;
}

func getString(v reflect.Value) (val string, ok bool) {
	switch v.Kind() {
	case reflect.StringKind:
		return v.(reflect.StringValue).Get(), true;
	}
	return "", false;
}

func getFloat(v reflect.Value) (val float64, ok bool) {
	switch v.Kind() {
	case reflect.FloatKind:
		return float64(v.(reflect.FloatValue).Get()), true;
	case reflect.Float32Kind:
		return float64(v.(reflect.Float32Value).Get()), true;
	case reflect.Float64Kind:
		return float64(v.(reflect.Float64Value).Get()), true;
	case reflect.Float80Kind:
		break;	// TODO: what to do here?
	}
	return 0.0, false;
}

func getPtr(v reflect.Value) (val uint64, ok bool) {
	switch v.Kind() {
	case reflect.PtrKind:
		return v.(reflect.PtrValue).Get(), true;
	}
	return 0, false;
}

// Convert ASCII to integer.

func parsenum(s string, start, end int) (n int, got bool, newi int) {
	if start >= end {
		return 0, false, end
	}
	if s[start] == '-' {
		a, b, c := parsenum(s, start+1, end);
		if b {
			return -a, b, c;
		}
	}
	isnum := false;
	num := 0;
	for '0' <= s[start] && s[start] <= '9' {
		num = num*10 + int(s[start] - '0');
		start++;
		isnum = true;
	}
	return num, isnum, start;
}

func (p *P) doprintf(format string, v reflect.StructValue) {
	p.ensure(len(format));	// a good starting size
	end := len(format) - 1;
	fieldnum := 0;	// we process one field per non-trivial format
	for i := 0; i <= end;  {
		c, w := sys.stringtorune(format, i);
		if c != '%' || i == end {
			p.add(c);
			i += w;
			continue;
		}
		var got bool;
		// saw % - do we have %20 (width)?
		w, got, i = parsenum(format, i+1, end);
		if got {
			p.fmt.w(w);
		}
		// do we have %.20 (precision)?
		if i < end && format[i] == '.' {
			w, got, i = parsenum(format, i+1, end);
			if got {
				p.fmt.p(w);
			}
		}
		c, w = sys.stringtorune(format, i);
		i += w;
		// percent is special - absorbs no operand
		if c == '%' {
			p.add('%');	// TODO: should we bother with width & prec?
			continue;
		}
		if fieldnum >= v.Len() {	// out of operands
			p.addstr("???");
			continue;
		}
		field := v.Field(fieldnum);
		fieldnum++;
		s := "";
		switch c {
			// bool
			case 't':
				if field.(reflect.BoolValue).Get() {
					s = "true";
				} else {
					s = "false";
				}

			// int
			case 'b':
				if v, signed, ok := getInt(field); ok {
					s = p.fmt.b64(uint64(v)).str()	// always unsigned
				} else {
					s = "%b%"
				}
			case 'd':
				if v, signed, ok := getInt(field); ok {
					if signed {
						s = p.fmt.d64(v).str()
					} else {
						s = p.fmt.ud64(uint64(v)).str()
					}
				} else {
					s = "%d%"
				}
			case 'o':
				if v, signed, ok := getInt(field); ok {
					if signed {
						s = p.fmt.o64(v).str()
					} else {
						s = p.fmt.uo64(uint64(v)).str()
					}
				} else {
					s= "%o%"
				}
			case 'x':
				if v, signed, ok := getInt(field); ok {
					if signed {
						s = p.fmt.x64(v).str()
					} else {
						s = p.fmt.ux64(uint64(v)).str()
					}
				} else {
					s = "%x%"
				}

			// float
			case 'e':
				if v, ok := getFloat(field); ok {
					s = p.fmt.e64(v).str()
				} else {
					s = "%e%"
				}
			case 'f':
				if v, ok := getFloat(field); ok {
					s = p.fmt.f64(v).str()
				} else {
					s = "%f%";
				}
			case 'g':
				if v, ok := getFloat(field); ok {
					s = p.fmt.g64(v).str()
				} else {
					s = "%g%"
				}

			// string
			case 's':
				if v, ok := getString(field); ok {
					s = p.fmt.s(v).str()
				} else {
					s = "%s%"
				}

			// pointer
			case 'p':
				if v, ok := getPtr(field); ok {
					s = "0x" + p.fmt.uX64(v).str()
				} else {
					s = "%p%"
				}

			default:
				s = "?" + string(c) + "?";
		}
		p.addstr(s);
	}
}

func (p *P) doprint(v reflect.StructValue, is_println bool) {
	prev_string := false;
	for fieldnum := 0; fieldnum < v.Len();  fieldnum++ {
		// always add spaces if we're doing println
		field := v.Field(fieldnum);
		s := "";
		if is_println {
			if fieldnum > 0 {
				p.add(' ')
			}
		} else if field.Kind() != reflect.StringKind && !prev_string{
			// if not doing println, add spaces if neither side is a string
			p.add(' ')
		}
		switch field.Kind() {
		case reflect.BoolKind:
			s = p.fmt.boolean(field.(reflect.BoolValue).Get()).str();
		case reflect.IntKind, reflect.Int8Kind, reflect.Int16Kind, reflect.Int32Kind, reflect.Int64Kind:
			v, signed, ok := getInt(field);
			s = p.fmt.d64(v).str();
		case reflect.UintKind, reflect.Uint8Kind, reflect.Uint16Kind, reflect.Uint32Kind, reflect.Uint64Kind:
			v, signed, ok := getInt(field);
			s = p.fmt.ud64(uint64(v)).str();
		case reflect.FloatKind, reflect.Float32Kind, reflect.Float64Kind, reflect.Float80Kind:
			v, ok := getFloat(field);
			s = p.fmt.g64(v).str();
		case reflect.StringKind:
			v, ok := getString(field);
			s = p.fmt.s(v).str();
		case reflect.PtrKind:
			v, ok := getPtr(field);
			p.add('0');
			p.add('x');
			s = p.fmt.uX64(v).str();
		default:
			s = "???";
		}
		p.addstr(s);
		prev_string = field.Kind() == reflect.StringKind;
	}
	if is_println {
		p.add('\n')
	}
}
<commit_msg>printf as we know and love it.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package fmt

/*
	C-like printf, but because of reflection knowledge does not need
	to be told about sizes and signedness (no %llud etc. - just %d).
*/

import (
	"fmt";
	"reflect";
	"os";
)

const Runeself = 0x80
const AllocSize = 32

export type P struct {
	n	int;
	buf	*[]byte;
	fmt	*Fmt;
}

export func Printer() *P {
	p := new(P);
	p.fmt = fmt.New();
	return p;
}

func (p *P) ensure(n int) {
	if p.buf == nil || len(p.buf) < n {
		newn := AllocSize;
		if p.buf != nil {
			newn += len(p.buf);
		}
		if newn < n {
			newn = n + AllocSize
		}
		b := new([]byte, newn);
		for i := 0; i < p.n; i++ {
			b[i] = p.buf[i];
		}
		p.buf = b;
	}
}

func (p *P) addstr(s string) {
	n := len(s);
	p.ensure(p.n + n);
	for i := 0; i < n; i++ {
		p.buf[p.n] = s[i];
		p.n++;
	}
}

func (p *P) addbytes(b *[]byte, start, end int) {
	p.ensure(p.n + end-start);
	for i := start; i < end; i++ {
		p.buf[p.n] = b[i];
		p.n++;
	}
}

func (p *P) add(c int) {
	p.ensure(p.n + 1);
	if c < Runeself {
		p.buf[p.n] = byte(c);
		p.n++;
	} else {
		p.addstr(string(c));
	}
}

func (p *P) reset() {
	p.n = 0;
}

export type Writer interface {
	Write(b *[]byte) (ret int, err *os.Error);
}

func (p *P) doprintf(format string, v reflect.StructValue);
func (p *P) doprint(v reflect.StructValue, addspace, addnewline bool);

// These routines end in 'f' and take a format string.

func (p *P) fprintf(w Writer, format string, a ...) (n int, error *os.Error) {
	v := reflect.NewValue(a).(reflect.PtrValue).Sub().(reflect.StructValue);
	p.doprintf(format, v);
	n, error = w.Write(p.buf[0:p.n]);
	p.reset();
	return n, error;
}

func (p *P) printf(format string, v ...) (n int, errno *os.Error) {
	n, errno = p.fprintf(os.Stdout, format, v);
	return n, errno;
}

func (p *P) sprintf(format string, v ...) string {
	p.doprintf(format, reflect.NewValue(v).(reflect.StructValue));
	s := string(p.buf)[0 : p.n];
	p.reset();
	return s;
}

// These routines do not take a format string and add spaces only
// when the operand on neither side is a string.

func (p *P) fprint(w Writer, a ...) (n int, error *os.Error) {
	v := reflect.NewValue(a).(reflect.PtrValue).Sub().(reflect.StructValue);
	p.doprint(v, false, false);
	n, error = w.Write(p.buf[0:p.n]);
	p.reset();
	return n, error;
}

func (p *P) print(v ...) (n int, errno *os.Error) {
	n, errno = p.fprint(os.Stdout, v);
	return n, errno;
}

func (p *P) sprint(v ...) string {
	p.doprint(reflect.NewValue(v).(reflect.StructValue), false, false);
	s := string(p.buf)[0 : p.n];
	p.reset();
	return s;
}

// These routines end in 'ln', do not take a format string,
// always add spaces between operands, and add a newline
// after the last operand.

func (p *P) fprintln(w Writer, a ...) (n int, error *os.Error) {
	v := reflect.NewValue(a).(reflect.PtrValue).Sub().(reflect.StructValue);
	p.doprint(v, true, true);
	n, error = w.Write(p.buf[0:p.n]);
	p.reset();
	return n, error;
}

func (p *P) println(v ...) (n int, errno *os.Error) {
	n, errno = p.fprintln(os.Stdout, v);
	return n, errno;
}

func (p *P) sprintln(v ...) string {
	p.doprint(reflect.NewValue(v).(reflect.StructValue), true, true);
	s := string(p.buf)[0 : p.n];
	p.reset();
	return s;
}

// Getters for the fields of the argument structure.

func getInt(v reflect.Value) (val int64, signed, ok bool) {
	switch v.Kind() {
	case reflect.IntKind:
		return int64(v.(reflect.IntValue).Get()), true, true;
	case reflect.Int8Kind:
		return int64(v.(reflect.Int8Value).Get()), true, true;
	case reflect.Int16Kind:
		return int64(v.(reflect.Int16Value).Get()), true, true;
	case reflect.Int32Kind:
		return int64(v.(reflect.Int32Value).Get()), true, true;
	case reflect.Int64Kind:
		return int64(v.(reflect.Int64Value).Get()), true, true;
	case reflect.UintKind:
		return int64(v.(reflect.UintValue).Get()), false, true;
	case reflect.Uint8Kind:
		return int64(v.(reflect.Uint8Value).Get()), false, true;
	case reflect.Uint16Kind:
		return int64(v.(reflect.Uint16Value).Get()), false, true;
	case reflect.Uint32Kind:
		return int64(v.(reflect.Uint32Value).Get()), false, true;
	case reflect.Uint64Kind:
		return int64(v.(reflect.Uint64Value).Get()), false, true;
	}
	return 0, false, false;
}

func getString(v reflect.Value) (val string, ok bool) {
	switch v.Kind() {
	case reflect.StringKind:
		return v.(reflect.StringValue).Get(), true;
	}
	return "", false;
}

func getFloat(v reflect.Value) (val float64, ok bool) {
	switch v.Kind() {
	case reflect.FloatKind:
		return float64(v.(reflect.FloatValue).Get()), true;
	case reflect.Float32Kind:
		return float64(v.(reflect.Float32Value).Get()), true;
	case reflect.Float64Kind:
		return float64(v.(reflect.Float64Value).Get()), true;
	case reflect.Float80Kind:
		break;	// TODO: what to do here?
	}
	return 0.0, false;
}

func getPtr(v reflect.Value) (val uint64, ok bool) {
	switch v.Kind() {
	case reflect.PtrKind:
		return v.(reflect.PtrValue).Get(), true;
	}
	return 0, false;
}

// Convert ASCII to integer.

func parsenum(s string, start, end int) (n int, got bool, newi int) {
	if start >= end {
		return 0, false, end
	}
	if s[start] == '-' {
		a, b, c := parsenum(s, start+1, end);
		if b {
			return -a, b, c;
		}
	}
	isnum := false;
	num := 0;
	for '0' <= s[start] && s[start] <= '9' {
		num = num*10 + int(s[start] - '0');
		start++;
		isnum = true;
	}
	return num, isnum, start;
}

func (p *P) doprintf(format string, v reflect.StructValue) {
	p.ensure(len(format));	// a good starting size
	end := len(format) - 1;
	fieldnum := 0;	// we process one field per non-trivial format
	for i := 0; i <= end;  {
		c, w := sys.stringtorune(format, i);
		if c != '%' || i == end {
			p.add(c);
			i += w;
			continue;
		}
		var got bool;
		// saw % - do we have %20 (width)?
		w, got, i = parsenum(format, i+1, end);
		if got {
			p.fmt.w(w);
		}
		// do we have %.20 (precision)?
		if i < end && format[i] == '.' {
			w, got, i = parsenum(format, i+1, end);
			if got {
				p.fmt.p(w);
			}
		}
		c, w = sys.stringtorune(format, i);
		i += w;
		// percent is special - absorbs no operand
		if c == '%' {
			p.add('%');	// TODO: should we bother with width & prec?
			continue;
		}
		if fieldnum >= v.Len() {	// out of operands
			p.addstr("???");
			continue;
		}
		field := v.Field(fieldnum);
		fieldnum++;
		s := "";
		switch c {
			// bool
			case 't':
				if field.(reflect.BoolValue).Get() {
					s = "true";
				} else {
					s = "false";
				}

			// int
			case 'b':
				if v, signed, ok := getInt(field); ok {
					s = p.fmt.b64(uint64(v)).str()	// always unsigned
				} else {
					s = "%b%"
				}
			case 'd':
				if v, signed, ok := getInt(field); ok {
					if signed {
						s = p.fmt.d64(v).str()
					} else {
						s = p.fmt.ud64(uint64(v)).str()
					}
				} else {
					s = "%d%"
				}
			case 'o':
				if v, signed, ok := getInt(field); ok {
					if signed {
						s = p.fmt.o64(v).str()
					} else {
						s = p.fmt.uo64(uint64(v)).str()
					}
				} else {
					s= "%o%"
				}
			case 'x':
				if v, signed, ok := getInt(field); ok {
					if signed {
						s = p.fmt.x64(v).str()
					} else {
						s = p.fmt.ux64(uint64(v)).str()
					}
				} else {
					s = "%x%"
				}

			// float
			case 'e':
				if v, ok := getFloat(field); ok {
					s = p.fmt.e64(v).str()
				} else {
					s = "%e%"
				}
			case 'f':
				if v, ok := getFloat(field); ok {
					s = p.fmt.f64(v).str()
				} else {
					s = "%f%";
				}
			case 'g':
				if v, ok := getFloat(field); ok {
					s = p.fmt.g64(v).str()
				} else {
					s = "%g%"
				}

			// string
			case 's':
				if v, ok := getString(field); ok {
					s = p.fmt.s(v).str()
				} else {
					s = "%s%"
				}

			// pointer
			case 'p':
				if v, ok := getPtr(field); ok {
					s = "0x" + p.fmt.uX64(v).str()
				} else {
					s = "%p%"
				}

			default:
				s = "?" + string(c) + "?";
		}
		p.addstr(s);
	}
}

func (p *P) doprint(v reflect.StructValue, addspace, addnewline bool) {
	prev_string := false;
	for fieldnum := 0; fieldnum < v.Len();  fieldnum++ {
		// always add spaces if we're doing println
		field := v.Field(fieldnum);
		s := "";
		if fieldnum > 0 {
			if addspace {
				p.add(' ')
			} else if field.Kind() != reflect.StringKind && !prev_string{
				// if not doing println, add spaces if neither side is a string
				p.add(' ')
			}
		}
		switch field.Kind() {
		case reflect.BoolKind:
			s = p.fmt.boolean(field.(reflect.BoolValue).Get()).str();
		case reflect.IntKind, reflect.Int8Kind, reflect.Int16Kind, reflect.Int32Kind, reflect.Int64Kind:
			v, signed, ok := getInt(field);
			s = p.fmt.d64(v).str();
		case reflect.UintKind, reflect.Uint8Kind, reflect.Uint16Kind, reflect.Uint32Kind, reflect.Uint64Kind:
			v, signed, ok := getInt(field);
			s = p.fmt.ud64(uint64(v)).str();
		case reflect.FloatKind, reflect.Float32Kind, reflect.Float64Kind, reflect.Float80Kind:
			v, ok := getFloat(field);
			s = p.fmt.g64(v).str();
		case reflect.StringKind:
			v, ok := getString(field);
			s = p.fmt.s(v).str();
		case reflect.PtrKind:
			v, ok := getPtr(field);
			p.add('0');
			p.add('x');
			s = p.fmt.uX64(v).str();
		case reflect.StructKind:
			p.add('{');
			p.doprint(field, true, false);
			p.add('}');
		default:
			s = "???";
		}
		p.addstr(s);
		prev_string = field.Kind() == reflect.StringKind;
	}
	if addnewline {
		p.add('\n')
	}
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build appengine

package build

import (
	"bytes"
	"encoding/gob"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"runtime"
	"sort"
	"text/template"

	"appengine"
	"appengine/datastore"
	"appengine/delay"
	"appengine/mail"
	"appengine/urlfetch"
)

const (
	mailFrom   = "builder@golang.org" // use this for sending any mail
	failMailTo = "golang-dev@googlegroups.com"
	domain     = "build.golang.org"
	gobotBase  = "http://research.swtch.com/gobot_codereview"
)

// ignoreFailure is a set of builders that we don't email about because
// they are not yet production-ready.
var ignoreFailure = map[string]bool{
	"dragonfly-386":         true,
	"dragonfly-amd64":       true,
	"netbsd-amd64-bsiegert": true,
	"netbsd-arm-rpi":        true,
	"plan9-amd64-aram":      true,
}

// notifyOnFailure checks whether the supplied Commit or the subsequent
// Commit (if present) breaks the build for this builder.
// If either of those commits break the build an email notification is sent
// from a delayed task. (We use a task because this way the mail won't be
// sent if the enclosing datastore transaction fails.)
//
// This must be run in a datastore transaction, and the provided *Commit must
// have been retrieved from the datastore within that transaction.
func notifyOnFailure(c appengine.Context, com *Commit, builder string) error {
	if ignoreFailure[builder] {
		return nil
	}

	// TODO(adg): implement notifications for packages
	if com.PackagePath != "" {
		return nil
	}

	p := &Package{Path: com.PackagePath}
	var broken *Commit
	cr := com.Result(builder, "")
	if cr == nil {
		return fmt.Errorf("no result for %s/%s", com.Hash, builder)
	}
	q := datastore.NewQuery("Commit").Ancestor(p.Key(c))
	if cr.OK {
		// This commit is OK. Notify if next Commit is broken.
		next := new(Commit)
		q = q.Filter("ParentHash=", com.Hash)
		if err := firstMatch(c, q, next); err != nil {
			if err == datastore.ErrNoSuchEntity {
				// OK at tip, no notification necessary.
				return nil
			}
			return err
		}
		if nr := next.Result(builder, ""); nr != nil && !nr.OK {
			c.Debugf("commit ok: %#v\nresult: %#v", com, cr)
			c.Debugf("next commit broken: %#v\nnext result:%#v", next, nr)
			broken = next
		}
	} else {
		// This commit is broken. Notify if the previous Commit is OK.
		prev := new(Commit)
		q = q.Filter("Hash=", com.ParentHash)
		if err := firstMatch(c, q, prev); err != nil {
			if err == datastore.ErrNoSuchEntity {
				// No previous result, let the backfill of
				// this result trigger the notification.
				return nil
			}
			return err
		}
		if pr := prev.Result(builder, ""); pr != nil && pr.OK {
			c.Debugf("commit broken: %#v\nresult: %#v", com, cr)
			c.Debugf("previous commit ok: %#v\nprevious result:%#v", prev, pr)
			broken = com
		}
	}
	if broken == nil {
		return nil
	}
	r := broken.Result(builder, "")
	if r == nil {
		return fmt.Errorf("finding result for %q: %+v", builder, com)
	}
	return commonNotify(c, broken, builder, r.LogHash)
}

// firstMatch executes the query q and loads the first entity into v.
func firstMatch(c appengine.Context, q *datastore.Query, v interface{}) error {
	t := q.Limit(1).Run(c)
	_, err := t.Next(v)
	if err == datastore.Done {
		err = datastore.ErrNoSuchEntity
	}
	return err
}

var notifyLater = delay.Func("notify", notify)

// notify tries to update the CL for the given Commit with a failure message.
// If it doesn't succeed, it sends a failure email to golang-dev.
func notify(c appengine.Context, com *Commit, builder, logHash string) {
	v := url.Values{"brokebuild": {builder}, "log": {logHash}}
	if !updateCL(c, com, v) {
		// Send a mail notification if the CL can't be found.
		sendFailMail(c, com, builder, logHash)
	}
}

// updateCL tells gobot to update the CL for the given Commit with
// the provided query values.
func updateCL(c appengine.Context, com *Commit, v url.Values) bool {
	cl, err := lookupCL(c, com)
	if err != nil {
		c.Errorf("could not find CL for %v: %v", com.Hash, err)
		return false
	}
	u := fmt.Sprintf("%v?cl=%v&%s", gobotBase, cl, v.Encode())
	r, err := urlfetch.Client(c).Post(u, "text/plain", nil)
	if err != nil {
		c.Errorf("could not update CL %v: %v", cl, err)
		return false
	}
	r.Body.Close()
	if r.StatusCode != http.StatusOK {
		c.Errorf("could not update CL %v: %v", cl, r.Status)
		return false
	}
	return true
}

var clURL = regexp.MustCompile(`https://codereview.appspot.com/([0-9]+)`)

// lookupCL consults code.google.com for the full change description for the
// provided Commit, and returns the relevant CL number.
func lookupCL(c appengine.Context, com *Commit) (string, error) {
	url := "https://code.google.com/p/go/source/detail?r=" + com.Hash
	r, err := urlfetch.Client(c).Get(url)
	if err != nil {
		return "", err
	}
	defer r.Body.Close()
	if r.StatusCode != http.StatusOK {
		return "", fmt.Errorf("retrieving %v: %v", url, r.Status)
	}
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return "", err
	}
	m := clURL.FindAllSubmatch(b, -1)
	if m == nil {
		return "", errors.New("no CL URL found on changeset page")
	}
	// Return the last visible codereview URL on the page,
	// in case the change description refers to another CL.
	return string(m[len(m)-1][1]), nil
}

var sendFailMailTmpl = template.Must(template.New("notify.txt").
	Funcs(template.FuncMap(tmplFuncs)).
	ParseFiles("build/notify.txt"))

func init() {
	gob.Register(&Commit{}) // for delay
}

var (
	sendPerfMailLater = delay.Func("sendPerfMail", sendPerfMailFunc)
	sendPerfMailTmpl  = template.Must(
		template.New("perf_notify.txt").
			Funcs(template.FuncMap(tmplFuncs)).
			ParseFiles("build/perf_notify.txt"),
	)
)

func sendPerfFailMail(c appengine.Context, builder string, res *PerfResult) error {
	com := &Commit{Hash: res.CommitHash}
	logHash := ""
	parsed := res.ParseData()
	for _, data := range parsed[builder] {
		if !data.OK {
			logHash = data.Artifacts["log"]
			break
		}
	}
	if logHash == "" {
		return fmt.Errorf("can not find failed result for commit %v on builder %v", com.Hash, builder)
	}
	return commonNotify(c, com, builder, logHash)
}

func commonNotify(c appengine.Context, com *Commit, builder, logHash string) error {
	if com.Num == 0 || com.Desc == "" {
		stk := make([]byte, 10000)
		n := runtime.Stack(stk, false)
		stk = stk[:n]
		c.Errorf("refusing to notify with com=%+v\n%s", *com, string(stk))
		return fmt.Errorf("misuse of commonNotify")
	}
	if com.FailNotificationSent {
		return nil
	}
	c.Infof("%s is broken commit; notifying", com.Hash)
	notifyLater.Call(c, com, builder, logHash) // add task to queue
	com.FailNotificationSent = true
	return putCommit(c, com)
}

// sendFailMail sends a mail notification that the build failed on the
// provided commit and builder.
func sendFailMail(c appengine.Context, com *Commit, builder, logHash string) {
	// get Log
	k := datastore.NewKey(c, "Log", logHash, 0, nil)
	l := new(Log)
	if err := datastore.Get(c, k, l); err != nil {
		c.Errorf("finding Log record %v: %v", logHash, err)
		return
	}
	logText, err := l.Text()
	if err != nil {
		c.Errorf("unpacking Log record %v: %v", logHash, err)
		return
	}

	// prepare mail message
	var body bytes.Buffer
	err = sendFailMailTmpl.Execute(&body, map[string]interface{}{
		"Builder": builder, "Commit": com, "LogHash": logHash, "LogText": logText,
		"Hostname": domain,
	})
	if err != nil {
		c.Errorf("rendering mail template: %v", err)
		return
	}
	subject := fmt.Sprintf("%s broken by %s", builder, shortDesc(com.Desc))
	msg := &mail.Message{
		Sender:  mailFrom,
		To:      []string{failMailTo},
		ReplyTo: failMailTo,
		Subject: subject,
		Body:    body.String(),
	}

	// send mail
	if err := mail.Send(c, msg); err != nil {
		c.Errorf("sending mail: %v", err)
	}
}

type PerfChangeBenchmark struct {
	Name    string
	Metrics []*PerfChangeMetric
}

type PerfChangeMetric struct {
	Name  string
	Old   uint64
	New   uint64
	Delta float64
}

type PerfChangeBenchmarkSlice []*PerfChangeBenchmark

func (l PerfChangeBenchmarkSlice) Len() int      { return len(l) }
func (l PerfChangeBenchmarkSlice) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
func (l PerfChangeBenchmarkSlice) Less(i, j int) bool {
	b1, p1 := splitBench(l[i].Name)
	b2, p2 := splitBench(l[j].Name)
	if b1 != b2 {
		return b1 < b2
	}
	return p1 < p2
}

type PerfChangeMetricSlice []*PerfChangeMetric

func (l PerfChangeMetricSlice) Len() int           { return len(l) }
func (l PerfChangeMetricSlice) Swap(i, j int)      { l[i], l[j] = l[j], l[i] }
func (l PerfChangeMetricSlice) Less(i, j int) bool { return l[i].Name < l[j].Name }

func sendPerfMailFunc(c appengine.Context, com *Commit, prevCommitHash, builder string, changes []*PerfChange) {
	// Sort the changes into the right order.
	var benchmarks []*PerfChangeBenchmark
	for _, ch := range changes {
		// Find the benchmark.
		var b *PerfChangeBenchmark
		for _, b1 := range benchmarks {
			if b1.Name == ch.Bench {
				b = b1
				break
			}
		}
		if b == nil {
			b = &PerfChangeBenchmark{Name: ch.Bench}
			benchmarks = append(benchmarks, b)
		}
		b.Metrics = append(b.Metrics, &PerfChangeMetric{Name: ch.Metric, Old: ch.Old, New: ch.New, Delta: ch.Diff})
	}
	for _, b := range benchmarks {
		sort.Sort(PerfChangeMetricSlice(b.Metrics))
	}
	sort.Sort(PerfChangeBenchmarkSlice(benchmarks))

	u := fmt.Sprintf("http://%v/perfdetail?commit=%v&commit0=%v&kind=builder&builder=%v", domain, com.Hash, prevCommitHash, builder)

	// Prepare mail message (without Commit, for updateCL).
	var body bytes.Buffer
	err := sendPerfMailTmpl.Execute(&body, map[string]interface{}{
		"Builder": builder, "Hostname": domain, "Url": u, "Benchmarks": benchmarks,
	})
	if err != nil {
		c.Errorf("rendering perf mail template: %v", err)
		return
	}

	// First, try to update the CL.
	v := url.Values{"textmsg": {body.String()}}
	if updateCL(c, com, v) {
		return
	}

	// Otherwise, send mail (with Commit, for independent mail message).
	body.Reset()
	err = sendPerfMailTmpl.Execute(&body, map[string]interface{}{
		"Builder": builder, "Commit": com, "Hostname": domain, "Url": u, "Benchmarks": benchmarks,
	})
	if err != nil {
		c.Errorf("rendering perf mail template: %v", err)
		return
	}
	subject := fmt.Sprintf("Perf changes on %s by %s", builder, shortDesc(com.Desc))
	msg := &mail.Message{
		Sender:  mailFrom,
		To:      []string{failMailTo},
		ReplyTo: failMailTo,
		Subject: subject,
		Body:    body.String(),
	}

	// send mail
	if err := mail.Send(c, msg); err != nil {
		c.Errorf("sending mail: %v", err)
	}
}
<commit_msg>go.tools/dashboard/app: update commit in transaction on perf regression<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build appengine

package build

import (
	"bytes"
	"encoding/gob"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"runtime"
	"sort"
	"text/template"

	"appengine"
	"appengine/datastore"
	"appengine/delay"
	"appengine/mail"
	"appengine/urlfetch"
)

const (
	mailFrom   = "builder@golang.org" // use this for sending any mail
	failMailTo = "golang-dev@googlegroups.com"
	domain     = "build.golang.org"
	gobotBase  = "http://research.swtch.com/gobot_codereview"
)

// ignoreFailure is a set of builders that we don't email about because
// they are not yet production-ready.
var ignoreFailure = map[string]bool{
	"dragonfly-386":         true,
	"dragonfly-amd64":       true,
	"netbsd-amd64-bsiegert": true,
	"netbsd-arm-rpi":        true,
	"plan9-amd64-aram":      true,
}

// notifyOnFailure checks whether the supplied Commit or the subsequent
// Commit (if present) breaks the build for this builder.
// If either of those commits break the build an email notification is sent
// from a delayed task. (We use a task because this way the mail won't be
// sent if the enclosing datastore transaction fails.)
//
// This must be run in a datastore transaction, and the provided *Commit must
// have been retrieved from the datastore within that transaction.
func notifyOnFailure(c appengine.Context, com *Commit, builder string) error {
	if ignoreFailure[builder] {
		return nil
	}

	// TODO(adg): implement notifications for packages
	if com.PackagePath != "" {
		return nil
	}

	p := &Package{Path: com.PackagePath}
	var broken *Commit
	cr := com.Result(builder, "")
	if cr == nil {
		return fmt.Errorf("no result for %s/%s", com.Hash, builder)
	}
	q := datastore.NewQuery("Commit").Ancestor(p.Key(c))
	if cr.OK {
		// This commit is OK. Notify if next Commit is broken.
		next := new(Commit)
		q = q.Filter("ParentHash=", com.Hash)
		if err := firstMatch(c, q, next); err != nil {
			if err == datastore.ErrNoSuchEntity {
				// OK at tip, no notification necessary.
				return nil
			}
			return err
		}
		if nr := next.Result(builder, ""); nr != nil && !nr.OK {
			c.Debugf("commit ok: %#v\nresult: %#v", com, cr)
			c.Debugf("next commit broken: %#v\nnext result:%#v", next, nr)
			broken = next
		}
	} else {
		// This commit is broken. Notify if the previous Commit is OK.
		prev := new(Commit)
		q = q.Filter("Hash=", com.ParentHash)
		if err := firstMatch(c, q, prev); err != nil {
			if err == datastore.ErrNoSuchEntity {
				// No previous result, let the backfill of
				// this result trigger the notification.
				return nil
			}
			return err
		}
		if pr := prev.Result(builder, ""); pr != nil && pr.OK {
			c.Debugf("commit broken: %#v\nresult: %#v", com, cr)
			c.Debugf("previous commit ok: %#v\nprevious result:%#v", prev, pr)
			broken = com
		}
	}
	if broken == nil {
		return nil
	}
	r := broken.Result(builder, "")
	if r == nil {
		return fmt.Errorf("finding result for %q: %+v", builder, com)
	}
	return commonNotify(c, broken, builder, r.LogHash)
}

// firstMatch executes the query q and loads the first entity into v.
func firstMatch(c appengine.Context, q *datastore.Query, v interface{}) error {
	t := q.Limit(1).Run(c)
	_, err := t.Next(v)
	if err == datastore.Done {
		err = datastore.ErrNoSuchEntity
	}
	return err
}

var notifyLater = delay.Func("notify", notify)

// notify tries to update the CL for the given Commit with a failure message.
// If it doesn't succeed, it sends a failure email to golang-dev.
func notify(c appengine.Context, com *Commit, builder, logHash string) {
	v := url.Values{"brokebuild": {builder}, "log": {logHash}}
	if !updateCL(c, com, v) {
		// Send a mail notification if the CL can't be found.
		sendFailMail(c, com, builder, logHash)
	}
}

// updateCL tells gobot to update the CL for the given Commit with
// the provided query values.
func updateCL(c appengine.Context, com *Commit, v url.Values) bool {
	cl, err := lookupCL(c, com)
	if err != nil {
		c.Errorf("could not find CL for %v: %v", com.Hash, err)
		return false
	}
	u := fmt.Sprintf("%v?cl=%v&%s", gobotBase, cl, v.Encode())
	r, err := urlfetch.Client(c).Post(u, "text/plain", nil)
	if err != nil {
		c.Errorf("could not update CL %v: %v", cl, err)
		return false
	}
	r.Body.Close()
	if r.StatusCode != http.StatusOK {
		c.Errorf("could not update CL %v: %v", cl, r.Status)
		return false
	}
	return true
}

var clURL = regexp.MustCompile(`https://codereview.appspot.com/([0-9]+)`)

// lookupCL consults code.google.com for the full change description for the
// provided Commit, and returns the relevant CL number.
func lookupCL(c appengine.Context, com *Commit) (string, error) {
	url := "https://code.google.com/p/go/source/detail?r=" + com.Hash
	r, err := urlfetch.Client(c).Get(url)
	if err != nil {
		return "", err
	}
	defer r.Body.Close()
	if r.StatusCode != http.StatusOK {
		return "", fmt.Errorf("retrieving %v: %v", url, r.Status)
	}
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return "", err
	}
	m := clURL.FindAllSubmatch(b, -1)
	if m == nil {
		return "", errors.New("no CL URL found on changeset page")
	}
	// Return the last visible codereview URL on the page,
	// in case the change description refers to another CL.
	return string(m[len(m)-1][1]), nil
}

var sendFailMailTmpl = template.Must(template.New("notify.txt").
	Funcs(template.FuncMap(tmplFuncs)).
	ParseFiles("build/notify.txt"))

func init() {
	gob.Register(&Commit{}) // for delay
}

var (
	sendPerfMailLater = delay.Func("sendPerfMail", sendPerfMailFunc)
	sendPerfMailTmpl  = template.Must(
		template.New("perf_notify.txt").
			Funcs(template.FuncMap(tmplFuncs)).
			ParseFiles("build/perf_notify.txt"),
	)
)

func sendPerfFailMail(c appengine.Context, builder string, res *PerfResult) error {
	return datastore.RunInTransaction(c, func(c appengine.Context) error {
		com := &Commit{Hash: res.CommitHash}
		if err := datastore.Get(c, com.Key(c), com); err != nil {
			return err
		}
		logHash := ""
		parsed := res.ParseData()
		for _, data := range parsed[builder] {
			if !data.OK {
				logHash = data.Artifacts["log"]
				break
			}
		}
		if logHash == "" {
			return fmt.Errorf("can not find failed result for commit %v on builder %v", com.Hash, builder)
		}
		return commonNotify(c, com, builder, logHash)
	}, nil)
}

// commonNotify MUST!!! be called from within a transaction inside which
// the provided Commit entity was retrieved from the datastore.
func commonNotify(c appengine.Context, com *Commit, builder, logHash string) error {
	if com.Num == 0 || com.Desc == "" {
		stk := make([]byte, 10000)
		n := runtime.Stack(stk, false)
		stk = stk[:n]
		c.Errorf("refusing to notify with com=%+v\n%s", *com, string(stk))
		return fmt.Errorf("misuse of commonNotify")
	}
	if com.FailNotificationSent {
		return nil
	}
	c.Infof("%s is broken commit; notifying", com.Hash)
	notifyLater.Call(c, com, builder, logHash) // add task to queue
	com.FailNotificationSent = true
	return putCommit(c, com)
}

// sendFailMail sends a mail notification that the build failed on the
// provided commit and builder.
func sendFailMail(c appengine.Context, com *Commit, builder, logHash string) {
	// get Log
	k := datastore.NewKey(c, "Log", logHash, 0, nil)
	l := new(Log)
	if err := datastore.Get(c, k, l); err != nil {
		c.Errorf("finding Log record %v: %v", logHash, err)
		return
	}
	logText, err := l.Text()
	if err != nil {
		c.Errorf("unpacking Log record %v: %v", logHash, err)
		return
	}

	// prepare mail message
	var body bytes.Buffer
	err = sendFailMailTmpl.Execute(&body, map[string]interface{}{
		"Builder": builder, "Commit": com, "LogHash": logHash, "LogText": logText,
		"Hostname": domain,
	})
	if err != nil {
		c.Errorf("rendering mail template: %v", err)
		return
	}
	subject := fmt.Sprintf("%s broken by %s", builder, shortDesc(com.Desc))
	msg := &mail.Message{
		Sender:  mailFrom,
		To:      []string{failMailTo},
		ReplyTo: failMailTo,
		Subject: subject,
		Body:    body.String(),
	}

	// send mail
	if err := mail.Send(c, msg); err != nil {
		c.Errorf("sending mail: %v", err)
	}
}

type PerfChangeBenchmark struct {
	Name    string
	Metrics []*PerfChangeMetric
}

type PerfChangeMetric struct {
	Name  string
	Old   uint64
	New   uint64
	Delta float64
}

type PerfChangeBenchmarkSlice []*PerfChangeBenchmark

func (l PerfChangeBenchmarkSlice) Len() int      { return len(l) }
func (l PerfChangeBenchmarkSlice) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
func (l PerfChangeBenchmarkSlice) Less(i, j int) bool {
	b1, p1 := splitBench(l[i].Name)
	b2, p2 := splitBench(l[j].Name)
	if b1 != b2 {
		return b1 < b2
	}
	return p1 < p2
}

type PerfChangeMetricSlice []*PerfChangeMetric

func (l PerfChangeMetricSlice) Len() int           { return len(l) }
func (l PerfChangeMetricSlice) Swap(i, j int)      { l[i], l[j] = l[j], l[i] }
func (l PerfChangeMetricSlice) Less(i, j int) bool { return l[i].Name < l[j].Name }

func sendPerfMailFunc(c appengine.Context, com *Commit, prevCommitHash, builder string, changes []*PerfChange) {
	// Sort the changes into the right order.
	var benchmarks []*PerfChangeBenchmark
	for _, ch := range changes {
		// Find the benchmark.
		var b *PerfChangeBenchmark
		for _, b1 := range benchmarks {
			if b1.Name == ch.Bench {
				b = b1
				break
			}
		}
		if b == nil {
			b = &PerfChangeBenchmark{Name: ch.Bench}
			benchmarks = append(benchmarks, b)
		}
		b.Metrics = append(b.Metrics, &PerfChangeMetric{Name: ch.Metric, Old: ch.Old, New: ch.New, Delta: ch.Diff})
	}
	for _, b := range benchmarks {
		sort.Sort(PerfChangeMetricSlice(b.Metrics))
	}
	sort.Sort(PerfChangeBenchmarkSlice(benchmarks))

	u := fmt.Sprintf("http://%v/perfdetail?commit=%v&commit0=%v&kind=builder&builder=%v", domain, com.Hash, prevCommitHash, builder)

	// Prepare mail message (without Commit, for updateCL).
	var body bytes.Buffer
	err := sendPerfMailTmpl.Execute(&body, map[string]interface{}{
		"Builder": builder, "Hostname": domain, "Url": u, "Benchmarks": benchmarks,
	})
	if err != nil {
		c.Errorf("rendering perf mail template: %v", err)
		return
	}

	// First, try to update the CL.
	v := url.Values{"textmsg": {body.String()}}
	if updateCL(c, com, v) {
		return
	}

	// Otherwise, send mail (with Commit, for independent mail message).
	body.Reset()
	err = sendPerfMailTmpl.Execute(&body, map[string]interface{}{
		"Builder": builder, "Commit": com, "Hostname": domain, "Url": u, "Benchmarks": benchmarks,
	})
	if err != nil {
		c.Errorf("rendering perf mail template: %v", err)
		return
	}
	subject := fmt.Sprintf("Perf changes on %s by %s", builder, shortDesc(com.Desc))
	msg := &mail.Message{
		Sender:  mailFrom,
		To:      []string{failMailTo},
		ReplyTo: failMailTo,
		Subject: subject,
		Body:    body.String(),
	}

	// send mail
	if err := mail.Send(c, msg); err != nil {
		c.Errorf("sending mail: %v", err)
	}
}
<|endoftext|>