<commit_before>package web

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"path/filepath"
	"sort"
	"strconv"
	"strings"

	"github.com/davecgh/go-spew/spew"
	"github.com/evepraisal/go-evepraisal"
	"github.com/evepraisal/go-evepraisal/legacy"
	"github.com/go-zoo/bone"
)

var (
	errInputTooBig = errors.New("Input value is too big")
	errInputEmpty  = errors.New("Input value is empty")
)

// AppraisalPage contains data used on the appraisal page
type AppraisalPage struct {
	Appraisal *evepraisal.Appraisal `json:"appraisal"`
	ShowFull  bool                  `json:"show_full,omitempty"`
	IsOwner   bool                  `json:"is_owner,omitempty"`
}

func appraisalLink(appraisal *evepraisal.Appraisal) string {
	if appraisal.Private {
		return fmt.Sprintf("/a/%s/%s", appraisal.ID, appraisal.PrivateToken)
	}
	return fmt.Sprintf("/a/%s", appraisal.ID)
}

func parseAppraisalBody(r *http.Request) (string, error) {
	// Parse body
	r.ParseMultipartForm(20 * 1000)

	var body string
	f, _, err := r.FormFile("uploadappraisal")
	if err == http.ErrNotMultipart || err == http.ErrMissingFile {
		body = r.FormValue("raw_textarea")
	} else if err != nil {
		return "", err
	} else {
		defer f.Close()
		bodyBytes, err := ioutil.ReadAll(f)
		if err != nil {
			return "", err
		}
		body = string(bodyBytes)
	}
	if len(body) > 200000 {
		return "", errInputTooBig
	}

	if len(body) == 0 {
		return "", errInputEmpty
	}
	return body, nil
}

// HandleAppraisal is the handler for POST /appraisal
func (ctx *Context) HandleAppraisal(w http.ResponseWriter, r *http.Request) {

	persist := r.FormValue("persist") != "no"

	body, err := parseAppraisalBody(r)
	if err != nil {
		ctx.renderErrorPage(r, w, http.StatusBadRequest, "Invalid input", err.Error())
		return
	}

	errorRoot := PageRoot{}
	errorRoot.UI.RawTextAreaDefault = body

	// Parse Market
	market := r.FormValue("market")

	// Legacy Market ID
	marketID, err := strconv.ParseInt(market, 10, 64)
	if err == nil {
		var ok bool
		market, ok = legacy.MarketIDToName[marketID]
		if !ok {
			ctx.renderErrorPage(r, w, http.StatusBadRequest, "Invalid input", "Market not found.")
			return
		}
	}

	// No market given
	if market == "" {
		ctx.renderErrorPageWithRoot(r, w, http.StatusBadRequest, "Invalid input", "A market is required.", errorRoot)
		return
	}

	// Invalid market given
	foundMarket := false
	for _, m := range selectableMarkets {
		if m.Name == market {
			foundMarket = true
			break
		}
	}
	if !foundMarket {
		ctx.renderErrorPageWithRoot(r, w, http.StatusBadRequest, "Invalid input", "Given market is not valid.", errorRoot)
		return
	}

	user := ctx.GetCurrentUser(r)

	visibility := r.FormValue("visibility")
	private := false
	if visibility == "private" && user != nil {
		private = true
	}

	// Actually do the appraisal
	appraisal, err := ctx.App.StringToAppraisal(market, body)
	if err == evepraisal.ErrNoValidLinesFound {
		log.Println("No valid lines found:", spew.Sdump(body))
		ctx.renderErrorPageWithRoot(r, w, http.StatusBadRequest, "Invalid input", err.Error(), errorRoot)
		return
	} else if err != nil {
		ctx.renderErrorPageWithRoot(r, w, http.StatusBadRequest, "Invalid input", err.Error(), errorRoot)
		return
	}

	appraisal.User = ctx.GetCurrentUser(r)
	appraisal.Private = private
	appraisal.PrivateToken = NewPrivateAppraisalToken()

	// Persist Appraisal to the database
	if persist {
		err = ctx.App.AppraisalDB.PutNewAppraisal(appraisal)
		if err != nil {
			ctx.renderServerErrorWithRoot(r, w, err, errorRoot)
			return
		}
	}

	// Log for later analyics
	log.Println(appraisal)

	// Set new session variable
	ctx.setSessionValue(r, w, "market", market)
	ctx.setSessionValue(r, w, "visibility", visibility)
	ctx.setSessionValue(r, w, "persist", persist)

	sort.Slice(appraisal.Items, func(i, j int) bool {
		return appraisal.Items[i].RepresentativePrice() > appraisal.Items[j].RepresentativePrice()
	})

	// Render the new appraisal to the screen (there is no redirect here, we set the URL using javascript later)
	w.Header().Add("X-Appraisal-ID", appraisal.ID)
	ctx.render(r, w, "appraisal.html",
		AppraisalPage{
			Appraisal: cleanAppraisal(appraisal),
			IsOwner:   IsAppraisalOwner(user, appraisal),
		},
	)
}

// HandleViewAppraisal is the handler for /a/[id]
func (ctx *Context) HandleViewAppraisal(w http.ResponseWriter, r *http.Request) {
	// Legacy Logic
	appraisalID := bone.GetValue(r, "appraisalID")
	if bone.GetValue(r, "legacyAppraisalID") != "" {
		legacyAppraisalIDStr := bone.GetValue(r, "legacyAppraisalID")
		suffix := filepath.Ext(legacyAppraisalIDStr)
		legacyAppraisalIDStr = strings.TrimSuffix(legacyAppraisalIDStr, suffix)
		legacyAppraisalID, err := strconv.ParseUint(legacyAppraisalIDStr, 10, 64)
		if err != nil {
			ctx.renderErrorPage(r, w, http.StatusNotFound, "Not Found", "I couldn't find what you're looking for")
			return
		}
		appraisalID = evepraisal.Uint64ToAppraisalID(legacyAppraisalID) + suffix
	}

	appraisal, err := ctx.App.AppraisalDB.GetAppraisal(appraisalID)
	if err == evepraisal.AppraisalNotFound {
		ctx.renderErrorPage(r, w, http.StatusNotFound, "Not Found", "I couldn't find what you're looking for")
		return
	} else if err != nil {
		ctx.renderServerError(r, w, err)
		return
	}

	user := ctx.GetCurrentUser(r)
	isOwner := IsAppraisalOwner(user, appraisal)

	if appraisal.Private {
		correctToken := appraisal.PrivateToken == bone.GetValue(r, "privateToken")
		if !(isOwner || correctToken) {
			ctx.renderErrorPage(r, w, http.StatusNotFound, "Not Found", "I couldn't find what you're looking for")
			return
		}
	} else if bone.GetValue(r, "privateToken") != "" {
		ctx.renderErrorPage(r, w, http.StatusNotFound, "Not Found", "I couldn't find what you're looking for")
		return
	}

	appraisal = cleanAppraisal(appraisal)

	sort.Slice(appraisal.Items, func(i, j int) bool {
		return appraisal.Items[i].RepresentativePrice() > appraisal.Items[j].RepresentativePrice()
	})

	if r.Header.Get("format") == "json" {
		w.Header().Add("Content-Type", "application/json")
		json.NewEncoder(w).Encode(appraisal)
		return
	}

	if r.Header.Get("format") == "raw" {
		io.WriteString(w, appraisal.Raw)
		return
	}

	ctx.render(r, w, "appraisal.html",
		AppraisalPage{
			Appraisal: appraisal,
			ShowFull:  r.FormValue("full") != "",
			IsOwner:   isOwner,
		})
}

// HandleDeleteAppraisal is the handler for POST /a/delete/[id]
func (ctx *Context) HandleDeleteAppraisal(w http.ResponseWriter, r *http.Request) {
	appraisalID := bone.GetValue(r, "appraisalID")
	appraisal, err := ctx.App.AppraisalDB.GetAppraisal(appraisalID)
	if err == evepraisal.AppraisalNotFound {
		ctx.renderErrorPage(r, w, http.StatusNotFound, "Not Found", "I couldn't find what you're looking for")
		return
	} else if err != nil {
		ctx.renderServerError(r, w, err)
		return
	}

	if !IsAppraisalOwner(ctx.GetCurrentUser(r), appraisal) {
		ctx.renderErrorPage(r, w, http.StatusNotFound, "Not Found", "I couldn't find what you're looking for")
		return
	}

	err = ctx.App.AppraisalDB.DeleteAppraisal(appraisalID)
	if err == evepraisal.AppraisalNotFound {
		ctx.renderErrorPage(r, w, http.StatusNotFound, "Not Found", "I couldn't find what you're looking for")
		return
	} else if err != nil {
		ctx.renderServerError(r, w, err)
		return
	}

	ctx.setFlashMessage(r, w, FlashMessage{Message: fmt.Sprintf("Appraisal %s has been deleted.", appraisalID), Severity: "success"})
	http.Redirect(w, r, "/", http.StatusTemporaryRedirect)
}

// NewPrivateAppraisalToken returns a new token to use for private appraisals
func NewPrivateAppraisalToken() string {
	const chars = "abcdefghijklmnopqrstuvwxyz0123456789"
	result := make([]byte, 16)
	for i := range result {
		result[i] = chars[rand.Intn(len(chars))]
	}
	return string(result)
}
<commit_msg>Fix showing of delete button right after posting<commit_after>package web

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"path/filepath"
	"sort"
	"strconv"
	"strings"

	"github.com/davecgh/go-spew/spew"
	"github.com/evepraisal/go-evepraisal"
	"github.com/evepraisal/go-evepraisal/legacy"
	"github.com/go-zoo/bone"
)

var (
	errInputTooBig = errors.New("Input value is too big")
	errInputEmpty  = errors.New("Input value is empty")
)

// AppraisalPage contains data used on the appraisal page
type AppraisalPage struct {
	Appraisal *evepraisal.Appraisal `json:"appraisal"`
	ShowFull  bool                  `json:"show_full,omitempty"`
	IsOwner   bool                  `json:"is_owner,omitempty"`
}

func appraisalLink(appraisal *evepraisal.Appraisal) string {
	if appraisal.Private {
		return fmt.Sprintf("/a/%s/%s", appraisal.ID, appraisal.PrivateToken)
	}
	return fmt.Sprintf("/a/%s", appraisal.ID)
}

func parseAppraisalBody(r *http.Request) (string, error) {
	// Parse body
	r.ParseMultipartForm(20 * 1000)

	var body string
	f, _, err := r.FormFile("uploadappraisal")
	if err == http.ErrNotMultipart || err == http.ErrMissingFile {
		body = r.FormValue("raw_textarea")
	} else if err != nil {
		return "", err
	} else {
		defer f.Close()
		bodyBytes, err := ioutil.ReadAll(f)
		if err != nil {
			return "", err
		}
		body = string(bodyBytes)
	}
	if len(body) > 200000 {
		return "", errInputTooBig
	}

	if len(body) == 0 {
		return "", errInputEmpty
	}
	return body, nil
}

// HandleAppraisal is the handler for POST /appraisal
func (ctx *Context) HandleAppraisal(w http.ResponseWriter, r *http.Request) {

	persist := r.FormValue("persist") != "no"

	body, err := parseAppraisalBody(r)
	if err != nil {
		ctx.renderErrorPage(r, w, http.StatusBadRequest, "Invalid input", err.Error())
		return
	}

	errorRoot := PageRoot{}
	errorRoot.UI.RawTextAreaDefault = body

	// Parse Market
	market := r.FormValue("market")

	// Legacy Market ID
	marketID, err := strconv.ParseInt(market, 10, 64)
	if err == nil {
		var ok bool
		market, ok = legacy.MarketIDToName[marketID]
		if !ok {
			ctx.renderErrorPage(r, w, http.StatusBadRequest, "Invalid input", "Market not found.")
			return
		}
	}

	// No market given
	if market == "" {
		ctx.renderErrorPageWithRoot(r, w, http.StatusBadRequest, "Invalid input", "A market is required.", errorRoot)
		return
	}

	// Invalid market given
	foundMarket := false
	for _, m := range selectableMarkets {
		if m.Name == market {
			foundMarket = true
			break
		}
	}
	if !foundMarket {
		ctx.renderErrorPageWithRoot(r, w, http.StatusBadRequest, "Invalid input", "Given market is not valid.", errorRoot)
		return
	}

	user := ctx.GetCurrentUser(r)

	visibility := r.FormValue("visibility")
	private := false
	if visibility == "private" && user != nil {
		private = true
	}

	// Actually do the appraisal
	appraisal, err := ctx.App.StringToAppraisal(market, body)
	if err == evepraisal.ErrNoValidLinesFound {
		log.Println("No valid lines found:", spew.Sdump(body))
		ctx.renderErrorPageWithRoot(r, w, http.StatusBadRequest, "Invalid input", err.Error(), errorRoot)
		return
	} else if err != nil {
		ctx.renderErrorPageWithRoot(r, w, http.StatusBadRequest, "Invalid input", err.Error(), errorRoot)
		return
	}

	appraisal.User = ctx.GetCurrentUser(r)
	appraisal.Private = private
	appraisal.PrivateToken = NewPrivateAppraisalToken()

	// Persist Appraisal to the database
	if persist {
		err = ctx.App.AppraisalDB.PutNewAppraisal(appraisal)
		if err != nil {
			ctx.renderServerErrorWithRoot(r, w, err, errorRoot)
			return
		}
	}

	// Log for later analyics
	log.Println(appraisal)

	// Set new session variable
	ctx.setSessionValue(r, w, "market", market)
	ctx.setSessionValue(r, w, "visibility", visibility)
	ctx.setSessionValue(r, w, "persist", persist)

	sort.Slice(appraisal.Items, func(i, j int) bool {
		return appraisal.Items[i].RepresentativePrice() > appraisal.Items[j].RepresentativePrice()
	})

	// Render the new appraisal to the screen (there is no redirect here, we set the URL using javascript later)
	w.Header().Add("X-Appraisal-ID", appraisal.ID)
	ctx.render(r, w, "appraisal.html",
		AppraisalPage{
			IsOwner:   IsAppraisalOwner(user, appraisal),
			Appraisal: cleanAppraisal(appraisal),
		},
	)
}

// HandleViewAppraisal is the handler for /a/[id]
func (ctx *Context) HandleViewAppraisal(w http.ResponseWriter, r *http.Request) {
	// Legacy Logic
	appraisalID := bone.GetValue(r, "appraisalID")
	if bone.GetValue(r, "legacyAppraisalID") != "" {
		legacyAppraisalIDStr := bone.GetValue(r, "legacyAppraisalID")
		suffix := filepath.Ext(legacyAppraisalIDStr)
		legacyAppraisalIDStr = strings.TrimSuffix(legacyAppraisalIDStr, suffix)
		legacyAppraisalID, err := strconv.ParseUint(legacyAppraisalIDStr, 10, 64)
		if err != nil {
			ctx.renderErrorPage(r, w, http.StatusNotFound, "Not Found", "I couldn't find what you're looking for")
			return
		}
		appraisalID = evepraisal.Uint64ToAppraisalID(legacyAppraisalID) + suffix
	}

	appraisal, err := ctx.App.AppraisalDB.GetAppraisal(appraisalID)
	if err == evepraisal.AppraisalNotFound {
		ctx.renderErrorPage(r, w, http.StatusNotFound, "Not Found", "I couldn't find what you're looking for")
		return
	} else if err != nil {
		ctx.renderServerError(r, w, err)
		return
	}

	user := ctx.GetCurrentUser(r)
	isOwner := IsAppraisalOwner(user, appraisal)

	if appraisal.Private {
		correctToken := appraisal.PrivateToken == bone.GetValue(r, "privateToken")
		if !(isOwner || correctToken) {
			ctx.renderErrorPage(r, w, http.StatusNotFound, "Not Found", "I couldn't find what you're looking for")
			return
		}
	} else if bone.GetValue(r, "privateToken") != "" {
		ctx.renderErrorPage(r, w, http.StatusNotFound, "Not Found", "I couldn't find what you're looking for")
		return
	}

	appraisal = cleanAppraisal(appraisal)

	sort.Slice(appraisal.Items, func(i, j int) bool {
		return appraisal.Items[i].RepresentativePrice() > appraisal.Items[j].RepresentativePrice()
	})

	if r.Header.Get("format") == "json" {
		w.Header().Add("Content-Type", "application/json")
		json.NewEncoder(w).Encode(appraisal)
		return
	}

	if r.Header.Get("format") == "raw" {
		io.WriteString(w, appraisal.Raw)
		return
	}

	ctx.render(r, w, "appraisal.html",
		AppraisalPage{
			Appraisal: appraisal,
			ShowFull:  r.FormValue("full") != "",
			IsOwner:   isOwner,
		})
}

// HandleDeleteAppraisal is the handler for POST /a/delete/[id]
func (ctx *Context) HandleDeleteAppraisal(w http.ResponseWriter, r *http.Request) {
	appraisalID := bone.GetValue(r, "appraisalID")
	appraisal, err := ctx.App.AppraisalDB.GetAppraisal(appraisalID)
	if err == evepraisal.AppraisalNotFound {
		ctx.renderErrorPage(r, w, http.StatusNotFound, "Not Found", "I couldn't find what you're looking for")
		return
	} else if err != nil {
		ctx.renderServerError(r, w, err)
		return
	}

	if !IsAppraisalOwner(ctx.GetCurrentUser(r), appraisal) {
		ctx.renderErrorPage(r, w, http.StatusNotFound, "Not Found", "I couldn't find what you're looking for")
		return
	}

	err = ctx.App.AppraisalDB.DeleteAppraisal(appraisalID)
	if err == evepraisal.AppraisalNotFound {
		ctx.renderErrorPage(r, w, http.StatusNotFound, "Not Found", "I couldn't find what you're looking for")
		return
	} else if err != nil {
		ctx.renderServerError(r, w, err)
		return
	}

	ctx.setFlashMessage(r, w, FlashMessage{Message: fmt.Sprintf("Appraisal %s has been deleted.", appraisalID), Severity: "success"})
	http.Redirect(w, r, "/", http.StatusTemporaryRedirect)
}

// NewPrivateAppraisalToken returns a new token to use for private appraisals
func NewPrivateAppraisalToken() string {
	const chars = "abcdefghijklmnopqrstuvwxyz0123456789"
	result := make([]byte, 16)
	for i := range result {
		result[i] = chars[rand.Intn(len(chars))]
	}
	return string(result)
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Fork, exec, wait, etc.

package syscall

import (
	"syscall"
	"unicode/utf16"
	"unsafe"
)

var zeroProcAttr syscall.ProcAttr
var zeroSysProcAttr syscall.SysProcAttr

func StartProcess(argv0 string, argv []string, attr *syscall.ProcAttr, username, password string) (pid int, handle uintptr, err error) {
	if len(argv0) == 0 {
		return 0, 0, syscall.EWINDOWS
	}
	if attr == nil {
		attr = &zeroProcAttr
	}
	sys := attr.Sys
	if sys == nil {
		sys = &zeroSysProcAttr
	}

	if len(attr.Files) > 3 {
		return 0, 0, syscall.EWINDOWS
	}
	if len(attr.Files) < 3 {
		return 0, 0, syscall.EINVAL
	}

	if len(attr.Dir) != 0 {
		// StartProcess assumes that argv0 is relative to attr.Dir,
		// because it implies Chdir(attr.Dir) before executing argv0.
		// Windows CreateProcess assumes the opposite: it looks for
		// argv0 relative to the current directory, and, only once the new
		// process is started, it does Chdir(attr.Dir). We are adjusting
		// for that difference here by making argv0 absolute.
		var err error
		argv0, err = joinExeDirAndFName(attr.Dir, argv0)
		if err != nil {
			return 0, 0, err
		}
	}
	argv0p, err := syscall.UTF16PtrFromString(argv0)
	if err != nil {
		return 0, 0, err
	}

	var cmdline string
	// Windows CreateProcess takes the command line as a single string:
	// use attr.CmdLine if set, else build the command line by escaping
	// and joining each argument with spaces
	if sys.CmdLine != "" {
		cmdline = sys.CmdLine
	} else {
		cmdline = makeCmdLine(argv)
	}

	var argvp *uint16
	if len(cmdline) != 0 {
		argvp, err = syscall.UTF16PtrFromString(cmdline)
		if err != nil {
			return 0, 0, err
		}
	}

	var dirp *uint16
	if len(attr.Dir) != 0 {
		dirp, err = syscall.UTF16PtrFromString(attr.Dir)
		if err != nil {
			return 0, 0, err
		}
	}

	// Acquire the fork lock so that no other threads
	// create new fds that are not yet close-on-exec
	// before we fork.
	syscall.ForkLock.Lock()
	defer syscall.ForkLock.Unlock()

	p, _ := syscall.GetCurrentProcess()
	fd := make([]syscall.Handle, len(attr.Files))
	for i := range attr.Files {
		if attr.Files[i] > 0 {
			err := syscall.DuplicateHandle(p, syscall.Handle(attr.Files[i]), p, &fd[i], 0, true, syscall.DUPLICATE_SAME_ACCESS)
			if err != nil {
				return 0, 0, err
			}
			defer syscall.CloseHandle(syscall.Handle(fd[i]))
		}
	}
	si := new(syscall.StartupInfo)
	si.Cb = uint32(unsafe.Sizeof(*si))
	si.Flags = syscall.STARTF_USESTDHANDLES
	if sys.HideWindow {
		si.Flags |= syscall.STARTF_USESHOWWINDOW
		si.ShowWindow = syscall.SW_HIDE
	}
	si.StdInput = fd[0]
	si.StdOutput = fd[1]
	si.StdErr = fd[2]

	pi := new(syscall.ProcessInformation)

	flags := sys.CreationFlags | syscall.CREATE_UNICODE_ENVIRONMENT
	if username+password == "" {
		err = CreateProcessWithLogon(
			syscall.StringToUTF16Ptr(username),
			syscall.StringToUTF16Ptr("."),
			syscall.StringToUTF16Ptr(password),
			LOGON_WITH_PROFILE,
			argv0p,
			argvp,
			flags,
			createEnvBlock(attr.Env),
			dirp,
			si,
			pi,
		)
	} else {
		err = CreateProcess(
			argv0p,
			argvp,
			nil,
			nil,
			true,
			flags,
			createEnvBlock(attr.Env),
			dirp,
			si,
			pi,
		)
	}
	if err != nil {
		return 0, 0, err
	}
	defer syscall.CloseHandle(syscall.Handle(pi.Thread))

	return int(pi.ProcessId), uintptr(pi.Process), nil
}

func joinExeDirAndFName(dir, p string) (name string, err error) {
	if len(p) == 0 {
		return "", syscall.EINVAL
	}
	if len(p) > 2 && isSlash(p[0]) && isSlash(p[1]) {
		// \\server\share\path form
		return p, nil
	}
	if len(p) > 1 && p[1] == ':' {
		// has drive letter
		if len(p) == 2 {
			return "", syscall.EINVAL
		}
		if isSlash(p[2]) {
			return p, nil
		} else {
			d, err := normalizeDir(dir)
			if err != nil {
				return "", err
			}
			if volToUpper(int(p[0])) == volToUpper(int(d[0])) {
				return syscall.FullPath(d + "\\" + p[2:])
			} else {
				return syscall.FullPath(p)
			}
		}
	} else {
		// no drive letter
		d, err := normalizeDir(dir)
		if err != nil {
			return "", err
		}
		if isSlash(p[0]) {
			return syscall.FullPath(d[:2] + p)
		} else {
			return syscall.FullPath(d + "\\" + p)
		}
	}
	// we shouldn't be here
	return "", syscall.EINVAL
}

func normalizeDir(dir string) (name string, err error) {
	ndir, err := syscall.FullPath(dir)
	if err != nil {
		return "", err
	}
	if len(ndir) > 2 && isSlash(ndir[0]) && isSlash(ndir[1]) {
		// dir cannot have \\server\share\path form
		return "", syscall.EINVAL
	}
	return ndir, nil
}

// makeCmdLine builds a command line out of args by escaping "special"
// characters and joining the arguments with spaces.
func makeCmdLine(args []string) string {
	var s string
	for _, v := range args {
		if s != "" {
			s += " "
		}
		s += syscall.EscapeArg(v)
	}
	return s
}

func isSlash(c uint8) bool {
	return c == '\\' || c == '/'
}

func volToUpper(ch int) int {
	if 'a' <= ch && ch <= 'z' {
		ch += 'A' - 'a'
	}
	return ch
}

// createEnvBlock converts an array of environment strings into
// the representation required by CreateProcess: a sequence of NUL
// terminated strings followed by a nil.
// Last bytes are two UCS-2 NULs, or four NUL bytes.
func createEnvBlock(envv []string) *uint16 {
	if len(envv) == 0 {
		return &utf16.Encode([]rune("\x00\x00"))[0]
	}
	length := 0
	for _, s := range envv {
		length += len(s) + 1
	}
	length += 1

	b := make([]byte, length)
	i := 0
	for _, s := range envv {
		l := len(s)
		copy(b[i:i+l], []byte(s))
		copy(b[i+l:i+l+1], []byte{0})
		i = i + l + 1
	}
	copy(b[i:i+1], []byte{0})

	return &utf16.Encode([]rune(string(b)))[0]
}
<commit_msg>Added debug<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Fork, exec, wait, etc.

package syscall

import (
	"fmt"
	"syscall"
	"unicode/utf16"
	"unsafe"
)

var zeroProcAttr syscall.ProcAttr
var zeroSysProcAttr syscall.SysProcAttr

func StartProcess(argv0 string, argv []string, attr *syscall.ProcAttr, username, password string) (pid int, handle uintptr, err error) {
	if len(argv0) == 0 {
		return 0, 0, syscall.EWINDOWS
	}
	if attr == nil {
		attr = &zeroProcAttr
	}
	sys := attr.Sys
	if sys == nil {
		sys = &zeroSysProcAttr
	}

	if len(attr.Files) > 3 {
		return 0, 0, syscall.EWINDOWS
	}
	if len(attr.Files) < 3 {
		return 0, 0, syscall.EINVAL
	}

	if len(attr.Dir) != 0 {
		// StartProcess assumes that argv0 is relative to attr.Dir,
		// because it implies Chdir(attr.Dir) before executing argv0.
		// Windows CreateProcess assumes the opposite: it looks for
		// argv0 relative to the current directory, and, only once the new
		// process is started, it does Chdir(attr.Dir). We are adjusting
		// for that difference here by making argv0 absolute.
		var err error
		argv0, err = joinExeDirAndFName(attr.Dir, argv0)
		if err != nil {
			return 0, 0, err
		}
	}
	argv0p, err := syscall.UTF16PtrFromString(argv0)
	if err != nil {
		return 0, 0, err
	}

	var cmdline string
	// Windows CreateProcess takes the command line as a single string:
	// use attr.CmdLine if set, else build the command line by escaping
	// and joining each argument with spaces
	if sys.CmdLine != "" {
		cmdline = sys.CmdLine
	} else {
		cmdline = makeCmdLine(argv)
	}

	var argvp *uint16
	if len(cmdline) != 0 {
		argvp, err = syscall.UTF16PtrFromString(cmdline)
		if err != nil {
			return 0, 0, err
		}
	}

	var dirp *uint16
	if len(attr.Dir) != 0 {
		dirp, err = syscall.UTF16PtrFromString(attr.Dir)
		if err != nil {
			return 0, 0, err
		}
	}

	// Acquire the fork lock so that no other threads
	// create new fds that are not yet close-on-exec
	// before we fork.
	syscall.ForkLock.Lock()
	defer syscall.ForkLock.Unlock()

	p, _ := syscall.GetCurrentProcess()
	fd := make([]syscall.Handle, len(attr.Files))
	for i := range attr.Files {
		if attr.Files[i] > 0 {
			err := syscall.DuplicateHandle(p, syscall.Handle(attr.Files[i]), p, &fd[i], 0, true, syscall.DUPLICATE_SAME_ACCESS)
			if err != nil {
				return 0, 0, err
			}
			defer syscall.CloseHandle(syscall.Handle(fd[i]))
		}
	}
	si := new(syscall.StartupInfo)
	si.Cb = uint32(unsafe.Sizeof(*si))
	si.Flags = syscall.STARTF_USESTDHANDLES
	if sys.HideWindow {
		si.Flags |= syscall.STARTF_USESHOWWINDOW
		si.ShowWindow = syscall.SW_HIDE
	}
	si.StdInput = fd[0]
	si.StdOutput = fd[1]
	si.StdErr = fd[2]

	pi := new(syscall.ProcessInformation)

	flags := sys.CreationFlags | syscall.CREATE_UNICODE_ENVIRONMENT
	if username+password != "" {
		err = CreateProcessWithLogon(
			syscall.StringToUTF16Ptr(username),
			syscall.StringToUTF16Ptr("."),
			syscall.StringToUTF16Ptr(password),
			LOGON_WITH_PROFILE,
			argv0p,
			argvp,
			flags,
			createEnvBlock(attr.Env),
			dirp,
			si,
			pi,
		)
	} else {
		err = CreateProcess(
			argv0p,
			argvp,
			nil,
			nil,
			true,
			flags,
			createEnvBlock(attr.Env),
			dirp,
			si,
			pi,
		)
	}
	if err != nil {
		fmt.Printf("\n\n\n\nusername: %q\npassword: %q\nLOGON_WITH_PROFILE: %q\nargv0p: %q\nargvp: %q\nflags: %q\nattr.Env: %q\ndirp: %q\nsi: %q\npi: %q\n\n\n\n\n", username, password, LOGON_WITH_PROFILE, argv0p, argvp, flags, attr.Env, dirp, si, pi)
		return 0, 0, err
	}
	defer syscall.CloseHandle(syscall.Handle(pi.Thread))

	return int(pi.ProcessId), uintptr(pi.Process), nil
}

func joinExeDirAndFName(dir, p string) (name string, err error) {
	if len(p) == 0 {
		return "", syscall.EINVAL
	}
	if len(p) > 2 && isSlash(p[0]) && isSlash(p[1]) {
		// \\server\share\path form
		return p, nil
	}
	if len(p) > 1 && p[1] == ':' {
		// has drive letter
		if len(p) == 2 {
			return "", syscall.EINVAL
		}
		if isSlash(p[2]) {
			return p, nil
		} else {
			d, err := normalizeDir(dir)
			if err != nil {
				return "", err
			}
			if volToUpper(int(p[0])) == volToUpper(int(d[0])) {
				return syscall.FullPath(d + "\\" + p[2:])
			} else {
				return syscall.FullPath(p)
			}
		}
	} else {
		// no drive letter
		d, err := normalizeDir(dir)
		if err != nil {
			return "", err
		}
		if isSlash(p[0]) {
			return syscall.FullPath(d[:2] + p)
		} else {
			return syscall.FullPath(d + "\\" + p)
		}
	}
	// we shouldn't be here
	return "", syscall.EINVAL
}

func normalizeDir(dir string) (name string, err error) {
	ndir, err := syscall.FullPath(dir)
	if err != nil {
		return "", err
	}
	if len(ndir) > 2 && isSlash(ndir[0]) && isSlash(ndir[1]) {
		// dir cannot have \\server\share\path form
		return "", syscall.EINVAL
	}
	return ndir, nil
}

// makeCmdLine builds a command line out of args by escaping "special"
// characters and joining the arguments with spaces.
func makeCmdLine(args []string) string {
	var s string
	for _, v := range args {
		if s != "" {
			s += " "
		}
		s += syscall.EscapeArg(v)
	}
	return s
}

func isSlash(c uint8) bool {
	return c == '\\' || c == '/'
}

func volToUpper(ch int) int {
	if 'a' <= ch && ch <= 'z' {
		ch += 'A' - 'a'
	}
	return ch
}

// createEnvBlock converts an array of environment strings into
// the representation required by CreateProcess: a sequence of NUL
// terminated strings followed by a nil.
// Last bytes are two UCS-2 NULs, or four NUL bytes.
func createEnvBlock(envv []string) *uint16 {
	if len(envv) == 0 {
		return &utf16.Encode([]rune("\x00\x00"))[0]
	}
	length := 0
	for _, s := range envv {
		length += len(s) + 1
	}
	length += 1

	b := make([]byte, length)
	i := 0
	for _, s := range envv {
		l := len(s)
		copy(b[i:i+l], []byte(s))
		copy(b[i+l:i+l+1], []byte{0})
		i = i + l + 1
	}
	copy(b[i:i+1], []byte{0})

	return &utf16.Encode([]rune(string(b)))[0]
}
<|endoftext|>


<commit_before>/*
 * This file is part of the UCB release of Plan 9. It is subject to the license
 * terms in the LICENSE file found in the top-level directory of this
 * distribution and at http://akaros.cs.berkeley.edu/files/Plan9License. No
 * part of the UCB release of Plan 9, including this file, may be copied,
 * modified, propagated, or distributed except according to the terms contained
 * in the LICENSE file.
 */

package harvey

import (
	"bytes"
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"path/filepath"
	"strings"
	"text/template"

	"bldy.build/build"
	"bldy.build/build/racy"
	"sevki.org/lib/prettyprint"
)

type Syscall struct {
	Ret     []string
	Args    []string
	Name    string
	Id      uint32
	Define  string
	Sysname string
	Libname string
	Fudge   string   `json:"-"`
	GoArgs  []string `json:"-"`
	Ret0    string   `json:"-"`
}

type Syserror struct {
	Name   string
	String string
	Id     uint32
}

type Bootmethods struct {
	Name    string
	Config  string
	Connect string
	Arg     string
}

type Sysconf struct {
	Syscalls    []Syscall
	Syserrors   []Syserror
	Bootmethods []Bootmethods
}

type MkSys struct {
	Name         string   `mk_sys:"name"`
	Mode         string   `mk_sys:"mode"`
	ARCH         string   `mk_sys:"arch"`
	SysConf      string   `mk_sys:"sysconf" build:"path"`
	Dependencies []string `mk_sys:"deps"`
	buf          bytes.Buffer
	Source       string
}

func (cl *MkSys) Hash() []byte {

	h := sha1.New()
	io.WriteString(h, cl.Mode)
	io.WriteString(h, cl.ARCH)
	io.WriteString(h, cl.Mode)
	racy.HashFiles(h, []string{cl.SysConf})

	return h.Sum(nil)
}

func (mkSys *MkSys) readSysconf() (*Sysconf, error) {
	buf, err := ioutil.ReadFile(mkSys.SysConf)
	if err != nil {
		return nil, fmt.Errorf("os readAll %s:%s", mkSys.SysConf, err.Error())
	}

	var sysconf Sysconf
	err = json.Unmarshal(buf, &sysconf)
	if err != nil {
		return nil, err
	}
	return &sysconf, nil

}
func (mkSys *MkSys) Installs() map[string]string {
	installs := make(map[string]string)
	switch mkSys.Mode {
	case "sys.h", "error.h", "sysdecl.h", "errstr.h":
		installs[filepath.Join("include", mkSys.Mode)] = mkSys.Mode
	case "syscallfiles":
		sysconf, err := mkSys.readSysconf()
		if err != nil {
			return nil
		}
		syscalls := sysconf.Syscalls
		for i := range syscalls {
			if syscalls[i].Define == "" {
				syscalls[i].Define = strings.ToUpper(syscalls[i].Name)
			}
			if syscalls[i].Sysname == "" {
				syscalls[i].Sysname = "sys" + syscalls[i].Name
			}
			if syscalls[i].Libname == "" {
				syscalls[i].Libname = syscalls[i].Name
			}
		}
		for i := range syscalls {
			installs[filepath.Join("9syscall", syscalls[i].Libname+".s")] = syscalls[i].Libname + ".s"
		}
	default:
		installs[mkSys.Mode] = mkSys.Mode
	}
	return installs
}

func (mkSys *MkSys) GetSource() string {
	return mkSys.Source
}

func (mkSys *MkSys) GetName() string {
	return mkSys.Name
}

func (mkSys *MkSys) GetDependencies() []string {
	return mkSys.Dependencies
}

func (mkSys *MkSys) Build(c *build.Context) error {

	c.Println(prettyprint.AsJSON(mkSys))

	sysconf, err := mkSys.readSysconf()
	if err != nil {
		return err
	}

	syscalls := sysconf.Syscalls
	syserrors := sysconf.Syserrors
	bootmethods := sysconf.Bootmethods
	for i := range syscalls {
		if syscalls[i].Define == "" {
			syscalls[i].Define = strings.ToUpper(syscalls[i].Name)
		}
		if syscalls[i].Sysname == "" {
			syscalls[i].Sysname = "sys" + syscalls[i].Name
		}
		if syscalls[i].Libname == "" {
			syscalls[i].Libname = syscalls[i].Name
		}
	}

	outfile, err := c.Create(mkSys.Mode)
	if err != nil {
		return err
	}
	switch mkSys.Mode {
	case "sys_harvey.s":
		if mkSys.ARCH != "amd64" {
			c.Println("ARCH unsupported or not set")
		}
		syscallargs := []string{"DI", "SI", "DX", "R10", "R8", "R9"}
		//funcallregs := []string{ "DI", "SI", "DX", "CX", "R8", "R9" };
		for i := range syscalls {
			goargs := []string{}
			fpoff := 0
			for k := range syscalls[i].Args {
				switch syscalls[i].Args[k] {
				case "int32_t", "uint32_t":
					goargs = append(goargs, fmt.Sprintf("MOVL	arg%d+%d(FP), %s", k, fpoff, syscallargs[k]))
					fpoff += 4
				case "void*", "char*", "char**", "uint8_t*", "int32_t*", "uint64_t*", "int64_t*", "int64_t":
					fpoff = (fpoff + 7) & ^7
					goargs = append(goargs, fmt.Sprintf("MOVQ	arg%d+%d(FP), %s", k, fpoff, syscallargs[k]))
					fpoff += 8
				default:
					return fmt.Errorf("unsupported arg %s in syscall: %v", syscalls[i].Args[k], syscalls[i])
				}
			}
			syscalls[i].GoArgs = goargs
			switch syscalls[i].Ret[0] {
			case "int32_t", "uint32_t":
				syscalls[i].Ret0 = fmt.Sprintf("MOVL	AX, ret+%d(FP)", fpoff)
				fpoff += 4
			case "void*", "char*", "char**", "uint8_t*", "int32_t*", "uint64_t*", "int64_t*", "int64_t":
				fpoff = (fpoff + 7) & ^7
				syscalls[i].Ret0 = fmt.Sprintf("MOVQ	AX, ret+%d(FP)", fpoff)
				fpoff += 8
			default:
				return fmt.Errorf("unsupported Ret[0] in syscall: %v", syscalls[i])
			}
		}
		tmpl, err := template.New("sys_harvey.s").Parse(`/* automatically generated by mksys */
/* System calls for AMD64, Harvey */
#include "go_asm.h"
#include "go_tls.h"
#include "textflag.h"
{{ range . }}
TEXT runtime·{{ .Libname }}(SB),NOSPLIT,$0
{{ range .GoArgs }}	{{ . }}
{{ end }}	MOVQ	${{ .Id }}, AX
	SYSCALL
	{{ .Ret0 }}
	RET
{{ end }}
`)
		if err != nil {
			return err
		}
		err = tmpl.Execute(outfile, syscalls)
		if err != nil {
			return err
		}

	case "syscallfiles":
		if mkSys.ARCH != "amd64" {
			return fmt.Errorf("ARCH unsupported or not set")
		}
		tmpl, err := template.New("syscall.s").Parse(`/* automatically generated by mksys */
.globl	{{ .Libname }}
{{ .Libname }}:
	movq %rcx, %r10 /* rcx gets smashed by systenter. Use r10.*/
	movq ${{ .Id }},%rax  /* Put the system call into rax, just like linux. */
	syscall
	ret
`)
		if err != nil {
			return err
		}

		for i := range syscalls {

			file, err := c.Create(syscalls[i].Libname + ".s")
			if err != nil {
				return err
			}

			err = tmpl.Execute(file, syscalls[i])
			if err != nil {
				return err
			}

			err = file.Close()
			if err != nil {
				return err
			}
		}
	case "sysnum.go":
		tmpl, err := template.New("sysnum.go").Parse(`// automatically generated by mksys
package syscall
const(
{{ range . }}	SYS_{{ .Define }} = {{ .Id }}
{{ end }}
)
`)
		err = tmpl.Execute(outfile, syscalls)
		if err != nil {
			return err
		}

	case "sys.h":
		tmpl, err := template.New("sys.h").Parse(`/* automatically generated by mksys */
{{ range . }}#define {{ .Define }} {{ .Id }}
{{ end }}
`)
		err = tmpl.Execute(outfile, syscalls)
		if err != nil {
			return err
		}

	case "sysdecl.h":
		tmpl, err := template.New("sysdecl.h").Parse(`/* automatically generated by mksys */
{{ range . }}extern {{ .Ret0 }} {{ .Libname }}({{ range $i, $e := .Args }}{{ if $i }}, {{ end }}{{ $e }}{{ end }});
{{ end }}
`)
		err = tmpl.Execute(outfile, syscalls)
		if err != nil {
			return err
		}

	case "systab.c":
		for i := range syscalls {
			var fudge string
			switch syscalls[i].Ret[0] {
			case "int32_t":
				fudge = "{ .i = -1 }"
			case "int64_t":
				fudge = "{ .vl = -1ll }"
			case "void*", "char*":
				fudge = "{ .v = (void*)-1ll }"
			default:
				return fmt.Errorf("unsupported Ret[0] in syscall: %v", syscalls[i])
			}
			if syscalls[i].Fudge == "" {
				syscalls[i].Fudge = fudge
			}

			syscalls[i].Ret0 = syscalls[i].Ret[0]
		}
		tmpl, err := template.New("systab.c").Parse(`/* automatically generated by mksys */
#include "u.h"
#include "../port/lib.h"
#include "mem.h"
#include "dat.h"
#include "fns.h"
#include <sys.h>
{{ range . }}extern void {{ .Sysname }}(Ar0*, ...);
{{ end }}
Systab systab[] = {
{{ range . }}[{{ .Define }}] { "{{ .Name }}", {{ .Sysname }}, {{ .Fudge }} },
{{ end }}
};
int nsyscall = nelem(systab);
`)
		err = tmpl.Execute(outfile, syscalls)
		if err != nil {
			return err
		}

	case "error.h":
		tmpl, err := template.New("error.h").Parse(`/* automatically generated by mksys */
{{ range . }}extern char {{ .Name }}[]; /* {{ .String }} */
{{ end }}
`)
		err = tmpl.Execute(outfile, syserrors)
		if err != nil {
			return err
		}

	case "errstr.h":
		tmpl, err := template.New("errstr.h").Parse(`/* automatically generated by mksys */
{{ range . }}char {{ .Name }}[] = "{{ .String }}";
{{ end }}
`)
		err = tmpl.Execute(outfile, syserrors)
		if err != nil {
			return err
		}
	case "bootamd64cpu.c":
		tmpl, err := template.New("bootamd64cpu.c").Parse(`/* automatically generated by mksys */
#include <u.h>
#include <libc.h>
#include <boot.h>
Method method[] = {
{{ range . }}{ "{{.Name}}", {{.Config}}, {{.Connect}}, "{{.Arg}}", },
{{ end }}
	{ nil },
};
int cpuflag = 1;
char* rootdir = "/root";
char* bootdisk = "#S/sdE0/";
extern void boot(int, char**);
void
main(int argc, char **argv)
{
		boot(argc, argv);
}
int (*cfs)(int) = 0;
`)
		err = tmpl.Execute(outfile, bootmethods)
		if err != nil {
			log.Fatal(err)
		}
	}
	return nil
}
<commit_msg>harvey: mksys fix the 'missing =' warning for clang<commit_after>/*
 * This file is part of the UCB release of Plan 9. It is subject to the license
 * terms in the LICENSE file found in the top-level directory of this
 * distribution and at http://akaros.cs.berkeley.edu/files/Plan9License. No
 * part of the UCB release of Plan 9, including this file, may be copied,
 * modified, propagated, or distributed except according to the terms contained
 * in the LICENSE file.
 */

package harvey

import (
	"bytes"
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"path/filepath"
	"strings"
	"text/template"

	"bldy.build/build"
	"bldy.build/build/racy"
	"sevki.org/lib/prettyprint"
)

type Syscall struct {
	Ret     []string
	Args    []string
	Name    string
	Id      uint32
	Define  string
	Sysname string
	Libname string
	Fudge   string   `json:"-"`
	GoArgs  []string `json:"-"`
	Ret0    string   `json:"-"`
}

type Syserror struct {
	Name   string
	String string
	Id     uint32
}

type Bootmethods struct {
	Name    string
	Config  string
	Connect string
	Arg     string
}

type Sysconf struct {
	Syscalls    []Syscall
	Syserrors   []Syserror
	Bootmethods []Bootmethods
}

type MkSys struct {
	Name         string   `mk_sys:"name"`
	Mode         string   `mk_sys:"mode"`
	ARCH         string   `mk_sys:"arch"`
	SysConf      string   `mk_sys:"sysconf" build:"path"`
	Dependencies []string `mk_sys:"deps"`
	buf          bytes.Buffer
	Source       string
}

func (cl *MkSys) Hash() []byte {

	h := sha1.New()
	io.WriteString(h, cl.Mode)
	io.WriteString(h, cl.ARCH)
	io.WriteString(h, cl.Mode)
	racy.HashFiles(h, []string{cl.SysConf})

	return h.Sum(nil)
}

func (mkSys *MkSys) readSysconf() (*Sysconf, error) {
	buf, err := ioutil.ReadFile(mkSys.SysConf)
	if err != nil {
		return nil, fmt.Errorf("os readAll %s:%s", mkSys.SysConf, err.Error())
	}

	var sysconf Sysconf
	err = json.Unmarshal(buf, &sysconf)
	if err != nil {
		return nil, err
	}
	return &sysconf, nil

}
func (mkSys *MkSys) Installs() map[string]string {
	installs := make(map[string]string)
	switch mkSys.Mode {
	case "sys.h", "error.h", "sysdecl.h", "errstr.h":
		installs[filepath.Join("include", mkSys.Mode)] = mkSys.Mode
	case "syscallfiles":
		sysconf, err := mkSys.readSysconf()
		if err != nil {
			return nil
		}
		syscalls := sysconf.Syscalls
		for i := range syscalls {
			if syscalls[i].Define == "" {
				syscalls[i].Define = strings.ToUpper(syscalls[i].Name)
			}
			if syscalls[i].Sysname == "" {
				syscalls[i].Sysname = "sys" + syscalls[i].Name
			}
			if syscalls[i].Libname == "" {
				syscalls[i].Libname = syscalls[i].Name
			}
		}
		for i := range syscalls {
			installs[filepath.Join("9syscall", syscalls[i].Libname+".s")] = syscalls[i].Libname + ".s"
		}
	default:
		installs[mkSys.Mode] = mkSys.Mode
	}
	return installs
}

func (mkSys *MkSys) GetSource() string {
	return mkSys.Source
}

func (mkSys *MkSys) GetName() string {
	return mkSys.Name
}

func (mkSys *MkSys) GetDependencies() []string {
	return mkSys.Dependencies
}

func (mkSys *MkSys) Build(c *build.Context) error {

	c.Println(prettyprint.AsJSON(mkSys))

	sysconf, err := mkSys.readSysconf()
	if err != nil {
		return err
	}

	syscalls := sysconf.Syscalls
	syserrors := sysconf.Syserrors
	bootmethods := sysconf.Bootmethods
	for i := range syscalls {
		if syscalls[i].Define == "" {
			syscalls[i].Define = strings.ToUpper(syscalls[i].Name)
		}
		if syscalls[i].Sysname == "" {
			syscalls[i].Sysname = "sys" + syscalls[i].Name
		}
		if syscalls[i].Libname == "" {
			syscalls[i].Libname = syscalls[i].Name
		}
	}

	outfile, err := c.Create(mkSys.Mode)
	if err != nil {
		return err
	}
	switch mkSys.Mode {
	case "sys_harvey.s":
		if mkSys.ARCH != "amd64" {
			c.Println("ARCH unsupported or not set")
		}
		syscallargs := []string{"DI", "SI", "DX", "R10", "R8", "R9"}
		//funcallregs := []string{ "DI", "SI", "DX", "CX", "R8", "R9" };
		for i := range syscalls {
			goargs := []string{}
			fpoff := 0
			for k := range syscalls[i].Args {
				switch syscalls[i].Args[k] {
				case "int32_t", "uint32_t":
					goargs = append(goargs, fmt.Sprintf("MOVL	arg%d+%d(FP), %s", k, fpoff, syscallargs[k]))
					fpoff += 4
				case "void*", "char*", "char**", "uint8_t*", "int32_t*", "uint64_t*", "int64_t*", "int64_t":
					fpoff = (fpoff + 7) & ^7
					goargs = append(goargs, fmt.Sprintf("MOVQ	arg%d+%d(FP), %s", k, fpoff, syscallargs[k]))
					fpoff += 8
				default:
					return fmt.Errorf("unsupported arg %s in syscall: %v", syscalls[i].Args[k], syscalls[i])
				}
			}
			syscalls[i].GoArgs = goargs
			switch syscalls[i].Ret[0] {
			case "int32_t", "uint32_t":
				syscalls[i].Ret0 = fmt.Sprintf("MOVL	AX, ret+%d(FP)", fpoff)
				fpoff += 4
			case "void*", "char*", "char**", "uint8_t*", "int32_t*", "uint64_t*", "int64_t*", "int64_t":
				fpoff = (fpoff + 7) & ^7
				syscalls[i].Ret0 = fmt.Sprintf("MOVQ	AX, ret+%d(FP)", fpoff)
				fpoff += 8
			default:
				return fmt.Errorf("unsupported Ret[0] in syscall: %v", syscalls[i])
			}
		}
		tmpl, err := template.New("sys_harvey.s").Parse(`/* automatically generated by mksys */
/* System calls for AMD64, Harvey */
#include "go_asm.h"
#include "go_tls.h"
#include "textflag.h"
{{ range . }}
TEXT runtime·{{ .Libname }}(SB),NOSPLIT,$0
{{ range .GoArgs }}	{{ . }}
{{ end }}	MOVQ	${{ .Id }}, AX
	SYSCALL
	{{ .Ret0 }}
	RET
{{ end }}
`)
		if err != nil {
			return err
		}
		err = tmpl.Execute(outfile, syscalls)
		if err != nil {
			return err
		}

	case "syscallfiles":
		if mkSys.ARCH != "amd64" {
			return fmt.Errorf("ARCH unsupported or not set")
		}
		tmpl, err := template.New("syscall.s").Parse(`/* automatically generated by mksys */
.globl	{{ .Libname }}
{{ .Libname }}:
	movq %rcx, %r10 /* rcx gets smashed by systenter. Use r10.*/
	movq ${{ .Id }},%rax  /* Put the system call into rax, just like linux. */
	syscall
	ret
`)
		if err != nil {
			return err
		}

		for i := range syscalls {

			file, err := c.Create(syscalls[i].Libname + ".s")
			if err != nil {
				return err
			}

			err = tmpl.Execute(file, syscalls[i])
			if err != nil {
				return err
			}

			err = file.Close()
			if err != nil {
				return err
			}
		}
	case "sysnum.go":
		tmpl, err := template.New("sysnum.go").Parse(`// automatically generated by mksys
package syscall
const(
{{ range . }}	SYS_{{ .Define }} = {{ .Id }}
{{ end }}
)
`)
		err = tmpl.Execute(outfile, syscalls)
		if err != nil {
			return err
		}

	case "sys.h":
		tmpl, err := template.New("sys.h").Parse(`/* automatically generated by mksys */
{{ range . }}#define {{ .Define }} {{ .Id }}
{{ end }}
`)
		err = tmpl.Execute(outfile, syscalls)
		if err != nil {
			return err
		}

	case "sysdecl.h":
		tmpl, err := template.New("sysdecl.h").Parse(`/* automatically generated by mksys */
{{ range . }}extern {{ .Ret0 }} {{ .Libname }}({{ range $i, $e := .Args }}{{ if $i }}, {{ end }}{{ $e }}{{ end }});
{{ end }}
`)
		err = tmpl.Execute(outfile, syscalls)
		if err != nil {
			return err
		}

	case "systab.c":
		for i := range syscalls {
			var fudge string
			switch syscalls[i].Ret[0] {
			case "int32_t":
				fudge = "{ .i = -1 }"
			case "int64_t":
				fudge = "{ .vl = -1ll }"
			case "void*", "char*":
				fudge = "{ .v = (void*)-1ll }"
			default:
				return fmt.Errorf("unsupported Ret[0] in syscall: %v", syscalls[i])
			}
			if syscalls[i].Fudge == "" {
				syscalls[i].Fudge = fudge
			}

			syscalls[i].Ret0 = syscalls[i].Ret[0]
		}
		tmpl, err := template.New("systab.c").Parse(`/* automatically generated by mksys */
#include "u.h"
#include "../port/lib.h"
#include "mem.h"
#include "dat.h"
#include "fns.h"
#include <sys.h>
{{ range . }}extern void {{ .Sysname }}(Ar0*, ...);
{{ end }}
Systab systab[] = {
{{ range . }}[{{ .Define }}] = { "{{ .Name }}", {{ .Sysname }}, {{ .Fudge }} },
{{ end }}
};
int nsyscall = nelem(systab);
`)
		err = tmpl.Execute(outfile, syscalls)
		if err != nil {
			return err
		}

	case "error.h":
		tmpl, err := template.New("error.h").Parse(`/* automatically generated by mksys */
{{ range . }}extern char {{ .Name }}[]; /* {{ .String }} */
{{ end }}
`)
		err = tmpl.Execute(outfile, syserrors)
		if err != nil {
			return err
		}

	case "errstr.h":
		tmpl, err := template.New("errstr.h").Parse(`/* automatically generated by mksys */
{{ range . }}char {{ .Name }}[] = "{{ .String }}";
{{ end }}
`)
		err = tmpl.Execute(outfile, syserrors)
		if err != nil {
			return err
		}
	case "bootamd64cpu.c":
		tmpl, err := template.New("bootamd64cpu.c").Parse(`/* automatically generated by mksys */
#include <u.h>
#include <libc.h>
#include <boot.h>
Method method[] = {
{{ range . }}{ "{{.Name}}", {{.Config}}, {{.Connect}}, "{{.Arg}}", },
{{ end }}
	{ nil },
};
int cpuflag = 1;
char* rootdir = "/root";
char* bootdisk = "#S/sdE0/";
extern void boot(int, char**);
void
main(int argc, char **argv)
{
		boot(argc, argv);
}
int (*cfs)(int) = 0;
`)
		err = tmpl.Execute(outfile, bootmethods)
		if err != nil {
			log.Fatal(err)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
)

var (
	serverloc = flag.String("serverloc", "http://static.xeserv.us/", "server to prepend to url paths")
	sourceloc = flag.String("sourceloc", "", "source URL for metadata generation")
)

func main() {
	flag.Parse()

	if flag.NArg() == 0 {
		fmt.Printf("%s: <dir>\n", os.Args[0])
		flag.Usage()
	}

	if *sourceloc == "" {
		log.Fatal("Need a source location")
	}

	images, err := ioutil.ReadDir(flag.Arg(0))
	if err != nil {
		panic(err)
	}

	for _, image := range images {
		if strings.HasSuffix(image.Name(), ".json") {
			log.Printf("Skipped %s...", image)
			continue
		}

		fout, err := os.Create(image.Name() + ".json")

		var i UploadImage
		i.Image.SourceURL = *sourceloc
		i.Image.ImageURL = *serverloc + filepath.Base(image.Name())

		outdata, err := json.MarshalIndent(&i, "", "\t")
		if err != nil {
			panic(err)
		}

		fout.Write(outdata)
		fout.Close()
	}
}
<commit_msg>fix dbupload stuff<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
)

var (
	serverloc = flag.String("serverloc", "http://static.xeserv.us/", "server to prepend to url paths")
	sourceloc = flag.String("sourceloc", "", "source URL for metadata generation")
	set       = flag.String("set", "", "comic set")
)

func main() {
	flag.Parse()

	if flag.NArg() == 0 {
		fmt.Printf("%s: <dir>\n", os.Args[0])
		flag.Usage()
	}

	if *sourceloc == "" {
		log.Fatal("Need a source location")
	}

	images, err := ioutil.ReadDir(flag.Arg(0))
	if err != nil {
		panic(err)
	}

	for _, image := range images {
		if strings.HasSuffix(image.Name(), ".json") {
			log.Printf("Skipped %s...", image)
			continue
		}

		fout, err := os.Create(image.Name() + ".json")

		var i UploadImage
		i.Image.SourceURL = *sourceloc
		i.Image.ImageURL = *serverloc + filepath.Base(image.Name())
		i.Image.Tags = "explicit"

		if *set != "" {
			i.Image.Tags = i.Image.Tags + ", comic:" + *set
		}

		outdata, err := json.MarshalIndent(&i, "", "\t")
		if err != nil {
			panic(err)
		}

		fout.Write(outdata)
		fout.Close()
	}
}
<|endoftext|>


<commit_before>package webhooks

import (
	"crypto/sha1"
	"encoding/base64"
	"net/url"
	"strings"

	"github.com/michaelklishin/rabbit-hole"
)

const sourceExchange = "amq.topic"

type RabbitStore struct {
	rb *rabbithole.Client
	vh string
}

func NewRabbitStore(r *rabbithole.Client, vh string) Store {
	setupVhost(r, vh)

	return &RabbitStore{
		rb: r,
		vh: vh,
	}
}

func (r *RabbitStore) All() (hooks []Webhook, err error) {
	hooks = make([]Webhook, 0)

	urls, err := r.getQueueUrls()

	if err != nil {
		return
	}

	bs, err := r.rb.ListBindingsIn(r.vh)

	if err != nil {
		return
	}

	for _, b := range bs {
		if b.DestinationType == "exchange" {
			if url, ok := urls[b.Destination]; ok {
				h := NewWebhook(b.PropertiesKey, url, b.RoutingKey)
				hooks = append(hooks, h)
			}
		}
	}

	return
}

func (r *RabbitStore) GetById(id string) Webhook {
	return Webhook{}
}

func (r *RabbitStore) Delete(id string) error {
	return nil
}

func (r *RabbitStore) Add(wh *Webhook) (err error) {
	qn, err := r.setupUrlQueue(wh.CallbackURL)

	if err != nil {
		return err
	}

	props, err := r.bindExchange(sourceExchange, qn, wh.Filter)

	wh.Id = props

	return
}

func setupVhost(r *rabbithole.Client, vh string) (err error) {
	permissions := rabbithole.Permissions{Configure: "*", Write: "*", Read: "*"}
	_, err = r.PutVhost(vh, rabbithole.VhostSettings{Tracing: false})

	if err != nil {
		return
	}

	_, err = r.UpdatePermissionsIn(vh, r.Username, permissions)

	return
}

func (r *RabbitStore) setupUrlQueue(url string) (qn string, err error) {
	qn, err = r.createQueue(url)

	if err != nil {
		return
	}

	err = r.createExchange(qn)

	if err != nil {
		return
	}

	err = r.bindQueue(qn)

	return
}

func generateEndpointQueueName(url string) string {
	h := sha1.New()
	h.Write([]byte(url))
	return base64.URLEncoding.EncodeToString(h.Sum(nil))
}

func (r *RabbitStore) createQueue(url string) (name string, err error) {
	name = generateEndpointQueueName(url)
	_, err = r.rb.DeclareQueue(r.vh, name, rabbithole.QueueSettings{
		Durable:    false,
		AutoDelete: false,
		Arguments:  map[string]interface{}{"url": url},
	})

	return
}

func (r *RabbitStore) createExchange(name string) (err error) {
	_, err = r.rb.DeclareExchange(r.vh, name, rabbithole.ExchangeSettings{
		Type: "direct",
	})

	return
}

func (r *RabbitStore) bindQueue(qn string) (err error) {
	_, err = r.rb.DeclareBinding(r.vh, rabbithole.BindingInfo{
		Source:          qn,
		Destination:     qn,
		DestinationType: "queue",
		RoutingKey:      "#",
	})

	return
}

func (r *RabbitStore) bindExchange(src, dst, filter string) (props string, err error) {
	res, err := r.rb.DeclareBinding(r.vh, rabbithole.BindingInfo{
		Source:          src,
		Destination:     dst,
		DestinationType: "exchange",
		RoutingKey:      filter,
	})

	if err != nil {
		return
	}

	props, err = url.QueryUnescape(strings.Split(res.Header.Get("Location"), "/")[1])

	return
}

func (r *RabbitStore) getQueueUrls() (urls map[string]string, err error) {
	urls = make(map[string]string)

	qs, err := r.rb.ListQueuesIn(r.vh)
	if err != nil {
		return
	}

	for _, q := range qs {
		if arg, ok := q.Arguments["url"]; ok {
			if url, ok := arg.(string); ok {
				urls[q.Name] = url
			}
		}
	}

	return
}
<commit_msg>caching endpoint data and webhooks pulled from rabbit<commit_after>package webhooks

import (
	"crypto/sha1"
	"encoding/base64"
	"net/url"
	"strings"

	"github.com/michaelklishin/rabbit-hole"
)

const sourceExchange = "amq.topic"

type RabbitStore struct {
	rb        *rabbithole.Client
	vh        string
	endpoints map[string]*endpoint
}

type endpoint struct {
	url   string
	hooks []Webhook
}

func NewRabbitStore(r *rabbithole.Client, vh string) Store {
	setupVhost(r, vh)

	s := &RabbitStore{
		rb: r,
		vh: vh,
	}

	s.reloadEndpoints()

	return s
}

func (r *RabbitStore) All() (hooks []Webhook, err error) {
	hooks = make([]Webhook, 0)
	for _, v := range r.endpoints {
		hooks = append(hooks, v.hooks...)
	}
	return
}

func (r RabbitStore) GetById(id string) Webhook {
	return Webhook{}
}

func (r *RabbitStore) Delete(id string) error {
	return nil
}

func (r *RabbitStore) Add(wh *Webhook) (err error) {
	qn, err := r.setupUrlQueue(wh.CallbackURL)

	if err != nil {
		return err
	}

	props, err := r.bindExchange(sourceExchange, qn, wh.Filter)

	wh.Id = props

	r.reloadEndpoints()

	return
}

func setupVhost(r *rabbithole.Client, vh string) (err error) {
	permissions := rabbithole.Permissions{Configure: "*", Write: "*", Read: "*"}
	_, err = r.PutVhost(vh, rabbithole.VhostSettings{Tracing: false})

	if err != nil {
		return
	}

	_, err = r.UpdatePermissionsIn(vh, r.Username, permissions)

	return
}

func (r *RabbitStore) setupUrlQueue(url string) (qn string, err error) {
	qn, err = r.createQueue(url)

	if err != nil {
		return
	}

	err = r.createExchange(qn)

	if err != nil {
		return
	}

	err = r.bindQueue(qn)

	return
}

func generateEndpointQueueName(url string) string {
	h := sha1.New()
	h.Write([]byte(url))
	return base64.URLEncoding.EncodeToString(h.Sum(nil))
}

func (r *RabbitStore) createQueue(url string) (name string, err error) {
	name = generateEndpointQueueName(url)
	_, err = r.rb.DeclareQueue(r.vh, name, rabbithole.QueueSettings{
		Durable:    false,
		AutoDelete: false,
		Arguments:  map[string]interface{}{"url": url},
	})

	return
}

func (r *RabbitStore) createExchange(name string) (err error) {
	_, err = r.rb.DeclareExchange(r.vh, name, rabbithole.ExchangeSettings{
		Type: "direct",
	})

	return
}

func (r *RabbitStore) bindQueue(qn string) (err error) {
	_, err = r.rb.DeclareBinding(r.vh, rabbithole.BindingInfo{
		Source:          qn,
		Destination:     qn,
		DestinationType: "queue",
		RoutingKey:      "#",
	})

	return
}

func (r *RabbitStore) bindExchange(src, dst, filter string) (props string, err error) {
	res, err := r.rb.DeclareBinding(r.vh, rabbithole.BindingInfo{
		Source:          src,
		Destination:     dst,
		DestinationType: "exchange",
		RoutingKey:      filter,
	})

	if err != nil {
		return
	}

	props, err = url.QueryUnescape(strings.Split(res.Header.Get("Location"), "/")[1])

	return
}

func (r *RabbitStore) getQueueUrls() (urls map[string]*endpoint, err error) {
	urls = make(map[string]*endpoint)

	qs, err := r.rb.ListQueuesIn(r.vh)
	if err != nil {
		return
	}

	for _, q := range qs {
		if arg, ok := q.Arguments["url"]; ok {
			if url, ok := arg.(string); ok {
				urls[q.Name] = &endpoint{url: url, hooks: make([]Webhook, 0)}
			}
		}
	}

	return
}

func (r *RabbitStore) reloadEndpoints() {
	endpoints, err := r.getQueueUrls()

	if err != nil {
		return
	}

	bs, err := r.rb.ListBindingsIn(r.vh)

	if err != nil {
		return
	}

	for _, b := range bs {
		if b.DestinationType == "exchange" {
			if ep, ok := endpoints[b.Destination]; ok {
				h := NewWebhook(b.PropertiesKey, ep.url, b.RoutingKey)
				ep.hooks = append(ep.hooks, h)
			}
		}
	}

	r.endpoints = endpoints
}
<|endoftext|>


<commit_before>package buildah

import (
	"encoding/json"
	"fmt"
	"path/filepath"

	"github.com/containers/storage/storage"
)

const (
	// Package is the name of this package, used in help output and to
	// identify working containers.
	Package       = "buildah"
	containerType = Package + " 0.0.0"
)

// Builder objects are used to represent containers which are being used to
// build images.  They also carry potential updates which will be applied to
// the image's configuration when the container's contents are used to build an
// image.
type Builder struct {
	store storage.Store

	// Type is used to help identify a build container's metadata.  It
	// should not be modified.
	Type string `json:"type"`
	// FromImage is the name of the source image which was used to create
	// the container, if one was used.  It should not be modified.
	FromImage string `json:"image,omitempty"`
	// Config is the source image's configuration.  It should not be
	// modified.
	Config []byte `json:"config,omitempty"`
	// Manifest is the source image's manifest.  It should not be modified.
	Manifest []byte `json:"manifest,omitempty"`

	// Container is the name of the build container.  It should not be modified.
	Container string `json:"container-name,omitempty"`
	// ContainerID is the ID of the build container.  It should not be modified.
	ContainerID string `json:"container-id,omitempty"`
	// MountPoint is the last location where the container's root
	// filesystem was mounted.  It should not be modified.
	MountPoint string `json:"mountpoint,omitempty"`
	// Mounts is a list of places where the container's root filesystem has
	// been mounted.  It should not be modified.
	Mounts []string `json:"mounts,omitempty"`
	// Links is a list of symbolic links to the currently-mounted container
	// root filesystem.  They will be removed when the container is
	// unmounted.  It should not be modified.
	Links []string `json:"links,omitempty"`

	// Annotations is a set of key-value pairs which is stored in the
	// image's manifest.
	Annotations map[string]string `json:"annotations,omitempty"`

	// CreatedBy is a description of how this container was built.
	CreatedBy string `json:"created-by,omitempty"`
	// OS is the operating system for which binaries in the image are
	// built.  The default is the current OS.
	OS string `json:"os,omitempty"`
	// Architecture is the type of processor for which binaries in the
	// image are built.  The default is the current architecture.
	Architecture string `json:"arch,omitempty"`
	// Maintainer is the point of contact for this container.
	Maintainer string `json:"maintainer,omitempty"`
	// User is the user as whom commands are run in the container.
	User string `json:"user,omitempty"`
	// Workdir is the default working directory for commands started in the
	// container.
	Workdir string `json:"workingdir,omitempty"`
	// Env is a list of environment variables to set for the container, in
	// the form NAME=VALUE.
	Env []string `json:"env,omitempty"`
	// Cmd sets a default command to run in containers based on the image.
	Cmd []string `json:"cmd,omitempty"`
	// Entrypoint is an entry point for containers based on the image.
	Entrypoint []string `json:"entrypoint,omitempty"`
	// Expose is a map keyed by specifications of ports to expose when a
	// container based on the image is run.
	Expose map[string]interface{} `json:"expose,omitempty"`
	// Labels is a set of key-value pairs which is stored in the
	// image's configuration.
	Labels map[string]string `json:"labels,omitempty"`
	// Volumes is a list of data volumes which will be created in
	// containers based on the image.
	Volumes []string `json:"volumes,omitempty"`
	// Arg is a set of build-time variables.
	Arg map[string]string `json:"arg,omitempty"`
}

// BuilderOptions are used to initialize a Builder.
type BuilderOptions struct {
	// FromImage is the name of the image which should be used as the
	// starting point for the container.  It can be set to an empty value
	// or "scratch" to indicate that the container should not be based on
	// an image.
	FromImage string
	// Container is a desired name for the build container.
	Container string
	// PullIfMissing signals to NewBuilder() that it should pull the image
	// if it is not present in the local Store.
	PullIfMissing bool
	// PullAlways signals to NewBuilder() that it should pull the source
	// image before creating the container.
	PullAlways bool
	// Registry is a value which is prepended to the image's name, if it
	// needs to be pulled and the image name alone can not be resolved to a
	// reference to a source image.
	Registry string
	// Mount signals to NewBuilder() that the container should be mounted
	// immediately.
	Mount bool
	// Link specifies a location for a symbolic link which should be
	// created if the container is mounted immediately.
	Link string
	// SignaturePolicyPath specifies an override location for the signature
	// policy which should be used for verifying the new image as it is
	// being written.  Except in specific circumstances, no value should be
	// specified, indicating that the shared, system-wide default policy
	// should be used.
	SignaturePolicyPath string
}

// NewBuilder creates a new build container.
func NewBuilder(store storage.Store, options BuilderOptions) (*Builder, error) {
	return newBuilder(store, options)
}

// OpenBuilder loads information about a build container given its name or ID.
func OpenBuilder(store storage.Store, container string) (*Builder, error) {
	c, err := store.GetContainer(container)
	if err != nil {
		return nil, err
	}
	buildstate, err := store.GetMetadata(c.ID)
	if err != nil {
		return nil, err
	}
	b := &Builder{}
	err = json.Unmarshal([]byte(buildstate), &b)
	if err != nil {
		return nil, err
	}
	if b.Type != containerType {
		return nil, fmt.Errorf("container is not a %s container", Package)
	}
	b.store = store
	return b, nil
}

// OpenBuilderByPath loads information about a build container given either a
// path to the container's root filesystem or a the path of a symbolic link
// which points to it.
func OpenBuilderByPath(store storage.Store, path string) (*Builder, error) {
	containers, err := store.Containers()
	if err != nil {
		return nil, err
	}
	abs, err := filepath.Abs(path)
	if err != nil {
		return nil, err
	}
	builderMatchesPath := func(b *Builder, path string) bool {
		if b.MountPoint == path {
			return true
		}
		for _, m := range b.Mounts {
			if m == path {
				return true
			}
		}
		for _, l := range b.Links {
			if l == path {
				return true
			}
		}
		return false
	}
	for _, container := range containers {
		buildstate, err := store.GetMetadata(container.ID)
		if err != nil {
			return nil, err
		}
		b := &Builder{}
		err = json.Unmarshal([]byte(buildstate), &b)
		if err == nil && b.Type == containerType && builderMatchesPath(b, abs) {
			b.store = store
			return b, nil
		}
	}
	return nil, storage.ErrContainerUnknown
}

// Save saves the builder's current state to the build container's metadata.
// This should not need to be called directly, as other methods of the Builder
// object take care of saving their state.
func (b *Builder) Save() error {
	buildstate, err := json.Marshal(b)
	if err != nil {
		return err
	}
	return b.store.SetMetadata(b.ContainerID, string(buildstate))
}
<commit_msg>Move our data to a file in the containerdir<commit_after>package buildah

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"path/filepath"

	"github.com/containers/storage/pkg/ioutils"
	"github.com/containers/storage/storage"
)

const (
	// Package is the name of this package, used in help output and to
	// identify working containers.
	Package       = "buildah"
	containerType = Package + " 0.0.0"
	stateFile     = Package + ".json"
)

// Builder objects are used to represent containers which are being used to
// build images.  They also carry potential updates which will be applied to
// the image's configuration when the container's contents are used to build an
// image.
type Builder struct {
	store storage.Store

	// Type is used to help identify a build container's metadata.  It
	// should not be modified.
	Type string `json:"type"`
	// FromImage is the name of the source image which was used to create
	// the container, if one was used.  It should not be modified.
	FromImage string `json:"image,omitempty"`
	// Config is the source image's configuration.  It should not be
	// modified.
	Config []byte `json:"config,omitempty"`
	// Manifest is the source image's manifest.  It should not be modified.
	Manifest []byte `json:"manifest,omitempty"`

	// Container is the name of the build container.  It should not be modified.
	Container string `json:"container-name,omitempty"`
	// ContainerID is the ID of the build container.  It should not be modified.
	ContainerID string `json:"container-id,omitempty"`
	// MountPoint is the last location where the container's root
	// filesystem was mounted.  It should not be modified.
	MountPoint string `json:"mountpoint,omitempty"`
	// Mounts is a list of places where the container's root filesystem has
	// been mounted.  It should not be modified.
	Mounts []string `json:"mounts,omitempty"`
	// Links is a list of symbolic links to the currently-mounted container
	// root filesystem.  They will be removed when the container is
	// unmounted.  It should not be modified.
	Links []string `json:"links,omitempty"`

	// Annotations is a set of key-value pairs which is stored in the
	// image's manifest.
	Annotations map[string]string `json:"annotations,omitempty"`

	// CreatedBy is a description of how this container was built.
	CreatedBy string `json:"created-by,omitempty"`
	// OS is the operating system for which binaries in the image are
	// built.  The default is the current OS.
	OS string `json:"os,omitempty"`
	// Architecture is the type of processor for which binaries in the
	// image are built.  The default is the current architecture.
	Architecture string `json:"arch,omitempty"`
	// Maintainer is the point of contact for this container.
	Maintainer string `json:"maintainer,omitempty"`
	// User is the user as whom commands are run in the container.
	User string `json:"user,omitempty"`
	// Workdir is the default working directory for commands started in the
	// container.
	Workdir string `json:"workingdir,omitempty"`
	// Env is a list of environment variables to set for the container, in
	// the form NAME=VALUE.
	Env []string `json:"env,omitempty"`
	// Cmd sets a default command to run in containers based on the image.
	Cmd []string `json:"cmd,omitempty"`
	// Entrypoint is an entry point for containers based on the image.
	Entrypoint []string `json:"entrypoint,omitempty"`
	// Expose is a map keyed by specifications of ports to expose when a
	// container based on the image is run.
	Expose map[string]interface{} `json:"expose,omitempty"`
	// Labels is a set of key-value pairs which is stored in the
	// image's configuration.
	Labels map[string]string `json:"labels,omitempty"`
	// Volumes is a list of data volumes which will be created in
	// containers based on the image.
	Volumes []string `json:"volumes,omitempty"`
	// Arg is a set of build-time variables.
	Arg map[string]string `json:"arg,omitempty"`
}

// BuilderOptions are used to initialize a Builder.
type BuilderOptions struct {
	// FromImage is the name of the image which should be used as the
	// starting point for the container.  It can be set to an empty value
	// or "scratch" to indicate that the container should not be based on
	// an image.
	FromImage string
	// Container is a desired name for the build container.
	Container string
	// PullIfMissing signals to NewBuilder() that it should pull the image
	// if it is not present in the local Store.
	PullIfMissing bool
	// PullAlways signals to NewBuilder() that it should pull the source
	// image before creating the container.
	PullAlways bool
	// Registry is a value which is prepended to the image's name, if it
	// needs to be pulled and the image name alone can not be resolved to a
	// reference to a source image.
	Registry string
	// Mount signals to NewBuilder() that the container should be mounted
	// immediately.
	Mount bool
	// Link specifies a location for a symbolic link which should be
	// created if the container is mounted immediately.
	Link string
	// SignaturePolicyPath specifies an override location for the signature
	// policy which should be used for verifying the new image as it is
	// being written.  Except in specific circumstances, no value should be
	// specified, indicating that the shared, system-wide default policy
	// should be used.
	SignaturePolicyPath string
}

// NewBuilder creates a new build container.
func NewBuilder(store storage.Store, options BuilderOptions) (*Builder, error) {
	return newBuilder(store, options)
}

// OpenBuilder loads information about a build container given its name or ID.
func OpenBuilder(store storage.Store, container string) (*Builder, error) {
	cdir, err := store.GetContainerDirectory(container)
	if err != nil {
		return nil, err
	}
	buildstate, err := ioutil.ReadFile(filepath.Join(cdir, stateFile))
	if err != nil {
		return nil, err
	}
	b := &Builder{}
	err = json.Unmarshal([]byte(buildstate), &b)
	if err != nil {
		return nil, err
	}
	if b.Type != containerType {
		return nil, fmt.Errorf("container is not a %s container", Package)
	}
	b.store = store
	return b, nil
}

// OpenBuilderByPath loads information about a build container given either a
// path to the container's root filesystem or a the path of a symbolic link
// which points to it.
func OpenBuilderByPath(store storage.Store, path string) (*Builder, error) {
	containers, err := store.Containers()
	if err != nil {
		return nil, err
	}
	abs, err := filepath.Abs(path)
	if err != nil {
		return nil, err
	}
	builderMatchesPath := func(b *Builder, path string) bool {
		if b.MountPoint == path {
			return true
		}
		for _, m := range b.Mounts {
			if m == path {
				return true
			}
		}
		for _, l := range b.Links {
			if l == path {
				return true
			}
		}
		return false
	}
	for _, container := range containers {
		buildstate, err := store.GetMetadata(container.ID)
		if err != nil {
			return nil, err
		}
		b := &Builder{}
		err = json.Unmarshal([]byte(buildstate), &b)
		if err == nil && b.Type == containerType && builderMatchesPath(b, abs) {
			b.store = store
			return b, nil
		}
	}
	return nil, storage.ErrContainerUnknown
}

// Save saves the builder's current state to the build container's metadata.
// This should not need to be called directly, as other methods of the Builder
// object take care of saving their state.
func (b *Builder) Save() error {
	buildstate, err := json.Marshal(b)
	if err != nil {
		return err
	}
	cdir, err := b.store.GetContainerDirectory(b.ContainerID)
	if err != nil {
		return err
	}
	return ioutils.AtomicWriteFile(filepath.Join(cdir, stateFile), buildstate, 0600)
}
<|endoftext|>


<commit_before>package ring

import (
	"math/rand"
	"time"
)

// Node is a target of a ring assignment. For example, it might represent a
// disk drive in a large storage cluster.
type Node struct {

	// Disabled is whether this node can currently be assigned replicas.
	// See Capacity below for a bit more information.
	Disabled bool

	// Capacity indicates how many replicas this node should be assigned; it is
	// relative to other nodes' capacities.
	//
	// Values less than 1 are treated as 0 and indicate the node should not
	// have any assignments, but if it already has some and they cannot yet be
	// moved due to some restrictions, that is okay for now.
	//
	// 0 capacity differs from Disabled in that Disabled means that no
	// assignments can be made, overriding the restrictions.
	//
	// 0 capacity might be used to "drain" a node for maintenance once enough
	// rebalances clear it of any assignments.
	Capacity int

	// TierIndexes specify which tiers a Node is within.
	//
	// Tiers indicate the layout of the node with respect to other nodes. For
	// example, the lowest tier, tier 0, might be the server (where each node
	// represents a drive on that server). The next tier, 1, might then be the
	// power zone the server is in. The number of tiers is flexible, so later
	// an additional tier for geographic region could be added.
	//
	// Different replicas of a given partition are attempted to be assigned to
	// distinct tiers at each level.
	//
	// If a tier index is less than 0, it will be assumed to be 0.
	//
	// If a tier index does not exist for a node (but exists for some other
	// node) it will be assumed to be 0.
	//
	// See the package documentation Example (Tiers) for in-depth code
	// discussion to better understand tiers.
	TierIndexes []int
}

// Builder encapsulates the information needed to best balance ring assignments.
type Builder struct {
	Ring

	// Nodes are the targets of each replica assignment.
	Nodes []*Node

	// LastMovedBase is the base time for the LastMoved delta values.
	LastMovedBase time.Time

	// LastMoved is the number of LastMovedUnits since LastMovedBase that
	// replicas were last moved; LastMoved[replica][partition] = elapsed.
	// This is used to restrict moving these replicas again.
	LastMoved [][]LastMovedType

	// LastMovedUnit is the duration that LastMoved values are measured in.
	//
	// 0 will be treated as the default, which currently is time.Minute.
	LastMovedUnit time.Duration

	// MoveWait is how much time that must elapse before a replica can be moved
	// again.
	//
	// 0 will be treated as the default, which currently is time.Hour.
	MoveWait time.Duration

	// MovesPerPartition is the highest number of replicas for a partition that
	// can be in motion within the MoveWait window.
	//
	// For full copy replicas, you probably want this set to no more than half
	// the replica count. For erasure coding, you probably want this set to no
	// more than "m", the maximum number of failure chunks.
	//
	// 0 (or less) will be treated as the default, which currently is
	// int(replicas/2), but at least 1.
	MovesPerPartition int

	// PointsAllowed is the number of percentage points over or under that the
	// Builder will try to keep replica assignments within. For example, if set
	// to 1% and a node should receive 100 replica assignments, the Builder
	// will target 99-101 assignments to that node. This is not guaranteed,
	// just a target.
	//
	// This setting will affect how quickly the partition count will increase.
	// If you feel your use case can handle an over/under of 10%, that could
	// end up with smaller rings if memory is more of a concern. However, just
	// setting the MaxPartitionCount is probably a better approach in such a
	// case.
	//
	// 0 will be treated as the default, which currently is 1 for one percent
	// extra or fewer replicas per node.
	PointsAllowed int

	// MaxPartitionCount is the maximum number of partitions the builder should
	// grow to. This keeps the rebalancer from running too long trying to
	// achieve desired balances and also caps memory usage.
	//
	// 0 (or less) will be treated as the default, which currently is 8388608.
	MaxPartitionCount int

	// rnd is used to add a some shuffling to the rebalancing algorithms. This
	// can help keep nodes from mirroring each other too much. Mirroring
	// meaning that two nodes share more partitions than they really should, so
	// if both fail at the same time more partitions lose multiple replicas
	// than they really should.
	//
	// If nil, this will be set to rand.New(rand.NewSource(0)) -- yes, this
	// means it's the same seed always, which is fine. We want the shuffling to
	// help with distribution, but providing repeatable results for users is
	// also beneficial.
	//
	// It can be set to noRnd to skip the shuffling, useful for tests comparing
	// shuffling versus not.
	rnd *rand.Rand
}

var noRnd = &rand.Rand{}

// Rebalance updates the b.Ring replica assignments, if needed and possible.
func (b *Builder) Rebalance() {
	validNodes := false
	for _, n := range b.Nodes {
		if !n.Disabled {
			validNodes = true
		}
	}
	if !validNodes {
		return
	}
	if b.rnd == nil {
		b.rnd = rand.New(rand.NewSource(0))
	}
	if b.LastMovedUnit == 0 {
		b.LastMovedUnit = time.Minute
	}
	if b.MoveWait == 0 {
		b.MoveWait = time.Hour
	}
	if len(b.Ring) == 0 {
		b.Ring = Ring{[]NodeIndexType{NodeIndexNil}}
		b.LastMoved = [][]LastMovedType{{LastMovedMax}}
	}
	newBase := time.Now()
	b.ShiftLastMoved(newBase.Sub(b.LastMovedBase))
	b.growIfNeeded()
	newRebalancer(b).rebalance()
}

// SetReplicaCount will add or remove replicas so that len(b.Ring) == count.
// If replicas are added they will be unassigned: b.Ring[newReplica][*] = NodeIndexNil
func (b *Builder) SetReplicaCount(count int) {
	if count < 1 {
		count = 1
	}
	if len(b.Ring) == 0 {
		b.Ring = Ring{[]NodeIndexType{NodeIndexNil}}
		b.LastMoved = [][]LastMovedType{{LastMovedMax}}
	}
	if count < len(b.Ring) {
		b.Ring = b.Ring[:count]
		b.LastMoved = b.LastMoved[:count]
	} else if count > len(b.Ring) {
		partitionCount := len(b.Ring[0])
		for count > len(b.Ring) {
			newPartitionToNodeIndex := make([]NodeIndexType, partitionCount)
			newPartitionToLastMove := make([]LastMovedType, partitionCount)
			for i := 0; i < partitionCount; i++ {
				newPartitionToNodeIndex[i] = NodeIndexNil
				newPartitionToLastMove[i] = LastMovedMax
			}
			b.Ring = append(b.Ring, newPartitionToNodeIndex)
			b.LastMoved = append(b.LastMoved, newPartitionToLastMove)
		}
	}
}

// ShiftLastMoved shifts the LastMoved records by the duration given, as if
// that much more time has passed since all replicas of all partitions have
// been moved.
//
// This is done automatically when Rebalance is called and therefore
// LastMovedBase is updated but can also be useful in testing, as the ring
// algorithms will restrict the movement of replicas based on this information.
func (b *Builder) ShiftLastMoved(d time.Duration) {
	if b.LastMovedUnit == 0 {
		b.LastMovedUnit = time.Minute
	}
	units := LastMovedType(LastMovedMax)
	if int64(d/b.LastMovedUnit) < int64(LastMovedMax) {
		units = LastMovedType(d / b.LastMovedUnit)
	}
	for _, partitionToLastMoved := range b.LastMoved {
		for partition := len(partitionToLastMoved) - 1; partition >= 0; partition-- {
			if LastMovedMax-partitionToLastMoved[partition] < units {
				partitionToLastMoved[partition] = LastMovedMax
			} else {
				partitionToLastMoved[partition] += units
			}
		}
	}
}

// RemoveNode will remove the node from the Builder.
//
// Note that this can be relatively expensive as all nodes that had been added
// after the removed node had been originally added will have their indexes
// shifted down one and many of the replica-to-partition-to-node assignments
// will have to be updated, as well as clearing any assignments that were to
// the removed node.
//
// Depending on the use case, it might be better to just leave a "dead" node in
// place and simply set it as disabled.
func (b *Builder) RemoveNode(nodeIndex NodeIndexType) {
	if nodeIndex == NodeIndexNil || int(nodeIndex) >= len(b.Nodes) {
		return
	}
	copy(b.Nodes[nodeIndex:], b.Nodes[nodeIndex+1:])
	b.Nodes = b.Nodes[:len(b.Nodes)-1]
	for _, partitionToNodeIndex := range b.Ring {
		for j := len(partitionToNodeIndex) - 1; j >= 0; j-- {
			if partitionToNodeIndex[j] == nodeIndex {
				partitionToNodeIndex[j] = NodeIndexNil
			} else if partitionToNodeIndex[j] > nodeIndex {
				partitionToNodeIndex[j]--
			}
		}
	}
}

func (b *Builder) growIfNeeded() {
	// Consider: Shrinking the partitionToNodeIndex slices doesn't happen
	// because it would normally cause more data movements than it's worth.
	// Perhaps in the future we can add detection of cases when shrinking makes
	// sense. Perhaps providing a way to do multiple rebalances over time to
	// distribute partitions where halving wouldn't move anything on the
	// shrinking rebalance.
	//
	// The current restriction of no shrinking does mean that if you change the
	// MaxPartitionCount and the current partition count is already over that,
	// it will not reduce at this time.
	if b.MaxPartitionCount < 1 {
		b.MaxPartitionCount = 8388608 // MaxPartitionCount default of 8388608 partitions.
	}
	if b.PartitionCount() >= b.MaxPartitionCount {
		return
	}
	replicaCount := len(b.Ring)
	// Calculate the partition count needed.
	// Each node is examined to see how much under or overweight it would be
	// and increasing the partition count until the difference is under the
	// points allowed.
	totalCapacity := float64(0)
	for _, n := range b.Nodes {
		if !n.Disabled {
			totalCapacity += float64(n.Capacity)
		}
	}
	partitionCount := len(b.Ring[0])
	doublings := uint(0)
	if b.PointsAllowed == 0 {
		b.PointsAllowed = 1 // PointsAllowed default of 1 percent.
	}
	pointsAllowed := float64(b.PointsAllowed) * 0.01
	keepTrying := true
	for keepTrying && partitionCount < b.MaxPartitionCount {
		keepTrying = false
		for _, n := range b.Nodes {
			if n.Disabled {
				continue
			}
			desiredPartitionCount := float64(partitionCount) * float64(replicaCount) * (float64(n.Capacity) / totalCapacity)
			under := (desiredPartitionCount - float64(int(desiredPartitionCount))) / desiredPartitionCount
			over := float64(0)
			if desiredPartitionCount > float64(int(desiredPartitionCount)) {
				over = (float64(int(desiredPartitionCount)+1) - desiredPartitionCount) / desiredPartitionCount
			}
			if under > pointsAllowed || over > pointsAllowed {
				partitionCount *= 2
				doublings++
				if partitionCount == b.MaxPartitionCount {
					break
				} else if partitionCount > b.MaxPartitionCount {
					partitionCount /= 2
					doublings--
					break
				}
				keepTrying = true
			}
		}
	}
	// Grow the partitionToNodeIndex slices if the partition count grew.
	if partitionCount > len(b.Ring[0]) {
		for replica := 0; replica < replicaCount; replica++ {
			partitionToNodeIndex := make([]NodeIndexType, partitionCount)
			partitionToLastMoved := make([]LastMovedType, partitionCount)
			for partition := 0; partition < partitionCount; partition++ {
				partitionToNodeIndex[partition] = b.Ring[replica][partition>>doublings]
				partitionToLastMoved[partition] = b.LastMoved[replica][partition>>doublings]
			}
			b.Ring[replica] = partitionToNodeIndex
			b.LastMoved[replica] = partitionToLastMoved
		}
	}
}

// Stats gives an overview of the state and health of the Builder.
// It is returned by the Builder.Stats() method.
type Stats struct {
	ReplicaCount      int
	EnabledNodeCount  int
	DisabledNodeCount int
	PartitionCount    int
	UnassignedCount   int
	EnabledCapacity   float64
	DisabledCapacity  float64
	// MaxUnderNodePercentage is the percentage a node is underweight, or has
	// less data assigned to it than its capacity would indicate it desires.
	MaxUnderNodePercentage float64
	MaxUnderNodeIndex      NodeIndexType
	// MaxOverNodePercentage is the percentage a node is overweight, or has
	// more data assigned to it than its capacity would indicate it desires.
	MaxOverNodePercentage float64
	MaxOverNodeIndex      NodeIndexType
}

func (b *Builder) Stats() *Stats {
	stats := &Stats{
		ReplicaCount:   b.ReplicaCount(),
		PartitionCount: b.PartitionCount(),
	}
	nodeIndexToPartitionCount := make([]int, len(b.Nodes))
	for _, partitionToNodeIndex := range b.Ring {
		for _, nodeIndex := range partitionToNodeIndex {
			if nodeIndex == NodeIndexNil {
				stats.UnassignedCount++
			} else {
				nodeIndexToPartitionCount[nodeIndex]++
			}
		}
	}
	for _, n := range b.Nodes {
		if n.Disabled {
			stats.DisabledNodeCount++
			if n.Capacity > 0 {
				stats.DisabledCapacity += float64(n.Capacity)
			}
		} else {
			stats.EnabledNodeCount++
			if n.Capacity > 0 {
				stats.EnabledCapacity += float64(n.Capacity)
			}
		}
	}
	for nodeIndex, n := range b.Nodes {
		if n.Disabled {
			continue
		}
		desiredPartitionCount := float64(n.Capacity) / stats.EnabledCapacity * float64(stats.PartitionCount) * float64(stats.ReplicaCount)
		actualPartitionCount := float64(nodeIndexToPartitionCount[nodeIndex])
		if desiredPartitionCount > actualPartitionCount {
			under := 100.0 * (desiredPartitionCount - actualPartitionCount) / desiredPartitionCount
			if under > stats.MaxUnderNodePercentage {
				stats.MaxUnderNodePercentage = under
				stats.MaxUnderNodeIndex = NodeIndexType(nodeIndex)
			}
		} else if desiredPartitionCount < actualPartitionCount {
			over := 100.0 * (actualPartitionCount - desiredPartitionCount) / desiredPartitionCount
			if over > stats.MaxOverNodePercentage {
				stats.MaxOverNodePercentage = over
				stats.MaxOverNodeIndex = NodeIndexType(nodeIndex)
			}
		}
	}
	return stats
}
<commit_msg>More doc info on Builder.Nodes<commit_after>package ring

import (
	"math/rand"
	"time"
)

// Node is a target of a ring assignment. For example, it might represent a
// disk drive in a large storage cluster.
type Node struct {

	// Disabled is whether this node can currently be assigned replicas.
	// See Capacity below for a bit more information.
	Disabled bool

	// Capacity indicates how many replicas this node should be assigned; it is
	// relative to other nodes' capacities.
	//
	// Values less than 1 are treated as 0 and indicate the node should not
	// have any assignments, but if it already has some and they cannot yet be
	// moved due to some restrictions, that is okay for now.
	//
	// 0 capacity differs from Disabled in that Disabled means that no
	// assignments can be made, overriding the restrictions.
	//
	// 0 capacity might be used to "drain" a node for maintenance once enough
	// rebalances clear it of any assignments.
	Capacity int

	// TierIndexes specify which tiers a Node is within.
	//
	// Tiers indicate the layout of the node with respect to other nodes. For
	// example, the lowest tier, tier 0, might be the server (where each node
	// represents a drive on that server). The next tier, 1, might then be the
	// power zone the server is in. The number of tiers is flexible, so later
	// an additional tier for geographic region could be added.
	//
	// Different replicas of a given partition are attempted to be assigned to
	// distinct tiers at each level.
	//
	// If a tier index is less than 0, it will be assumed to be 0.
	//
	// If a tier index does not exist for a node (but exists for some other
	// node) it will be assumed to be 0.
	//
	// See the package documentation Example (Tiers) for in-depth code
	// discussion to better understand tiers.
	TierIndexes []int
}

// Builder encapsulates the information needed to best balance ring assignments.
type Builder struct {
	Ring

	// Nodes are the targets of each replica assignment.
	//
	// You can add nodes to this slice directly but if you wish to remove
	// nodes, use the Builder's RemoveNode method. Indexes to this slice are
	// used elsewhere and need to be updated if they change. Currently there is
	// no method to help with completely changing the order of Nodes. If you
	// really want to do that, you will have to update the Builder's Ring
	// assignments yourself.
	Nodes []*Node

	// LastMovedBase is the base time for the LastMoved delta values.
	LastMovedBase time.Time

	// LastMoved is the number of LastMovedUnits since LastMovedBase that
	// replicas were last moved; LastMoved[replica][partition] = elapsed.
	// This is used to restrict moving these replicas again.
	LastMoved [][]LastMovedType

	// LastMovedUnit is the duration that LastMoved values are measured in.
	//
	// 0 will be treated as the default, which currently is time.Minute.
	LastMovedUnit time.Duration

	// MoveWait is how much time that must elapse before a replica can be moved
	// again.
	//
	// 0 will be treated as the default, which currently is time.Hour.
	MoveWait time.Duration

	// MovesPerPartition is the highest number of replicas for a partition that
	// can be in motion within the MoveWait window.
	//
	// For full copy replicas, you probably want this set to no more than half
	// the replica count. For erasure coding, you probably want this set to no
	// more than "m", the maximum number of failure chunks.
	//
	// 0 (or less) will be treated as the default, which currently is
	// int(replicas/2), but at least 1.
	MovesPerPartition int

	// PointsAllowed is the number of percentage points over or under that the
	// Builder will try to keep replica assignments within. For example, if set
	// to 1% and a node should receive 100 replica assignments, the Builder
	// will target 99-101 assignments to that node. This is not guaranteed,
	// just a target.
	//
	// This setting will affect how quickly the partition count will increase.
	// If you feel your use case can handle an over/under of 10%, that could
	// end up with smaller rings if memory is more of a concern. However, just
	// setting the MaxPartitionCount is probably a better approach in such a
	// case.
	//
	// 0 will be treated as the default, which currently is 1 for one percent
	// extra or fewer replicas per node.
	PointsAllowed int

	// MaxPartitionCount is the maximum number of partitions the builder should
	// grow to. This keeps the rebalancer from running too long trying to
	// achieve desired balances and also caps memory usage.
	//
	// 0 (or less) will be treated as the default, which currently is 8388608.
	MaxPartitionCount int

	// rnd is used to add a some shuffling to the rebalancing algorithms. This
	// can help keep nodes from mirroring each other too much. Mirroring
	// meaning that two nodes share more partitions than they really should, so
	// if both fail at the same time more partitions lose multiple replicas
	// than they really should.
	//
	// If nil, this will be set to rand.New(rand.NewSource(0)) -- yes, this
	// means it's the same seed always, which is fine. We want the shuffling to
	// help with distribution, but providing repeatable results for users is
	// also beneficial.
	//
	// It can be set to noRnd to skip the shuffling, useful for tests comparing
	// shuffling versus not.
	rnd *rand.Rand
}

var noRnd = &rand.Rand{}

// Rebalance updates the b.Ring replica assignments, if needed and possible.
func (b *Builder) Rebalance() {
	validNodes := false
	for _, n := range b.Nodes {
		if !n.Disabled {
			validNodes = true
		}
	}
	if !validNodes {
		return
	}
	if b.rnd == nil {
		b.rnd = rand.New(rand.NewSource(0))
	}
	if b.LastMovedUnit == 0 {
		b.LastMovedUnit = time.Minute
	}
	if b.MoveWait == 0 {
		b.MoveWait = time.Hour
	}
	if len(b.Ring) == 0 {
		b.Ring = Ring{[]NodeIndexType{NodeIndexNil}}
		b.LastMoved = [][]LastMovedType{{LastMovedMax}}
	}
	newBase := time.Now()
	b.ShiftLastMoved(newBase.Sub(b.LastMovedBase))
	b.growIfNeeded()
	newRebalancer(b).rebalance()
}

// SetReplicaCount will add or remove replicas so that len(b.Ring) == count.
// If replicas are added they will be unassigned: b.Ring[newReplica][*] = NodeIndexNil
func (b *Builder) SetReplicaCount(count int) {
	if count < 1 {
		count = 1
	}
	if len(b.Ring) == 0 {
		b.Ring = Ring{[]NodeIndexType{NodeIndexNil}}
		b.LastMoved = [][]LastMovedType{{LastMovedMax}}
	}
	if count < len(b.Ring) {
		b.Ring = b.Ring[:count]
		b.LastMoved = b.LastMoved[:count]
	} else if count > len(b.Ring) {
		partitionCount := len(b.Ring[0])
		for count > len(b.Ring) {
			newPartitionToNodeIndex := make([]NodeIndexType, partitionCount)
			newPartitionToLastMove := make([]LastMovedType, partitionCount)
			for i := 0; i < partitionCount; i++ {
				newPartitionToNodeIndex[i] = NodeIndexNil
				newPartitionToLastMove[i] = LastMovedMax
			}
			b.Ring = append(b.Ring, newPartitionToNodeIndex)
			b.LastMoved = append(b.LastMoved, newPartitionToLastMove)
		}
	}
}

// ShiftLastMoved shifts the LastMoved records by the duration given, as if
// that much more time has passed since all replicas of all partitions have
// been moved.
//
// This is done automatically when Rebalance is called and therefore
// LastMovedBase is updated but can also be useful in testing, as the ring
// algorithms will restrict the movement of replicas based on this information.
func (b *Builder) ShiftLastMoved(d time.Duration) {
	if b.LastMovedUnit == 0 {
		b.LastMovedUnit = time.Minute
	}
	units := LastMovedType(LastMovedMax)
	if int64(d/b.LastMovedUnit) < int64(LastMovedMax) {
		units = LastMovedType(d / b.LastMovedUnit)
	}
	for _, partitionToLastMoved := range b.LastMoved {
		for partition := len(partitionToLastMoved) - 1; partition >= 0; partition-- {
			if LastMovedMax-partitionToLastMoved[partition] < units {
				partitionToLastMoved[partition] = LastMovedMax
			} else {
				partitionToLastMoved[partition] += units
			}
		}
	}
}

// RemoveNode will remove the node from the Builder.
//
// Note that this can be relatively expensive as all nodes that had been added
// after the removed node had been originally added will have their indexes
// shifted down one and many of the replica-to-partition-to-node assignments
// will have to be updated, as well as clearing any assignments that were to
// the removed node.
//
// Depending on the use case, it might be better to just leave a "dead" node in
// place and simply set it as disabled.
func (b *Builder) RemoveNode(nodeIndex NodeIndexType) {
	if nodeIndex == NodeIndexNil || int(nodeIndex) >= len(b.Nodes) {
		return
	}
	copy(b.Nodes[nodeIndex:], b.Nodes[nodeIndex+1:])
	b.Nodes = b.Nodes[:len(b.Nodes)-1]
	for _, partitionToNodeIndex := range b.Ring {
		for j := len(partitionToNodeIndex) - 1; j >= 0; j-- {
			if partitionToNodeIndex[j] == nodeIndex {
				partitionToNodeIndex[j] = NodeIndexNil
			} else if partitionToNodeIndex[j] > nodeIndex {
				partitionToNodeIndex[j]--
			}
		}
	}
}

func (b *Builder) growIfNeeded() {
	// Consider: Shrinking the partitionToNodeIndex slices doesn't happen
	// because it would normally cause more data movements than it's worth.
	// Perhaps in the future we can add detection of cases when shrinking makes
	// sense. Perhaps providing a way to do multiple rebalances over time to
	// distribute partitions where halving wouldn't move anything on the
	// shrinking rebalance.
	//
	// The current restriction of no shrinking does mean that if you change the
	// MaxPartitionCount and the current partition count is already over that,
	// it will not reduce at this time.
	if b.MaxPartitionCount < 1 {
		b.MaxPartitionCount = 8388608 // MaxPartitionCount default of 8388608 partitions.
	}
	if b.PartitionCount() >= b.MaxPartitionCount {
		return
	}
	replicaCount := len(b.Ring)
	// Calculate the partition count needed.
	// Each node is examined to see how much under or overweight it would be
	// and increasing the partition count until the difference is under the
	// points allowed.
	totalCapacity := float64(0)
	for _, n := range b.Nodes {
		if !n.Disabled {
			totalCapacity += float64(n.Capacity)
		}
	}
	partitionCount := len(b.Ring[0])
	doublings := uint(0)
	if b.PointsAllowed == 0 {
		b.PointsAllowed = 1 // PointsAllowed default of 1 percent.
	}
	pointsAllowed := float64(b.PointsAllowed) * 0.01
	keepTrying := true
	for keepTrying && partitionCount < b.MaxPartitionCount {
		keepTrying = false
		for _, n := range b.Nodes {
			if n.Disabled {
				continue
			}
			desiredPartitionCount := float64(partitionCount) * float64(replicaCount) * (float64(n.Capacity) / totalCapacity)
			under := (desiredPartitionCount - float64(int(desiredPartitionCount))) / desiredPartitionCount
			over := float64(0)
			if desiredPartitionCount > float64(int(desiredPartitionCount)) {
				over = (float64(int(desiredPartitionCount)+1) - desiredPartitionCount) / desiredPartitionCount
			}
			if under > pointsAllowed || over > pointsAllowed {
				partitionCount *= 2
				doublings++
				if partitionCount == b.MaxPartitionCount {
					break
				} else if partitionCount > b.MaxPartitionCount {
					partitionCount /= 2
					doublings--
					break
				}
				keepTrying = true
			}
		}
	}
	// Grow the partitionToNodeIndex slices if the partition count grew.
	if partitionCount > len(b.Ring[0]) {
		for replica := 0; replica < replicaCount; replica++ {
			partitionToNodeIndex := make([]NodeIndexType, partitionCount)
			partitionToLastMoved := make([]LastMovedType, partitionCount)
			for partition := 0; partition < partitionCount; partition++ {
				partitionToNodeIndex[partition] = b.Ring[replica][partition>>doublings]
				partitionToLastMoved[partition] = b.LastMoved[replica][partition>>doublings]
			}
			b.Ring[replica] = partitionToNodeIndex
			b.LastMoved[replica] = partitionToLastMoved
		}
	}
}

// Stats gives an overview of the state and health of the Builder.
// It is returned by the Builder.Stats() method.
type Stats struct {
	ReplicaCount      int
	EnabledNodeCount  int
	DisabledNodeCount int
	PartitionCount    int
	UnassignedCount   int
	EnabledCapacity   float64
	DisabledCapacity  float64
	// MaxUnderNodePercentage is the percentage a node is underweight, or has
	// less data assigned to it than its capacity would indicate it desires.
	MaxUnderNodePercentage float64
	MaxUnderNodeIndex      NodeIndexType
	// MaxOverNodePercentage is the percentage a node is overweight, or has
	// more data assigned to it than its capacity would indicate it desires.
	MaxOverNodePercentage float64
	MaxOverNodeIndex      NodeIndexType
}

func (b *Builder) Stats() *Stats {
	stats := &Stats{
		ReplicaCount:   b.ReplicaCount(),
		PartitionCount: b.PartitionCount(),
	}
	nodeIndexToPartitionCount := make([]int, len(b.Nodes))
	for _, partitionToNodeIndex := range b.Ring {
		for _, nodeIndex := range partitionToNodeIndex {
			if nodeIndex == NodeIndexNil {
				stats.UnassignedCount++
			} else {
				nodeIndexToPartitionCount[nodeIndex]++
			}
		}
	}
	for _, n := range b.Nodes {
		if n.Disabled {
			stats.DisabledNodeCount++
			if n.Capacity > 0 {
				stats.DisabledCapacity += float64(n.Capacity)
			}
		} else {
			stats.EnabledNodeCount++
			if n.Capacity > 0 {
				stats.EnabledCapacity += float64(n.Capacity)
			}
		}
	}
	for nodeIndex, n := range b.Nodes {
		if n.Disabled {
			continue
		}
		desiredPartitionCount := float64(n.Capacity) / stats.EnabledCapacity * float64(stats.PartitionCount) * float64(stats.ReplicaCount)
		actualPartitionCount := float64(nodeIndexToPartitionCount[nodeIndex])
		if desiredPartitionCount > actualPartitionCount {
			under := 100.0 * (desiredPartitionCount - actualPartitionCount) / desiredPartitionCount
			if under > stats.MaxUnderNodePercentage {
				stats.MaxUnderNodePercentage = under
				stats.MaxUnderNodeIndex = NodeIndexType(nodeIndex)
			}
		} else if desiredPartitionCount < actualPartitionCount {
			over := 100.0 * (actualPartitionCount - desiredPartitionCount) / desiredPartitionCount
			if over > stats.MaxOverNodePercentage {
				stats.MaxOverNodePercentage = over
				stats.MaxOverNodeIndex = NodeIndexType(nodeIndex)
			}
		}
	}
	return stats
}
<|endoftext|>