<commit_before>// This is free and unencumbered software released into the public
// domain.  For more information, see <http://unlicense.org> or the
// accompanying UNLICENSE file.

package editor

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"time"

	"github.com/go-fsnotify/fsnotify"
	"github.com/nelsam/gxui"
	"github.com/nelsam/gxui/math"
	"github.com/nelsam/gxui/mixins"
	"github.com/nelsam/gxui/themes/basic"
	"github.com/nelsam/vidar/suggestions"
	"github.com/nelsam/vidar/syntax"
)

type CodeEditor struct {
	mixins.CodeEditor
	adapter          *suggestions.Adapter
	suggestions      gxui.List
	suggestionsChild *gxui.Child

	theme   *basic.Theme
	driver  gxui.Driver
	history *History

	lastModified time.Time
	hasChanges   bool
	filepath     string

	watcher *fsnotify.Watcher

	// loading is a channel keeping track of a count of
	// threads that are (re)loading the file.
	loading chan bool
}

func (e *CodeEditor) Init(driver gxui.Driver, theme *basic.Theme, font gxui.Font, file string) {
	e.theme = theme
	e.driver = driver
	e.loading = make(chan bool, 5)
	e.history = NewHistory()

	e.adapter = &suggestions.Adapter{}
	e.suggestions = e.CreateSuggestionList()
	e.suggestions.SetAdapter(e.adapter)

	e.CodeEditor.Init(e, driver, theme, font)
	e.CodeEditor.SetScrollBarEnabled(true)
	e.SetDesiredWidth(math.MaxSize.W)

	e.OnTextChanged(func(changes []gxui.TextBoxEdit) {
		e.hasChanges = true
		// TODO: only update layers that changed.
		newLayers, err := syntax.Layers(e.filepath, e.Text())
		e.SetSyntaxLayers(newLayers)
		// TODO: display the error in some pane of the editor
		_ = err
		e.history.Add(changes...)
	})
	e.filepath = file
	e.open()

	e.SetTextColor(theme.TextBoxDefaultStyle.FontColor)
	e.SetMargin(math.Spacing{L: 3, T: 3, R: 3, B: 3})
	e.SetPadding(math.Spacing{L: 3, T: 3, R: 3, B: 3})
	e.SetBorderPen(gxui.TransparentPen)
}

func (e *CodeEditor) open() {
	if e.filepath == "" {
		e.SetText(`// Scratch
// This buffer is for jotting down quick notes, but is not saved to disk.
// Use at your own risk!`)
		return
	}
	go e.watch()
	e.load()
}

func (e *CodeEditor) watch() {
	var err error
	e.watcher, err = fsnotify.NewWatcher()
	if err != nil {
		log.Printf("Error creating new fsnotify watcher: %s", err)
		return
	}
	err = e.watcher.Add(e.filepath)
	if os.IsNotExist(err) {
		err = e.waitForFileCreate()
	}
	if err != nil {
		log.Printf("Error trying to watch %s for changes: %s", e.filepath, err)
		return
	}
	err = e.inotifyWait(func(event fsnotify.Event) bool {
		if event.Op&fsnotify.Write == fsnotify.Write {
			e.load()
		}
		return false
	})
	if err != nil {
		log.Printf("Failed to wait on events for %s: %s", e.filepath, err)
		return
	}
}

func (e *CodeEditor) waitForFileCreate() error {
	dir := filepath.Dir(e.filepath)
	if err := os.MkdirAll(dir, 0750|os.ModeDir); err != nil {
		return err
	}
	if err := e.watcher.Add(dir); err != nil {
		return err
	}
	defer e.watcher.Remove(dir)

	return e.inotifyWait(func(event fsnotify.Event) bool {
		return event.Name == e.filepath && event.Op&fsnotify.Create == fsnotify.Create
	})
}

func (e *CodeEditor) inotifyWait(eventFunc func(fsnotify.Event) (done bool)) error {
	for {
		select {
		case event := <-e.watcher.Events:
			if eventFunc(event) {
				return nil
			}
		case err := <-e.watcher.Errors:
			return err
		}
	}
}

func (e *CodeEditor) load() {
	e.loading <- true
	defer func() {
		<-e.loading
	}()
	f, err := os.Open(e.filepath)
	if os.IsNotExist(err) {
		e.SetText("")
		return
	}
	if err != nil {
		log.Printf("Error opening file %s: %s", e.filepath, err)
		return
	}
	defer f.Close()
	finfo, err := f.Stat()
	if err != nil {
		log.Printf("Error stating file %s: %s", e.filepath, err)
		return
	}
	e.lastModified = finfo.ModTime()
	b, err := ioutil.ReadAll(f)
	if err != nil {
		log.Printf("Error reading file %s: %s", e.filepath, err)
		return
	}
	e.driver.Call(func() {
		if len(e.loading) > 1 {
			return
		}
		if e.Text() == string(b) {
			return
		}
		location := e.Controller().FirstCaret()
		e.SetText(string(b))
		e.Controller().SetCaret(location)
	})
}

func (e *CodeEditor) History() *History {
	return e.history
}

func (e *CodeEditor) HasChanges() bool {
	return e.hasChanges
}

func (e *CodeEditor) LastKnownMTime() time.Time {
	return e.lastModified
}

func (e *CodeEditor) Filepath() string {
	return e.filepath
}

func (e *CodeEditor) FlushedChanges() {
	e.hasChanges = false
	e.lastModified = time.Now()
}

func (e *CodeEditor) Paint(c gxui.Canvas) {
	e.CodeEditor.Paint(c)

	if e.HasFocus() {
		r := e.Size().Rect()
		c.DrawRoundedRect(r, 3, 3, 3, 3, e.theme.FocusedStyle.Pen, e.theme.FocusedStyle.Brush)
	}
}

func (e *CodeEditor) CreateSuggestionList() gxui.List {
	l := e.theme.CreateList()
	l.SetBackgroundBrush(e.theme.CodeSuggestionListStyle.Brush)
	l.SetBorderPen(e.theme.CodeSuggestionListStyle.Pen)
	return l
}

func (e *CodeEditor) SetSuggestionProvider(provider gxui.CodeSuggestionProvider) {
	if e.SuggestionProvider() != provider {
		e.CodeEditor.SetSuggestionProvider(provider)
		if e.IsSuggestionListShowing() {
			e.updateSuggestionList()
		}
	}
}

func (e *CodeEditor) IsSuggestionListShowing() bool {
	return e.suggestionsChild != nil
}

func (e *CodeEditor) SortSuggestionList() {
	e.updateSuggestionList()
}

func (e *CodeEditor) ShowSuggestionList() {
	if e.SuggestionProvider() == nil || e.IsSuggestionListShowing() {
		return
	}
	e.showSuggestionList()
	e.updateSuggestionList()
}

func (e *CodeEditor) showSuggestionList() {
	e.suggestionsChild = e.AddChild(e.suggestions)
}

func (e *CodeEditor) updateSuggestionList() {
	caret := e.Controller().LastCaret()

	suggestions := e.SuggestionProvider().SuggestionsAt(caret)
	if len(suggestions) == 0 {
		e.HideSuggestionList()
		return
	}
	// TODO: if len(suggestions) == 1, show the completion in-line
	// instead of in a completion box.
	longest := 0
	for _, suggestion := range suggestions {
		suggestionText := suggestion.(fmt.Stringer).String()
		if len(suggestionText) > longest {
			longest = len(suggestionText)
		}
	}
	size := e.Font().GlyphMaxSize()
	size.W *= longest
	e.adapter.SetSize(size)

	e.adapter.SetSuggestions(suggestions)
	partial := e.WordAt(caret)
	e.adapter.Sort(partial)
	e.suggestions.Select(e.suggestions.Adapter().ItemAt(0))

	// Position the suggestion list below the last caret
	lineIdx := e.LineIndex(caret)
	bounds := e.Size().Rect().Contract(e.Padding())
	line := e.Line(lineIdx)
	lineOffset := gxui.ChildToParent(math.ZeroPoint, line, e)
	target := line.PositionAt(caret).Add(lineOffset)
	cs := e.suggestions.DesiredSize(math.ZeroSize, bounds.Size())
	e.suggestions.SetSize(cs)
	e.suggestionsChild.Layout(cs.Rect().Offset(target).Intersect(bounds))

	e.suggestions.Redraw()
	e.Redraw()
}

func (e *CodeEditor) HideSuggestionList() {
	if e.IsSuggestionListShowing() {
		e.RemoveChild(e.suggestions)
		e.Redraw()
		e.suggestionsChild = nil
	}
}

func (e *CodeEditor) KeyPress(event gxui.KeyboardEvent) bool {
	if event.Modifier.Control() || event.Modifier.Super() {
		switch event.Key {
		case gxui.KeySpace:
			e.ShowSuggestionList()
			return true
		case gxui.KeyTab:
			return false
		}
	}
	switch event.Key {
	case gxui.KeyPeriod:
		e.ShowSuggestionList()
		return true
	case gxui.KeyBackspace:
		result := e.TextBox.KeyPress(event)
		if e.IsSuggestionListShowing() {
			e.SortSuggestionList()
		}
		return result
	case gxui.KeyHome, gxui.KeyEnd, gxui.KeyPageUp, gxui.KeyPageDown, gxui.KeyDelete, gxui.KeyA:
		// These are all bindings that the TextBox handles fine.
		return e.TextBox.KeyPress(event)
	case gxui.KeyTab:
		// TODO: Gain knowledge about scope, so we know how much to indent.
		switch {
		case event.Modifier.Shift():
			e.Controller().UnindentSelection()
		default:
			e.Controller().IndentSelection()
		}
		return true
	case gxui.KeyUp, gxui.KeyDown:
		if e.IsSuggestionListShowing() {
			return e.suggestions.KeyPress(event)
		}
		return e.TextBox.KeyPress(event)
	case gxui.KeyLeft, gxui.KeyRight:
		if e.IsSuggestionListShowing() {
			e.HideSuggestionList()
		}
		return e.TextBox.KeyPress(event)
	case gxui.KeyEnter:
		controller := e.Controller()
		if e.IsSuggestionListShowing() {
			text := e.adapter.Suggestion(e.suggestions.Selected()).Code()
			start, end := controller.WordAt(controller.LastCaret())
			controller.SetSelection(gxui.CreateTextSelection(start, end, false))
			controller.ReplaceAll(text)
			controller.Deselect(false)
			e.HideSuggestionList()
		} else {
			// TODO: implement electric braces.  See
			// http://www.emacswiki.org/emacs/AutoPairs under
			// "Electric-RET".
			e.Controller().ReplaceWithNewlineKeepIndent()
		}
		return true
	case gxui.KeyEscape:
		if e.IsSuggestionListShowing() {
			e.HideSuggestionList()
			return true
		}
	}
	return false
}

func (e *CodeEditor) KeyStroke(event gxui.KeyStrokeEvent) (consume bool) {
	consume = e.TextBox.KeyStroke(event)
	if e.IsSuggestionListShowing() {
		e.SortSuggestionList()
	}
	return
}

func (e *CodeEditor) CreateLine(theme gxui.Theme, index int) (mixins.TextBoxLine, gxui.Control) {
	lineNumber := theme.CreateLabel()
	lineNumber.SetText(fmt.Sprintf("%4d", index+1))

	line := &mixins.CodeEditorLine{}
	line.Init(line, theme, &e.CodeEditor, index)

	layout := theme.CreateLinearLayout()
	layout.SetDirection(gxui.LeftToRight)
	layout.AddChild(lineNumber)
	layout.AddChild(line)

	return line, layout
}
<commit_msg>Deselect all selections on escape key press<commit_after>// This is free and unencumbered software released into the public
// domain.  For more information, see <http://unlicense.org> or the
// accompanying UNLICENSE file.

package editor

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"time"

	"github.com/go-fsnotify/fsnotify"
	"github.com/nelsam/gxui"
	"github.com/nelsam/gxui/math"
	"github.com/nelsam/gxui/mixins"
	"github.com/nelsam/gxui/themes/basic"
	"github.com/nelsam/vidar/suggestions"
	"github.com/nelsam/vidar/syntax"
)

type CodeEditor struct {
	mixins.CodeEditor
	adapter          *suggestions.Adapter
	suggestions      gxui.List
	suggestionsChild *gxui.Child

	theme   *basic.Theme
	driver  gxui.Driver
	history *History

	lastModified time.Time
	hasChanges   bool
	filepath     string

	watcher *fsnotify.Watcher

	// loading is a channel keeping track of a count of
	// threads that are (re)loading the file.
	loading chan bool
}

func (e *CodeEditor) Init(driver gxui.Driver, theme *basic.Theme, font gxui.Font, file string) {
	e.theme = theme
	e.driver = driver
	e.loading = make(chan bool, 5)
	e.history = NewHistory()

	e.adapter = &suggestions.Adapter{}
	e.suggestions = e.CreateSuggestionList()
	e.suggestions.SetAdapter(e.adapter)

	e.CodeEditor.Init(e, driver, theme, font)
	e.CodeEditor.SetScrollBarEnabled(true)
	e.SetDesiredWidth(math.MaxSize.W)

	e.OnTextChanged(func(changes []gxui.TextBoxEdit) {
		e.hasChanges = true
		// TODO: only update layers that changed.
		newLayers, err := syntax.Layers(e.filepath, e.Text())
		e.SetSyntaxLayers(newLayers)
		// TODO: display the error in some pane of the editor
		_ = err
		e.history.Add(changes...)
	})
	e.filepath = file
	e.open()

	e.SetTextColor(theme.TextBoxDefaultStyle.FontColor)
	e.SetMargin(math.Spacing{L: 3, T: 3, R: 3, B: 3})
	e.SetPadding(math.Spacing{L: 3, T: 3, R: 3, B: 3})
	e.SetBorderPen(gxui.TransparentPen)
}

func (e *CodeEditor) open() {
	if e.filepath == "" {
		e.SetText(`// Scratch
// This buffer is for jotting down quick notes, but is not saved to disk.
// Use at your own risk!`)
		return
	}
	go e.watch()
	e.load()
}

func (e *CodeEditor) watch() {
	var err error
	e.watcher, err = fsnotify.NewWatcher()
	if err != nil {
		log.Printf("Error creating new fsnotify watcher: %s", err)
		return
	}
	err = e.watcher.Add(e.filepath)
	if os.IsNotExist(err) {
		err = e.waitForFileCreate()
	}
	if err != nil {
		log.Printf("Error trying to watch %s for changes: %s", e.filepath, err)
		return
	}
	err = e.inotifyWait(func(event fsnotify.Event) bool {
		if event.Op&fsnotify.Write == fsnotify.Write {
			e.load()
		}
		return false
	})
	if err != nil {
		log.Printf("Failed to wait on events for %s: %s", e.filepath, err)
		return
	}
}

func (e *CodeEditor) waitForFileCreate() error {
	dir := filepath.Dir(e.filepath)
	if err := os.MkdirAll(dir, 0750|os.ModeDir); err != nil {
		return err
	}
	if err := e.watcher.Add(dir); err != nil {
		return err
	}
	defer e.watcher.Remove(dir)

	return e.inotifyWait(func(event fsnotify.Event) bool {
		return event.Name == e.filepath && event.Op&fsnotify.Create == fsnotify.Create
	})
}

func (e *CodeEditor) inotifyWait(eventFunc func(fsnotify.Event) (done bool)) error {
	for {
		select {
		case event := <-e.watcher.Events:
			if eventFunc(event) {
				return nil
			}
		case err := <-e.watcher.Errors:
			return err
		}
	}
}

func (e *CodeEditor) load() {
	e.loading <- true
	defer func() {
		<-e.loading
	}()
	f, err := os.Open(e.filepath)
	if os.IsNotExist(err) {
		e.SetText("")
		return
	}
	if err != nil {
		log.Printf("Error opening file %s: %s", e.filepath, err)
		return
	}
	defer f.Close()
	finfo, err := f.Stat()
	if err != nil {
		log.Printf("Error stating file %s: %s", e.filepath, err)
		return
	}
	e.lastModified = finfo.ModTime()
	b, err := ioutil.ReadAll(f)
	if err != nil {
		log.Printf("Error reading file %s: %s", e.filepath, err)
		return
	}
	e.driver.Call(func() {
		if len(e.loading) > 1 {
			return
		}
		if e.Text() == string(b) {
			return
		}
		location := e.Controller().FirstCaret()
		e.SetText(string(b))
		e.Controller().SetCaret(location)
	})
}

func (e *CodeEditor) History() *History {
	return e.history
}

func (e *CodeEditor) HasChanges() bool {
	return e.hasChanges
}

func (e *CodeEditor) LastKnownMTime() time.Time {
	return e.lastModified
}

func (e *CodeEditor) Filepath() string {
	return e.filepath
}

func (e *CodeEditor) FlushedChanges() {
	e.hasChanges = false
	e.lastModified = time.Now()
}

func (e *CodeEditor) Paint(c gxui.Canvas) {
	e.CodeEditor.Paint(c)

	if e.HasFocus() {
		r := e.Size().Rect()
		c.DrawRoundedRect(r, 3, 3, 3, 3, e.theme.FocusedStyle.Pen, e.theme.FocusedStyle.Brush)
	}
}

func (e *CodeEditor) CreateSuggestionList() gxui.List {
	l := e.theme.CreateList()
	l.SetBackgroundBrush(e.theme.CodeSuggestionListStyle.Brush)
	l.SetBorderPen(e.theme.CodeSuggestionListStyle.Pen)
	return l
}

func (e *CodeEditor) SetSuggestionProvider(provider gxui.CodeSuggestionProvider) {
	if e.SuggestionProvider() != provider {
		e.CodeEditor.SetSuggestionProvider(provider)
		if e.IsSuggestionListShowing() {
			e.updateSuggestionList()
		}
	}
}

func (e *CodeEditor) IsSuggestionListShowing() bool {
	return e.suggestionsChild != nil
}

func (e *CodeEditor) SortSuggestionList() {
	e.updateSuggestionList()
}

func (e *CodeEditor) ShowSuggestionList() {
	if e.SuggestionProvider() == nil || e.IsSuggestionListShowing() {
		return
	}
	e.showSuggestionList()
	e.updateSuggestionList()
}

func (e *CodeEditor) showSuggestionList() {
	e.suggestionsChild = e.AddChild(e.suggestions)
}

func (e *CodeEditor) updateSuggestionList() {
	caret := e.Controller().LastCaret()

	suggestions := e.SuggestionProvider().SuggestionsAt(caret)
	if len(suggestions) == 0 {
		e.HideSuggestionList()
		return
	}
	// TODO: if len(suggestions) == 1, show the completion in-line
	// instead of in a completion box.
	longest := 0
	for _, suggestion := range suggestions {
		suggestionText := suggestion.(fmt.Stringer).String()
		if len(suggestionText) > longest {
			longest = len(suggestionText)
		}
	}
	size := e.Font().GlyphMaxSize()
	size.W *= longest
	e.adapter.SetSize(size)

	e.adapter.SetSuggestions(suggestions)
	partial := e.WordAt(caret)
	e.adapter.Sort(partial)
	e.suggestions.Select(e.suggestions.Adapter().ItemAt(0))

	// Position the suggestion list below the last caret
	lineIdx := e.LineIndex(caret)
	bounds := e.Size().Rect().Contract(e.Padding())
	line := e.Line(lineIdx)
	lineOffset := gxui.ChildToParent(math.ZeroPoint, line, e)
	target := line.PositionAt(caret).Add(lineOffset)
	cs := e.suggestions.DesiredSize(math.ZeroSize, bounds.Size())
	e.suggestions.SetSize(cs)
	e.suggestionsChild.Layout(cs.Rect().Offset(target).Intersect(bounds))

	e.suggestions.Redraw()
	e.Redraw()
}

func (e *CodeEditor) HideSuggestionList() {
	if e.IsSuggestionListShowing() {
		e.RemoveChild(e.suggestions)
		e.Redraw()
		e.suggestionsChild = nil
	}
}

func (e *CodeEditor) KeyPress(event gxui.KeyboardEvent) bool {
	if event.Modifier.Control() || event.Modifier.Super() {
		switch event.Key {
		case gxui.KeySpace:
			e.ShowSuggestionList()
			return true
		case gxui.KeyTab:
			return false
		}
	}
	switch event.Key {
	case gxui.KeyPeriod:
		e.ShowSuggestionList()
		return true
	case gxui.KeyBackspace:
		result := e.TextBox.KeyPress(event)
		if e.IsSuggestionListShowing() {
			e.SortSuggestionList()
		}
		return result
	case gxui.KeyHome, gxui.KeyEnd, gxui.KeyPageUp, gxui.KeyPageDown, gxui.KeyDelete, gxui.KeyA:
		// These are all bindings that the TextBox handles fine.
		return e.TextBox.KeyPress(event)
	case gxui.KeyTab:
		// TODO: Gain knowledge about scope, so we know how much to indent.
		switch {
		case event.Modifier.Shift():
			e.Controller().UnindentSelection()
		default:
			e.Controller().IndentSelection()
		}
		return true
	case gxui.KeyUp, gxui.KeyDown:
		if e.IsSuggestionListShowing() {
			return e.suggestions.KeyPress(event)
		}
		return e.TextBox.KeyPress(event)
	case gxui.KeyLeft, gxui.KeyRight:
		if e.IsSuggestionListShowing() {
			e.HideSuggestionList()
		}
		return e.TextBox.KeyPress(event)
	case gxui.KeyEnter:
		controller := e.Controller()
		if e.IsSuggestionListShowing() {
			text := e.adapter.Suggestion(e.suggestions.Selected()).Code()
			start, end := controller.WordAt(controller.LastCaret())
			controller.SetSelection(gxui.CreateTextSelection(start, end, false))
			controller.ReplaceAll(text)
			controller.Deselect(false)
			e.HideSuggestionList()
		} else {
			// TODO: implement electric braces.  See
			// http://www.emacswiki.org/emacs/AutoPairs under
			// "Electric-RET".
			e.Controller().ReplaceWithNewlineKeepIndent()
		}
		return true
	case gxui.KeyEscape:
		if e.IsSuggestionListShowing() {
			e.HideSuggestionList()
			return true
		}
		// TODO: Keep track of some sort of concept of a "focused" caret and
		// focus that.
		e.Controller().SetCaret(e.Controller().FirstCaret())
	}
	return false
}

func (e *CodeEditor) KeyStroke(event gxui.KeyStrokeEvent) (consume bool) {
	consume = e.TextBox.KeyStroke(event)
	if e.IsSuggestionListShowing() {
		e.SortSuggestionList()
	}
	return
}

func (e *CodeEditor) CreateLine(theme gxui.Theme, index int) (mixins.TextBoxLine, gxui.Control) {
	lineNumber := theme.CreateLabel()
	lineNumber.SetText(fmt.Sprintf("%4d", index+1))

	line := &mixins.CodeEditorLine{}
	line.Init(line, theme, &e.CodeEditor, index)

	layout := theme.CreateLinearLayout()
	layout.SetDirection(gxui.LeftToRight)
	layout.AddChild(lineNumber)
	layout.AddChild(line)

	return line, layout
}
<|endoftext|>


<commit_before>package api

import (
	"fmt"
	"net/http"

	"github.com/chromium/hstspreload"
	"github.com/chromium/hstspreload.org/database"
)

// DebugAllStates allows preloading a domain without any checks.
// This should only be exposed for test servers.
func (api API) DebugAllStates(w http.ResponseWriter, r *http.Request) {
	states, err := api.database.AllDomainStates()
	if err != nil {
		msg := fmt.Sprintf("Internal error: could not get domain states. (%s)\n", err)
		http.Error(w, msg, http.StatusInternalServerError)
		return
	}

	writeJSONOrBust(w, states)
}

// DebugSetPreloaded allows preloading a domain without any checks.
// This should only be exposed for test servers.
func (api API) DebugSetPreloaded(w http.ResponseWriter, r *http.Request) {
	domain, ok := getASCIIDomain(http.MethodPost, w, r)
	if !ok {
		return
	}

	issues := hstspreload.Issues{}

	putErr := api.database.PutState(database.DomainState{
		Name:   domain,
		Status: database.StatusPreloaded,
	})

	if putErr != nil {
		issue := hstspreload.Issue{
			Code:    "internal.server.remove.set_preloaded_failed",
			Summary: "Internal error",
			Message: "Unable to save to set as preloaded.",
		}
		issues = hstspreload.Issues{
			Errors:   append(issues.Errors, issue),
			Warnings: issues.Warnings,
		}
	}

	writeJSONOrBust(w, issues)
}

// DebugSetRejected allows rejecting a domain without any checks.
// This should only be exposed for test servers.
func (api API) DebugSetRejected(w http.ResponseWriter, r *http.Request) {
	domain, ok := getASCIIDomain(http.MethodPost, w, r)
	if !ok {
		return
	}

	issues := hstspreload.Issues{}

	putErr := api.database.PutState(database.DomainState{
		Name:    domain,
		Status:  database.StatusRejected,
		Message: "Domain failed to satisfy continued requirements while pending.",
	})

	if putErr != nil {
		issue := hstspreload.Issue{
			Code:    "internal.server.remove.set_rejected_failed",
			Summary: "Internal error",
			Message: fmt.Sprintf("Unable to save to set as rejected: %s", putErr),
		}
		issues = hstspreload.Issues{
			Errors:   append(issues.Errors, issue),
			Warnings: issues.Warnings,
		}
	}

	writeJSONOrBust(w, issues)
}
<commit_msg>Print error in DebugSetPreloaded.<commit_after>package api

import (
	"fmt"
	"net/http"

	"github.com/chromium/hstspreload"
	"github.com/chromium/hstspreload.org/database"
)

// DebugAllStates allows preloading a domain without any checks.
// This should only be exposed for test servers.
func (api API) DebugAllStates(w http.ResponseWriter, r *http.Request) {
	states, err := api.database.AllDomainStates()
	if err != nil {
		msg := fmt.Sprintf("Internal error: could not get domain states. (%s)\n", err)
		http.Error(w, msg, http.StatusInternalServerError)
		return
	}

	writeJSONOrBust(w, states)
}

// DebugSetPreloaded allows preloading a domain without any checks.
// This should only be exposed for test servers.
func (api API) DebugSetPreloaded(w http.ResponseWriter, r *http.Request) {
	domain, ok := getASCIIDomain(http.MethodPost, w, r)
	if !ok {
		return
	}

	issues := hstspreload.Issues{}

	putErr := api.database.PutState(database.DomainState{
		Name:   domain,
		Status: database.StatusPreloaded,
	})

	if putErr != nil {
		issue := hstspreload.Issue{
			Code:    "internal.server.remove.set_preloaded_failed",
			Summary: "Internal error",
			Message: fmt.Sprintf("Unable to save to set as preloaded: %s", putErr),
		}
		issues = hstspreload.Issues{
			Errors:   append(issues.Errors, issue),
			Warnings: issues.Warnings,
		}
	}

	writeJSONOrBust(w, issues)
}

// DebugSetRejected allows rejecting a domain without any checks.
// This should only be exposed for test servers.
func (api API) DebugSetRejected(w http.ResponseWriter, r *http.Request) {
	domain, ok := getASCIIDomain(http.MethodPost, w, r)
	if !ok {
		return
	}

	issues := hstspreload.Issues{}

	putErr := api.database.PutState(database.DomainState{
		Name:    domain,
		Status:  database.StatusRejected,
		Message: "Domain failed to satisfy continued requirements while pending.",
	})

	if putErr != nil {
		issue := hstspreload.Issue{
			Code:    "internal.server.remove.set_rejected_failed",
			Summary: "Internal error",
			Message: fmt.Sprintf("Unable to save to set as rejected: %s", putErr),
		}
		issues = hstspreload.Issues{
			Errors:   append(issues.Errors, issue),
			Warnings: issues.Warnings,
		}
	}

	writeJSONOrBust(w, issues)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"log"
	"net/http"
	"time"
)

const OPEN311_API_URI = "http://311api.cityofchicago.org/open311/v2/requests.json?extensions=true&page_size=500"

type Worker struct {
	// Db         *sql.DB
	LastRunAt time.Time
}

var worker Worker
var srdb ServiceRequestDB
var env Environment

//  command line flags
var (
	version     string // set at compile time, will be the current git hash
	environment = flag.String("environment", "", "Environment to run in, e.g. staging, production")
	config      = flag.String("config", "./config/database.yml", "database configuration file")
	// sr_number     = flag.String("sr-number", "", "SR number to fetch")
	backfill      = flag.Bool("backfill", false, "run in reverse and backfill data")
	backfill_date = flag.String("backfill-from", time.Now().Format(time.RFC3339), "date to start backfilling data from. Use RFC3339 format. Default will be the current time.")
)

func init() {
	flag.Parse()

	log.Printf("CWFY Fetch worker version %s running in %s environment, configuration file %s", version, *environment, *config)

	srdb.Init(env.Load(config, environment))
	// todo: error handling
}

func main() {
	defer srdb.Close()

	// if *sr_number != "" {
	// 	sr := fetchSingleRequest(*sr_number)
	// 	srdb.Save(sr)
	// 	return
	// }

	start_backfill_from := *backfill_date
	for {
		switch {
		case *backfill:
			requests := backFillRequests(start_backfill_from)
			for _, request := range requests {
				srdb.Save(&request)
			}

			start_backfill_from = requests[len(requests)-1].Updated_datetime.Format(time.RFC3339)

		case time.Since(worker.LastRunAt) > (30 * time.Second):
			// load requests from open311
			for _, request := range fetchRequests() {
				srdb.Save(&request)
			}
			worker.LastRunAt = time.Now()
		default:
			log.Print("sleeping for 10 seconds")
			time.Sleep(10 * time.Second)
		}
	}
}

// func fetchSingleRequest(sr_number string) (request ServiceRequest) {
// 	// given an SR, fetch the record
// 	log.Printf("fetching single SR %s", sr_number)
// 	open311_api_endpoint := fmt.Sprintf("http://311api.cityofchicago.org/open311/v2/requests/%s.json?extensions=true", sr_number)
//
// 	log.Printf("fetching from %s", open311_api_endpoint)
// 	resp, err := http.Get(open311_api_endpoint)
// 	if err != nil {
// 		log.Fatal("error fetching from Open311 endpoint", err)
// 	}
// 	defer resp.Body.Close()
//
// 	// load response body
// 	body, err := ioutil.ReadAll(resp.Body)
// 	if err != nil {
// 		log.Fatal("error loading response body", err)
// 	}
//
// 	// parse JSON and load into an array of ServiceRequest objects
// 	var requests []ServiceRequest
//
// 	err = json.Unmarshal(body, &requests)
// 	if err != nil {
// 		log.Fatal("error parsing JSON:", err)
// 	}
//
// 	log.Printf("received %d requests from Open311", len(requests))
//
// 	return requests[0]
// }

func fetchRequests() (requests []ServiceRequest) {
	last_updated_at := time.Now()

	newest, _ := srdb.Newest() // FIXME: error handling
	if newest != nil {
		// override with the most recent SR available
		last_updated_at = newest.Updated_datetime
	}

	log.Print("[fetchRequests] most recent SR timestamp ", last_updated_at.Format(time.RFC3339))

	// construct the request URI using base params and the proper time
	open311_api_endpoint := OPEN311_API_URI + "&updated_after=" + last_updated_at.Format(time.RFC3339)

	log.Printf("[fetchRequests] fetching from %s", open311_api_endpoint)

	http.DefaultTransport.(*http.Transport).ResponseHeaderTimeout = time.Second * 60

	resp, err := http.Get(open311_api_endpoint)

	if err != nil {
		log.Fatalln("[fetchRequests] error fetching from Open311 endpoint", err)
	}

	defer resp.Body.Close()

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("[fetchRequests] error loading response body", err)
	}

	// parse JSON and load into an array of ServiceRequest objects
	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("[fetchRequests] error parsing JSON:", err)
	}

	log.Printf("[fetchRequests] received %d requests from Open311", len(requests))

	return requests
}

func backFillRequests(start_from string) (requests []ServiceRequest) {
	var fetch_from time.Time

	if start_from == "" {
		oldest, _ := srdb.Oldest() // FIXME: error handling
		fetch_from = oldest.Updated_datetime
		log.Printf("no start_from value provided, so falling back to oldest (by last update) SR in the database: %s", fetch_from)
	} else {
		t, err := time.Parse(time.RFC3339, start_from)
		if err != nil {
			log.Fatal("[backfill] error parsing date to start from", err)
		}
		fetch_from = t
	}

	formatted_date_string_with_tz := fetch_from.Format(time.RFC3339)

	// construct the request URI using base params and the proper time
	open311_api_endpoint := OPEN311_API_URI + "&updated_before=" + formatted_date_string_with_tz

	log.Printf("[backfill] fetching from %s", open311_api_endpoint)
	http.DefaultTransport.(*http.Transport).ResponseHeaderTimeout = time.Second * 60

	resp, err := http.Get(open311_api_endpoint)
	if err != nil {
		log.Fatalln("[backfill] error fetching from Open311 endpoint", err)
	}
	defer resp.Body.Close()

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("[backfill] error loading response body", err)
	}

	// parse JSON and load into an array of ServiceRequest objects
	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("[backfill] error parsing JSON:", err)
	}

	log.Printf("[backfill] received %d requests from Open311", len(requests))

	return requests
}
<commit_msg>gracefully shutdown fetch worker on SIGKILL and SIGINT<commit_after>package main

import (
	"os"
	"os/signal"
	"encoding/json"
	"flag"
	"io/ioutil"
	"log"
	"net/http"
	"time"
)

const OPEN311_API_URI = "http://311api.cityofchicago.org/open311/v2/requests.json?extensions=true&page_size=500"

type Worker struct {
	// Db         *sql.DB
	LastRunAt time.Time
}

var worker Worker
var srdb ServiceRequestDB
var env Environment

//  command line flags
var (
	version     string // set at compile time, will be the current git hash
	environment = flag.String("environment", "", "Environment to run in, e.g. staging, production")
	config      = flag.String("config", "./config/database.yml", "database configuration file")
	// sr_number     = flag.String("sr-number", "", "SR number to fetch")
	backfill      = flag.Bool("backfill", false, "run in reverse and backfill data")
	backfill_date = flag.String("backfill-from", time.Now().Format(time.RFC3339), "date to start backfilling data from. Use RFC3339 format. Default will be the current time.")
)

func init() {
	flag.Parse()

	log.Printf("CWFY Fetch worker version %s running in %s environment, configuration file %s", version, *environment, *config)

	srdb.Init(env.Load(config, environment))
	// todo: error handling
}

func main() {
	defer srdb.Close()

	// if *sr_number != "" {
	// 	sr := fetchSingleRequest(*sr_number)
	// 	srdb.Save(sr)
	// 	return
	// }

	// listen for SIGINT (h/t http://stackoverflow.com/a/12571099/1247272)
	notify_channel := make(chan os.Signal, 1)
	signal.Notify(notify_channel, os.Interrupt, os.Kill)
	go func() {
		for _ = range notify_channel {
			log.Printf("stopping ChicagoWorksForYou.com worker")
			srdb.Close()
			os.Exit(1)
		}
	}()

	start_backfill_from := *backfill_date
	for {
		switch {
		case *backfill:
			requests := backFillRequests(start_backfill_from)
			for _, request := range requests {
				srdb.Save(&request)
			}

			start_backfill_from = requests[len(requests)-1].Updated_datetime.Format(time.RFC3339)

		case time.Since(worker.LastRunAt) > (30 * time.Second):
			// load requests from open311
			for _, request := range fetchRequests() {
				srdb.Save(&request)
			}
			worker.LastRunAt = time.Now()
		default:
			log.Print("sleeping for 10 seconds")
			time.Sleep(10 * time.Second)
		}
	}
}

// func fetchSingleRequest(sr_number string) (request ServiceRequest) {
// 	// given an SR, fetch the record
// 	log.Printf("fetching single SR %s", sr_number)
// 	open311_api_endpoint := fmt.Sprintf("http://311api.cityofchicago.org/open311/v2/requests/%s.json?extensions=true", sr_number)
//
// 	log.Printf("fetching from %s", open311_api_endpoint)
// 	resp, err := http.Get(open311_api_endpoint)
// 	if err != nil {
// 		log.Fatal("error fetching from Open311 endpoint", err)
// 	}
// 	defer resp.Body.Close()
//
// 	// load response body
// 	body, err := ioutil.ReadAll(resp.Body)
// 	if err != nil {
// 		log.Fatal("error loading response body", err)
// 	}
//
// 	// parse JSON and load into an array of ServiceRequest objects
// 	var requests []ServiceRequest
//
// 	err = json.Unmarshal(body, &requests)
// 	if err != nil {
// 		log.Fatal("error parsing JSON:", err)
// 	}
//
// 	log.Printf("received %d requests from Open311", len(requests))
//
// 	return requests[0]
// }

func fetchRequests() (requests []ServiceRequest) {
	last_updated_at := time.Now()

	newest, _ := srdb.Newest() // FIXME: error handling
	if newest != nil {
		// override with the most recent SR available
		last_updated_at = newest.Updated_datetime
	}

	log.Print("[fetchRequests] most recent SR timestamp ", last_updated_at.Format(time.RFC3339))

	// construct the request URI using base params and the proper time
	open311_api_endpoint := OPEN311_API_URI + "&updated_after=" + last_updated_at.Format(time.RFC3339)

	log.Printf("[fetchRequests] fetching from %s", open311_api_endpoint)

	http.DefaultTransport.(*http.Transport).ResponseHeaderTimeout = time.Second * 60

	resp, err := http.Get(open311_api_endpoint)

	if err != nil {
		log.Fatalln("[fetchRequests] error fetching from Open311 endpoint", err)
	}

	defer resp.Body.Close()

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("[fetchRequests] error loading response body", err)
	}

	// parse JSON and load into an array of ServiceRequest objects
	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("[fetchRequests] error parsing JSON:", err)
	}

	log.Printf("[fetchRequests] received %d requests from Open311", len(requests))

	return requests
}

func backFillRequests(start_from string) (requests []ServiceRequest) {
	var fetch_from time.Time

	if start_from == "" {
		oldest, _ := srdb.Oldest() // FIXME: error handling
		fetch_from = oldest.Updated_datetime
		log.Printf("no start_from value provided, so falling back to oldest (by last update) SR in the database: %s", fetch_from)
	} else {
		t, err := time.Parse(time.RFC3339, start_from)
		if err != nil {
			log.Fatal("[backfill] error parsing date to start from", err)
		}
		fetch_from = t
	}

	formatted_date_string_with_tz := fetch_from.Format(time.RFC3339)

	// construct the request URI using base params and the proper time
	open311_api_endpoint := OPEN311_API_URI + "&updated_before=" + formatted_date_string_with_tz

	log.Printf("[backfill] fetching from %s", open311_api_endpoint)
	http.DefaultTransport.(*http.Transport).ResponseHeaderTimeout = time.Second * 60

	resp, err := http.Get(open311_api_endpoint)
	if err != nil {
		log.Fatalln("[backfill] error fetching from Open311 endpoint", err)
	}
	defer resp.Body.Close()

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("[backfill] error loading response body", err)
	}

	// parse JSON and load into an array of ServiceRequest objects
	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("[backfill] error parsing JSON:", err)
	}

	log.Printf("[backfill] received %d requests from Open311", len(requests))

	return requests
}
<|endoftext|>


<commit_before>package api

import (
	"sort"
	"strconv"
)

// Nodes is used to query node-related API endpoints
type Nodes struct {
	client *Client
}

// Nodes returns a handle on the node endpoints.
func (c *Client) Nodes() *Nodes {
	return &Nodes{client: c}
}

// List is used to list out all of the nodes
func (n *Nodes) List(q *QueryOptions) ([]*NodeListStub, *QueryMeta, error) {
	var resp NodeIndexSort
	qm, err := n.client.query("/v1/nodes", &resp, q)
	if err != nil {
		return nil, nil, err
	}
	sort.Sort(NodeIndexSort(resp))
	return resp, qm, nil
}

// Info is used to query a specific node by its ID.
func (n *Nodes) Info(nodeID string, q *QueryOptions) (*Node, *QueryMeta, error) {
	var resp Node
	qm, err := n.client.query("/v1/node/"+nodeID, &resp, q)
	if err != nil {
		return nil, nil, err
	}
	return &resp, qm, nil
}

// ToggleDrain is used to toggle drain mode on/off for a given node.
func (n *Nodes) ToggleDrain(nodeID string, drain bool, q *WriteOptions) (*WriteMeta, error) {
	drainArg := strconv.FormatBool(drain)
	wm, err := n.client.write("/v1/node/"+nodeID+"/drain?enable="+drainArg, nil, nil, q)
	if err != nil {
		return nil, err
	}
	return wm, nil
}

// Allocations is used to return the allocations associated with a node.
func (n *Nodes) Allocations(nodeID string, q *QueryOptions) ([]*Allocation, *QueryMeta, error) {
	var resp []*Allocation
	qm, err := n.client.query("/v1/node/"+nodeID+"/allocations", &resp, q)
	if err != nil {
		return nil, nil, err
	}
	sort.Sort(AllocationSort(resp))
	return resp, qm, nil
}

// ForceEvaluate is used to force-evaluate an existing node.
func (n *Nodes) ForceEvaluate(nodeID string, q *WriteOptions) (string, *WriteMeta, error) {
	var resp nodeEvalResponse
	wm, err := n.client.write("/v1/node/"+nodeID+"/evaluate", nil, &resp, q)
	if err != nil {
		return "", nil, err
	}
	return resp.EvalID, wm, nil
}

// Node is used to deserialize a node entry.
type Node struct {
	ID                string
	Datacenter        string
	Name              string
	Attributes        map[string]string
	Resources         *Resources
	Reserved          *Resources
	Links             map[string]string
	NodeClass         string
	Drain             bool
	Status            string
	StatusDescription string
	CreateIndex       uint64
	ModifyIndex       uint64
}

// NodeListStub is a subset of information returned during
// node list operations.
type NodeListStub struct {
	ID                string
	Datacenter        string
	Name              string
	NodeClass         string
	Drain             bool
	Status            string
	StatusDescription string
	CreateIndex       uint64
	ModifyIndex       uint64
}

// NodeIndexSort reverse sorts nodes by CreateIndex
type NodeIndexSort []*NodeListStub

func (n NodeIndexSort) Len() int {
	return len(n)
}

func (n NodeIndexSort) Less(i, j int) bool {
	return n[i].CreateIndex > n[j].CreateIndex
}

func (n NodeIndexSort) Swap(i, j int) {
	n[i], n[j] = n[j], n[i]
}

// nodeEvalResponse is used to decode a force-eval.
type nodeEvalResponse struct {
	EvalID string
}

// AllocationSort reverse sorts allocs by CreateIndex.
type AllocationSort []*Allocation

func (a AllocationSort) Len() int {
	return len(a)
}

func (a AllocationSort) Less(i, j int) bool {
	return a[i].CreateIndex > a[j].CreateIndex
}

func (a AllocationSort) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}
<commit_msg>Add Meta field for Node struct<commit_after>package api

import (
	"sort"
	"strconv"
)

// Nodes is used to query node-related API endpoints
type Nodes struct {
	client *Client
}

// Nodes returns a handle on the node endpoints.
func (c *Client) Nodes() *Nodes {
	return &Nodes{client: c}
}

// List is used to list out all of the nodes
func (n *Nodes) List(q *QueryOptions) ([]*NodeListStub, *QueryMeta, error) {
	var resp NodeIndexSort
	qm, err := n.client.query("/v1/nodes", &resp, q)
	if err != nil {
		return nil, nil, err
	}
	sort.Sort(NodeIndexSort(resp))
	return resp, qm, nil
}

// Info is used to query a specific node by its ID.
func (n *Nodes) Info(nodeID string, q *QueryOptions) (*Node, *QueryMeta, error) {
	var resp Node
	qm, err := n.client.query("/v1/node/"+nodeID, &resp, q)
	if err != nil {
		return nil, nil, err
	}
	return &resp, qm, nil
}

// ToggleDrain is used to toggle drain mode on/off for a given node.
func (n *Nodes) ToggleDrain(nodeID string, drain bool, q *WriteOptions) (*WriteMeta, error) {
	drainArg := strconv.FormatBool(drain)
	wm, err := n.client.write("/v1/node/"+nodeID+"/drain?enable="+drainArg, nil, nil, q)
	if err != nil {
		return nil, err
	}
	return wm, nil
}

// Allocations is used to return the allocations associated with a node.
func (n *Nodes) Allocations(nodeID string, q *QueryOptions) ([]*Allocation, *QueryMeta, error) {
	var resp []*Allocation
	qm, err := n.client.query("/v1/node/"+nodeID+"/allocations", &resp, q)
	if err != nil {
		return nil, nil, err
	}
	sort.Sort(AllocationSort(resp))
	return resp, qm, nil
}

// ForceEvaluate is used to force-evaluate an existing node.
func (n *Nodes) ForceEvaluate(nodeID string, q *WriteOptions) (string, *WriteMeta, error) {
	var resp nodeEvalResponse
	wm, err := n.client.write("/v1/node/"+nodeID+"/evaluate", nil, &resp, q)
	if err != nil {
		return "", nil, err
	}
	return resp.EvalID, wm, nil
}

// Node is used to deserialize a node entry.
type Node struct {
	ID                string
	Datacenter        string
	Name              string
	Attributes        map[string]string
	Resources         *Resources
	Reserved          *Resources
	Links             map[string]string
	Meta              map[string]string
	NodeClass         string
	Drain             bool
	Status            string
	StatusDescription string
	CreateIndex       uint64
	ModifyIndex       uint64
}

// NodeListStub is a subset of information returned during
// node list operations.
type NodeListStub struct {
	ID                string
	Datacenter        string
	Name              string
	NodeClass         string
	Drain             bool
	Status            string
	StatusDescription string
	CreateIndex       uint64
	ModifyIndex       uint64
}

// NodeIndexSort reverse sorts nodes by CreateIndex
type NodeIndexSort []*NodeListStub

func (n NodeIndexSort) Len() int {
	return len(n)
}

func (n NodeIndexSort) Less(i, j int) bool {
	return n[i].CreateIndex > n[j].CreateIndex
}

func (n NodeIndexSort) Swap(i, j int) {
	n[i], n[j] = n[j], n[i]
}

// nodeEvalResponse is used to decode a force-eval.
type nodeEvalResponse struct {
	EvalID string
}

// AllocationSort reverse sorts allocs by CreateIndex.
type AllocationSort []*Allocation

func (a AllocationSort) Len() int {
	return len(a)
}

func (a AllocationSort) Less(i, j int) bool {
	return a[i].CreateIndex > a[j].CreateIndex
}

func (a AllocationSort) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}
<|endoftext|>


<commit_before>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package main

import (
	"fmt"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/google/syzkaller/pkg/compiler"
	"github.com/google/syzkaller/pkg/osutil"
)

type linux struct{}

func (*linux) prepare(sourcedir string, build bool, arches []string) error {
	if sourcedir == "" {
		return fmt.Errorf("provide path to kernel checkout via -sourcedir flag (or make extract SOURCEDIR)")
	}
	if build {
		// Run 'make mrproper', otherwise out-of-tree build fails.
		// However, it takes unreasonable amount of time,
		// so first check few files and if they are missing hope for best.
		if osutil.IsExist(filepath.Join(sourcedir, ".config")) ||
			osutil.IsExist(filepath.Join(sourcedir, "init/main.o")) ||
			osutil.IsExist(filepath.Join(sourcedir, "include/generated/compile.h")) {
			fmt.Printf("make mrproper\n")
			out, err := osutil.RunCmd(time.Hour, sourcedir, "make", "mrproper",
				"-j", fmt.Sprint(runtime.NumCPU()))
			if err != nil {
				return fmt.Errorf("make mrproper failed: %v\n%s", err, out)
			}
		}
	} else {
		if len(arches) > 1 {
			return fmt.Errorf("more than 1 arch is invalid without -build")
		}
	}
	return nil
}

func (*linux) prepareArch(arch *Arch) error {
	// Kernel misses these headers on all arches.
	// So we create empty stubs in buildDir/syzkaller and add -IbuildDir/syzkaller
	// as the last flag so it won't override real kernel headers.
	for _, hdr := range []string{
		"asm/a.out.h",
		"asm/prctl.h",
		"asm/mce.h",
	} {
		fullPath := filepath.Join(arch.buildDir, "syzkaller", hdr)
		if err := osutil.MkdirAll(filepath.Dir(fullPath)); err != nil {
			return err
		}
		if err := osutil.WriteFile(fullPath, nil); err != nil {
			return nil
		}
	}
	if !arch.build {
		return nil
	}
	target := arch.target
	var cflags []string
	for _, flag := range target.CrossCFlags {
		if !strings.HasPrefix(flag, "-W") {
			cflags = append(cflags, flag)
		}
	}
	kernelDir := arch.sourceDir
	buildDir := arch.buildDir
	makeArgs := []string{
		"ARCH=" + target.KernelArch,
		"CROSS_COMPILE=" + target.CCompilerPrefix,
		"CFLAGS=" + strings.Join(cflags, " "),
		"O=" + buildDir,
		"-j", fmt.Sprint(runtime.NumCPU()),
	}
	out, err := osutil.RunCmd(time.Hour, kernelDir, "make", append(makeArgs, "defconfig")...)
	if err != nil {
		return fmt.Errorf("make defconfig failed: %v\n%s", err, out)
	}
	_, err = osutil.RunCmd(time.Minute, buildDir, filepath.Join(kernelDir, "scripts", "config"),
		// powerpc arch is configured to be big-endian by default, but we want little-endian powerpc.
		// Since all of our archs are little-endian for now, we just blindly switch it.
		"-d", "CPU_BIG_ENDIAN", "-e", "CPU_LITTLE_ENDIAN",
		// Without CONFIG_NETFILTER kernel does not build.
		"-e", "NETFILTER",
		// include/net/mptcp.h is the only header in kernel that guards some
		// of the consts with own config, so we need to enable CONFIG_MPTCP.
		"-e", "MPTCP",
		// security/smack/smack.h requires this to build.
		"-e", "SECURITY",
		"-e", "SECURITY_SMACK",
	)
	if err != nil {
		return err
	}
	out, err = osutil.RunCmd(time.Hour, kernelDir, "make", append(makeArgs, "olddefconfig")...)
	if err != nil {
		return fmt.Errorf("make olddefconfig failed: %v\n%s", err, out)
	}
	out, err = osutil.RunCmd(time.Hour, kernelDir, "make", append(makeArgs, "init/main.o")...)
	if err != nil {
		return fmt.Errorf("make failed: %v\n%s", err, out)
	}
	return nil
}

func (*linux) processFile(arch *Arch, info *compiler.ConstInfo) (map[string]uint64, map[string]bool, error) {
	headerArch := arch.target.KernelHeaderArch
	sourceDir := arch.sourceDir
	buildDir := arch.buildDir
	args := []string{
		// This would be useful to ensure that we don't include any host headers,
		// but kernel includes at least <stdarg.h>
		// "-nostdinc",
		"-w", "-fmessage-length=0",
		"-O3", // required to get expected values for some __builtin_constant_p
		"-I.",
		"-D__KERNEL__",
		"-DKBUILD_MODNAME=\"-\"",
		"-I" + sourceDir + "/arch/" + headerArch + "/include",
		"-I" + buildDir + "/arch/" + headerArch + "/include/generated/uapi",
		"-I" + buildDir + "/arch/" + headerArch + "/include/generated",
		"-I" + buildDir + "/include",
		"-I" + sourceDir + "/include",
		"-I" + sourceDir + "/arch/" + headerArch + "/include/uapi",
		"-I" + buildDir + "/arch/" + headerArch + "/include/generated/uapi",
		"-I" + sourceDir + "/include/uapi",
		"-I" + buildDir + "/include/generated/uapi",
		"-I" + sourceDir,
		"-I" + buildDir + "/syzkaller",
		"-include", sourceDir + "/include/linux/kconfig.h",
	}
	args = append(args, arch.target.CrossCFlags...)
	for _, incdir := range info.Incdirs {
		args = append(args, "-I"+sourceDir+"/"+incdir)
	}
	if arch.includeDirs != "" {
		for _, dir := range strings.Split(arch.includeDirs, ",") {
			args = append(args, "-I"+dir)
		}
	}
	params := &extractParams{
		AddSource:      "#include <asm/unistd.h>",
		ExtractFromELF: true,
	}
	cc := arch.target.CCompilerPrefix + "gcc"
	res, undeclared, err := extract(info, cc, args, params)
	if err != nil {
		return nil, nil, err
	}
	if arch.target.PtrSize == 4 {
		// mmap syscall on i386/arm is translated to old_mmap and has different signature.
		// As a workaround fix it up to mmap2, which has signature that we expect.
		// pkg/csource has the same hack.
		const mmap = "__NR_mmap"
		const mmap2 = "__NR_mmap2"
		if res[mmap] != 0 || undeclared[mmap] {
			if res[mmap2] == 0 {
				return nil, nil, fmt.Errorf("%v is missing", mmap2)
			}
			res[mmap] = res[mmap2]
			delete(undeclared, mmap)
		}
	}
	return res, undeclared, nil
}
<commit_msg>sys/syz-extract: Add mips64le specific include paths<commit_after>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package main

import (
	"fmt"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/google/syzkaller/pkg/compiler"
	"github.com/google/syzkaller/pkg/osutil"
)

type linux struct{}

func (*linux) prepare(sourcedir string, build bool, arches []string) error {
	if sourcedir == "" {
		return fmt.Errorf("provide path to kernel checkout via -sourcedir flag (or make extract SOURCEDIR)")
	}
	if build {
		// Run 'make mrproper', otherwise out-of-tree build fails.
		// However, it takes unreasonable amount of time,
		// so first check few files and if they are missing hope for best.
		if osutil.IsExist(filepath.Join(sourcedir, ".config")) ||
			osutil.IsExist(filepath.Join(sourcedir, "init/main.o")) ||
			osutil.IsExist(filepath.Join(sourcedir, "include/generated/compile.h")) {
			fmt.Printf("make mrproper\n")
			out, err := osutil.RunCmd(time.Hour, sourcedir, "make", "mrproper",
				"-j", fmt.Sprint(runtime.NumCPU()))
			if err != nil {
				return fmt.Errorf("make mrproper failed: %v\n%s", err, out)
			}
		}
	} else {
		if len(arches) > 1 {
			return fmt.Errorf("more than 1 arch is invalid without -build")
		}
	}
	return nil
}

func (*linux) prepareArch(arch *Arch) error {
	// Kernel misses these headers on all arches.
	// So we create empty stubs in buildDir/syzkaller and add -IbuildDir/syzkaller
	// as the last flag so it won't override real kernel headers.
	for _, hdr := range []string{
		"asm/a.out.h",
		"asm/prctl.h",
		"asm/mce.h",
	} {
		fullPath := filepath.Join(arch.buildDir, "syzkaller", hdr)
		if err := osutil.MkdirAll(filepath.Dir(fullPath)); err != nil {
			return err
		}
		if err := osutil.WriteFile(fullPath, nil); err != nil {
			return nil
		}
	}
	if !arch.build {
		return nil
	}
	target := arch.target
	var cflags []string
	for _, flag := range target.CrossCFlags {
		if !strings.HasPrefix(flag, "-W") {
			cflags = append(cflags, flag)
		}
	}
	kernelDir := arch.sourceDir
	buildDir := arch.buildDir
	makeArgs := []string{
		"ARCH=" + target.KernelArch,
		"CROSS_COMPILE=" + target.CCompilerPrefix,
		"CFLAGS=" + strings.Join(cflags, " "),
		"O=" + buildDir,
		"-j", fmt.Sprint(runtime.NumCPU()),
	}
	out, err := osutil.RunCmd(time.Hour, kernelDir, "make", append(makeArgs, "defconfig")...)
	if err != nil {
		return fmt.Errorf("make defconfig failed: %v\n%s", err, out)
	}
	_, err = osutil.RunCmd(time.Minute, buildDir, filepath.Join(kernelDir, "scripts", "config"),
		// powerpc arch is configured to be big-endian by default, but we want little-endian powerpc.
		// Since all of our archs are little-endian for now, we just blindly switch it.
		"-d", "CPU_BIG_ENDIAN", "-e", "CPU_LITTLE_ENDIAN",
		// Without CONFIG_NETFILTER kernel does not build.
		"-e", "NETFILTER",
		// include/net/mptcp.h is the only header in kernel that guards some
		// of the consts with own config, so we need to enable CONFIG_MPTCP.
		"-e", "MPTCP",
		// security/smack/smack.h requires this to build.
		"-e", "SECURITY",
		"-e", "SECURITY_SMACK",
	)
	if err != nil {
		return err
	}
	out, err = osutil.RunCmd(time.Hour, kernelDir, "make", append(makeArgs, "olddefconfig")...)
	if err != nil {
		return fmt.Errorf("make olddefconfig failed: %v\n%s", err, out)
	}
	out, err = osutil.RunCmd(time.Hour, kernelDir, "make", append(makeArgs, "init/main.o")...)
	if err != nil {
		return fmt.Errorf("make failed: %v\n%s", err, out)
	}
	return nil
}

func (*linux) processFile(arch *Arch, info *compiler.ConstInfo) (map[string]uint64, map[string]bool, error) {
	headerArch := arch.target.KernelHeaderArch
	sourceDir := arch.sourceDir
	buildDir := arch.buildDir
	args := []string{
		// This would be useful to ensure that we don't include any host headers,
		// but kernel includes at least <stdarg.h>
		// "-nostdinc",
		"-w", "-fmessage-length=0",
		"-O3", // required to get expected values for some __builtin_constant_p
		"-I.",
		"-D__KERNEL__",
		"-DKBUILD_MODNAME=\"-\"",
		"-I" + sourceDir + "/arch/" + headerArch + "/include",
		"-I" + buildDir + "/arch/" + headerArch + "/include/generated/uapi",
		"-I" + buildDir + "/arch/" + headerArch + "/include/generated",
		"-I" + buildDir + "/include",
		"-I" + sourceDir + "/include",
		"-I" + sourceDir + "/arch/" + headerArch + "/include/uapi",
		"-I" + buildDir + "/arch/" + headerArch + "/include/generated/uapi",
		"-I" + sourceDir + "/include/uapi",
		"-I" + buildDir + "/include/generated/uapi",
		"-I" + sourceDir,
		"-I" + buildDir + "/syzkaller",
		"-I" + sourceDir + "/arch/" + headerArch + "/include/asm/mach-malta",
		"-I" + sourceDir + "/arch/" + headerArch + "/include/asm/mach-generic",
		"-include", sourceDir + "/include/linux/kconfig.h",
	}
	args = append(args, arch.target.CrossCFlags...)
	for _, incdir := range info.Incdirs {
		args = append(args, "-I"+sourceDir+"/"+incdir)
	}
	if arch.includeDirs != "" {
		for _, dir := range strings.Split(arch.includeDirs, ",") {
			args = append(args, "-I"+dir)
		}
	}
	params := &extractParams{
		AddSource:      "#include <asm/unistd.h>",
		ExtractFromELF: true,
	}
	cc := arch.target.CCompilerPrefix + "gcc"
	res, undeclared, err := extract(info, cc, args, params)
	if err != nil {
		return nil, nil, err
	}
	if arch.target.PtrSize == 4 {
		// mmap syscall on i386/arm is translated to old_mmap and has different signature.
		// As a workaround fix it up to mmap2, which has signature that we expect.
		// pkg/csource has the same hack.
		const mmap = "__NR_mmap"
		const mmap2 = "__NR_mmap2"
		if res[mmap] != 0 || undeclared[mmap] {
			if res[mmap2] == 0 {
				return nil, nil, fmt.Errorf("%v is missing", mmap2)
			}
			res[mmap] = res[mmap2]
			delete(undeclared, mmap)
		}
	}
	return res, undeclared, nil
}
<|endoftext|>


<commit_before>package system

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"

	"errors"
	bosherr "github.com/cloudfoundry/bosh-utils/errors"
	boshlog "github.com/cloudfoundry/bosh-utils/logger"
)

type osFileSystem struct {
	logger           boshlog.Logger
	logTag           string
	tempRoot         string
	requiresTempRoot bool
}

func NewOsFileSystem(logger boshlog.Logger) FileSystem {
	return &osFileSystem{logger: logger, logTag: "File System", requiresTempRoot: false}
}

func NewOsFileSystemWithStrictTempRoot(logger boshlog.Logger) FileSystem {
	return &osFileSystem{logger: logger, logTag: "File System", requiresTempRoot: true}
}

func (fs *osFileSystem) HomeDir(username string) (string, error) {
	fs.logger.Debug(fs.logTag, "Getting HomeDir for %s", username)

	homeDir, err := fs.runCommand(fmt.Sprintf("echo ~%s", username))
	if err != nil {
		return "", bosherr.WrapErrorf(err, "Shelling out to get user '%s' home directory", username)
	}

	if strings.HasPrefix(homeDir, "~") {
		return "", bosherr.Errorf("Failed to get user '%s' home directory", username)
	}

	fs.logger.Debug(fs.logTag, "HomeDir is %s", homeDir)
	return homeDir, nil
}

func (fs *osFileSystem) ExpandPath(path string) (string, error) {
	fs.logger.Debug(fs.logTag, "Expanding path for '%s'", path)

	var err error
	if strings.IndexRune(path, '~') == 0 {
		currentUserHome, err := fs.HomeDir("")
		if err != nil {
			return "", bosherr.WrapError(err, "Getting current user home dir")
		}

		path = filepath.Clean(strings.Replace(path, "~", currentUserHome, 1))
	}

	path, err = filepath.Abs(path)
	if err != nil {
		return "", bosherr.WrapError(err, "Getting absolute path")
	}

	return path, nil
}

func (fs *osFileSystem) MkdirAll(path string, perm os.FileMode) (err error) {
	fs.logger.Debug(fs.logTag, "Making dir %s with perm %d", path, perm)
	return os.MkdirAll(path, perm)
}

func (fs *osFileSystem) Chown(path, username string) error {
	fs.logger.Debug(fs.logTag, "Chown %s to user %s", path, username)

	uid, err := fs.runCommand(fmt.Sprintf("id -u %s", username))
	if err != nil {
		return bosherr.WrapErrorf(err, "Getting user id for '%s'", username)
	}

	uidAsInt, err := strconv.Atoi(uid)
	if err != nil {
		return bosherr.WrapError(err, "Converting UID to integer")
	}

	gid, err := fs.runCommand(fmt.Sprintf("id -g %s", username))
	if err != nil {
		return bosherr.WrapErrorf(err, "Getting group id for '%s'", username)
	}

	gidAsInt, err := strconv.Atoi(gid)
	if err != nil {
		return bosherr.WrapError(err, "Converting GID to integer")
	}

	err = os.Chown(path, uidAsInt, gidAsInt)
	if err != nil {
		return bosherr.WrapError(err, "Doing Chown")
	}

	return nil
}

func (fs *osFileSystem) Chmod(path string, perm os.FileMode) (err error) {
	fs.logger.Debug(fs.logTag, "Chmod %s to %d", path, perm)
	return os.Chmod(path, perm)
}

func (fs *osFileSystem) OpenFile(path string, flag int, perm os.FileMode) (File, error) {
	return os.OpenFile(path, flag, perm)
}

func (fs *osFileSystem) WriteFileString(path, content string) (err error) {
	return fs.WriteFile(path, []byte(content))
}

func (fs *osFileSystem) WriteFile(path string, content []byte) error {
	fs.logger.Debug(fs.logTag, "Writing %s", path)

	err := fs.MkdirAll(filepath.Dir(path), os.ModePerm)
	if err != nil {
		return bosherr.WrapError(err, "Creating dir to write file")
	}

	file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
	if err != nil {
		return bosherr.WrapErrorf(err, "Creating file %s", path)
	}

	defer file.Close()

	fs.logger.DebugWithDetails(fs.logTag, "Write content", content)

	_, err = file.Write(content)
	if err != nil {
		return bosherr.WrapErrorf(err, "Writing content to file %s", path)
	}

	return nil
}

func (fs *osFileSystem) ConvergeFileContents(path string, content []byte) (bool, error) {
	if fs.filesAreIdentical(content, path) {
		fs.logger.Debug(fs.logTag, "Skipping writing %s because contents are identical", path)
		return false, nil
	}

	fs.logger.Debug(fs.logTag, "File %s will be overwritten", path)

	err := fs.MkdirAll(filepath.Dir(path), os.ModePerm)
	if err != nil {
		return true, bosherr.WrapErrorf(err, "Making dir for file %s", path)
	}

	file, err := os.Create(path)
	if err != nil {
		return true, bosherr.WrapErrorf(err, "Creating file %s", path)
	}

	defer file.Close()

	_, err = file.Write(content)
	if err != nil {
		return true, bosherr.WrapErrorf(err, "Writing content to file %s", path)
	}

	return true, nil
}

func (fs *osFileSystem) ReadFileString(path string) (content string, err error) {
	bytes, err := fs.ReadFile(path)
	if err != nil {
		return
	}

	content = string(bytes)
	return
}

func (fs *osFileSystem) ReadFile(path string) (content []byte, err error) {
	fs.logger.Debug(fs.logTag, "Reading file %s", path)

	file, err := os.Open(path)
	if err != nil {
		err = bosherr.WrapErrorf(err, "Opening file %s", path)
		return
	}

	defer file.Close()

	bytes, err := ioutil.ReadAll(file)
	if err != nil {
		err = bosherr.WrapErrorf(err, "Reading file content %s", path)
		return
	}

	content = bytes

	fs.logger.DebugWithDetails(fs.logTag, "Read content", content)
	return
}

func (fs *osFileSystem) FileExists(path string) bool {
	fs.logger.Debug(fs.logTag, "Checking if file exists %s", path)

	_, err := os.Stat(path)
	if err != nil {
		return !os.IsNotExist(err)
	}
	return true
}

func (fs *osFileSystem) Rename(oldPath, newPath string) (err error) {
	fs.logger.Debug(fs.logTag, "Renaming %s to %s", oldPath, newPath)

	fs.RemoveAll(newPath)
	return os.Rename(oldPath, newPath)
}

func (fs *osFileSystem) Symlink(oldPath, newPath string) error {
	fs.logger.Debug(fs.logTag, "Symlinking oldPath %s with newPath %s", oldPath, newPath)

	actualOldPath, err := filepath.EvalSymlinks(oldPath)
	if err != nil {
		return bosherr.WrapErrorf(err, "Evaluating symlinks for %s", oldPath)
	}

	existingTargetedPath, err := filepath.EvalSymlinks(newPath)
	if err == nil {
		if existingTargetedPath == actualOldPath {
			return nil
		}

		err = os.Remove(newPath)
		if err != nil {
			return bosherr.WrapErrorf(err, "Failed to delete symlimk at %s", newPath)
		}
	}

	containingDir := filepath.Dir(newPath)
	if !fs.FileExists(containingDir) {
		fs.MkdirAll(containingDir, os.FileMode(0700))
	}

	return os.Symlink(oldPath, newPath)
}

func (fs *osFileSystem) ReadLink(symlinkPath string) (targetPath string, err error) {
	targetPath, err = filepath.EvalSymlinks(symlinkPath)
	return
}

func (fs *osFileSystem) CopyFile(srcPath, dstPath string) error {
	fs.logger.Debug(fs.logTag, "Copying file '%s' to '%s'", srcPath, dstPath)
	srcFile, err := os.Open(srcPath)
	if err != nil {
		return bosherr.WrapError(err, "Opening source path")
	}

	defer srcFile.Close()

	dstFile, err := os.Create(dstPath)
	if err != nil {
		return bosherr.WrapError(err, "Creating destination file")
	}

	defer dstFile.Close()

	_, err = io.Copy(dstFile, srcFile)
	if err != nil {
		return bosherr.WrapError(err, "Copying file")
	}

	return nil
}

func (fs *osFileSystem) CopyDir(srcPath, dstPath string) error {
	fs.logger.Debug(fs.logTag, "Copying dir '%s' to '%s'", srcPath, dstPath)

	sourceInfo, err := os.Stat(srcPath)
	if err != nil {
		return bosherr.WrapErrorf(err, "Reading dir stats for '%s'", srcPath)
	}

	// create destination dir with same permissions as source dir
	err = os.MkdirAll(dstPath, sourceInfo.Mode())
	if err != nil {
		return bosherr.WrapErrorf(err, "Making destination dir '%s'", dstPath)
	}

	files, err := fs.listDirContents(srcPath)
	if err != nil {
		return bosherr.WrapErrorf(err, "Listing contents of source dir '%s", srcPath)
	}

	for _, file := range files {
		fileSrcPath := filepath.Join(srcPath, file.Name())
		fileDstPath := filepath.Join(dstPath, file.Name())

		if file.IsDir() {
			err = fs.CopyDir(fileSrcPath, fileDstPath)
			if err != nil {
				return bosherr.WrapErrorf(err, "Copying sub-dir '%s' to '%s'", fileSrcPath, fileDstPath)
			}
		} else {
			err = fs.CopyFile(fileSrcPath, fileDstPath)
			if err != nil {
				return bosherr.WrapErrorf(err, "Copying file '%s' to '%s'", fileSrcPath, fileDstPath)
			}
		}
	}

	return nil
}

func (fs *osFileSystem) listDirContents(dirPath string) ([]os.FileInfo, error) {
	directory, err := os.Open(dirPath)
	if err != nil {
		return nil, bosherr.WrapErrorf(err, "Openning dir '%s' for reading", dirPath)
	}
	defer directory.Close()

	files, err := directory.Readdir(-1)
	if err != nil {
		return nil, bosherr.WrapErrorf(err, "Reading dir '%s' contents", dirPath)
	}

	return files, nil
}

func (fs *osFileSystem) TempFile(prefix string) (file File, err error) {
	fs.logger.Debug(fs.logTag, "Creating temp file with prefix %s", prefix)
	if fs.tempRoot == "" && fs.requiresTempRoot {
		return nil, errors.New("Set a temp directory root with ChangeTempRoot before making temp files")
	}
	return ioutil.TempFile(fs.tempRoot, prefix)
}

func (fs *osFileSystem) TempDir(prefix string) (path string, err error) {
	fs.logger.Debug(fs.logTag, "Creating temp dir with prefix %s", prefix)
	if fs.tempRoot == "" && fs.requiresTempRoot {
		return "", errors.New("Set a temp directory root with ChangeTempRoot before making temp directories")
	}
	return ioutil.TempDir(fs.tempRoot, prefix)
}

func (f *osFileSystem) ChangeTempRoot(tempRootPath string) error {
	err := f.MkdirAll(tempRootPath, os.ModePerm)
	if err != nil {
		return err
	}
	f.tempRoot = tempRootPath
	return nil
}

func (fs *osFileSystem) RemoveAll(fileOrDir string) (err error) {
	fs.logger.Debug(fs.logTag, "Remove all %s", fileOrDir)
	err = os.RemoveAll(fileOrDir)
	return
}

func (fs *osFileSystem) Glob(pattern string) (matches []string, err error) {
	fs.logger.Debug(fs.logTag, "Glob '%s'", pattern)
	return filepath.Glob(pattern)
}

func (fs *osFileSystem) Walk(root string, walkFunc filepath.WalkFunc) error {
	return filepath.Walk(root, walkFunc)
}

func (fs *osFileSystem) filesAreIdentical(newContent []byte, filePath string) bool {
	existingStat, err := os.Stat(filePath)
	if err != nil || int64(len(newContent)) != existingStat.Size() {
		return false
	}

	existingContent, err := fs.ReadFile(filePath)
	if err != nil {
		return false
	}

	return bytes.Compare(newContent, existingContent) == 0
}

func (fs *osFileSystem) runCommand(cmd string) (string, error) {
	var stdout bytes.Buffer
	shCmd := exec.Command("sh", "-c", cmd)
	shCmd.Stdout = &stdout
	if err := shCmd.Run(); err != nil {
		return "", err
	}

	return strings.TrimSpace(stdout.String()), nil
}
<commit_msg>Display file permissions as octal number, because decimal doesn't make any sense<commit_after>package system

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"

	"errors"
	bosherr "github.com/cloudfoundry/bosh-utils/errors"
	boshlog "github.com/cloudfoundry/bosh-utils/logger"
)

type osFileSystem struct {
	logger           boshlog.Logger
	logTag           string
	tempRoot         string
	requiresTempRoot bool
}

func NewOsFileSystem(logger boshlog.Logger) FileSystem {
	return &osFileSystem{logger: logger, logTag: "File System", requiresTempRoot: false}
}

func NewOsFileSystemWithStrictTempRoot(logger boshlog.Logger) FileSystem {
	return &osFileSystem{logger: logger, logTag: "File System", requiresTempRoot: true}
}

func (fs *osFileSystem) HomeDir(username string) (string, error) {
	fs.logger.Debug(fs.logTag, "Getting HomeDir for %s", username)

	homeDir, err := fs.runCommand(fmt.Sprintf("echo ~%s", username))
	if err != nil {
		return "", bosherr.WrapErrorf(err, "Shelling out to get user '%s' home directory", username)
	}

	if strings.HasPrefix(homeDir, "~") {
		return "", bosherr.Errorf("Failed to get user '%s' home directory", username)
	}

	fs.logger.Debug(fs.logTag, "HomeDir is %s", homeDir)
	return homeDir, nil
}

func (fs *osFileSystem) ExpandPath(path string) (string, error) {
	fs.logger.Debug(fs.logTag, "Expanding path for '%s'", path)

	var err error
	if strings.IndexRune(path, '~') == 0 {
		currentUserHome, err := fs.HomeDir("")
		if err != nil {
			return "", bosherr.WrapError(err, "Getting current user home dir")
		}

		path = filepath.Clean(strings.Replace(path, "~", currentUserHome, 1))
	}

	path, err = filepath.Abs(path)
	if err != nil {
		return "", bosherr.WrapError(err, "Getting absolute path")
	}

	return path, nil
}

func (fs *osFileSystem) MkdirAll(path string, perm os.FileMode) (err error) {
	fs.logger.Debug(fs.logTag, "Making dir %s with perm %#o", path, perm)
	return os.MkdirAll(path, perm)
}

func (fs *osFileSystem) Chown(path, username string) error {
	fs.logger.Debug(fs.logTag, "Chown %s to user %s", path, username)

	uid, err := fs.runCommand(fmt.Sprintf("id -u %s", username))
	if err != nil {
		return bosherr.WrapErrorf(err, "Getting user id for '%s'", username)
	}

	uidAsInt, err := strconv.Atoi(uid)
	if err != nil {
		return bosherr.WrapError(err, "Converting UID to integer")
	}

	gid, err := fs.runCommand(fmt.Sprintf("id -g %s", username))
	if err != nil {
		return bosherr.WrapErrorf(err, "Getting group id for '%s'", username)
	}

	gidAsInt, err := strconv.Atoi(gid)
	if err != nil {
		return bosherr.WrapError(err, "Converting GID to integer")
	}

	err = os.Chown(path, uidAsInt, gidAsInt)
	if err != nil {
		return bosherr.WrapError(err, "Doing Chown")
	}

	return nil
}

func (fs *osFileSystem) Chmod(path string, perm os.FileMode) (err error) {
	fs.logger.Debug(fs.logTag, "Chmod %s to %d", path, perm)
	return os.Chmod(path, perm)
}

func (fs *osFileSystem) OpenFile(path string, flag int, perm os.FileMode) (File, error) {
	return os.OpenFile(path, flag, perm)
}

func (fs *osFileSystem) WriteFileString(path, content string) (err error) {
	return fs.WriteFile(path, []byte(content))
}

func (fs *osFileSystem) WriteFile(path string, content []byte) error {
	fs.logger.Debug(fs.logTag, "Writing %s", path)

	err := fs.MkdirAll(filepath.Dir(path), os.ModePerm)
	if err != nil {
		return bosherr.WrapError(err, "Creating dir to write file")
	}

	file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
	if err != nil {
		return bosherr.WrapErrorf(err, "Creating file %s", path)
	}

	defer file.Close()

	fs.logger.DebugWithDetails(fs.logTag, "Write content", content)

	_, err = file.Write(content)
	if err != nil {
		return bosherr.WrapErrorf(err, "Writing content to file %s", path)
	}

	return nil
}

func (fs *osFileSystem) ConvergeFileContents(path string, content []byte) (bool, error) {
	if fs.filesAreIdentical(content, path) {
		fs.logger.Debug(fs.logTag, "Skipping writing %s because contents are identical", path)
		return false, nil
	}

	fs.logger.Debug(fs.logTag, "File %s will be overwritten", path)

	err := fs.MkdirAll(filepath.Dir(path), os.ModePerm)
	if err != nil {
		return true, bosherr.WrapErrorf(err, "Making dir for file %s", path)
	}

	file, err := os.Create(path)
	if err != nil {
		return true, bosherr.WrapErrorf(err, "Creating file %s", path)
	}

	defer file.Close()

	_, err = file.Write(content)
	if err != nil {
		return true, bosherr.WrapErrorf(err, "Writing content to file %s", path)
	}

	return true, nil
}

func (fs *osFileSystem) ReadFileString(path string) (content string, err error) {
	bytes, err := fs.ReadFile(path)
	if err != nil {
		return
	}

	content = string(bytes)
	return
}

func (fs *osFileSystem) ReadFile(path string) (content []byte, err error) {
	fs.logger.Debug(fs.logTag, "Reading file %s", path)

	file, err := os.Open(path)
	if err != nil {
		err = bosherr.WrapErrorf(err, "Opening file %s", path)
		return
	}

	defer file.Close()

	bytes, err := ioutil.ReadAll(file)
	if err != nil {
		err = bosherr.WrapErrorf(err, "Reading file content %s", path)
		return
	}

	content = bytes

	fs.logger.DebugWithDetails(fs.logTag, "Read content", content)
	return
}

func (fs *osFileSystem) FileExists(path string) bool {
	fs.logger.Debug(fs.logTag, "Checking if file exists %s", path)

	_, err := os.Stat(path)
	if err != nil {
		return !os.IsNotExist(err)
	}
	return true
}

func (fs *osFileSystem) Rename(oldPath, newPath string) (err error) {
	fs.logger.Debug(fs.logTag, "Renaming %s to %s", oldPath, newPath)

	fs.RemoveAll(newPath)
	return os.Rename(oldPath, newPath)
}

func (fs *osFileSystem) Symlink(oldPath, newPath string) error {
	fs.logger.Debug(fs.logTag, "Symlinking oldPath %s with newPath %s", oldPath, newPath)

	actualOldPath, err := filepath.EvalSymlinks(oldPath)
	if err != nil {
		return bosherr.WrapErrorf(err, "Evaluating symlinks for %s", oldPath)
	}

	existingTargetedPath, err := filepath.EvalSymlinks(newPath)
	if err == nil {
		if existingTargetedPath == actualOldPath {
			return nil
		}

		err = os.Remove(newPath)
		if err != nil {
			return bosherr.WrapErrorf(err, "Failed to delete symlimk at %s", newPath)
		}
	}

	containingDir := filepath.Dir(newPath)
	if !fs.FileExists(containingDir) {
		fs.MkdirAll(containingDir, os.FileMode(0700))
	}

	return os.Symlink(oldPath, newPath)
}

func (fs *osFileSystem) ReadLink(symlinkPath string) (targetPath string, err error) {
	targetPath, err = filepath.EvalSymlinks(symlinkPath)
	return
}

func (fs *osFileSystem) CopyFile(srcPath, dstPath string) error {
	fs.logger.Debug(fs.logTag, "Copying file '%s' to '%s'", srcPath, dstPath)
	srcFile, err := os.Open(srcPath)
	if err != nil {
		return bosherr.WrapError(err, "Opening source path")
	}

	defer srcFile.Close()

	dstFile, err := os.Create(dstPath)
	if err != nil {
		return bosherr.WrapError(err, "Creating destination file")
	}

	defer dstFile.Close()

	_, err = io.Copy(dstFile, srcFile)
	if err != nil {
		return bosherr.WrapError(err, "Copying file")
	}

	return nil
}

func (fs *osFileSystem) CopyDir(srcPath, dstPath string) error {
	fs.logger.Debug(fs.logTag, "Copying dir '%s' to '%s'", srcPath, dstPath)

	sourceInfo, err := os.Stat(srcPath)
	if err != nil {
		return bosherr.WrapErrorf(err, "Reading dir stats for '%s'", srcPath)
	}

	// create destination dir with same permissions as source dir
	err = os.MkdirAll(dstPath, sourceInfo.Mode())
	if err != nil {
		return bosherr.WrapErrorf(err, "Making destination dir '%s'", dstPath)
	}

	files, err := fs.listDirContents(srcPath)
	if err != nil {
		return bosherr.WrapErrorf(err, "Listing contents of source dir '%s", srcPath)
	}

	for _, file := range files {
		fileSrcPath := filepath.Join(srcPath, file.Name())
		fileDstPath := filepath.Join(dstPath, file.Name())

		if file.IsDir() {
			err = fs.CopyDir(fileSrcPath, fileDstPath)
			if err != nil {
				return bosherr.WrapErrorf(err, "Copying sub-dir '%s' to '%s'", fileSrcPath, fileDstPath)
			}
		} else {
			err = fs.CopyFile(fileSrcPath, fileDstPath)
			if err != nil {
				return bosherr.WrapErrorf(err, "Copying file '%s' to '%s'", fileSrcPath, fileDstPath)
			}
		}
	}

	return nil
}

func (fs *osFileSystem) listDirContents(dirPath string) ([]os.FileInfo, error) {
	directory, err := os.Open(dirPath)
	if err != nil {
		return nil, bosherr.WrapErrorf(err, "Openning dir '%s' for reading", dirPath)
	}
	defer directory.Close()

	files, err := directory.Readdir(-1)
	if err != nil {
		return nil, bosherr.WrapErrorf(err, "Reading dir '%s' contents", dirPath)
	}

	return files, nil
}

func (fs *osFileSystem) TempFile(prefix string) (file File, err error) {
	fs.logger.Debug(fs.logTag, "Creating temp file with prefix %s", prefix)
	if fs.tempRoot == "" && fs.requiresTempRoot {
		return nil, errors.New("Set a temp directory root with ChangeTempRoot before making temp files")
	}
	return ioutil.TempFile(fs.tempRoot, prefix)
}

func (fs *osFileSystem) TempDir(prefix string) (path string, err error) {
	fs.logger.Debug(fs.logTag, "Creating temp dir with prefix %s", prefix)
	if fs.tempRoot == "" && fs.requiresTempRoot {
		return "", errors.New("Set a temp directory root with ChangeTempRoot before making temp directories")
	}
	return ioutil.TempDir(fs.tempRoot, prefix)
}

func (f *osFileSystem) ChangeTempRoot(tempRootPath string) error {
	err := f.MkdirAll(tempRootPath, os.ModePerm)
	if err != nil {
		return err
	}
	f.tempRoot = tempRootPath
	return nil
}

func (fs *osFileSystem) RemoveAll(fileOrDir string) (err error) {
	fs.logger.Debug(fs.logTag, "Remove all %s", fileOrDir)
	err = os.RemoveAll(fileOrDir)
	return
}

func (fs *osFileSystem) Glob(pattern string) (matches []string, err error) {
	fs.logger.Debug(fs.logTag, "Glob '%s'", pattern)
	return filepath.Glob(pattern)
}

func (fs *osFileSystem) Walk(root string, walkFunc filepath.WalkFunc) error {
	return filepath.Walk(root, walkFunc)
}

func (fs *osFileSystem) filesAreIdentical(newContent []byte, filePath string) bool {
	existingStat, err := os.Stat(filePath)
	if err != nil || int64(len(newContent)) != existingStat.Size() {
		return false
	}

	existingContent, err := fs.ReadFile(filePath)
	if err != nil {
		return false
	}

	return bytes.Compare(newContent, existingContent) == 0
}

func (fs *osFileSystem) runCommand(cmd string) (string, error) {
	var stdout bytes.Buffer
	shCmd := exec.Command("sh", "-c", cmd)
	shCmd.Stdout = &stdout
	if err := shCmd.Run(); err != nil {
		return "", err
	}

	return strings.TrimSpace(stdout.String()), nil
}
<|endoftext|>


<commit_before>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build linux darwin

// Package checks implements code which checks permissions of socket files to
// mitigate the possibility of a non-root attacker messing with socketservice
// communications channel.
package checks

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"syscall"
)

// CheckSocketFile ensures the naming, mode (perms, filetype) and ownership
// (uid, gid) of a Unix socket match what we create in a Fleetspeak socket
// service. This gives us some extra security. Note that using os.Lstat here
// prevents confusing FS with symlink tricks.
//
// If the returned error is a os.IsNotExist, then a Fleetspeak socket didn't exist. This
// condition may be retriable.
func CheckSocketFile(socketPath string) error {
	if !strings.HasPrefix(socketPath, "/") {
		return fmt.Errorf("expected a Unix domain socket address starting with \"/\", got %q", socketPath)
	}

	parent := filepath.Dir(socketPath)

	// Lstat prevents using a symlink as the Unix socket or the parent dir.
	parentFI, err := os.Lstat(parent)
	// Allow the caller to distinguish between two scenarios:
	// 1) os.IsNotExist(err) => fleetspeak is not running, but may start later
	// 2) otherwise fleetspeak is running, but there is a permissions error
	if os.IsNotExist(err) {
		return err
	}
	if err != nil {
		return fmt.Errorf("can't stat the given socketPath's (%q) parent directory: %v", socketPath, err)
	}

	// Note that providing such specific mode is deliberately much stricter than
	// just checking the permission bits and os.FileMode.IsDir .
	//
	// It's also beneficial to have an enclosing directory with a strict mode
	// because some implementations of Unix domain sockets apparently ignore
	// the permissions entirely.
	if parentFI.Mode() != 0700|os.ModeDir {
		return fmt.Errorf("the given socketPath's (%q) parent directory has unexpected mode", socketPath)
	}

	if err := checkUnixOwnership(parentFI); err != nil {
		return fmt.Errorf("unexpected ownership of socketPath's (%q) parent directory: %v", socketPath, err)
	}

	fi, err := os.Lstat(socketPath)
	if err != nil {
		return fmt.Errorf("can't stat the given socketPath (%q): %v", socketPath, err)
	}

	if fi.Mode() != 0600|os.ModeSocket {
		return fmt.Errorf("the given socketPath (%q) has unexpected mode", socketPath)
	}

	if err := checkUnixOwnership(fi); err != nil {
		return fmt.Errorf("unexpected ownership of socketPath (%q): %v", socketPath, err)
	}

	return nil
}

func checkUnixOwnership(fi os.FileInfo) error {
	st, ok := fi.Sys().(*syscall.Stat_t)
	if !ok {
		return errors.New("failed to cast os.FileInfo.Sys() to syscall.Stat_t; is this a non-Unix system?")
	}

	uid := os.Getuid()
	if uid != int(st.Uid) {
		return fmt.Errorf("unexpected uid %d (wanted %d)", st.Uid, uid)
	}

	gid := os.Getgid()
	// On some platforms (e.g. Darwin) groups 1 and 0 are mostly interchangeable.
	if gid != int(st.Gid) && st.Gid > 1 && runtime.GOOS == "darwin" {
		return fmt.Errorf("unexpected gid %d (wanted %d or <1 (system groups))", st.Gid, gid)
	}

	return nil
}
<commit_msg>Update sock_checks_unix.go<commit_after>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build linux darwin

// Package checks implements code which checks permissions of socket files to
// mitigate the possibility of a non-root attacker messing with socketservice
// communications channel.
package checks

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"syscall"
)

// CheckSocketFile ensures the naming, mode (perms, filetype) and ownership
// (uid, gid) of a Unix socket match what we create in a Fleetspeak socket
// service. This gives us some extra security. Note that using os.Lstat here
// prevents confusing FS with symlink tricks.
//
// If the returned error is a os.IsNotExist, then a Fleetspeak socket didn't exist. This
// condition may be retriable.
func CheckSocketFile(socketPath string) error {
	if !strings.HasPrefix(socketPath, "/") {
		return fmt.Errorf("expected a Unix domain socket address starting with \"/\", got %q", socketPath)
	}

	parent := filepath.Dir(socketPath)

	// Lstat prevents using a symlink as the Unix socket or the parent dir.
	parentFI, err := os.Lstat(parent)
	// Allow the caller to distinguish between two scenarios:
	// 1) os.IsNotExist(err) => fleetspeak is not running, but may start later
	// 2) otherwise fleetspeak is running, but there is a permissions error
	if os.IsNotExist(err) {
		return err
	}
	if err != nil {
		return fmt.Errorf("can't stat the given socketPath's (%q) parent directory: %v", socketPath, err)
	}

	// Note that providing such specific mode is deliberately much stricter than
	// just checking the permission bits and os.FileMode.IsDir .
	//
	// It's also beneficial to have an enclosing directory with a strict mode
	// because some implementations of Unix domain sockets apparently ignore
	// the permissions entirely.
	if parentFI.Mode() != 0700|os.ModeDir {
		return fmt.Errorf("the given socketPath's (%q) parent directory has unexpected mode", socketPath)
	}

	if err := checkUnixOwnership(parentFI); err != nil {
		return fmt.Errorf("unexpected ownership of socketPath's (%q) parent directory: %v", socketPath, err)
	}

	fi, err := os.Lstat(socketPath)
	if err != nil {
		return fmt.Errorf("can't stat the given socketPath (%q): %v", socketPath, err)
	}

	if fi.Mode() != 0600|os.ModeSocket {
		return fmt.Errorf("the given socketPath (%q) has unexpected mode", socketPath)
	}

	if err := checkUnixOwnership(fi); err != nil {
		return fmt.Errorf("unexpected ownership of socketPath (%q): %v", socketPath, err)
	}

	return nil
}

func checkUnixOwnership(fi os.FileInfo) error {
	st, ok := fi.Sys().(*syscall.Stat_t)
	if !ok {
		return errors.New("failed to cast os.FileInfo.Sys() to syscall.Stat_t; is this a non-Unix system?")
	}

	uid := os.Getuid()
	if uid != int(st.Uid) {
		return fmt.Errorf("unexpected uid %d (wanted %d)", st.Uid, uid)
	}

	gid := os.Getgid()
	switch runtime.GOOS {
	case "darwin":
		// On macOS groups 1 and 0 are mostly interchangeable.
		if gid != int(st.Gid) && st.Gid > 1 {
			return fmt.Errorf("unexpected gid %d (wanted %d or <1 (system groups))", st.Gid, gid)
		}
	default:
		if gid != int(st.Gid) {
			return fmt.Errorf("unexpected gid %d (wanted %d)", st.Gid, gid)
		}
	}

	return nil
}

<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This is a blog server for articles written in present format.
// It powers blog.golang.org.
package main

import (
	"bytes"
	"encoding/xml"
	"html/template"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"code.google.com/p/go.blog/pkg/atom"
	_ "code.google.com/p/go.talks/pkg/playground"
	"code.google.com/p/go.talks/pkg/present"
)

const (
	hostname     = "blog.golang.org"
	baseURL      = "http://" + hostname
	homeArticles = 5  // number of articles to display on the home page
	feedArticles = 10 // number of articles to include in Atom feed
)

// Doc represents an article, adorned with presentation data:
// its absolute path and related articles.
type Doc struct {
	*present.Doc
	Permalink    string
	Path         string
	Related      []*Doc
	Newer, Older *Doc
	HTML         template.HTML // rendered article
}

// Server implements an http.Handler that serves blog articles.
type Server struct {
	pathPrefix string
	docs       []*Doc
	tags       []string
	docPaths   map[string]*Doc
	docTags    map[string][]*Doc
	template   struct {
		home, index, article, doc *template.Template
	}
	atomFeed []byte // pre-rendered Atom feed
	content  http.Handler
}

// NewServer constructs a new Server, serving articles from the specified
// contentPath generated from templates from templatePath, under the prefix
// specified by pathPrefix.
func NewServer(pathPrefix, contentPath, templatePath string) (*Server, error) {
	present.PlayEnabled = true

	root := filepath.Join(templatePath, "root.tmpl")
	parse := func(name string) (*template.Template, error) {
		t := template.New("").Funcs(funcMap)
		return t.ParseFiles(root, filepath.Join(templatePath, name))
	}

	s := &Server{pathPrefix: pathPrefix}

	// Parse templates.
	var err error
	s.template.home, err = parse("home.tmpl")
	if err != nil {
		return nil, err
	}
	s.template.index, err = parse("index.tmpl")
	if err != nil {
		return nil, err
	}
	s.template.article, err = parse("article.tmpl")
	if err != nil {
		return nil, err
	}
	p := present.Template().Funcs(funcMap)
	s.template.doc, err = p.ParseFiles(filepath.Join(templatePath, "doc.tmpl"))
	if err != nil {
		return nil, err
	}

	// Load content.
	err = s.loadDocs(filepath.Clean(contentPath))
	if err != nil {
		return nil, err
	}

	err = s.renderAtomFeed()
	if err != nil {
		return nil, err
	}

	// Set up content file server.
	s.content = http.StripPrefix(pathPrefix, http.FileServer(http.Dir(contentPath)))

	return s, nil
}

var funcMap = template.FuncMap{
	"sectioned": sectioned,
	"authors":   authors,
}

// sectioned returns true if the provided Doc contains more than one section.
// This is used to control whether to display the table of contents and headings.
func sectioned(d *present.Doc) bool {
	return len(d.Sections) > 1
}

// authors returns a comma-separated list of author names.
func authors(authors []present.Author) string {
	var b bytes.Buffer
	last := len(authors) - 1
	for i, a := range authors {
		if i > 0 {
			if i == last {
				b.WriteString(" and ")
			} else {
				b.WriteString(", ")
			}
		}
		b.WriteString(authorName(a))
	}
	return b.String()
}

// authorName returns the first line of the Author text: the author's name.
func authorName(a present.Author) string {
	el := a.TextElem()
	if len(el) == 0 {
		return ""
	}
	text, ok := el[0].(present.Text)
	if !ok || len(text.Lines) == 0 {
		return ""
	}
	return text.Lines[0]
}

// loadDocs reads all content from the provided file system root, renders all
// the articles it finds, adds them to the Server's docs field, computes the
// denormalized docPaths, docTags, and tags fields, and populates the various
// helper fields (Next, Previous, Related) for each Doc.
func (s *Server) loadDocs(root string) error {
	// Read content into docs field.
	const ext = ".article"
	fn := func(p string, info os.FileInfo, err error) error {
		if filepath.Ext(p) != ext {
			return nil
		}
		f, err := os.Open(p)
		if err != nil {
			return err
		}
		defer f.Close()
		d, err := present.Parse(f, p, 0)
		if err != nil {
			return err
		}
		html := new(bytes.Buffer)
		err = d.Render(html, s.template.doc)
		if err != nil {
			return err
		}
		p = p[len(root) : len(p)-len(ext)] // trim root and extension
		s.docs = append(s.docs, &Doc{
			Doc:  d,
			Path: path.Join(s.pathPrefix, p),
			HTML: template.HTML(html.String()),
		})
		return nil
	}
	err := filepath.Walk(root, fn)
	if err != nil {
		return err
	}
	sort.Sort(docsByTime(s.docs))

	// Pull out doc paths and tags and put in reverse-associating maps.
	s.docPaths = make(map[string]*Doc)
	s.docTags = make(map[string][]*Doc)
	for _, d := range s.docs {
		s.docPaths[d.Path] = d
		for _, t := range d.Tags {
			s.docTags[t] = append(s.docTags[t], d)
		}
	}

	// Pull out unique sorted list of tags.
	for t := range s.docTags {
		s.tags = append(s.tags, t)
	}
	sort.Strings(s.tags)

	// Set up presentation-related fields, Newer, Older, and Related.
	for _, doc := range s.docs {
		// Newer, Older: docs adjacent to doc
		for i := range s.docs {
			if s.docs[i] != doc {
				continue
			}
			if i > 0 {
				doc.Newer = s.docs[i-1]
			}
			if i+1 < len(s.docs) {
				doc.Older = s.docs[i+1]
			}
			break
		}

		// Related: all docs that share tags with doc.
		related := make(map[*Doc]bool)
		for _, t := range doc.Tags {
			for _, d := range s.docTags[t] {
				if d != doc {
					related[d] = true
				}
			}
		}
		for d := range related {
			doc.Related = append(doc.Related, d)
		}
		sort.Sort(docsByTime(doc.Related))
	}

	return nil
}

// renderAtomFeed generates an XML Atom feed and stores it in the Server's
// atomFeed field.
func (s *Server) renderAtomFeed() error {
	var updated time.Time
	if len(s.docs) > 1 {
		updated = s.docs[0].Time
	}
	feed := atom.Feed{
		Title:   "The Go Programming Language Blog",
		ID:      "tag:" + hostname + ",2013:" + hostname,
		Updated: atom.Time(updated),
		Link: []atom.Link{{
			Rel:  "self",
			Href: baseURL + path.Join(s.pathPrefix, "/feed.atom"),
		}},
	}
	for i, doc := range s.docs {
		if i >= feedArticles {
			break
		}
		e := &atom.Entry{
			Title: doc.Title,
			ID:    feed.ID + doc.Path,
			Link: []atom.Link{{
				Rel:  "alternate",
				Href: baseURL + doc.Path,
			}},
			Published: atom.Time(doc.Time),
			Updated:   atom.Time(doc.Time),
			Summary: &atom.Text{
				Type: "html",
				Body: summary(doc),
			},
			Content: &atom.Text{
				Type: "html",
				Body: string(doc.HTML),
			},
			Author: &atom.Person{
				Name: authors(doc.Authors),
			},
		}
		feed.Entry = append(feed.Entry, e)
	}
	data, err := xml.Marshal(&feed)
	if err != nil {
		return err
	}
	s.atomFeed = data
	return nil
}

// summary returns the first paragraph of text from the provided Doc.
func summary(d *Doc) string {
	if len(d.Sections) == 0 {
		return ""
	}
	for _, elem := range d.Sections[0].Elem {
		text, ok := elem.(present.Text)
		if !ok || text.Pre {
			// skip everything but non-text elements
			continue
		}
		var buf bytes.Buffer
		for _, s := range text.Lines {
			buf.WriteString(string(present.Style(s)))
			buf.WriteByte('\n')
		}
		return buf.String()
	}
	return ""
}

// rootData encapsulates data destined for the root template.
type rootData struct {
	Doc  *Doc
	Data interface{}
}

// ServeHTTP servers either an article list or a single article.
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	p := r.URL.Path
	if !strings.HasPrefix(p, s.pathPrefix) {
		http.Error(w, "not found", http.StatusNotFound)
		return
	}
	var (
		d rootData
		t *template.Template
	)
	switch p[len(s.pathPrefix):] {
	case "":
		d.Data = s.docs
		if len(s.docs) > homeArticles {
			d.Data = s.docs[:homeArticles]
		}
		t = s.template.home
	case "index":
		d.Data = s.docs
		t = s.template.index
	case "feed.atom":
		w.Header().Set("Content-type", "application/atom+xml")
		w.Write(s.atomFeed)
		return
	default:
		doc, ok := s.docPaths[p]
		if !ok {
			// Not a doc; try to just serve static content.
			s.content.ServeHTTP(w, r)
			return
		}
		d.Doc = doc
		t = s.template.article
	}
	err := t.ExecuteTemplate(w, "root", d)
	if err != nil {
		log.Println(err)
	}
}

// docsByTime implements sort.Interface, sorting Docs by their Time field.
type docsByTime []*Doc

func (s docsByTime) Len() int           { return len(s) }
func (s docsByTime) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s docsByTime) Less(i, j int) bool { return s[i].Time.After(s[j].Time) }
<commit_msg>go.blog/cmd/blog: serve Atom feed from old blog's feed URL<commit_after>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This is a blog server for articles written in present format.
// It powers blog.golang.org.
package main

import (
	"bytes"
	"encoding/xml"
	"html/template"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"code.google.com/p/go.blog/pkg/atom"
	_ "code.google.com/p/go.talks/pkg/playground"
	"code.google.com/p/go.talks/pkg/present"
)

const (
	hostname     = "blog.golang.org"
	baseURL      = "http://" + hostname
	homeArticles = 5  // number of articles to display on the home page
	feedArticles = 10 // number of articles to include in Atom feed
)

// Doc represents an article, adorned with presentation data:
// its absolute path and related articles.
type Doc struct {
	*present.Doc
	Permalink    string
	Path         string
	Related      []*Doc
	Newer, Older *Doc
	HTML         template.HTML // rendered article
}

// Server implements an http.Handler that serves blog articles.
type Server struct {
	pathPrefix string
	docs       []*Doc
	tags       []string
	docPaths   map[string]*Doc
	docTags    map[string][]*Doc
	template   struct {
		home, index, article, doc *template.Template
	}
	atomFeed []byte // pre-rendered Atom feed
	content  http.Handler
}

// NewServer constructs a new Server, serving articles from the specified
// contentPath generated from templates from templatePath, under the prefix
// specified by pathPrefix.
func NewServer(pathPrefix, contentPath, templatePath string) (*Server, error) {
	present.PlayEnabled = true

	root := filepath.Join(templatePath, "root.tmpl")
	parse := func(name string) (*template.Template, error) {
		t := template.New("").Funcs(funcMap)
		return t.ParseFiles(root, filepath.Join(templatePath, name))
	}

	s := &Server{pathPrefix: pathPrefix}

	// Parse templates.
	var err error
	s.template.home, err = parse("home.tmpl")
	if err != nil {
		return nil, err
	}
	s.template.index, err = parse("index.tmpl")
	if err != nil {
		return nil, err
	}
	s.template.article, err = parse("article.tmpl")
	if err != nil {
		return nil, err
	}
	p := present.Template().Funcs(funcMap)
	s.template.doc, err = p.ParseFiles(filepath.Join(templatePath, "doc.tmpl"))
	if err != nil {
		return nil, err
	}

	// Load content.
	err = s.loadDocs(filepath.Clean(contentPath))
	if err != nil {
		return nil, err
	}

	err = s.renderAtomFeed()
	if err != nil {
		return nil, err
	}

	// Set up content file server.
	s.content = http.StripPrefix(pathPrefix, http.FileServer(http.Dir(contentPath)))

	return s, nil
}

var funcMap = template.FuncMap{
	"sectioned": sectioned,
	"authors":   authors,
}

// sectioned returns true if the provided Doc contains more than one section.
// This is used to control whether to display the table of contents and headings.
func sectioned(d *present.Doc) bool {
	return len(d.Sections) > 1
}

// authors returns a comma-separated list of author names.
func authors(authors []present.Author) string {
	var b bytes.Buffer
	last := len(authors) - 1
	for i, a := range authors {
		if i > 0 {
			if i == last {
				b.WriteString(" and ")
			} else {
				b.WriteString(", ")
			}
		}
		b.WriteString(authorName(a))
	}
	return b.String()
}

// authorName returns the first line of the Author text: the author's name.
func authorName(a present.Author) string {
	el := a.TextElem()
	if len(el) == 0 {
		return ""
	}
	text, ok := el[0].(present.Text)
	if !ok || len(text.Lines) == 0 {
		return ""
	}
	return text.Lines[0]
}

// loadDocs reads all content from the provided file system root, renders all
// the articles it finds, adds them to the Server's docs field, computes the
// denormalized docPaths, docTags, and tags fields, and populates the various
// helper fields (Next, Previous, Related) for each Doc.
func (s *Server) loadDocs(root string) error {
	// Read content into docs field.
	const ext = ".article"
	fn := func(p string, info os.FileInfo, err error) error {
		if filepath.Ext(p) != ext {
			return nil
		}
		f, err := os.Open(p)
		if err != nil {
			return err
		}
		defer f.Close()
		d, err := present.Parse(f, p, 0)
		if err != nil {
			return err
		}
		html := new(bytes.Buffer)
		err = d.Render(html, s.template.doc)
		if err != nil {
			return err
		}
		p = p[len(root) : len(p)-len(ext)] // trim root and extension
		s.docs = append(s.docs, &Doc{
			Doc:  d,
			Path: path.Join(s.pathPrefix, p),
			HTML: template.HTML(html.String()),
		})
		return nil
	}
	err := filepath.Walk(root, fn)
	if err != nil {
		return err
	}
	sort.Sort(docsByTime(s.docs))

	// Pull out doc paths and tags and put in reverse-associating maps.
	s.docPaths = make(map[string]*Doc)
	s.docTags = make(map[string][]*Doc)
	for _, d := range s.docs {
		s.docPaths[d.Path] = d
		for _, t := range d.Tags {
			s.docTags[t] = append(s.docTags[t], d)
		}
	}

	// Pull out unique sorted list of tags.
	for t := range s.docTags {
		s.tags = append(s.tags, t)
	}
	sort.Strings(s.tags)

	// Set up presentation-related fields, Newer, Older, and Related.
	for _, doc := range s.docs {
		// Newer, Older: docs adjacent to doc
		for i := range s.docs {
			if s.docs[i] != doc {
				continue
			}
			if i > 0 {
				doc.Newer = s.docs[i-1]
			}
			if i+1 < len(s.docs) {
				doc.Older = s.docs[i+1]
			}
			break
		}

		// Related: all docs that share tags with doc.
		related := make(map[*Doc]bool)
		for _, t := range doc.Tags {
			for _, d := range s.docTags[t] {
				if d != doc {
					related[d] = true
				}
			}
		}
		for d := range related {
			doc.Related = append(doc.Related, d)
		}
		sort.Sort(docsByTime(doc.Related))
	}

	return nil
}

// renderAtomFeed generates an XML Atom feed and stores it in the Server's
// atomFeed field.
func (s *Server) renderAtomFeed() error {
	var updated time.Time
	if len(s.docs) > 1 {
		updated = s.docs[0].Time
	}
	feed := atom.Feed{
		Title:   "The Go Programming Language Blog",
		ID:      "tag:" + hostname + ",2013:" + hostname,
		Updated: atom.Time(updated),
		Link: []atom.Link{{
			Rel:  "self",
			Href: baseURL + path.Join(s.pathPrefix, "/feed.atom"),
		}},
	}
	for i, doc := range s.docs {
		if i >= feedArticles {
			break
		}
		e := &atom.Entry{
			Title: doc.Title,
			ID:    feed.ID + doc.Path,
			Link: []atom.Link{{
				Rel:  "alternate",
				Href: baseURL + doc.Path,
			}},
			Published: atom.Time(doc.Time),
			Updated:   atom.Time(doc.Time),
			Summary: &atom.Text{
				Type: "html",
				Body: summary(doc),
			},
			Content: &atom.Text{
				Type: "html",
				Body: string(doc.HTML),
			},
			Author: &atom.Person{
				Name: authors(doc.Authors),
			},
		}
		feed.Entry = append(feed.Entry, e)
	}
	data, err := xml.Marshal(&feed)
	if err != nil {
		return err
	}
	s.atomFeed = data
	return nil
}

// summary returns the first paragraph of text from the provided Doc.
func summary(d *Doc) string {
	if len(d.Sections) == 0 {
		return ""
	}
	for _, elem := range d.Sections[0].Elem {
		text, ok := elem.(present.Text)
		if !ok || text.Pre {
			// skip everything but non-text elements
			continue
		}
		var buf bytes.Buffer
		for _, s := range text.Lines {
			buf.WriteString(string(present.Style(s)))
			buf.WriteByte('\n')
		}
		return buf.String()
	}
	return ""
}

// rootData encapsulates data destined for the root template.
type rootData struct {
	Doc  *Doc
	Data interface{}
}

// ServeHTTP servers either an article list or a single article.
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	p := r.URL.Path
	if !strings.HasPrefix(p, s.pathPrefix) {
		http.Error(w, "not found", http.StatusNotFound)
		return
	}
	var (
		d rootData
		t *template.Template
	)
	switch p[len(s.pathPrefix):] {
	case "":
		d.Data = s.docs
		if len(s.docs) > homeArticles {
			d.Data = s.docs[:homeArticles]
		}
		t = s.template.home
	case "index":
		d.Data = s.docs
		t = s.template.index
	case "feed.atom", "feeds/posts/default":
		w.Header().Set("Content-type", "application/atom+xml")
		w.Write(s.atomFeed)
		return
	default:
		doc, ok := s.docPaths[p]
		if !ok {
			// Not a doc; try to just serve static content.
			s.content.ServeHTTP(w, r)
			return
		}
		d.Doc = doc
		t = s.template.article
	}
	err := t.ExecuteTemplate(w, "root", d)
	if err != nil {
		log.Println(err)
	}
}

// docsByTime implements sort.Interface, sorting Docs by their Time field.
type docsByTime []*Doc

func (s docsByTime) Len() int           { return len(s) }
func (s docsByTime) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s docsByTime) Less(i, j int) bool { return s[i].Time.After(s[j].Time) }
<|endoftext|>