<commit_before>// Copyright (c) 2016-2017 Daniel Oaks <daniel@danieloaks.net>
// released under the MIT license

package ircbnc

import (
	"fmt"
	"net"
	"runtime/debug"
	"strings"
	"sync"
	"time"

	"code.cloudfoundry.org/bytefmt"

	"log"

	"github.com/goshuirc/bnc/lib/ircclient"
	"github.com/goshuirc/irc-go/ircmsg"
)

type RegistrationLocks struct {
	Lock sync.Mutex
	Cap  bool
	Nick bool
	User bool
	Pass bool
}

func (locks *RegistrationLocks) Set(lockName string, val bool) {
	locks.Lock.Lock()

	switch lockName {
	case "cap":
		locks.Cap = val
	case "nick":
		locks.Nick = val
	case "user":
		locks.User = val
	case "pass":
		locks.Pass = val
	}

	locks.Lock.Unlock()
}

func (locks *RegistrationLocks) Completed() bool {
	locks.Lock.Lock()
	completed := locks.Cap && locks.Pass && locks.Nick && locks.User
	locks.Lock.Unlock()
	return completed
}

// Listener is a listener for a client connected directly to us.
type Listener struct {
	Socket Socket

	Manager          *Manager
	ConnectTime      time.Time
	Caps             map[string]string
	ExtraISupports   map[string]string
	TagsEnabled      bool
	ClientNick       string
	Source           string
	Registered       bool
	regLocks         *RegistrationLocks
	User             *User
	ServerConnection *ServerConnection
}

// NewListener creates a new Listener.
func NewListener(m *Manager, conn net.Conn) {
	now := time.Now()
	listener := &Listener{
		Manager:     m,
		ClientNick:  "*",
		ConnectTime: now,
		Source:      m.Source,
		regLocks: &RegistrationLocks{
			Cap:  true,
			Nick: false,
			User: false,
			Pass: false,
		},
		Caps:           make(map[string]string),
		ExtraISupports: make(map[string]string),
	}

	maxSendQBytes, _ := bytefmt.ToBytes("32k")
	listener.Socket = NewSocket(conn, maxSendQBytes)

	hook := &HookNewListener{
		Listener: listener,
	}
	m.Bus.Dispatch(HookNewListenerName, hook)
	if hook.Halt {
		listener.Socket.Close()
		return
	}

	go listener.Socket.RunSocketWriter()
	listener.RunSocketReader()
}

func (listener *Listener) IsCapEnabled(cap string) bool {
	_, enabled := listener.Caps[cap]
	return enabled
}

// tryRegistration dumps the registration blob and all if it hasn't been sent already.
func (listener *Listener) tryRegistration() {
	if listener.Registered {
		return
	}

	if listener.regLocks.Completed() {
		listener.DumpRegistration()
		listener.Registered = true
		listener.DumpChannels()

		listener.Manager.Bus.Dispatch(HookStateSentName, &HookStateSent{
			Listener: listener,
			Server:   listener.ServerConnection,
		})
	}
}

// DumpRegistration dumps the registration numerics/replies to the listener.
func (listener *Listener) DumpRegistration() {
	if listener.ServerConnection == nil {
		listener.SendNilConnect()
	} else {
		listener.ServerConnection.DumpRegistration(listener)
	}
}

func (listener *Listener) SendExtraISupports() {
	params := []string{listener.ClientNick}

	for token, val := range listener.ExtraISupports {
		if val != "" {
			params = append(params, fmt.Sprintf("%s=%s", token, val))
		} else {
			params = append(params, token)
		}
	}

	params = append(params, "are supported by this server")

	isupportMessage := ircmsg.MakeMessage(nil, listener.Manager.Source, "005", params...)
	listener.SendMessage(&isupportMessage)
}

// SendNilConnect sends a connection init (001+ERR_NOMOTD) to the listener when they are not connected to a server.
func (listener *Listener) SendNilConnect() {
	listener.Send(nil, listener.Source, "001", listener.ClientNick, "- Welcome to GoshuBNC -")
	listener.SendExtraISupports()
	listener.Send(nil, listener.Source, "422", listener.ClientNick, "MOTD File is missing")
	listener.Send(nil, listener.Manager.StatusSource, "NOTICE", listener.ClientNick, "You are not connected to any specific network")
	listener.Send(nil, listener.Manager.StatusSource, "NOTICE", listener.ClientNick, fmt.Sprintf("If you want to connect to a network, connect with the server password %s/<network>:<password>", "<username>"))
}

// DumpChannels dumps the active channels to the listener.
func (listener *Listener) DumpChannels() {
	if listener.ServerConnection != nil {
		listener.ServerConnection.DumpChannels(listener)
	}
}

// RunSocketReader reads lines from the listener socket and dispatches them as appropriate.
func (listener *Listener) RunSocketReader() {
	for {
		line, err := listener.Socket.Read()
		if err != nil {
			break
		}

		listener.processIncomingLine(line)
	}

	listener.Manager.Bus.Dispatch(HookListenerCloseName, &HookListenerClose{
		Listener: listener,
	})
	if listener.ServerConnection != nil {
		listener.ServerConnection.RemoveListener(listener)
	}
}

// processIncomingLine splits and handles the given command line.
// Returns true if client is exiting (sent a QUIT command, etc).
func (listener *Listener) processIncomingLine(line string) {
	// Don't let a single users error kill the entire bnc for everyone
	defer func() {
		if r := recover(); r != nil {
			log.Printf("[ERROR] Recovered from %s\n%s", r, debug.Stack())
		}
	}()

	msg, parseLineErr := ircmsg.ParseLine(line)

	// Trigger the event if the line parsed or not just incase something else wants to
	// deal with them
	hook := &HookIrcRaw{
		FromClient: true,
		Listener:   listener,
		User:       listener.User,
		Server:     listener.ServerConnection,
		Raw:        line,
		Message:    msg,
	}
	listener.Manager.Bus.Dispatch(HookIrcRawName, hook)
	if hook.Halt {
		return
	}

	if parseLineErr != nil {
		listener.Send(nil, "", "ERROR", "Your client sent a malformed line")
		return
	}

	shouldHalt := Capabilities.MessageFromClient(listener, &msg)
	if shouldHalt {
		return
	}

	command, commandExists := ClientCommands[strings.ToUpper(msg.Command)]
	if commandExists {
		shouldHalt := command.Run(listener, msg)
		if shouldHalt {
			return
		}
	}

	// Forward the data
	if listener.Registered && listener.ServerConnection != nil {
		line, _ := msg.Line()
		_, err := listener.ServerConnection.Foo.WriteLine(line)
		if err != nil {
			log.Println(err.Error())
		}
	}

	//TODO(dan): This is an error+disconnect purely for reasons of testing.
	// Later it may be downgraded to not-that-bad.
	// listener.Send(nil, "", "ERROR", fmt.Sprintf("Your client sent a command that could not be parsed [%s]", msg.Command))
	// return true
}

// SendMessage sends an IrcMessage to the listener
func (listener *Listener) SendMessage(msg *ircmsg.IrcMessage) error {
	return listener.Send(&msg.Tags, msg.Prefix, msg.Command, msg.Params...)
}

// Send sends an IRC line to the listener.
func (listener *Listener) Send(tags *map[string]ircmsg.TagValue, prefix string, command string, params ...string) error {
	var message ircmsg.IrcMessage
	if listener.TagsEnabled {
		message = ircmsg.MakeMessage(tags, prefix, command, params...)
	} else {
		message = ircmsg.MakeMessage(nil, prefix, command, params...)
	}

	shouldHalt := Capabilities.MessageToClient(listener, &message)
	if shouldHalt {
		return nil
	}

	line, err := message.Line()
	if err != nil {
		// try not to fail quietly - especially useful when running tests, as a note to dig deeper
		// log.Println("Error assembling message:")
		// spew.Dump(message)
		// debug.PrintStack()

		message = ircmsg.MakeMessage(nil, "", ircclient.ERR_UNKNOWNERROR, "*", "Error assembling message for sending")
		line, _ := message.Line()
		listener.Socket.Write(line)
		return err
	}

	listener.SendLine(line)
	return nil
}

// SendLine sends a raw string line to the listener
func (listener *Listener) SendLine(line string) {
	listener.Socket.WriteLine(line)
}

func (listener *Listener) SendStatus(line string) {
	listener.Send(nil, listener.Manager.StatusSource, "PRIVMSG", listener.ClientNick, line)
}
<commit_msg>listener: Clean up RegistrationLocks, don't expose internal mutex<commit_after>// Copyright (c) 2016-2017 Daniel Oaks <daniel@danieloaks.net>
// released under the MIT license

package ircbnc

import (
	"fmt"
	"net"
	"runtime/debug"
	"strings"
	"sync"
	"time"

	"code.cloudfoundry.org/bytefmt"

	"log"

	"github.com/goshuirc/bnc/lib/ircclient"
	"github.com/goshuirc/irc-go/ircmsg"
)

// RegistrationLocks ensure the user can't complete registration until they've finished the reg process.
type RegistrationLocks struct {
	sync.Mutex
	Cap  bool
	Nick bool
	User bool
	Pass bool
}

// Set sets the given registration lock.
func (rl *RegistrationLocks) Set(lockName string, val bool) {
	rl.Lock()
	defer rl.Unlock()

	switch lockName {
	case "cap":
		rl.Cap = val
	case "nick":
		rl.Nick = val
	case "user":
		rl.User = val
	case "pass":
		rl.Pass = val
	}
}

// Completed returns true if all of our registration locks have been completed.
func (rl *RegistrationLocks) Completed() bool {
	rl.Lock()
	completed := rl.Cap && rl.Pass && rl.Nick && rl.User
	rl.Unlock()
	return completed
}

// Listener is a listener for a client connected directly to us.
type Listener struct {
	Socket Socket

	Manager          *Manager
	ConnectTime      time.Time
	Caps             map[string]string
	ExtraISupports   map[string]string
	TagsEnabled      bool
	ClientNick       string
	Source           string
	Registered       bool
	regLocks         *RegistrationLocks
	User             *User
	ServerConnection *ServerConnection
}

// NewListener creates a new Listener.
func NewListener(m *Manager, conn net.Conn) {
	now := time.Now()
	listener := &Listener{
		Manager:     m,
		ClientNick:  "*",
		ConnectTime: now,
		Source:      m.Source,
		regLocks: &RegistrationLocks{
			Cap:  true,
			Nick: false,
			User: false,
			Pass: false,
		},
		Caps:           make(map[string]string),
		ExtraISupports: make(map[string]string),
	}

	maxSendQBytes, _ := bytefmt.ToBytes("32k")
	listener.Socket = NewSocket(conn, maxSendQBytes)

	hook := &HookNewListener{
		Listener: listener,
	}
	m.Bus.Dispatch(HookNewListenerName, hook)
	if hook.Halt {
		listener.Socket.Close()
		return
	}

	go listener.Socket.RunSocketWriter()
	listener.RunSocketReader()
}

func (listener *Listener) IsCapEnabled(cap string) bool {
	_, enabled := listener.Caps[cap]
	return enabled
}

// tryRegistration dumps the registration blob and all if it hasn't been sent already.
func (listener *Listener) tryRegistration() {
	if listener.Registered {
		return
	}

	if listener.regLocks.Completed() {
		listener.DumpRegistration()
		listener.Registered = true
		listener.DumpChannels()

		listener.Manager.Bus.Dispatch(HookStateSentName, &HookStateSent{
			Listener: listener,
			Server:   listener.ServerConnection,
		})
	}
}

// DumpRegistration dumps the registration numerics/replies to the listener.
func (listener *Listener) DumpRegistration() {
	if listener.ServerConnection == nil {
		listener.SendNilConnect()
	} else {
		listener.ServerConnection.DumpRegistration(listener)
	}
}

func (listener *Listener) SendExtraISupports() {
	params := []string{listener.ClientNick}

	for token, val := range listener.ExtraISupports {
		if val != "" {
			params = append(params, fmt.Sprintf("%s=%s", token, val))
		} else {
			params = append(params, token)
		}
	}

	params = append(params, "are supported by this server")

	isupportMessage := ircmsg.MakeMessage(nil, listener.Manager.Source, "005", params...)
	listener.SendMessage(&isupportMessage)
}

// SendNilConnect sends a connection init (001+ERR_NOMOTD) to the listener when they are not connected to a server.
func (listener *Listener) SendNilConnect() {
	listener.Send(nil, listener.Source, "001", listener.ClientNick, "- Welcome to GoshuBNC -")
	listener.SendExtraISupports()
	listener.Send(nil, listener.Source, "422", listener.ClientNick, "MOTD File is missing")
	listener.Send(nil, listener.Manager.StatusSource, "NOTICE", listener.ClientNick, "You are not connected to any specific network")
	listener.Send(nil, listener.Manager.StatusSource, "NOTICE", listener.ClientNick, fmt.Sprintf("If you want to connect to a network, connect with the server password %s/<network>:<password>", "<username>"))
}

// DumpChannels dumps the active channels to the listener.
func (listener *Listener) DumpChannels() {
	if listener.ServerConnection != nil {
		listener.ServerConnection.DumpChannels(listener)
	}
}

// RunSocketReader reads lines from the listener socket and dispatches them as appropriate.
func (listener *Listener) RunSocketReader() {
	for {
		line, err := listener.Socket.Read()
		if err != nil {
			break
		}

		listener.processIncomingLine(line)
	}

	listener.Manager.Bus.Dispatch(HookListenerCloseName, &HookListenerClose{
		Listener: listener,
	})
	if listener.ServerConnection != nil {
		listener.ServerConnection.RemoveListener(listener)
	}
}

// processIncomingLine splits and handles the given command line.
// Returns true if client is exiting (sent a QUIT command, etc).
func (listener *Listener) processIncomingLine(line string) {
	// Don't let a single users error kill the entire bnc for everyone
	defer func() {
		if r := recover(); r != nil {
			log.Printf("[ERROR] Recovered from %s\n%s", r, debug.Stack())
		}
	}()

	msg, parseLineErr := ircmsg.ParseLine(line)

	// Trigger the event if the line parsed or not just incase something else wants to
	// deal with them
	hook := &HookIrcRaw{
		FromClient: true,
		Listener:   listener,
		User:       listener.User,
		Server:     listener.ServerConnection,
		Raw:        line,
		Message:    msg,
	}
	listener.Manager.Bus.Dispatch(HookIrcRawName, hook)
	if hook.Halt {
		return
	}

	if parseLineErr != nil {
		listener.Send(nil, "", "ERROR", "Your client sent a malformed line")
		return
	}

	shouldHalt := Capabilities.MessageFromClient(listener, &msg)
	if shouldHalt {
		return
	}

	command, commandExists := ClientCommands[strings.ToUpper(msg.Command)]
	if commandExists {
		shouldHalt := command.Run(listener, msg)
		if shouldHalt {
			return
		}
	}

	// Forward the data
	if listener.Registered && listener.ServerConnection != nil {
		line, _ := msg.Line()
		_, err := listener.ServerConnection.Foo.WriteLine(line)
		if err != nil {
			log.Println(err.Error())
		}
	}

	//TODO(dan): This is an error+disconnect purely for reasons of testing.
	// Later it may be downgraded to not-that-bad.
	// listener.Send(nil, "", "ERROR", fmt.Sprintf("Your client sent a command that could not be parsed [%s]", msg.Command))
	// return true
}

// SendMessage sends an IrcMessage to the user
func (listener *Listener) SendMessage(msg *ircmsg.IrcMessage) error {
	return listener.Send(&msg.Tags, msg.Prefix, msg.Command, msg.Params...)
}

// Send sends an IRC line to the user.
func (listener *Listener) Send(tags *map[string]ircmsg.TagValue, prefix string, command string, params ...string) error {
	var message ircmsg.IrcMessage
	if listener.TagsEnabled {
		message = ircmsg.MakeMessage(tags, prefix, command, params...)
	} else {
		message = ircmsg.MakeMessage(nil, prefix, command, params...)
	}

	shouldHalt := Capabilities.MessageToClient(listener, &message)
	if shouldHalt {
		return nil
	}

	line, err := message.Line()
	if err != nil {
		// try not to fail quietly - especially useful when running tests, as a note to dig deeper
		// log.Println("Error assembling message:")
		// spew.Dump(message)
		// debug.PrintStack()

		message = ircmsg.MakeMessage(nil, "", ircclient.ERR_UNKNOWNERROR, "*", "Error assembling message for sending")
		line, _ := message.Line()
		listener.Socket.Write(line)
		return err
	}

	listener.SendLine(line)
	return nil
}

// SendLine sends a raw string line to the user.
func (listener *Listener) SendLine(line string) {
	listener.Socket.WriteLine(line)
}

// SendStatus sends a status PRIVMSG to the user.
func (listener *Listener) SendStatus(line string) {
	listener.Send(nil, listener.Manager.StatusSource, "PRIVMSG", listener.ClientNick, line)
}
<|endoftext|>


<commit_before>/*
   Implements the Google omaha protocol.

   Omaha is a request/response protocol using XML. Requests are made by
   clients and responses are given by the Omaha server.
   http://code.google.com/p/omaha/wiki/ServerProtocol
*/
package omaha

import (
	"encoding/xml"
)

type Request struct {
	XMLName        xml.Name `xml:"request" datastore:"-"`
	Os             Os       `xml:"os"`
	Apps           []*App   `xml:"app"`
	Protocol       string   `xml:"protocol,attr"`
	Version        string   `xml:"version,attr,omitempty"`
	IsMachine      string   `xml:"ismachine,attr,omitempty"`
	SessionId      string   `xml:"sessionid,attr,omitempty"`
	UserId         string   `xml:"userid,attr,omitempty"`
	InstallSource  string   `xml:"installsource,attr,omitempty"`
	TestSource     string   `xml:"testsource,attr,omitempty"`
	RequestId      string   `xml:"requestid,attr,omitempty"`
	UpdaterVersion string   `xml:"updaterversion,attr,omitempty"`
}

func NewRequest(version string, platform string, sp string, arch string) *Request {
	r := new(Request)
	r.Protocol = "3.0"
	r.Os = Os{Version: version, Platform: platform, Sp: sp, Arch: arch}
	return r
}

func (r *Request) AddApp(id string, version string) *App {
	a := NewApp(id)
	a.Version = version
	r.Apps = append(r.Apps, a)
	return a
}

/* Response
 */
type Response struct {
	XMLName  xml.Name `xml:"response" datastore:"-" json:"-"`
	DayStart DayStart `xml:"daystart"`
	Apps     []*App   `xml:"app"`
	Protocol string   `xml:"protocol,attr"`
	Server   string   `xml:"server,attr"`
}

func NewResponse(server string) *Response {
	r := &Response{Server: server, Protocol: "3.0"}
	r.DayStart.ElapsedSeconds = "0"
	return r
}

type DayStart struct {
	ElapsedSeconds string `xml:"elapsed_seconds,attr"`
}

func (r *Response) AddApp(id string) *App {
	a := NewApp(id)
	r.Apps = append(r.Apps, a)
	return a
}

type App struct {
	XMLName     xml.Name     `xml:"app" datastore"-" json:"-"`
	Ping        *Ping        `xml:"ping"`
	UpdateCheck *UpdateCheck `xml:"updatecheck"`
	Events      []*Event     `xml:"event" json:",omitempty"`
	Id          string       `xml:"appid,attr,omitempty"`
	Version     string       `xml:"version,attr,omitempty"`
	NextVersion string       `xml:"nextversion,attr,omitempty"`
	Lang        string       `xml:"lang,attr,omitempty"`
	Client      string       `xml:"client,attr,omitempty"`
	InstallAge  string       `xml:"installage,attr,omitempty"`
	Status      string       `xml:"status,attr,omitempty"`

	// update engine extensions
	Track     string `xml:"track,attr,omitempty"`
	FromTrack string `xml:"from_track,attr,omitempty"`

	// coreos update engine extensions
	BootId    string `xml:"bootid,attr,omitempty"`
	MachineID string `xml:"machineid,attr,omitempty"`
	OEM       string `xml:"oem,attr,omitempty"`
}

func NewApp(id string) *App {
	a := &App{Id: id}
	return a
}

func (a *App) AddUpdateCheck() *UpdateCheck {
	a.UpdateCheck = new(UpdateCheck)
	return a.UpdateCheck
}

func (a *App) AddPing() *Ping {
	a.Ping = new(Ping)
	return a.Ping
}

func (a *App) AddEvent() *Event {
	event := new(Event)
	a.Events = append(a.Events, event)
	return event
}

type UpdateCheck struct {
	XMLName             xml.Name  `xml:"updatecheck" datastore:"-" json:"-"`
	Urls                *Urls     `xml:"urls"`
	Manifest            *Manifest `xml:"manifest"`
	TargetVersionPrefix string    `xml:"targetversionprefix,attr,omitempty"`
	Status              string    `xml:"status,attr,omitempty"`
}

func (u *UpdateCheck) AddUrl(codebase string) *Url {
	if u.Urls == nil {
		u.Urls = new(Urls)
	}
	url := new(Url)
	url.CodeBase = codebase
	u.Urls.Urls = append(u.Urls.Urls, *url)
	return url
}

func (u *UpdateCheck) AddManifest(version string) *Manifest {
	u.Manifest = &Manifest{Version: version}
	return u.Manifest
}

type Ping struct {
	XMLName        xml.Name `xml:"ping" datastore:"-" json:"-"`
	LastReportDays string   `xml:"r,attr,omitempty"`
	Status         string   `xml:"status,attr,omitempty"`
}

type Os struct {
	XMLName  xml.Name `xml:"os" datastore:"-" json:"-"`
	Platform string   `xml:"platform,attr,omitempty"`
	Version  string   `xml:"version,attr,omitempty"`
	Sp       string   `xml:"sp,attr,omitempty"`
	Arch     string   `xml:"arch,attr,omitempty"`
}

func NewOs(platform string, version string, sp string, arch string) *Os {
	o := &Os{Version: version, Platform: platform, Sp: sp, Arch: arch}
	return o
}

type Event struct {
	XMLName         xml.Name `xml:"event" datastore:"-" json:"-"`
	Type            string   `xml:"eventtype,attr,omitempty"`
	Result          string   `xml:"eventresult,attr,omitempty"`
	PreviousVersion string   `xml:"previousversion,attr,omitempty"`
}

type Urls struct {
	XMLName xml.Name `xml:"urls" datastore:"-" json:"-"`
	Urls    []Url    `xml:"url" json:",omitempty"`
}

type Url struct {
	XMLName  xml.Name `xml:"url" datastore:"-" json:"-"`
	CodeBase string   `xml:"codebase,attr"`
}

type Manifest struct {
	XMLName  xml.Name `xml:"manifest" datastore:"-" json:"-"`
	Packages Packages `xml:"packages"`
	Actions  Actions  `xml:"actions"`
	Version  string   `xml:"version,attr"`
}

type Packages struct {
	XMLName  xml.Name  `xml:"packages" datastore:"-" json:"-"`
	Packages []Package `xml:"package" json:",omitempty"`
}

type Package struct {
	XMLName  xml.Name `xml:"package" datastore:"-" json:"-"`
	Hash     string   `xml:"hash,attr"`
	Name     string   `xml:"name,attr"`
	Size     string   `xml:"size,attr"`
	Required bool     `xml:"required,attr"`
}

func (m *Manifest) AddPackage(hash string, name string, size string, required bool) *Package {
	p := &Package{Hash: hash, Name: name, Size: size, Required: required}
	m.Packages.Packages = append(m.Packages.Packages, *p)
	return p
}

type Actions struct {
	XMLName xml.Name  `xml:"actions" datastore:"-" json:"-"`
	Actions []*Action `xml:"action" json:",omitempty"`
}

type Action struct {
	XMLName xml.Name `xml:"action" datastore:"-" json:"-"`
	Event   string   `xml:"event,attr"`

	// Extensions added by update_engine
	ChromeOSVersion       string `xml:"ChromeOSVersion,attr"`
	Sha256                string `xml:"sha256,attr"`
	NeedsAdmin            bool   `xml:"needsadmin,attr"`
	IsDelta               bool   `xml:"IsDelta,attr"`
	DisablePayloadBackoff bool   `xml:"DisablePayloadBackoff,attr,omitempty"`
	MetadataSignatureRsa  string `xml:"MetadataSignatureRsa,attr,omitempty"`
	MetadataSize          string `xml:"MetadataSize,attr,omitempty"`
	Deadline              string `xml:"deadline,attr,omitempty"`
}

func (m *Manifest) AddAction(event string) *Action {
	a := &Action{Event: event}
	m.Actions.Actions = append(m.Actions.Actions, a)
	return a
}

var EventTypes = map[int]string{
	0:   "unknown",
	1:   "download complete",
	2:   "install complete",
	3:   "update complete",
	4:   "uninstall",
	5:   "download started",
	6:   "install started",
	9:   "new application install started",
	10:  "setup started",
	11:  "setup finished",
	12:  "update application started",
	13:  "update download started",
	14:  "update download finished",
	15:  "update installer started",
	16:  "setup update begin",
	17:  "setup update complete",
	20:  "register product complete",
	30:  "OEM install first check",
	40:  "app-specific command started",
	41:  "app-specific command ended",
	100: "setup failure",
	102: "COM server failure",
	103: "setup update failure",
}

var EventResults = map[int]string{
	0:  "error",
	1:  "success",
	2:  "success reboot",
	3:  "success restart browser",
	4:  "cancelled",
	5:  "error installer MSI",
	6:  "error installer other",
	7:  "noupdate",
	8:  "error installer system",
	9:  "update deferred",
	10: "handoff error",
}
<commit_msg>feat(omaha): add "ping" event type.<commit_after>/*
   Implements the Google omaha protocol.

   Omaha is a request/response protocol using XML. Requests are made by
   clients and responses are given by the Omaha server.
   http://code.google.com/p/omaha/wiki/ServerProtocol
*/
package omaha

import (
	"encoding/xml"
)

type Request struct {
	XMLName        xml.Name `xml:"request" datastore:"-"`
	Os             Os       `xml:"os"`
	Apps           []*App   `xml:"app"`
	Protocol       string   `xml:"protocol,attr"`
	Version        string   `xml:"version,attr,omitempty"`
	IsMachine      string   `xml:"ismachine,attr,omitempty"`
	SessionId      string   `xml:"sessionid,attr,omitempty"`
	UserId         string   `xml:"userid,attr,omitempty"`
	InstallSource  string   `xml:"installsource,attr,omitempty"`
	TestSource     string   `xml:"testsource,attr,omitempty"`
	RequestId      string   `xml:"requestid,attr,omitempty"`
	UpdaterVersion string   `xml:"updaterversion,attr,omitempty"`
}

func NewRequest(version string, platform string, sp string, arch string) *Request {
	r := new(Request)
	r.Protocol = "3.0"
	r.Os = Os{Version: version, Platform: platform, Sp: sp, Arch: arch}
	return r
}

func (r *Request) AddApp(id string, version string) *App {
	a := NewApp(id)
	a.Version = version
	r.Apps = append(r.Apps, a)
	return a
}

/* Response
 */
type Response struct {
	XMLName  xml.Name `xml:"response" datastore:"-" json:"-"`
	DayStart DayStart `xml:"daystart"`
	Apps     []*App   `xml:"app"`
	Protocol string   `xml:"protocol,attr"`
	Server   string   `xml:"server,attr"`
}

func NewResponse(server string) *Response {
	r := &Response{Server: server, Protocol: "3.0"}
	r.DayStart.ElapsedSeconds = "0"
	return r
}

type DayStart struct {
	ElapsedSeconds string `xml:"elapsed_seconds,attr"`
}

func (r *Response) AddApp(id string) *App {
	a := NewApp(id)
	r.Apps = append(r.Apps, a)
	return a
}

type App struct {
	XMLName     xml.Name     `xml:"app" datastore"-" json:"-"`
	Ping        *Ping        `xml:"ping"`
	UpdateCheck *UpdateCheck `xml:"updatecheck"`
	Events      []*Event     `xml:"event" json:",omitempty"`
	Id          string       `xml:"appid,attr,omitempty"`
	Version     string       `xml:"version,attr,omitempty"`
	NextVersion string       `xml:"nextversion,attr,omitempty"`
	Lang        string       `xml:"lang,attr,omitempty"`
	Client      string       `xml:"client,attr,omitempty"`
	InstallAge  string       `xml:"installage,attr,omitempty"`
	Status      string       `xml:"status,attr,omitempty"`

	// update engine extensions
	Track     string `xml:"track,attr,omitempty"`
	FromTrack string `xml:"from_track,attr,omitempty"`

	// coreos update engine extensions
	BootId    string `xml:"bootid,attr,omitempty"`
	MachineID string `xml:"machineid,attr,omitempty"`
	OEM       string `xml:"oem,attr,omitempty"`
}

func NewApp(id string) *App {
	a := &App{Id: id}
	return a
}

func (a *App) AddUpdateCheck() *UpdateCheck {
	a.UpdateCheck = new(UpdateCheck)
	return a.UpdateCheck
}

func (a *App) AddPing() *Ping {
	a.Ping = new(Ping)
	return a.Ping
}

func (a *App) AddEvent() *Event {
	event := new(Event)
	a.Events = append(a.Events, event)
	return event
}

type UpdateCheck struct {
	XMLName             xml.Name  `xml:"updatecheck" datastore:"-" json:"-"`
	Urls                *Urls     `xml:"urls"`
	Manifest            *Manifest `xml:"manifest"`
	TargetVersionPrefix string    `xml:"targetversionprefix,attr,omitempty"`
	Status              string    `xml:"status,attr,omitempty"`
}

func (u *UpdateCheck) AddUrl(codebase string) *Url {
	if u.Urls == nil {
		u.Urls = new(Urls)
	}
	url := new(Url)
	url.CodeBase = codebase
	u.Urls.Urls = append(u.Urls.Urls, *url)
	return url
}

func (u *UpdateCheck) AddManifest(version string) *Manifest {
	u.Manifest = &Manifest{Version: version}
	return u.Manifest
}

type Ping struct {
	XMLName        xml.Name `xml:"ping" datastore:"-" json:"-"`
	LastReportDays string   `xml:"r,attr,omitempty"`
	Status         string   `xml:"status,attr,omitempty"`
}

type Os struct {
	XMLName  xml.Name `xml:"os" datastore:"-" json:"-"`
	Platform string   `xml:"platform,attr,omitempty"`
	Version  string   `xml:"version,attr,omitempty"`
	Sp       string   `xml:"sp,attr,omitempty"`
	Arch     string   `xml:"arch,attr,omitempty"`
}

func NewOs(platform string, version string, sp string, arch string) *Os {
	o := &Os{Version: version, Platform: platform, Sp: sp, Arch: arch}
	return o
}

type Event struct {
	XMLName         xml.Name `xml:"event" datastore:"-" json:"-"`
	Type            string   `xml:"eventtype,attr,omitempty"`
	Result          string   `xml:"eventresult,attr,omitempty"`
	PreviousVersion string   `xml:"previousversion,attr,omitempty"`
}

type Urls struct {
	XMLName xml.Name `xml:"urls" datastore:"-" json:"-"`
	Urls    []Url    `xml:"url" json:",omitempty"`
}

type Url struct {
	XMLName  xml.Name `xml:"url" datastore:"-" json:"-"`
	CodeBase string   `xml:"codebase,attr"`
}

type Manifest struct {
	XMLName  xml.Name `xml:"manifest" datastore:"-" json:"-"`
	Packages Packages `xml:"packages"`
	Actions  Actions  `xml:"actions"`
	Version  string   `xml:"version,attr"`
}

type Packages struct {
	XMLName  xml.Name  `xml:"packages" datastore:"-" json:"-"`
	Packages []Package `xml:"package" json:",omitempty"`
}

type Package struct {
	XMLName  xml.Name `xml:"package" datastore:"-" json:"-"`
	Hash     string   `xml:"hash,attr"`
	Name     string   `xml:"name,attr"`
	Size     string   `xml:"size,attr"`
	Required bool     `xml:"required,attr"`
}

func (m *Manifest) AddPackage(hash string, name string, size string, required bool) *Package {
	p := &Package{Hash: hash, Name: name, Size: size, Required: required}
	m.Packages.Packages = append(m.Packages.Packages, *p)
	return p
}

type Actions struct {
	XMLName xml.Name  `xml:"actions" datastore:"-" json:"-"`
	Actions []*Action `xml:"action" json:",omitempty"`
}

type Action struct {
	XMLName xml.Name `xml:"action" datastore:"-" json:"-"`
	Event   string   `xml:"event,attr"`

	// Extensions added by update_engine
	ChromeOSVersion       string `xml:"ChromeOSVersion,attr"`
	Sha256                string `xml:"sha256,attr"`
	NeedsAdmin            bool   `xml:"needsadmin,attr"`
	IsDelta               bool   `xml:"IsDelta,attr"`
	DisablePayloadBackoff bool   `xml:"DisablePayloadBackoff,attr,omitempty"`
	MetadataSignatureRsa  string `xml:"MetadataSignatureRsa,attr,omitempty"`
	MetadataSize          string `xml:"MetadataSize,attr,omitempty"`
	Deadline              string `xml:"deadline,attr,omitempty"`
}

func (m *Manifest) AddAction(event string) *Action {
	a := &Action{Event: event}
	m.Actions.Actions = append(m.Actions.Actions, a)
	return a
}

var EventTypes = map[int]string{
	0:   "unknown",
	1:   "download complete",
	2:   "install complete",
	3:   "update complete",
	4:   "uninstall",
	5:   "download started",
	6:   "install started",
	9:   "new application install started",
	10:  "setup started",
	11:  "setup finished",
	12:  "update application started",
	13:  "update download started",
	14:  "update download finished",
	15:  "update installer started",
	16:  "setup update begin",
	17:  "setup update complete",
	20:  "register product complete",
	30:  "OEM install first check",
	40:  "app-specific command started",
	41:  "app-specific command ended",
	100: "setup failure",
	102: "COM server failure",
	103: "setup update failure",
	800: "ping",
}

var EventResults = map[int]string{
	0:  "error",
	1:  "success",
	2:  "success reboot",
	3:  "success restart browser",
	4:  "cancelled",
	5:  "error installer MSI",
	6:  "error installer other",
	7:  "noupdate",
	8:  "error installer system",
	9:  "update deferred",
	10: "handoff error",
}
<|endoftext|>


<commit_before>package webhook

import (
	"crypto/hmac"
	"crypto/sha1"
	"encoding/hex"
	"encoding/json"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/google/go-github/github"
	"github.com/matrix-org/go-neb/errors"
	"github.com/matrix-org/go-neb/matrix"
	"html"
	"io/ioutil"
	"net/http"
	"strings"
)

// OnReceiveRequest processes incoming github webhook requests and returns a
// matrix message to send, along with parsed repo information.
// The secretToken, if supplied, will be used to verify the request is from
// Github. If it isn't, an error is returned.
func OnReceiveRequest(r *http.Request, secretToken string) (string, *github.Repository, *matrix.HTMLMessage, *errors.HTTPError) {
	// Verify the HMAC signature if NEB was configured with a secret token
	eventType := r.Header.Get("X-GitHub-Event")
	signatureSHA1 := r.Header.Get("X-Hub-Signature")
	content, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.WithError(err).Print("Failed to read Github webhook body")
		return "", nil, nil, &errors.HTTPError{nil, "Failed to parse body", 400}
	}
	// Verify request if a secret token has been supplied.
	if secretToken != "" {
		sigHex := strings.Split(signatureSHA1, "=")[1]
		var sigBytes []byte
		sigBytes, err = hex.DecodeString(sigHex)
		if err != nil {
			log.WithError(err).WithField("X-Hub-Signature", sigHex).Print(
				"Failed to decode signature as hex.")
			return "", nil, nil, &errors.HTTPError{nil, "Failed to decode signature", 400}
		}

		if !checkMAC([]byte(content), sigBytes, []byte(secretToken)) {
			log.WithFields(log.Fields{
				"X-Hub-Signature": signatureSHA1,
			}).Print("Received Github event which failed MAC check.")
			return "", nil, nil, &errors.HTTPError{nil, "Bad signature", 403}
		}
	}

	log.WithFields(log.Fields{
		"event_type": eventType,
		"signature":  signatureSHA1,
	}).Print("Received Github event")

	if eventType == "ping" {
		// Github will send a "ping" event when the webhook is first created. We need
		// to return a 200 in order for the webhook to be marked as "up" (this doesn't
		// affect delivery, just the tick/cross status flag).
		return "", nil, nil, &errors.HTTPError{nil, "pong", 200}
	}

	htmlStr, repo, err := parseGithubEvent(eventType, content)
	if err != nil {
		log.WithError(err).Print("Failed to parse github event")
		return "", nil, nil, &errors.HTTPError{nil, "Failed to parse github event", 500}
	}

	msg := matrix.GetHTMLMessage("m.notice", htmlStr)
	return eventType, repo, &msg, nil
}

// checkMAC reports whether messageMAC is a valid HMAC tag for message.
func checkMAC(message, messageMAC, key []byte) bool {
	mac := hmac.New(sha1.New, key)
	mac.Write(message)
	expectedMAC := mac.Sum(nil)
	return hmac.Equal(messageMAC, expectedMAC)
}

// parseGithubEvent parses a github event type and JSON data and returns an explanatory
// HTML string and the github repository this event affects, or an error.
func parseGithubEvent(eventType string, data []byte) (string, *github.Repository, error) {
	if eventType == "pull_request" {
		var ev github.PullRequestEvent
		if err := json.Unmarshal(data, &ev); err != nil {
			return "", nil, err
		}
		return pullRequestHTMLMessage(ev), ev.Repo, nil
	} else if eventType == "issues" {
		var ev github.IssuesEvent
		if err := json.Unmarshal(data, &ev); err != nil {
			return "", nil, err
		}
		return issueHTMLMessage(ev), ev.Repo, nil
	} else if eventType == "push" {
		var ev github.PushEvent
		if err := json.Unmarshal(data, &ev); err != nil {
			return "", nil, err
		}

		// The 'push' event repository format is subtly different from normal, so munge the bits we need.
		fullName := *ev.Repo.Owner.Name + "/" + *ev.Repo.Name
		repo := github.Repository{
			Owner: &github.User{
				Login: ev.Repo.Owner.Name,
			},
			Name:     ev.Repo.Name,
			FullName: &fullName,
		}
		return pushHTMLMessage(ev), &repo, nil
	} else if eventType == "issue_comment" {
		var ev github.IssueCommentEvent
		if err := json.Unmarshal(data, &ev); err != nil {
			return "", nil, err
		}
		return issueCommentHTMLMessage(ev), ev.Repo, nil
	} else if eventType == "pull_request_review_comment" {
		var ev github.PullRequestReviewCommentEvent
		if err := json.Unmarshal(data, &ev); err != nil {
			return "", nil, err
		}
		return prReviewCommentHTMLMessage(ev), ev.Repo, nil
	}
	return "", nil, fmt.Errorf("Unrecognized event type")
}

func pullRequestHTMLMessage(p github.PullRequestEvent) string {
	var actionTarget string
	if p.PullRequest.Assignee != nil && p.PullRequest.Assignee.Login != nil {
		actionTarget = fmt.Sprintf(" to %s", *p.PullRequest.Assignee.Login)
	}
	return fmt.Sprintf(
		"[%s] %s %s pull request #%d: %s [%s]%s - %s",
		html.EscapeString(*p.Repo.FullName),
		html.EscapeString(*p.Sender.Login),
		html.EscapeString(*p.Action),
		*p.Number,
		html.EscapeString(*p.PullRequest.Title),
		html.EscapeString(*p.PullRequest.State),
		html.EscapeString(actionTarget),
		html.EscapeString(*p.PullRequest.HTMLURL),
	)
}

func issueHTMLMessage(p github.IssuesEvent) string {
	var actionTarget string
	if p.Issue.Assignee != nil && p.Issue.Assignee.Login != nil {
		actionTarget = fmt.Sprintf(" to %s", *p.Issue.Assignee.Login)
	}
	return fmt.Sprintf(
		"[%s] %s %s issue #%d: %s [%s]%s - %s",
		html.EscapeString(*p.Repo.FullName),
		html.EscapeString(*p.Sender.Login),
		html.EscapeString(*p.Action),
		*p.Issue.Number,
		html.EscapeString(*p.Issue.Title),
		html.EscapeString(*p.Issue.State),
		html.EscapeString(actionTarget),
		html.EscapeString(*p.Issue.HTMLURL),
	)
}

func issueCommentHTMLMessage(p github.IssueCommentEvent) string {
	var kind string
	if p.Issue.PullRequestLinks == nil {
		kind = "issue"
	} else {
		kind = "pull request"
	}

	return fmt.Sprintf(
		"[%s] %s commented on %s's %s #%d: %s - %s",
		html.EscapeString(*p.Repo.FullName),
		html.EscapeString(*p.Comment.User.Login),
		html.EscapeString(*p.Issue.User.Login),
		kind,
		*p.Issue.Number,
		html.EscapeString(*p.Issue.Title),
		html.EscapeString(*p.Issue.HTMLURL),
	)
}

func prReviewCommentHTMLMessage(p github.PullRequestReviewCommentEvent) string {
	assignee := "None"
	if p.PullRequest.Assignee != nil {
		assignee = html.EscapeString(*p.PullRequest.Assignee.Login)
	}
	return fmt.Sprintf(
		"[%s] %s made a line comment on %s's pull request #%d (assignee: %s): %s - %s",
		html.EscapeString(*p.Repo.FullName),
		html.EscapeString(*p.Sender.Login),
		html.EscapeString(*p.PullRequest.User.Login),
		*p.PullRequest.Number,
		assignee,
		html.EscapeString(*p.PullRequest.Title),
		html.EscapeString(*p.Comment.HTMLURL),
	)
}

func pushHTMLMessage(p github.PushEvent) string {
	// /refs/heads/alice/branch-name => alice/branch-name
	branch := strings.Replace(*p.Ref, "refs/heads/", "", -1)

	// this branch was deleted, no HeadCommit object and deleted=true
	if p.HeadCommit == nil && p.Deleted != nil && *p.Deleted {
		return fmt.Sprintf(
			`[<u>%s</u>] %s <font color="red"><b>deleted</font> %s</b>`,
			html.EscapeString(*p.Repo.FullName),
			html.EscapeString(*p.Pusher.Name),
			html.EscapeString(branch),
		)
	}

	if p.Commits != nil && len(p.Commits) > 1 {
		// multi-commit message
		// [<repo>] <username> pushed <num> commits to <branch>: <git.io link>
		// <up to 3 commits>
		var cList []string
		for _, c := range p.Commits {
			cList = append(cList, fmt.Sprintf(
				`%s: %s`,
				html.EscapeString(nameForAuthor(c.Author)),
				html.EscapeString(*c.Message),
			))
		}
		return fmt.Sprintf(
			`[<u>%s</u>] %s pushed %d commits to <b>%s</b>: %s<br>%s`,
			html.EscapeString(*p.Repo.FullName),
			html.EscapeString(nameForAuthor(p.HeadCommit.Committer)),
			len(p.Commits),
			html.EscapeString(branch),
			html.EscapeString(*p.HeadCommit.URL),
			strings.Join(cList, "
"),
		)
	}

	// single commit message
	// [<repo>] <username> pushed to <branch>: <msg> - <git.io link>
	return fmt.Sprintf(
		`[<u>%s</u>] %s pushed to <b>%s</b>: %s  - %s`,
		html.EscapeString(*p.Repo.FullName),
		html.EscapeString(nameForAuthor(p.HeadCommit.Committer)),
		html.EscapeString(branch),
		html.EscapeString(*p.HeadCommit.Message),
		html.EscapeString(*p.HeadCommit.URL),
	)
}

func nameForAuthor(a *github.CommitAuthor) string {
	if a == nil {
		return ""
	}
	if a.Login != nil { // prefer to use their GH username than the name they commited as
		return *a.Login
	}
	return *a.Name
}
<commit_msg>Fix #105 : Mention which label was added/removed<commit_after>package webhook

import (
	"crypto/hmac"
	"crypto/sha1"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"html"
	"io/ioutil"
	"net/http"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/google/go-github/github"
	"github.com/matrix-org/go-neb/errors"
	"github.com/matrix-org/go-neb/matrix"
)

// OnReceiveRequest processes incoming github webhook requests and returns a
// matrix message to send, along with parsed repo information.
// The secretToken, if supplied, will be used to verify the request is from
// Github. If it isn't, an error is returned.
func OnReceiveRequest(r *http.Request, secretToken string) (string, *github.Repository, *matrix.HTMLMessage, *errors.HTTPError) {
	// Verify the HMAC signature if NEB was configured with a secret token
	eventType := r.Header.Get("X-GitHub-Event")
	signatureSHA1 := r.Header.Get("X-Hub-Signature")
	content, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.WithError(err).Print("Failed to read Github webhook body")
		return "", nil, nil, &errors.HTTPError{nil, "Failed to parse body", 400}
	}
	// Verify request if a secret token has been supplied.
	if secretToken != "" {
		sigHex := strings.Split(signatureSHA1, "=")[1]
		var sigBytes []byte
		sigBytes, err = hex.DecodeString(sigHex)
		if err != nil {
			log.WithError(err).WithField("X-Hub-Signature", sigHex).Print(
				"Failed to decode signature as hex.")
			return "", nil, nil, &errors.HTTPError{nil, "Failed to decode signature", 400}
		}

		if !checkMAC([]byte(content), sigBytes, []byte(secretToken)) {
			log.WithFields(log.Fields{
				"X-Hub-Signature": signatureSHA1,
			}).Print("Received Github event which failed MAC check.")
			return "", nil, nil, &errors.HTTPError{nil, "Bad signature", 403}
		}
	}

	log.WithFields(log.Fields{
		"event_type": eventType,
		"signature":  signatureSHA1,
	}).Print("Received Github event")

	if eventType == "ping" {
		// Github will send a "ping" event when the webhook is first created. We need
		// to return a 200 in order for the webhook to be marked as "up" (this doesn't
		// affect delivery, just the tick/cross status flag).
		return "", nil, nil, &errors.HTTPError{nil, "pong", 200}
	}

	htmlStr, repo, err := parseGithubEvent(eventType, content)
	if err != nil {
		log.WithError(err).Print("Failed to parse github event")
		return "", nil, nil, &errors.HTTPError{nil, "Failed to parse github event", 500}
	}

	msg := matrix.GetHTMLMessage("m.notice", htmlStr)
	return eventType, repo, &msg, nil
}

// checkMAC reports whether messageMAC is a valid HMAC tag for message.
func checkMAC(message, messageMAC, key []byte) bool {
	mac := hmac.New(sha1.New, key)
	mac.Write(message)
	expectedMAC := mac.Sum(nil)
	return hmac.Equal(messageMAC, expectedMAC)
}

// parseGithubEvent parses a github event type and JSON data and returns an explanatory
// HTML string and the github repository this event affects, or an error.
func parseGithubEvent(eventType string, data []byte) (string, *github.Repository, error) {
	if eventType == "pull_request" {
		var ev github.PullRequestEvent
		if err := json.Unmarshal(data, &ev); err != nil {
			return "", nil, err
		}
		return pullRequestHTMLMessage(ev), ev.Repo, nil
	} else if eventType == "issues" {
		var ev github.IssuesEvent
		if err := json.Unmarshal(data, &ev); err != nil {
			return "", nil, err
		}
		return issueHTMLMessage(ev), ev.Repo, nil
	} else if eventType == "push" {
		var ev github.PushEvent
		if err := json.Unmarshal(data, &ev); err != nil {
			return "", nil, err
		}

		// The 'push' event repository format is subtly different from normal, so munge the bits we need.
		fullName := *ev.Repo.Owner.Name + "/" + *ev.Repo.Name
		repo := github.Repository{
			Owner: &github.User{
				Login: ev.Repo.Owner.Name,
			},
			Name:     ev.Repo.Name,
			FullName: &fullName,
		}
		return pushHTMLMessage(ev), &repo, nil
	} else if eventType == "issue_comment" {
		var ev github.IssueCommentEvent
		if err := json.Unmarshal(data, &ev); err != nil {
			return "", nil, err
		}
		return issueCommentHTMLMessage(ev), ev.Repo, nil
	} else if eventType == "pull_request_review_comment" {
		var ev github.PullRequestReviewCommentEvent
		if err := json.Unmarshal(data, &ev); err != nil {
			return "", nil, err
		}
		return prReviewCommentHTMLMessage(ev), ev.Repo, nil
	}
	return "", nil, fmt.Errorf("Unrecognized event type")
}

func pullRequestHTMLMessage(p github.PullRequestEvent) string {
	var actionTarget string
	if p.PullRequest.Assignee != nil && p.PullRequest.Assignee.Login != nil {
		actionTarget = fmt.Sprintf(" to %s", *p.PullRequest.Assignee.Login)
	}
	return fmt.Sprintf(
		"[%s] %s %s pull request #%d: %s [%s]%s - %s",
		html.EscapeString(*p.Repo.FullName),
		html.EscapeString(*p.Sender.Login),
		html.EscapeString(*p.Action),
		*p.Number,
		html.EscapeString(*p.PullRequest.Title),
		html.EscapeString(*p.PullRequest.State),
		html.EscapeString(actionTarget),
		html.EscapeString(*p.PullRequest.HTMLURL),
	)
}

func issueHTMLMessage(p github.IssuesEvent) string {
	var actionTarget string
	if p.Issue.Assignee != nil && p.Issue.Assignee.Login != nil {
		actionTarget = fmt.Sprintf(" to %s", *p.Issue.Assignee.Login)
	}
	action := html.EscapeString(*p.Action)
	if p.Label != nil && (*p.Action == "labeled" || *p.Action == "unlabeled") {
		action = *p.Action + " [" + html.EscapeString(*p.Label.Name) + "] to"
	}
	return fmt.Sprintf(
		"[%s] %s %s issue #%d: %s [%s]%s - %s",
		html.EscapeString(*p.Repo.FullName),
		html.EscapeString(*p.Sender.Login),
		action,
		*p.Issue.Number,
		html.EscapeString(*p.Issue.Title),
		html.EscapeString(*p.Issue.State),
		html.EscapeString(actionTarget),
		html.EscapeString(*p.Issue.HTMLURL),
	)
}

func issueCommentHTMLMessage(p github.IssueCommentEvent) string {
	var kind string
	if p.Issue.PullRequestLinks == nil {
		kind = "issue"
	} else {
		kind = "pull request"
	}

	return fmt.Sprintf(
		"[%s] %s commented on %s's %s #%d: %s - %s",
		html.EscapeString(*p.Repo.FullName),
		html.EscapeString(*p.Comment.User.Login),
		html.EscapeString(*p.Issue.User.Login),
		kind,
		*p.Issue.Number,
		html.EscapeString(*p.Issue.Title),
		html.EscapeString(*p.Issue.HTMLURL),
	)
}

func prReviewCommentHTMLMessage(p github.PullRequestReviewCommentEvent) string {
	assignee := "None"
	if p.PullRequest.Assignee != nil {
		assignee = html.EscapeString(*p.PullRequest.Assignee.Login)
	}
	return fmt.Sprintf(
		"[%s] %s made a line comment on %s's pull request #%d (assignee: %s): %s - %s",
		html.EscapeString(*p.Repo.FullName),
		html.EscapeString(*p.Sender.Login),
		html.EscapeString(*p.PullRequest.User.Login),
		*p.PullRequest.Number,
		assignee,
		html.EscapeString(*p.PullRequest.Title),
		html.EscapeString(*p.Comment.HTMLURL),
	)
}

func pushHTMLMessage(p github.PushEvent) string {
	// /refs/heads/alice/branch-name => alice/branch-name
	branch := strings.Replace(*p.Ref, "refs/heads/", "", -1)

	// this branch was deleted, no HeadCommit object and deleted=true
	if p.HeadCommit == nil && p.Deleted != nil && *p.Deleted {
		return fmt.Sprintf(
			`[<u>%s</u>] %s <font color="red"><b>deleted</font> %s</b>`,
			html.EscapeString(*p.Repo.FullName),
			html.EscapeString(*p.Pusher.Name),
			html.EscapeString(branch),
		)
	}

	if p.Commits != nil && len(p.Commits) > 1 {
		// multi-commit message
		// [<repo>] <username> pushed <num> commits to <branch>: <git.io link>
		// <up to 3 commits>
		var cList []string
		for _, c := range p.Commits {
			cList = append(cList, fmt.Sprintf(
				`%s: %s`,
				html.EscapeString(nameForAuthor(c.Author)),
				html.EscapeString(*c.Message),
			))
		}
		return fmt.Sprintf(
			`[<u>%s</u>] %s pushed %d commits to <b>%s</b>: %s<br>%s`,
			html.EscapeString(*p.Repo.FullName),
			html.EscapeString(nameForAuthor(p.HeadCommit.Committer)),
			len(p.Commits),
			html.EscapeString(branch),
			html.EscapeString(*p.HeadCommit.URL),
			strings.Join(cList, "
"),
		)
	}

	// single commit message
	// [<repo>] <username> pushed to <branch>: <msg> - <git.io link>
	return fmt.Sprintf(
		`[<u>%s</u>] %s pushed to <b>%s</b>: %s  - %s`,
		html.EscapeString(*p.Repo.FullName),
		html.EscapeString(nameForAuthor(p.HeadCommit.Committer)),
		html.EscapeString(branch),
		html.EscapeString(*p.HeadCommit.Message),
		html.EscapeString(*p.HeadCommit.URL),
	)
}

func nameForAuthor(a *github.CommitAuthor) string {
	if a == nil {
		return ""
	}
	if a.Login != nil { // prefer to use their GH username than the name they commited as
		return *a.Login
	}
	return *a.Name
}
<|endoftext|>


<commit_before>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// Package tailer provides a class that is responsible for tailing log files
// and extracting new log lines to be passed into the virtual machines.
package tailer

// For regular files, mtail gets notified on modifications (i.e. appends) to
// log files that are being watched, in order to read the new lines. Log files
// can also be rotated, so mtail is also notified of creates in the log file
// directory.

import (
	"errors"
	"expvar"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"sync"
	"syscall"
	"time"
	"unicode/utf8"

	"github.com/golang/glog"

	"github.com/google/mtail/watcher"

	"github.com/spf13/afero"
)

var (
	logCount     = expvar.NewInt("log_count")
	logErrors    = expvar.NewMap("log_errors_total")
	logRotations = expvar.NewMap("log_rotations_total")
)

// Tailer receives notification of changes from a Watcher and extracts new log
// lines from files. It also handles new log file creation events and log
// rotations.
type Tailer struct {
	w watcher.Watcher

	watched      map[string]struct{}   // Names of logs being watched.
	watchedLock  sync.RWMutex          // protects `watched'
	lines        chan<- string         // Logfile lines being emitted.
	files        map[string]afero.File // File handles for each pathname.
	filesLock    sync.Mutex            // protects `files'
	partials     map[string]string     // Accumulator for the currently read line for each pathname.
	partialsLock sync.Mutex            // protects 'partials'

	fs afero.Fs // mockable filesystem interface
}

// Options configures a Tailer
type Options struct {
	Lines chan<- string
	W     watcher.Watcher // Not required, will use watcher.LogWatcher if it is zero.
	FS    afero.Fs        // Not required, will use afero.OsFs if it is zero.
}

// New returns a new Tailer, configured with the supplied Options
func New(o Options) (*Tailer, error) {
	if o.Lines == nil {
		return nil, errors.New("tailer needs lines")
	}
	fs := o.FS
	if fs == nil {
		fs = &afero.OsFs{}
	}
	w := o.W
	if w == nil {
		var err error
		w, err = watcher.NewLogWatcher()
		if err != nil {
			return nil, fmt.Errorf("Couldn't create a watcher for tailer: %s", err)
		}
	}
	t := &Tailer{
		w:        w,
		watched:  make(map[string]struct{}),
		lines:    o.Lines,
		files:    make(map[string]afero.File),
		partials: make(map[string]string),
		fs:       fs,
	}
	go t.run()
	return t, nil
}

// addWatched adds a path to the list of watched items.
func (t *Tailer) addWatched(path string) {
	t.watchedLock.Lock()
	defer t.watchedLock.Unlock()
	t.watched[path] = struct{}{}
}

// isWatching indicates if the path is being watched. It includes both
// filenames and directories.
func (t *Tailer) isWatching(path string) bool {
	t.watchedLock.RLock()
	defer t.watchedLock.RUnlock()
	_, ok := t.watched[path]
	return ok
}

// Tail registers a file path to be tailed.
func (t *Tailer) Tail(pathname string) {
	fullpath, err := filepath.Abs(pathname)
	if err != nil {
		glog.Infof("Failed to find absolute path for %q: %s\n", pathname, err)
		return
	}
	if !t.isWatching(fullpath) {
		t.addWatched(fullpath)
		logCount.Add(1)
		t.openLogPath(fullpath, false)
	}
}

// TailFile registers a file handle to be tailed.  There is no filesystem to
// watch, so no watches are registered, and no file paths are opened.
func (t *Tailer) TailFile(f afero.File) error {
	logCount.Add(1)
	return t.startNewFile(f, false)
}

// handleLogUpdate reads all available bytes from an already opened file
// identified by pathname, and sends them to be processed on the lines channel.
func (t *Tailer) handleLogUpdate(pathname string) {
	t.filesLock.Lock()
	fd, ok := t.files[pathname]
	t.filesLock.Unlock()
	if !ok {
		glog.Warningf("No file descriptor found for %q, but is being watched", pathname)
		return
	}
	var err error
	t.partialsLock.Lock()
	t.partials[pathname], err = t.read(fd, t.partials[pathname])
	t.partialsLock.Unlock()
	if err != nil && err != io.EOF {
		glog.Info(err)
	}
}

// read reads blocks of 4096 bytes from the File, sending lines to the
// channel as it encounters newlines.  If EOF is encountered, the partial line
// is returned to be concatenated with on the next call.
func (t *Tailer) read(f afero.File, partialIn string) (partialOut string, err error) {
	partial := partialIn
	b := make([]byte, 0, 4096)
	for {
		n, err := f.Read(b[:cap(b)])
		b = b[:n]
		if err != nil {
			return partial, err
		}

		for i, width := 0, 0; i < len(b) && i < n; i += width {
			var rune rune
			rune, width = utf8.DecodeRune(b[i:])
			switch {
			case rune != '\n':
				partial += string(rune)
			default:
				// send off line for processing
				t.lines <- partial
				// reset accumulator
				partial = ""
			}
		}
	}
	return partial, fmt.Errorf("reader shutdown requested")
}

// inode returns the inode number of a file, or 0 if the file has no underlying Sys implementation.
func inode(f os.FileInfo) uint64 {
	s := f.Sys()
	if s == nil {
		return 0
	}
	switch s := s.(type) {
	case *syscall.Stat_t:
		return s.Ino
	default:
		return 0
	}
}

// handleLogCreate handles both new and rotated log files.
func (t *Tailer) handleLogCreate(pathname string) {
	t.filesLock.Lock()
	fd, ok := t.files[pathname]
	t.filesLock.Unlock()
	if ok {
		s1, err := fd.Stat()
		if err != nil {
			glog.Infof("Stat failed on %q: %s", t.files[pathname].Name(), err)
			return
		}
		s2, err := t.fs.Stat(pathname)
		if err != nil {
			glog.Infof("Stat failed on %q: %s", pathname, err)
			return
		}
		if inode(s1) != inode(s2) {
			glog.V(1).Infof("New inode detected for %s, treating as rotation.", pathname)
			logRotations.Add(pathname, 1)
			// flush the old log, pathname is still an index into t.files with the old inode.
			t.handleLogUpdate(pathname)
			fd.Close()
			go func() {
				// Run in goroutine as Remove may block waiting on event processing.
				err := t.w.Remove(pathname)
				if err != nil {
					glog.Infof("Failed removing watches on %s: %s", pathname, err)
				}
				// openLogPath readds the file to the watcher, so must be strictly after the Remove succeeds.
				t.openLogPath(pathname, true)
			}()
		} else {
			glog.V(1).Infof("Path %s already being watched, and inode not changed.",
				pathname)
		}
	} else {
		// Freshly opened log file, never seen before.
		t.openLogPath(pathname, true)
	}
}

// openLogPath opens a log file named by pathname.
func (t *Tailer) openLogPath(pathname string, seenBefore bool) {
	d := path.Dir(pathname)
	if !t.isWatching(d) {
		err := t.w.Add(d)
		if err != nil {
			glog.Infof("Failed to create new watch on directory %q: %s", d, err)
		}
		t.addWatched(d)
	}

	retries := 3
	retryDelay := 1 * time.Millisecond
	var f afero.File
	var err error
	for retries > 0 {
		f, err = t.fs.Open(pathname)
		if err == nil {
			break
		}
		// Doesn't exist yet. We're watching the directory, so we'll pick it up
		// again on create; return successfully.
		if os.IsNotExist(err) {
			return
		}
		glog.Infof("Failed to open %q for reading: %s", pathname, err)
		logErrors.Add(pathname, 1)
		// seenBefore indicates also that we're rotating a file that previously worked, so retry.
		if seenBefore {
			retries = retries - 1
			time.Sleep(retryDelay)
			retryDelay = retryDelay + retryDelay
		} else {
			return
		}
	}
	err = t.startNewFile(f, seenBefore)
	if err != nil {
		glog.Error(err)
	}
}

// startNewFile optionally seeks to the start or end of the file, then starts
// the consumption of log lines. Rotated logs should read from the start, but
// logs opened for the first time read from the end.
func (t *Tailer) startNewFile(f afero.File, seekStart bool) error {
	fi, err := f.Stat()
	if err != nil {
		// Stat failed, log error and return.
		logErrors.Add(f.Name(), 1)
		return fmt.Errorf("Failed to stat %q: %s", f.Name(), err)
	}
	switch m := fi.Mode(); {
	case m&os.ModeType == 0:
		if seekStart {
			f.Seek(0, os.SEEK_SET)
		} else {
			f.Seek(0, os.SEEK_END)
		}
		err = t.w.Add(f.Name())
		if err != nil {
			return fmt.Errorf("Adding a change watch failed on %q: %s", f.Name(), err)
		}
		// In case the new log has been written to already, attempt to read the
		// first lines.
		t.partialsLock.Lock()
		t.partials[f.Name()], err = t.read(f, "")
		t.partialsLock.Unlock()
		if err != nil {
			if err == io.EOF {
				// Don't worry about EOF on first read, that's expected.
				break
			}
			return err
		}
	case m&os.ModeType == os.ModeNamedPipe:
		go t.readForever(f)
	default:
		return fmt.Errorf("Can't open files with mode %v: %s", m&os.ModeType, f.Name())
	}
	t.filesLock.Lock()
	t.files[f.Name()] = f
	t.filesLock.Unlock()
	glog.Infof("Tailing %s", f.Name())

	return nil
}

// start is the main event loop for the Tailer.
// It receives notification of log file changes from the watcher channel, and
// handles them.
func (t *Tailer) run() {
	for e := range t.w.Events() {
		switch e := e.(type) {
		case watcher.UpdateEvent:
			if t.isWatching(e.Pathname) {
				t.handleLogUpdate(e.Pathname)
			}
		case watcher.CreateEvent:
			if t.isWatching(e.Pathname) {
				t.handleLogCreate(e.Pathname)
			}
		case watcher.DeleteEvent:
		default:
			glog.Infof("Unexpected event %#v", e)
		}
	}
	glog.Infof("Shutting down tailer.")
	close(t.lines)
}

// readForever handles non-logfile inputs by reading from the File until it is closed.
func (t *Tailer) readForever(f afero.File) {
	var err error
	partial := ""
	for {
		partial, err = t.read(f, partial)
		// We want to exit at EOF, because the FD has been closed.
		if err != nil {
			glog.Infof("%s: %s", err, f.Name())
			return
		}
		// TODO(jaq): nonblocking read, handle eagain, and do a little sleep if so, so the read can be interrupted
	}
}

// Close signals termination to the watcher.
func (t *Tailer) Close() {
	t.w.Close()
}
<commit_msg>convert type for other platforms<commit_after>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// Package tailer provides a class that is responsible for tailing log files
// and extracting new log lines to be passed into the virtual machines.
package tailer

// For regular files, mtail gets notified on modifications (i.e. appends) to
// log files that are being watched, in order to read the new lines. Log files
// can also be rotated, so mtail is also notified of creates in the log file
// directory.

import (
	"errors"
	"expvar"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"sync"
	"syscall"
	"time"
	"unicode/utf8"

	"github.com/golang/glog"

	"github.com/google/mtail/watcher"

	"github.com/spf13/afero"
)

var (
	logCount     = expvar.NewInt("log_count")
	logErrors    = expvar.NewMap("log_errors_total")
	logRotations = expvar.NewMap("log_rotations_total")
)

// Tailer receives notification of changes from a Watcher and extracts new log
// lines from files. It also handles new log file creation events and log
// rotations.
type Tailer struct {
	w watcher.Watcher

	watched      map[string]struct{}   // Names of logs being watched.
	watchedLock  sync.RWMutex          // protects `watched'
	lines        chan<- string         // Logfile lines being emitted.
	files        map[string]afero.File // File handles for each pathname.
	filesLock    sync.Mutex            // protects `files'
	partials     map[string]string     // Accumulator for the currently read line for each pathname.
	partialsLock sync.Mutex            // protects 'partials'

	fs afero.Fs // mockable filesystem interface
}

// Options configures a Tailer
type Options struct {
	Lines chan<- string
	W     watcher.Watcher // Not required, will use watcher.LogWatcher if it is zero.
	FS    afero.Fs        // Not required, will use afero.OsFs if it is zero.
}

// New returns a new Tailer, configured with the supplied Options
func New(o Options) (*Tailer, error) {
	if o.Lines == nil {
		return nil, errors.New("tailer needs lines")
	}
	fs := o.FS
	if fs == nil {
		fs = &afero.OsFs{}
	}
	w := o.W
	if w == nil {
		var err error
		w, err = watcher.NewLogWatcher()
		if err != nil {
			return nil, fmt.Errorf("Couldn't create a watcher for tailer: %s", err)
		}
	}
	t := &Tailer{
		w:        w,
		watched:  make(map[string]struct{}),
		lines:    o.Lines,
		files:    make(map[string]afero.File),
		partials: make(map[string]string),
		fs:       fs,
	}
	go t.run()
	return t, nil
}

// addWatched adds a path to the list of watched items.
func (t *Tailer) addWatched(path string) {
	t.watchedLock.Lock()
	defer t.watchedLock.Unlock()
	t.watched[path] = struct{}{}
}

// isWatching indicates if the path is being watched. It includes both
// filenames and directories.
func (t *Tailer) isWatching(path string) bool {
	t.watchedLock.RLock()
	defer t.watchedLock.RUnlock()
	_, ok := t.watched[path]
	return ok
}

// Tail registers a file path to be tailed.
func (t *Tailer) Tail(pathname string) {
	fullpath, err := filepath.Abs(pathname)
	if err != nil {
		glog.Infof("Failed to find absolute path for %q: %s\n", pathname, err)
		return
	}
	if !t.isWatching(fullpath) {
		t.addWatched(fullpath)
		logCount.Add(1)
		t.openLogPath(fullpath, false)
	}
}

// TailFile registers a file handle to be tailed.  There is no filesystem to
// watch, so no watches are registered, and no file paths are opened.
func (t *Tailer) TailFile(f afero.File) error {
	logCount.Add(1)
	return t.startNewFile(f, false)
}

// handleLogUpdate reads all available bytes from an already opened file
// identified by pathname, and sends them to be processed on the lines channel.
func (t *Tailer) handleLogUpdate(pathname string) {
	t.filesLock.Lock()
	fd, ok := t.files[pathname]
	t.filesLock.Unlock()
	if !ok {
		glog.Warningf("No file descriptor found for %q, but is being watched", pathname)
		return
	}
	var err error
	t.partialsLock.Lock()
	t.partials[pathname], err = t.read(fd, t.partials[pathname])
	t.partialsLock.Unlock()
	if err != nil && err != io.EOF {
		glog.Info(err)
	}
}

// read reads blocks of 4096 bytes from the File, sending lines to the
// channel as it encounters newlines.  If EOF is encountered, the partial line
// is returned to be concatenated with on the next call.
func (t *Tailer) read(f afero.File, partialIn string) (partialOut string, err error) {
	partial := partialIn
	b := make([]byte, 0, 4096)
	for {
		n, err := f.Read(b[:cap(b)])
		b = b[:n]
		if err != nil {
			return partial, err
		}

		for i, width := 0, 0; i < len(b) && i < n; i += width {
			var rune rune
			rune, width = utf8.DecodeRune(b[i:])
			switch {
			case rune != '\n':
				partial += string(rune)
			default:
				// send off line for processing
				t.lines <- partial
				// reset accumulator
				partial = ""
			}
		}
	}
	return partial, fmt.Errorf("reader shutdown requested")
}

// inode returns the inode number of a file, or 0 if the file has no underlying Sys implementation.
func inode(f os.FileInfo) uint64 {
	s := f.Sys()
	if s == nil {
		return 0
	}
	switch s := s.(type) {
	case *syscall.Stat_t:
		return uint64(s.Ino)
	default:
		return 0
	}
}

// handleLogCreate handles both new and rotated log files.
func (t *Tailer) handleLogCreate(pathname string) {
	t.filesLock.Lock()
	fd, ok := t.files[pathname]
	t.filesLock.Unlock()
	if ok {
		s1, err := fd.Stat()
		if err != nil {
			glog.Infof("Stat failed on %q: %s", t.files[pathname].Name(), err)
			return
		}
		s2, err := t.fs.Stat(pathname)
		if err != nil {
			glog.Infof("Stat failed on %q: %s", pathname, err)
			return
		}
		if inode(s1) != inode(s2) {
			glog.V(1).Infof("New inode detected for %s, treating as rotation.", pathname)
			logRotations.Add(pathname, 1)
			// flush the old log, pathname is still an index into t.files with the old inode.
			t.handleLogUpdate(pathname)
			fd.Close()
			go func() {
				// Run in goroutine as Remove may block waiting on event processing.
				err := t.w.Remove(pathname)
				if err != nil {
					glog.Infof("Failed removing watches on %s: %s", pathname, err)
				}
				// openLogPath readds the file to the watcher, so must be strictly after the Remove succeeds.
				t.openLogPath(pathname, true)
			}()
		} else {
			glog.V(1).Infof("Path %s already being watched, and inode not changed.",
				pathname)
		}
	} else {
		// Freshly opened log file, never seen before.
		t.openLogPath(pathname, true)
	}
}

// openLogPath opens a log file named by pathname.
func (t *Tailer) openLogPath(pathname string, seenBefore bool) {
	d := path.Dir(pathname)
	if !t.isWatching(d) {
		err := t.w.Add(d)
		if err != nil {
			glog.Infof("Failed to create new watch on directory %q: %s", d, err)
		}
		t.addWatched(d)
	}

	retries := 3
	retryDelay := 1 * time.Millisecond
	var f afero.File
	var err error
	for retries > 0 {
		f, err = t.fs.Open(pathname)
		if err == nil {
			break
		}
		// Doesn't exist yet. We're watching the directory, so we'll pick it up
		// again on create; return successfully.
		if os.IsNotExist(err) {
			return
		}
		glog.Infof("Failed to open %q for reading: %s", pathname, err)
		logErrors.Add(pathname, 1)
		// seenBefore indicates also that we're rotating a file that previously worked, so retry.
		if seenBefore {
			retries = retries - 1
			time.Sleep(retryDelay)
			retryDelay = retryDelay + retryDelay
		} else {
			return
		}
	}
	err = t.startNewFile(f, seenBefore)
	if err != nil {
		glog.Error(err)
	}
}

// startNewFile optionally seeks to the start or end of the file, then starts
// the consumption of log lines. Rotated logs should read from the start, but
// logs opened for the first time read from the end.
func (t *Tailer) startNewFile(f afero.File, seekStart bool) error {
	fi, err := f.Stat()
	if err != nil {
		// Stat failed, log error and return.
		logErrors.Add(f.Name(), 1)
		return fmt.Errorf("Failed to stat %q: %s", f.Name(), err)
	}
	switch m := fi.Mode(); {
	case m&os.ModeType == 0:
		if seekStart {
			f.Seek(0, os.SEEK_SET)
		} else {
			f.Seek(0, os.SEEK_END)
		}
		err = t.w.Add(f.Name())
		if err != nil {
			return fmt.Errorf("Adding a change watch failed on %q: %s", f.Name(), err)
		}
		// In case the new log has been written to already, attempt to read the
		// first lines.
		t.partialsLock.Lock()
		t.partials[f.Name()], err = t.read(f, "")
		t.partialsLock.Unlock()
		if err != nil {
			if err == io.EOF {
				// Don't worry about EOF on first read, that's expected.
				break
			}
			return err
		}
	case m&os.ModeType == os.ModeNamedPipe:
		go t.readForever(f)
	default:
		return fmt.Errorf("Can't open files with mode %v: %s", m&os.ModeType, f.Name())
	}
	t.filesLock.Lock()
	t.files[f.Name()] = f
	t.filesLock.Unlock()
	glog.Infof("Tailing %s", f.Name())

	return nil
}

// start is the main event loop for the Tailer.
// It receives notification of log file changes from the watcher channel, and
// handles them.
func (t *Tailer) run() {
	for e := range t.w.Events() {
		switch e := e.(type) {
		case watcher.UpdateEvent:
			if t.isWatching(e.Pathname) {
				t.handleLogUpdate(e.Pathname)
			}
		case watcher.CreateEvent:
			if t.isWatching(e.Pathname) {
				t.handleLogCreate(e.Pathname)
			}
		case watcher.DeleteEvent:
		default:
			glog.Infof("Unexpected event %#v", e)
		}
	}
	glog.Infof("Shutting down tailer.")
	close(t.lines)
}

// readForever handles non-logfile inputs by reading from the File until it is closed.
func (t *Tailer) readForever(f afero.File) {
	var err error
	partial := ""
	for {
		partial, err = t.read(f, partial)
		// We want to exit at EOF, because the FD has been closed.
		if err != nil {
			glog.Infof("%s: %s", err, f.Name())
			return
		}
		// TODO(jaq): nonblocking read, handle eagain, and do a little sleep if so, so the read can be interrupted
	}
}

// Close signals termination to the watcher.
func (t *Tailer) Close() {
	t.w.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/bitly/go-nsq"
	"github.com/codegangsta/cli"
	. "github.com/xtaci/nsq-logger"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"
)

const (
	LAYOUT = "2006/1/2 15:04:05"
)

var (
	logTemplate = make(map[byte]string)
)

func init() {
	logTemplate[FINEST] = "\033[0;34m%v [FINEST] %v %v %v %v %v \033[0m"
	logTemplate[FINE] = "\033[0;34m%v [FINE] %v %v %v %v %v \033[0m"
	logTemplate[DEBUG] = "\033[1;32m%v [DEBUG] %v %v %v %v %v \033[0m"
	logTemplate[TRACE] = "\033[0;37m%v [TRACE] %v %v %v %v %v \033[0m"
	logTemplate[WARNING] = "\033[7;33m%v [WARNING] %v %v %v %v %v \033[0m"
	logTemplate[INFO] = "\033[0;32m%v [INFO] %v %v %v %v %v  \033[0m"
	logTemplate[ERROR] = "\033[7;31m%v [ERROR] %v %v %v %v %v \033[0m"
	logTemplate[CRITICAL] = "\033[0;31m%v [CRITICAL] %v %v %v %v %v \033[0m"
}

var (
	tailFlag = []cli.Flag{
		cli.StringFlag{
			Name:  "topic,t",
			Value: "LOG",
			Usage: "NSQ topic , defalut is LOG",
		},
		cli.StringFlag{
			Name:  "channel, c",
			Value: "tailn",
			Usage: "NSQ channel, defalut is tailn",
		},
		cli.StringFlag{
			Name:  "number, n",
			Value: "0",
			Usage: "Line to show, defalut no limit",
		},
		cli.StringFlag{
			Name:  "nsqd-tcp-address, a",
			Value: "localhost:4150",
			Usage: "nsqd TCP address",
		},
		cli.StringFlag{
			Name:  "lookupd-http-address, l",
			Value: "localhost:4160",
			Usage: "lookupd HTTP address",
		},
		cli.StringFlag{
			Name:  "timeout, o",
			Value: "5",
			Usage: "Dial timeout, default 5s",
		},
		cli.StringFlag{
			Name:  "type",
			Value: "NSQLOG",
			Usage: "Tail type , default NSQLOG (others print json )",
		},
		cli.StringFlag{
			Name:  "log",
			Value: "false",
			Usage: "whether open inner log",
		},
	}
)

type TailHandler struct {
	totalMessages int
	messagesShown int
	printMessage  func(m *nsq.Message) error
}

func NSQLog(m *nsq.Message) error {
	info := &LogFormat{}
	err := json.Unmarshal(m.Body, &info)
	if err != nil {
		fmt.Printf("err %v\n", err)
		return err
	}
	fmt.Println(fmt.Sprintf(logTemplate[info.Level], info.Time.Format(LAYOUT), info.Prefix, info.Host, info.Msg, info.Caller, info.LineNo))
	return nil
}

func Log(m *nsq.Message) error {
	_, err := os.Stdout.Write(m.Body)
	if err != nil {
		return err
	}
	_, err = os.Stdout.WriteString("\n")
	return err
}

func main() {
	myApp := cli.NewApp()
	myApp.Name = "tailn"
	myApp.Usage = "Tail log from nsq !"
	myApp.Version = "0.0.1"
	myApp.Flags = tailFlag
	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
	myApp.Action = func(c *cli.Context) {
		// TODO CheckFlag
		CheckFlag(c)
		cfg := nsq.NewConfig()
		// dail timeout
		cfg.DialTimeout = time.Duration(c.Int("timeout")) * time.Second
		cfg.UserAgent = fmt.Sprint("go-nsq version:%v", nsq.VERSION)
		consumer, err := nsq.NewConsumer(c.String("topic"), c.String("channel"), cfg)
		if err != nil {
			fmt.Printf("error %v\n", err)
			os.Exit(0)
		}

		f := NSQLog
		if c.String("type") != "NSQLOG" {
			f = Log
		}

		// 屏蔽内部日志
		if !c.Bool("log") {
			consumer.SetLogger(nil, 0)
		}

		consumer.AddHandler(&TailHandler{c.Int("number"), 0, f})
		err = consumer.ConnectToNSQDs(strings.Split(c.String("nsqd-tcp-address"), ","))
		if err != nil {
			fmt.Printf("error %v\n", err)
			os.Exit(0)

		}
		err = consumer.ConnectToNSQLookupds(strings.Split(c.String("lookupd-http-address"), ","))
		if err != nil {
			fmt.Printf("error %v\n", err)
			os.Exit(0)
		}
		for {
			select {
			case <-consumer.StopChan:
				return
			case <-sigChan:
				consumer.Stop()
			}
		}
	}
	myApp.Run(os.Args)
}

func CheckFlag(c *cli.Context) {
	if c.String("channel") == "" || c.String("topic") == "" {
		cli.ShowAppHelp(c)
		os.Exit(0)
	}
}

func (th *TailHandler) HandleMessage(m *nsq.Message) error {
	th.messagesShown++
	if err := th.printMessage(m); err != nil {
		fmt.Printf("err %v\n", err)
		return err
	}
	if th.totalMessages > 0 && th.totalMessages < th.messagesShown {
		os.Exit(0)
	}
	return nil
}
<commit_msg>fix tailn dep<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/bitly/go-nsq"
	"github.com/codegangsta/cli"
	. "github.com/gonet2/libs/nsq-logger"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"
)

const (
	LAYOUT = "2006/1/2 15:04:05"
)

var (
	logTemplate = make(map[byte]string)
)

func init() {
	logTemplate[FINEST] = "\033[0;34m%v [FINEST] %v %v %v %v %v \033[0m"
	logTemplate[FINE] = "\033[0;34m%v [FINE] %v %v %v %v %v \033[0m"
	logTemplate[DEBUG] = "\033[1;32m%v [DEBUG] %v %v %v %v %v \033[0m"
	logTemplate[TRACE] = "\033[0;37m%v [TRACE] %v %v %v %v %v \033[0m"
	logTemplate[WARNING] = "\033[7;33m%v [WARNING] %v %v %v %v %v \033[0m"
	logTemplate[INFO] = "\033[0;32m%v [INFO] %v %v %v %v %v  \033[0m"
	logTemplate[ERROR] = "\033[7;31m%v [ERROR] %v %v %v %v %v \033[0m"
	logTemplate[CRITICAL] = "\033[0;31m%v [CRITICAL] %v %v %v %v %v \033[0m"
}

var (
	tailFlag = []cli.Flag{
		cli.StringFlag{
			Name:  "topic,t",
			Value: "LOG",
			Usage: "NSQ topic , defalut is LOG",
		},
		cli.StringFlag{
			Name:  "channel, c",
			Value: "tailn",
			Usage: "NSQ channel, defalut is tailn",
		},
		cli.StringFlag{
			Name:  "number, n",
			Value: "0",
			Usage: "Line to show, defalut no limit",
		},
		cli.StringFlag{
			Name:  "nsqd-tcp-address, a",
			Value: "localhost:4150",
			Usage: "nsqd TCP address",
		},
		cli.StringFlag{
			Name:  "lookupd-http-address, l",
			Value: "localhost:4160",
			Usage: "lookupd HTTP address",
		},
		cli.StringFlag{
			Name:  "timeout, o",
			Value: "5",
			Usage: "Dial timeout, default 5s",
		},
		cli.StringFlag{
			Name:  "type",
			Value: "NSQLOG",
			Usage: "Tail type , default NSQLOG (others print json )",
		},
		cli.StringFlag{
			Name:  "log",
			Value: "false",
			Usage: "whether open inner log",
		},
	}
)

type TailHandler struct {
	totalMessages int
	messagesShown int
	printMessage  func(m *nsq.Message) error
}

func NSQLog(m *nsq.Message) error {
	info := &LogFormat{}
	err := json.Unmarshal(m.Body, &info)
	if err != nil {
		fmt.Printf("err %v\n", err)
		return err
	}
	fmt.Println(fmt.Sprintf(logTemplate[info.Level], info.Time.Format(LAYOUT), info.Prefix, info.Host, info.Msg, info.Caller, info.LineNo))
	return nil
}

func Log(m *nsq.Message) error {
	_, err := os.Stdout.Write(m.Body)
	if err != nil {
		return err
	}
	_, err = os.Stdout.WriteString("\n")
	return err
}

func main() {
	myApp := cli.NewApp()
	myApp.Name = "tailn"
	myApp.Usage = "Tail log from nsq !"
	myApp.Version = "0.0.1"
	myApp.Flags = tailFlag
	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
	myApp.Action = func(c *cli.Context) {
		// TODO CheckFlag
		CheckFlag(c)
		cfg := nsq.NewConfig()
		// dail timeout
		cfg.DialTimeout = time.Duration(c.Int("timeout")) * time.Second
		cfg.UserAgent = fmt.Sprint("go-nsq version:%v", nsq.VERSION)
		consumer, err := nsq.NewConsumer(c.String("topic"), c.String("channel"), cfg)
		if err != nil {
			fmt.Printf("error %v\n", err)
			os.Exit(0)
		}

		f := NSQLog
		if c.String("type") != "NSQLOG" {
			f = Log
		}

		// 屏蔽内部日志
		if !c.Bool("log") {
			consumer.SetLogger(nil, 0)
		}

		consumer.AddHandler(&TailHandler{c.Int("number"), 0, f})
		err = consumer.ConnectToNSQDs(strings.Split(c.String("nsqd-tcp-address"), ","))
		if err != nil {
			fmt.Printf("error %v\n", err)
			os.Exit(0)

		}
		err = consumer.ConnectToNSQLookupds(strings.Split(c.String("lookupd-http-address"), ","))
		if err != nil {
			fmt.Printf("error %v\n", err)
			os.Exit(0)
		}
		for {
			select {
			case <-consumer.StopChan:
				return
			case <-sigChan:
				consumer.Stop()
			}
		}
	}
	myApp.Run(os.Args)
}

func CheckFlag(c *cli.Context) {
	if c.String("channel") == "" || c.String("topic") == "" {
		cli.ShowAppHelp(c)
		os.Exit(0)
	}
}

func (th *TailHandler) HandleMessage(m *nsq.Message) error {
	th.messagesShown++
	if err := th.printMessage(m); err != nil {
		fmt.Printf("err %v\n", err)
		return err
	}
	if th.totalMessages > 0 && th.totalMessages < th.messagesShown {
		os.Exit(0)
	}
	return nil
}
<|endoftext|>


<commit_before>package gotop

import (
	// "fmt"
	"strconv"
	"strings"
	"time"
)

var (
	memInfoFile = "/proc/meminfo"
	memInfoFields = []string {"MemTotal", "MemFree", "Buffers", "Cached", "SwapTotal", "SwapFree"}
)

type MemInfo struct {
	MemTotal  int
	MemFree   int
	Buffers   int
	Cached    int
	SwapTotal int
	SwapFree  int
}

func MemoryUsage(memInfoChan chan MemInfo, interval time.Duration) {
	defer close(memInfoChan)
	memInfoMap := make(map[string]int)
	for {
		memoryData, err := readFile(memInfoFile)
		if err != nil {
			return
		}
		for _, line := range strings.Split(memoryData, "\n") {
			field := fieldName(line)
			if isMemInfoField(field) {
				memInfoMap[field] = fieldValue(line)
			}
		}
		memInfo := getMemInfo(memInfoMap)
		memInfoChan <- memInfo
		time.Sleep(interval)
	}
}

func getMemInfo(data map[string]int) MemInfo {
	var memInfo MemInfo
	for key, value := range data {
		switch(key) {
			case "MemTotal":
				memInfo.MemTotal = value
			case "MemFree":
				memInfo.MemFree = value
			case "Buffers":
				memInfo.Buffers = value
			case "Cached":
				memInfo.Cached = value
			case "SwapTotal":
				memInfo.SwapTotal = value
			case "SwapFree":
				memInfo.SwapFree = value
		}
	}
	return memInfo
}

func fieldName(line string) string {
	index := strings.Index(line, ":")
	if index >= 0 {
		return line[:index]
	}
	return ""
}

func fieldValue(line string) int {
	indexOne := strings.IndexAny(line, "0123456789")
	indexTwo := strings.LastIndex(line, " ")
	val, _ := strconv.Atoi(line[indexOne:indexTwo])
	return val
}

func isMemInfoField(field string) bool {
	for _, val := range memInfoFields {
		if field == val {
			return true
		}
	}
	return false
}<commit_msg>Change logic of finding fieldValue()<commit_after>package gotop

import (
	// "fmt"
	"strconv"
	"strings"
	"time"
)

var (
	memInfoFile   = "/proc/meminfo"
	memInfoFields = []string{"MemTotal", "MemFree", "Buffers", "Cached", "SwapTotal", "SwapFree"}
)

type MemInfo struct {
	MemTotal  int
	MemFree   int
	Buffers   int
	Cached    int
	SwapTotal int
	SwapFree  int
}

func MemoryUsage(memInfoChan chan MemInfo, interval time.Duration) {
	defer close(memInfoChan)
	memInfoMap := make(map[string]int)
	for {
		memoryData, err := readFile(memInfoFile)
		if err != nil {
			return
		}
		for _, line := range strings.Split(memoryData, "\n") {
			field := fieldName(line)
			if isMemInfoField(field) {
				memInfoMap[field] = fieldValue(line)
			}
		}
		memInfo := getMemInfo(memInfoMap)
		memInfoChan <- memInfo
		time.Sleep(interval)
	}
}

func getMemInfo(data map[string]int) MemInfo {
	var memInfo MemInfo
	for key, value := range data {
		switch key {
		case "MemTotal":
			memInfo.MemTotal = value
		case "MemFree":
			memInfo.MemFree = value
		case "Buffers":
			memInfo.Buffers = value
		case "Cached":
			memInfo.Cached = value
		case "SwapTotal":
			memInfo.SwapTotal = value
		case "SwapFree":
			memInfo.SwapFree = value
		}
	}
	return memInfo
}

func fieldName(line string) string {
	index := strings.Index(line, ":")
	if index >= 0 {
		return line[:index]
	}
	return ""
}

func fieldValue(line string) int {
	indexOne := strings.IndexAny(line, "0123456789")
	indexTwo := strings.LastIndexAny(line, "0123456789")
	val, _ := strconv.Atoi(line[indexOne : indexTwo+1])
	return val
}

func isMemInfoField(field string) bool {
	for _, val := range memInfoFields {
		if field == val {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This binary generates fake raw conversions for aggregation experiments on the hierarchical DPF expansion.
// The distribution of the conversion IDs are controlled by a prefix tree structure,
// which also creates DPF parameters that determine the hierarchy how the DPF keys are generated and expanded.
package main

import (
	"context"
	"flag"
	"fmt"

	log "github.com/golang/glog"
	"github.com/google/privacy-sandbox-aggregation-service/pipeline/cryptoio"
	"github.com/google/privacy-sandbox-aggregation-service/pipeline/dpfbrowsersimulator"
)

var (
	sumParamsOutputFile     = flag.String("sum_params_output_file", "", "Output file for the DPF parameters for SUM.")
	prefixesOutPutFile      = flag.String("prefixes_output_file", "", "Output file for the prefixes.")
	rawConversionOutputFile = flag.String("raw_conversion_output_file", "", "Output file for the fake raw conversions.")
	totalCount              = flag.Uint64("total_count", 1000000, "Total count of raw conversions.")

	logN                = flag.Uint64("log_n", 20, "Bits of the aggregation domain size.")
	logElementSizeSum   = flag.Uint64("log_element_size_sum", 6, "Bits of element size for SUM aggregation.")
	logElementSizeCount = flag.Uint64("log_element_size_count", 6, "Bits of element size for COUNT aggregation.")
)

func writeConversions(filename string, conversions []dpfbrowsersimulator.RawConversion) error {
	lines := make([]string, len(conversions))
	for i, conversion := range conversions {
		lines[i] = fmt.Sprintf("%d,%d", conversion.Index, conversion.Value)
	}
	return cryptoio.SaveLines(filename, lines)
}

func main() {
	flag.Parse()

	// Create the prefix tree.
	root := &dpfbrowsersimulator.PrefixNode{Class: "root"}
	// Suppose the first 12 bits represent the campaign ID, and only 2^5 IDs have data.
	for i := 0; i < 1<<5; i++ {
		child := root.AddChildNode("campaignid", 12 /*bitSize*/, uint64(i) /*value*/)
		// Following 5 bits representing geo, and only 2^3 locations have data.
		for j := 0; j < 1<<3; j++ {
			child.AddChildNode("geo", 5 /*bitSize*/, uint64(j) /*value*/)
		}
	}

	prefixes, prefixDomainBits := dpfbrowsersimulator.CalculatePrefixes(root)
	sumParams := dpfbrowsersimulator.CalculateParameters(prefixDomainBits, int32(*logN), 1<<*logElementSizeSum)
	ctx := context.Background()
	if err := cryptoio.SavePrefixes(ctx, *prefixesOutPutFile, prefixes); err != nil {
		log.Exit(err)
	}
	if err := cryptoio.SaveDPFParameters(ctx, *sumParamsOutputFile, sumParams); err != nil {
		log.Exit(err)
	}

	var conversions []dpfbrowsersimulator.RawConversion
	for i := uint64(0); i < *totalCount; i++ {
		index, err := dpfbrowsersimulator.CreateConversionIndex(prefixes.Prefixes[len(prefixes.Prefixes)-1].Prefix, prefixDomainBits[len(prefixDomainBits)-1], *logN, true /*hasPrefix*/)
		if err != nil {
			log.Exit(err)
		}
		conversions = append(conversions, dpfbrowsersimulator.RawConversion{Index: index, Value: 1})
	}
	if err := writeConversions(*rawConversionOutputFile, conversions); err != nil {
		log.Exit(err)
	}
}
<commit_msg>Remove unused flag in dpf_generate_raw_conversion.<commit_after>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This binary generates fake raw conversions for aggregation experiments on the hierarchical DPF expansion.
// The distribution of the conversion IDs are controlled by a prefix tree structure,
// which also creates DPF parameters that determine the hierarchy how the DPF keys are generated and expanded.
package main

import (
	"context"
	"flag"
	"fmt"

	log "github.com/golang/glog"
	"github.com/google/privacy-sandbox-aggregation-service/pipeline/cryptoio"
	"github.com/google/privacy-sandbox-aggregation-service/pipeline/dpfbrowsersimulator"
)

var (
	sumParamsOutputFile     = flag.String("sum_params_output_file", "", "Output file for the DPF parameters for SUM.")
	prefixesOutPutFile      = flag.String("prefixes_output_file", "", "Output file for the prefixes.")
	rawConversionOutputFile = flag.String("raw_conversion_output_file", "", "Output file for the fake raw conversions.")
	totalCount              = flag.Uint64("total_count", 1000000, "Total count of raw conversions.")

	logN                = flag.Uint64("log_n", 20, "Bits of the aggregation domain size.")
	logElementSizeSum   = flag.Uint64("log_element_size_sum", 6, "Bits of element size for SUM aggregation.")
)

func writeConversions(filename string, conversions []dpfbrowsersimulator.RawConversion) error {
	lines := make([]string, len(conversions))
	for i, conversion := range conversions {
		lines[i] = fmt.Sprintf("%d,%d", conversion.Index, conversion.Value)
	}
	return cryptoio.SaveLines(filename, lines)
}

func main() {
	flag.Parse()

	// Create the prefix tree.
	root := &dpfbrowsersimulator.PrefixNode{Class: "root"}
	// Suppose the first 12 bits represent the campaign ID, and only 2^5 IDs have data.
	for i := 0; i < 1<<5; i++ {
		child := root.AddChildNode("campaignid", 12 /*bitSize*/, uint64(i) /*value*/)
		// Following 5 bits representing geo, and only 2^3 locations have data.
		for j := 0; j < 1<<3; j++ {
			child.AddChildNode("geo", 5 /*bitSize*/, uint64(j) /*value*/)
		}
	}

	prefixes, prefixDomainBits := dpfbrowsersimulator.CalculatePrefixes(root)
	sumParams := dpfbrowsersimulator.CalculateParameters(prefixDomainBits, int32(*logN), 1<<*logElementSizeSum)
	ctx := context.Background()
	if err := cryptoio.SavePrefixes(ctx, *prefixesOutPutFile, prefixes); err != nil {
		log.Exit(err)
	}
	if err := cryptoio.SaveDPFParameters(ctx, *sumParamsOutputFile, sumParams); err != nil {
		log.Exit(err)
	}

	var conversions []dpfbrowsersimulator.RawConversion
	for i := uint64(0); i < *totalCount; i++ {
		index, err := dpfbrowsersimulator.CreateConversionIndex(prefixes.Prefixes[len(prefixes.Prefixes)-1].Prefix, prefixDomainBits[len(prefixDomainBits)-1], *logN, true /*hasPrefix*/)
		if err != nil {
			log.Exit(err)
		}
		conversions = append(conversions, dpfbrowsersimulator.RawConversion{Index: index, Value: 1})
	}
	if err := writeConversions(*rawConversionOutputFile, conversions); err != nil {
		log.Exit(err)
	}
}
<|endoftext|>


<commit_before>package sections

import (
	"io/ioutil"
	"path/filepath"
	"sync"

	"github.com/hashicorp/go-multierror"
)

type Collection interface {
	// write
	Persist() error
	Erase() error

	// read
	Sections() []string
	ContentDir() string
}

func OpenCollection(locations locationProvider) (Collection, error) {
	var errs *multierror.Error

	contentDir := locations.ContentDir()
	secs := []section{}
	for _, sectionFilename := range locations.Sections() {
		contentBytes, readErr := ioutil.ReadFile(filepath.Join(contentDir, sectionFilename))
		if readErr != nil {
			errs = multierror.Append(errs, readErr)
			continue // try and read the rest of the sections
		}

		s := &readOnlySection{
			commonSectionInfo: &commonSectionInfo{
				filename:   sectionFilename,
				content:    contentBytes,
				name:       sectionNameForFilename(sectionFilename), // this will be a best effort guess for any non top level sections
				contentDir: contentDir,
			},
		}

		secs = append(secs, s)
	}

	return NewCollection(contentDir, secs...), errs.ErrorOrNil()
}

// NewCollection creates a Collection from a variable number of sections along with the content directory where the sections reside
// NewCollection is a [variadic function](https://gobyexample.com/variadic-functions) in order to facilitate testing since we can't pass a slice of a non exported interface into this function from test
func NewCollection(contentDir string, sections ...section) Collection {
	return &collection{
		contentDir: contentDir,
		sections:   sections,
	}
}

type locationProvider interface {
	// Sections are expected to be section file names
	ContentDir() string
	Sections() []string
}

type collection struct {
	sections   []section
	contentDir string
	locker     sync.RWMutex
}

func (c *collection) Persist() error {
	c.locker.Lock()
	defer c.locker.Unlock()

	err := c.persist()
	if err != nil {
		return nil
	}

	return nil
}

func (c *collection) Erase() error {
	c.locker.Lock()
	defer c.locker.Unlock()

	return c.erase()
}

func (c *collection) Sections() []string {
	c.locker.RLock()
	defer c.locker.RUnlock()

	sectionFilenames := []string{}
	for i := range c.sections {
		sectionFilenames = append(sectionFilenames, c.sections[i].Filename())
	}

	return sectionFilenames
}

func (c *collection) ContentDir() string {
	c.locker.RLock()
	defer c.locker.RUnlock()

	return c.contentDir
}

func (c *collection) persist() error {
	var errs *multierror.Error

	for _, s := range c.sections {
		errs = multierror.Append(errs, s.Persist())
	}

	return errs.ErrorOrNil()
}

func (c *collection) erase() error {
	var errs *multierror.Error

	for _, s := range c.sections {
		err := s.Erase()
		errs = multierror.Append(errs, err)
	}

	return errs.ErrorOrNil()
}
<commit_msg>cleanup: erase partially written section content if an error occurs<commit_after>package sections

import (
	"io/ioutil"
	"path/filepath"
	"sync"

	"github.com/hashicorp/go-multierror"
)

type Collection interface {
	// write
	Persist() error
	Erase() error

	// read
	Sections() []string
	ContentDir() string
}

func OpenCollection(locations locationProvider) (Collection, error) {
	var errs *multierror.Error

	contentDir := locations.ContentDir()
	secs := []section{}
	for _, sectionFilename := range locations.Sections() {
		contentBytes, readErr := ioutil.ReadFile(filepath.Join(contentDir, sectionFilename))
		if readErr != nil {
			errs = multierror.Append(errs, readErr)
			continue // try and read the rest of the sections
		}

		s := &readOnlySection{
			commonSectionInfo: &commonSectionInfo{
				filename:   sectionFilename,
				content:    contentBytes,
				name:       sectionNameForFilename(sectionFilename), // this will be a best effort guess for any non top level sections
				contentDir: contentDir,
			},
		}

		secs = append(secs, s)
	}

	return NewCollection(contentDir, secs...), errs.ErrorOrNil()
}

// NewCollection creates a Collection from a variable number of sections along with the content directory where the sections reside
// NewCollection is a [variadic function](https://gobyexample.com/variadic-functions) in order to facilitate testing since we can't pass a slice of a non exported interface into this function from test
func NewCollection(contentDir string, sections ...section) Collection {
	return &collection{
		contentDir: contentDir,
		sections:   sections,
	}
}

type locationProvider interface {
	// Sections are expected to be section file names
	ContentDir() string
	Sections() []string
}

type collection struct {
	sections   []section
	contentDir string
	locker     sync.RWMutex
}

func (c *collection) Persist() error {
	c.locker.Lock()
	defer c.locker.Unlock()

	var errs *multierror.Error

	errs = multierror.Append(errs, c.persist())
	if errs.ErrorOrNil() != nil {
		errs = multierror.Append(errs, c.erase())
	}

	return errs.ErrorOrNil()
}

func (c *collection) Erase() error {
	c.locker.Lock()
	defer c.locker.Unlock()

	return c.erase()
}

func (c *collection) Sections() []string {
	c.locker.RLock()
	defer c.locker.RUnlock()

	sectionFilenames := []string{}
	for i := range c.sections {
		sectionFilenames = append(sectionFilenames, c.sections[i].Filename())
	}

	return sectionFilenames
}

func (c *collection) ContentDir() string {
	c.locker.RLock()
	defer c.locker.RUnlock()

	return c.contentDir
}

func (c *collection) persist() error {
	var errs *multierror.Error

	for _, s := range c.sections {
		errs = multierror.Append(errs, s.Persist())
	}

	return errs.ErrorOrNil()
}

func (c *collection) erase() error {
	var errs *multierror.Error

	for _, s := range c.sections {
		errs = multierror.Append(errs, s.Erase())
	}

	return errs.ErrorOrNil()
}
<|endoftext|>