<commit_before>//
// Copyright 2014-2021 Cristian Maglie. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//

// +build linux darwin freebsd openbsd

package serial

import (
	"io/ioutil"
	"regexp"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"go.bug.st/serial/unixutils"
	"golang.org/x/sys/unix"
)

type unixPort struct {
	handle int

	readTimeout time.Duration
	closeLock   sync.RWMutex
	closeSignal *unixutils.Pipe
	opened      uint32
}

func (port *unixPort) Close() error {
	if !atomic.CompareAndSwapUint32(&port.opened, 1, 0) {
		return nil
	}

	// Close port
	port.releaseExclusiveAccess()
	if err := unix.Close(port.handle); err != nil {
		return err
	}

	if port.closeSignal != nil {
		// Send close signal to all pending reads (if any)
		port.closeSignal.Write([]byte{0})

		// Wait for all readers to complete
		port.closeLock.Lock()
		defer port.closeLock.Unlock()

		// Close signaling pipe
		if err := port.closeSignal.Close(); err != nil {
			return err
		}
	}
	return nil
}

func (port *unixPort) Read(p []byte) (int, error) {
	port.closeLock.RLock()
	defer port.closeLock.RUnlock()
	if atomic.LoadUint32(&port.opened) != 1 {
		return 0, &PortError{code: PortClosed}
	}

	var deadline time.Time
	if port.readTimeout != NoTimeout {
		deadline = time.Now().Add(port.readTimeout)
	}

	fds := unixutils.NewFDSet(port.handle, port.closeSignal.ReadFD())
	for {
		timeout := time.Duration(-1)
		if port.readTimeout != NoTimeout {
			timeout = deadline.Sub(time.Now())
		}
		res, err := unixutils.Select(fds, nil, fds, timeout)
		if err == unix.EINTR {
			continue
		}
		if err != nil {
			return 0, err
		}
		if res.IsReadable(port.closeSignal.ReadFD()) {
			return 0, &PortError{code: PortClosed}
		}
		if !res.IsReadable(port.handle) {
			// Timeout happened
			return 0, nil
		}
		n, err := unix.Read(port.handle, p)
		if err == unix.EINTR {
			continue
		}
		// Linux: when the port is disconnected during a read operation
		// the port is left in a "readable with zero-length-data" state.
		// https://stackoverflow.com/a/34945814/1655275
		if n == 0 && err == nil {
			return 0, &PortError{code: PortClosed}
		}
		if n < 0 { // Do not return -1 unix errors
			n = 0
		}
		return n, err
	}
}

func (port *unixPort) Write(p []byte) (n int, err error) {
	n, err = unix.Write(port.handle, p)
	if n < 0 { // Do not return -1 unix errors
		n = 0
	}
	return
}

func (port *unixPort) ResetInputBuffer() error {
	return unix.IoctlSetInt(port.handle, ioctlTcflsh, unix.TCIFLUSH)
}

func (port *unixPort) ResetOutputBuffer() error {
	return unix.IoctlSetInt(port.handle, ioctlTcflsh, unix.TCOFLUSH)
}

func (port *unixPort) SetMode(mode *Mode) error {
	settings, err := port.getTermSettings()
	if err != nil {
		return err
	}
	if err := setTermSettingsParity(mode.Parity, settings); err != nil {
		return err
	}
	if err := setTermSettingsDataBits(mode.DataBits, settings); err != nil {
		return err
	}
	if err := setTermSettingsStopBits(mode.StopBits, settings); err != nil {
		return err
	}
	requireSpecialBaudrate := false
	if err, special := setTermSettingsBaudrate(mode.BaudRate, settings); err != nil {
		return err
	} else if special {
		requireSpecialBaudrate = true
	}
	if err := port.setTermSettings(settings); err != nil {
		return err
	}
	if requireSpecialBaudrate {
		// MacOSX require this one to be the last operation otherwise an
		// 'Invalid serial port' error is produced.
		if err := port.setSpecialBaudrate(uint32(mode.BaudRate)); err != nil {
			return err
		}
	}
	return nil
}

func (port *unixPort) SetDTR(dtr bool) error {
	status, err := port.getModemBitsStatus()
	if err != nil {
		return err
	}
	if dtr {
		status |= unix.TIOCM_DTR
	} else {
		status &^= unix.TIOCM_DTR
	}
	return port.setModemBitsStatus(status)
}

func (port *unixPort) SetRTS(rts bool) error {
	status, err := port.getModemBitsStatus()
	if err != nil {
		return err
	}
	if rts {
		status |= unix.TIOCM_RTS
	} else {
		status &^= unix.TIOCM_RTS
	}
	return port.setModemBitsStatus(status)
}

func (port *unixPort) SetReadTimeout(timeout time.Duration) error {
	if timeout < 0 && timeout != NoTimeout {
		return &PortError{code: InvalidTimeoutValue}
	}
	port.readTimeout = timeout
	return nil
}

func (port *unixPort) GetModemStatusBits() (*ModemStatusBits, error) {
	status, err := port.getModemBitsStatus()
	if err != nil {
		return nil, err
	}
	return &ModemStatusBits{
		CTS: (status & unix.TIOCM_CTS) != 0,
		DCD: (status & unix.TIOCM_CD) != 0,
		DSR: (status & unix.TIOCM_DSR) != 0,
		RI:  (status & unix.TIOCM_RI) != 0,
	}, nil
}

func nativeOpen(portName string, mode *Mode) (*unixPort, error) {
	h, err := unix.Open(portName, unix.O_RDWR|unix.O_NOCTTY|unix.O_NDELAY, 0)
	if err != nil {
		switch err {
		case unix.EBUSY:
			return nil, &PortError{code: PortBusy}
		case unix.EACCES:
			return nil, &PortError{code: PermissionDenied}
		}
		return nil, err
	}
	port := &unixPort{
		handle:      h,
		opened:      1,
		readTimeout: NoTimeout,
	}

	// Setup serial port
	settings, err := port.getTermSettings()
	if err != nil {
		port.Close()
		return nil, &PortError{code: InvalidSerialPort}
	}

	// Set raw mode
	setRawMode(settings)

	// Explicitly disable RTS/CTS flow control
	setTermSettingsCtsRts(false, settings)

	if port.setTermSettings(settings) != nil {
		port.Close()
		return nil, &PortError{code: InvalidSerialPort}
	}

	// MacOSX require that this operation is the last one otherwise an
	// 'Invalid serial port' error is returned... don't know why...
	if port.SetMode(mode) != nil {
		port.Close()
		return nil, &PortError{code: InvalidSerialPort}
	}

	unix.SetNonblock(h, false)

	port.acquireExclusiveAccess()

	// This pipe is used as a signal to cancel blocking Read
	pipe := &unixutils.Pipe{}
	if err := pipe.Open(); err != nil {
		port.Close()
		return nil, &PortError{code: InvalidSerialPort, causedBy: err}
	}
	port.closeSignal = pipe

	return port, nil
}

func nativeGetPortsList() ([]string, error) {
	files, err := ioutil.ReadDir(devFolder)
	if err != nil {
		return nil, err
	}

	ports := make([]string, 0, len(files))
	for _, f := range files {
		// Skip folders
		if f.IsDir() {
			continue
		}

		// Keep only devices with the correct name
		match, err := regexp.MatchString(regexFilter, f.Name())
		if err != nil {
			return nil, err
		}
		if !match {
			continue
		}

		portName := devFolder + "/" + f.Name()

		// Check if serial port is real or is a placeholder serial port "ttySxx"
		if strings.HasPrefix(f.Name(), "ttyS") {
			port, err := nativeOpen(portName, &Mode{})
			if err != nil {
				serr, ok := err.(*PortError)
				if ok && serr.Code() == InvalidSerialPort {
					continue
				}
			} else {
				port.Close()
			}
		}

		// Save serial port in the resulting list
		ports = append(ports, portName)
	}

	return ports, nil
}

// termios manipulation functions

func setTermSettingsParity(parity Parity, settings *unix.Termios) error {
	switch parity {
	case NoParity:
		settings.Cflag &^= unix.PARENB
		settings.Cflag &^= unix.PARODD
		settings.Cflag &^= tcCMSPAR
		settings.Iflag &^= unix.INPCK
	case OddParity:
		settings.Cflag |= unix.PARENB
		settings.Cflag |= unix.PARODD
		settings.Cflag &^= tcCMSPAR
		settings.Iflag |= unix.INPCK
	case EvenParity:
		settings.Cflag |= unix.PARENB
		settings.Cflag &^= unix.PARODD
		settings.Cflag &^= tcCMSPAR
		settings.Iflag |= unix.INPCK
	case MarkParity:
		if tcCMSPAR == 0 {
			return &PortError{code: InvalidParity}
		}
		settings.Cflag |= unix.PARENB
		settings.Cflag |= unix.PARODD
		settings.Cflag |= tcCMSPAR
		settings.Iflag |= unix.INPCK
	case SpaceParity:
		if tcCMSPAR == 0 {
			return &PortError{code: InvalidParity}
		}
		settings.Cflag |= unix.PARENB
		settings.Cflag &^= unix.PARODD
		settings.Cflag |= tcCMSPAR
		settings.Iflag |= unix.INPCK
	default:
		return &PortError{code: InvalidParity}
	}
	return nil
}

func setTermSettingsDataBits(bits int, settings *unix.Termios) error {
	databits, ok := databitsMap[bits]
	if !ok {
		return &PortError{code: InvalidDataBits}
	}
	// Remove previous databits setting
	settings.Cflag &^= unix.CSIZE
	// Set requested databits
	settings.Cflag |= databits
	return nil
}

func setTermSettingsStopBits(bits StopBits, settings *unix.Termios) error {
	switch bits {
	case OneStopBit:
		settings.Cflag &^= unix.CSTOPB
	case OnePointFiveStopBits:
		return &PortError{code: InvalidStopBits}
	case TwoStopBits:
		settings.Cflag |= unix.CSTOPB
	default:
		return &PortError{code: InvalidStopBits}
	}
	return nil
}

func setTermSettingsCtsRts(enable bool, settings *unix.Termios) {
	if enable {
		settings.Cflag |= tcCRTSCTS
	} else {
		settings.Cflag &^= tcCRTSCTS
	}
}

func setRawMode(settings *unix.Termios) {
	// Set local mode
	settings.Cflag |= unix.CREAD
	settings.Cflag |= unix.CLOCAL

	// Set raw mode
	settings.Lflag &^= unix.ICANON
	settings.Lflag &^= unix.ECHO
	settings.Lflag &^= unix.ECHOE
	settings.Lflag &^= unix.ECHOK
	settings.Lflag &^= unix.ECHONL
	settings.Lflag &^= unix.ECHOCTL
	settings.Lflag &^= unix.ECHOPRT
	settings.Lflag &^= unix.ECHOKE
	settings.Lflag &^= unix.ISIG
	settings.Lflag &^= unix.IEXTEN

	settings.Iflag &^= unix.IXON
	settings.Iflag &^= unix.IXOFF
	settings.Iflag &^= unix.IXANY
	settings.Iflag &^= unix.INPCK
	settings.Iflag &^= unix.IGNPAR
	settings.Iflag &^= unix.PARMRK
	settings.Iflag &^= unix.ISTRIP
	settings.Iflag &^= unix.IGNBRK
	settings.Iflag &^= unix.BRKINT
	settings.Iflag &^= unix.INLCR
	settings.Iflag &^= unix.IGNCR
	settings.Iflag &^= unix.ICRNL
	settings.Iflag &^= tcIUCLC

	settings.Oflag &^= unix.OPOST

	// Block reads until at least one char is available (no timeout)
	settings.Cc[unix.VMIN] = 1
	settings.Cc[unix.VTIME] = 0
}

// native syscall wrapper functions

func (port *unixPort) getTermSettings() (*unix.Termios, error) {
	return unix.IoctlGetTermios(port.handle, ioctlTcgetattr)
}

func (port *unixPort) setTermSettings(settings *unix.Termios) error {
	return unix.IoctlSetTermios(port.handle, ioctlTcsetattr, settings)
}

func (port *unixPort) getModemBitsStatus() (int, error) {
	return unix.IoctlGetInt(port.handle, unix.TIOCMGET)
}

func (port *unixPort) setModemBitsStatus(status int) error {
	return unix.IoctlSetPointerInt(port.handle, unix.TIOCMSET, status)
}

func (port *unixPort) acquireExclusiveAccess() error {
	return unix.IoctlSetInt(port.handle, unix.TIOCEXCL, 0)
}

func (port *unixPort) releaseExclusiveAccess() error {
	return unix.IoctlSetInt(port.handle, unix.TIOCNXCL, 0)
}
<commit_msg>Skip enumerating ports that are not accessible for any reason<commit_after>//
// Copyright 2014-2021 Cristian Maglie. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//

// +build linux darwin freebsd openbsd

package serial

import (
	"io/ioutil"
	"regexp"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"go.bug.st/serial/unixutils"
	"golang.org/x/sys/unix"
)

type unixPort struct {
	handle int

	readTimeout time.Duration
	closeLock   sync.RWMutex
	closeSignal *unixutils.Pipe
	opened      uint32
}

func (port *unixPort) Close() error {
	if !atomic.CompareAndSwapUint32(&port.opened, 1, 0) {
		return nil
	}

	// Close port
	port.releaseExclusiveAccess()
	if err := unix.Close(port.handle); err != nil {
		return err
	}

	if port.closeSignal != nil {
		// Send close signal to all pending reads (if any)
		port.closeSignal.Write([]byte{0})

		// Wait for all readers to complete
		port.closeLock.Lock()
		defer port.closeLock.Unlock()

		// Close signaling pipe
		if err := port.closeSignal.Close(); err != nil {
			return err
		}
	}
	return nil
}

func (port *unixPort) Read(p []byte) (int, error) {
	port.closeLock.RLock()
	defer port.closeLock.RUnlock()
	if atomic.LoadUint32(&port.opened) != 1 {
		return 0, &PortError{code: PortClosed}
	}

	var deadline time.Time
	if port.readTimeout != NoTimeout {
		deadline = time.Now().Add(port.readTimeout)
	}

	fds := unixutils.NewFDSet(port.handle, port.closeSignal.ReadFD())
	for {
		timeout := time.Duration(-1)
		if port.readTimeout != NoTimeout {
			timeout = deadline.Sub(time.Now())
		}
		res, err := unixutils.Select(fds, nil, fds, timeout)
		if err == unix.EINTR {
			continue
		}
		if err != nil {
			return 0, err
		}
		if res.IsReadable(port.closeSignal.ReadFD()) {
			return 0, &PortError{code: PortClosed}
		}
		if !res.IsReadable(port.handle) {
			// Timeout happened
			return 0, nil
		}
		n, err := unix.Read(port.handle, p)
		if err == unix.EINTR {
			continue
		}
		// Linux: when the port is disconnected during a read operation
		// the port is left in a "readable with zero-length-data" state.
		// https://stackoverflow.com/a/34945814/1655275
		if n == 0 && err == nil {
			return 0, &PortError{code: PortClosed}
		}
		if n < 0 { // Do not return -1 unix errors
			n = 0
		}
		return n, err
	}
}

func (port *unixPort) Write(p []byte) (n int, err error) {
	n, err = unix.Write(port.handle, p)
	if n < 0 { // Do not return -1 unix errors
		n = 0
	}
	return
}

func (port *unixPort) ResetInputBuffer() error {
	return unix.IoctlSetInt(port.handle, ioctlTcflsh, unix.TCIFLUSH)
}

func (port *unixPort) ResetOutputBuffer() error {
	return unix.IoctlSetInt(port.handle, ioctlTcflsh, unix.TCOFLUSH)
}

func (port *unixPort) SetMode(mode *Mode) error {
	settings, err := port.getTermSettings()
	if err != nil {
		return err
	}
	if err := setTermSettingsParity(mode.Parity, settings); err != nil {
		return err
	}
	if err := setTermSettingsDataBits(mode.DataBits, settings); err != nil {
		return err
	}
	if err := setTermSettingsStopBits(mode.StopBits, settings); err != nil {
		return err
	}
	requireSpecialBaudrate := false
	if err, special := setTermSettingsBaudrate(mode.BaudRate, settings); err != nil {
		return err
	} else if special {
		requireSpecialBaudrate = true
	}
	if err := port.setTermSettings(settings); err != nil {
		return err
	}
	if requireSpecialBaudrate {
		// MacOSX require this one to be the last operation otherwise an
		// 'Invalid serial port' error is produced.
		if err := port.setSpecialBaudrate(uint32(mode.BaudRate)); err != nil {
			return err
		}
	}
	return nil
}

func (port *unixPort) SetDTR(dtr bool) error {
	status, err := port.getModemBitsStatus()
	if err != nil {
		return err
	}
	if dtr {
		status |= unix.TIOCM_DTR
	} else {
		status &^= unix.TIOCM_DTR
	}
	return port.setModemBitsStatus(status)
}

func (port *unixPort) SetRTS(rts bool) error {
	status, err := port.getModemBitsStatus()
	if err != nil {
		return err
	}
	if rts {
		status |= unix.TIOCM_RTS
	} else {
		status &^= unix.TIOCM_RTS
	}
	return port.setModemBitsStatus(status)
}

func (port *unixPort) SetReadTimeout(timeout time.Duration) error {
	if timeout < 0 && timeout != NoTimeout {
		return &PortError{code: InvalidTimeoutValue}
	}
	port.readTimeout = timeout
	return nil
}

func (port *unixPort) GetModemStatusBits() (*ModemStatusBits, error) {
	status, err := port.getModemBitsStatus()
	if err != nil {
		return nil, err
	}
	return &ModemStatusBits{
		CTS: (status & unix.TIOCM_CTS) != 0,
		DCD: (status & unix.TIOCM_CD) != 0,
		DSR: (status & unix.TIOCM_DSR) != 0,
		RI:  (status & unix.TIOCM_RI) != 0,
	}, nil
}

func nativeOpen(portName string, mode *Mode) (*unixPort, error) {
	h, err := unix.Open(portName, unix.O_RDWR|unix.O_NOCTTY|unix.O_NDELAY, 0)
	if err != nil {
		switch err {
		case unix.EBUSY:
			return nil, &PortError{code: PortBusy}
		case unix.EACCES:
			return nil, &PortError{code: PermissionDenied}
		}
		return nil, err
	}
	port := &unixPort{
		handle:      h,
		opened:      1,
		readTimeout: NoTimeout,
	}

	// Setup serial port
	settings, err := port.getTermSettings()
	if err != nil {
		port.Close()
		return nil, &PortError{code: InvalidSerialPort}
	}

	// Set raw mode
	setRawMode(settings)

	// Explicitly disable RTS/CTS flow control
	setTermSettingsCtsRts(false, settings)

	if port.setTermSettings(settings) != nil {
		port.Close()
		return nil, &PortError{code: InvalidSerialPort}
	}

	// MacOSX require that this operation is the last one otherwise an
	// 'Invalid serial port' error is returned... don't know why...
	if port.SetMode(mode) != nil {
		port.Close()
		return nil, &PortError{code: InvalidSerialPort}
	}

	unix.SetNonblock(h, false)

	port.acquireExclusiveAccess()

	// This pipe is used as a signal to cancel blocking Read
	pipe := &unixutils.Pipe{}
	if err := pipe.Open(); err != nil {
		port.Close()
		return nil, &PortError{code: InvalidSerialPort, causedBy: err}
	}
	port.closeSignal = pipe

	return port, nil
}

func nativeGetPortsList() ([]string, error) {
	files, err := ioutil.ReadDir(devFolder)
	if err != nil {
		return nil, err
	}

	ports := make([]string, 0, len(files))
	for _, f := range files {
		// Skip folders
		if f.IsDir() {
			continue
		}

		// Keep only devices with the correct name
		match, err := regexp.MatchString(regexFilter, f.Name())
		if err != nil {
			return nil, err
		}
		if !match {
			continue
		}

		portName := devFolder + "/" + f.Name()

		// Check if serial port is real or is a placeholder serial port "ttySxx"
		if strings.HasPrefix(f.Name(), "ttyS") {
			port, err := nativeOpen(portName, &Mode{})
			if err != nil {
				continue
			} else {
				port.Close()
			}
		}

		// Save serial port in the resulting list
		ports = append(ports, portName)
	}

	return ports, nil
}

// termios manipulation functions

func setTermSettingsParity(parity Parity, settings *unix.Termios) error {
	switch parity {
	case NoParity:
		settings.Cflag &^= unix.PARENB
		settings.Cflag &^= unix.PARODD
		settings.Cflag &^= tcCMSPAR
		settings.Iflag &^= unix.INPCK
	case OddParity:
		settings.Cflag |= unix.PARENB
		settings.Cflag |= unix.PARODD
		settings.Cflag &^= tcCMSPAR
		settings.Iflag |= unix.INPCK
	case EvenParity:
		settings.Cflag |= unix.PARENB
		settings.Cflag &^= unix.PARODD
		settings.Cflag &^= tcCMSPAR
		settings.Iflag |= unix.INPCK
	case MarkParity:
		if tcCMSPAR == 0 {
			return &PortError{code: InvalidParity}
		}
		settings.Cflag |= unix.PARENB
		settings.Cflag |= unix.PARODD
		settings.Cflag |= tcCMSPAR
		settings.Iflag |= unix.INPCK
	case SpaceParity:
		if tcCMSPAR == 0 {
			return &PortError{code: InvalidParity}
		}
		settings.Cflag |= unix.PARENB
		settings.Cflag &^= unix.PARODD
		settings.Cflag |= tcCMSPAR
		settings.Iflag |= unix.INPCK
	default:
		return &PortError{code: InvalidParity}
	}
	return nil
}

func setTermSettingsDataBits(bits int, settings *unix.Termios) error {
	databits, ok := databitsMap[bits]
	if !ok {
		return &PortError{code: InvalidDataBits}
	}
	// Remove previous databits setting
	settings.Cflag &^= unix.CSIZE
	// Set requested databits
	settings.Cflag |= databits
	return nil
}

func setTermSettingsStopBits(bits StopBits, settings *unix.Termios) error {
	switch bits {
	case OneStopBit:
		settings.Cflag &^= unix.CSTOPB
	case OnePointFiveStopBits:
		return &PortError{code: InvalidStopBits}
	case TwoStopBits:
		settings.Cflag |= unix.CSTOPB
	default:
		return &PortError{code: InvalidStopBits}
	}
	return nil
}

func setTermSettingsCtsRts(enable bool, settings *unix.Termios) {
	if enable {
		settings.Cflag |= tcCRTSCTS
	} else {
		settings.Cflag &^= tcCRTSCTS
	}
}

func setRawMode(settings *unix.Termios) {
	// Set local mode
	settings.Cflag |= unix.CREAD
	settings.Cflag |= unix.CLOCAL

	// Set raw mode
	settings.Lflag &^= unix.ICANON
	settings.Lflag &^= unix.ECHO
	settings.Lflag &^= unix.ECHOE
	settings.Lflag &^= unix.ECHOK
	settings.Lflag &^= unix.ECHONL
	settings.Lflag &^= unix.ECHOCTL
	settings.Lflag &^= unix.ECHOPRT
	settings.Lflag &^= unix.ECHOKE
	settings.Lflag &^= unix.ISIG
	settings.Lflag &^= unix.IEXTEN

	settings.Iflag &^= unix.IXON
	settings.Iflag &^= unix.IXOFF
	settings.Iflag &^= unix.IXANY
	settings.Iflag &^= unix.INPCK
	settings.Iflag &^= unix.IGNPAR
	settings.Iflag &^= unix.PARMRK
	settings.Iflag &^= unix.ISTRIP
	settings.Iflag &^= unix.IGNBRK
	settings.Iflag &^= unix.BRKINT
	settings.Iflag &^= unix.INLCR
	settings.Iflag &^= unix.IGNCR
	settings.Iflag &^= unix.ICRNL
	settings.Iflag &^= tcIUCLC

	settings.Oflag &^= unix.OPOST

	// Block reads until at least one char is available (no timeout)
	settings.Cc[unix.VMIN] = 1
	settings.Cc[unix.VTIME] = 0
}

// native syscall wrapper functions

func (port *unixPort) getTermSettings() (*unix.Termios, error) {
	return unix.IoctlGetTermios(port.handle, ioctlTcgetattr)
}

func (port *unixPort) setTermSettings(settings *unix.Termios) error {
	return unix.IoctlSetTermios(port.handle, ioctlTcsetattr, settings)
}

func (port *unixPort) getModemBitsStatus() (int, error) {
	return unix.IoctlGetInt(port.handle, unix.TIOCMGET)
}

func (port *unixPort) setModemBitsStatus(status int) error {
	return unix.IoctlSetPointerInt(port.handle, unix.TIOCMSET, status)
}

func (port *unixPort) acquireExclusiveAccess() error {
	return unix.IoctlSetInt(port.handle, unix.TIOCEXCL, 0)
}

func (port *unixPort) releaseExclusiveAccess() error {
	return unix.IoctlSetInt(port.handle, unix.TIOCNXCL, 0)
}
<|endoftext|>


<commit_before>// +build darwin,amd64 linux,amd64

// Package fswalker provides the checker.Comparer interface using FSWalker
// walker and reporter.
package fswalker

import (
	"context"
	"encoding/json"
	"io"
	"log"
	"path/filepath"
	"strings"

	// nolint:staticcheck
	"github.com/golang/protobuf/proto"
	"github.com/google/fswalker"
	fspb "github.com/google/fswalker/proto/fswalker"
	"github.com/pkg/errors"

	"github.com/kopia/kopia/tests/robustness/checker"
	"github.com/kopia/kopia/tests/tools/fswalker/reporter"
	"github.com/kopia/kopia/tests/tools/fswalker/walker"
)

var _ checker.Comparer = &WalkCompare{}

// WalkCompare is a checker.Comparer that utilizes the fswalker
// libraries to perform the data consistency check.
type WalkCompare struct {
	GlobalFilterFuncs []func(string, fswalker.ActionData) bool
}

// NewWalkCompare instantiates a new WalkCompare and returns its pointer.
func NewWalkCompare() *WalkCompare {
	return &WalkCompare{
		GlobalFilterFuncs: []func(string, fswalker.ActionData) bool{
			filterFileTimeDiffs,
			isRootDirectoryRename,
			dirSizeMightBeOffByBlockSizeMultiple,
		},
	}
}

// Gather meets the checker.Comparer interface. It performs a fswalker Walk
// and returns the resulting Walk as a protobuf Marshaled buffer.
func (chk *WalkCompare) Gather(ctx context.Context, path string) ([]byte, error) {
	walkData, err := walker.WalkPathHash(ctx, path)
	if err != nil {
		return nil, errors.Wrap(err, "walk with hashing error during gather phase")
	}

	err = rerootWalkDataPaths(walkData, path)
	if err != nil {
		return nil, errors.Wrap(err, "reroot walk paths error during gather phase")
	}

	// Store the walk data along with the snapshot ID
	b, err := proto.Marshal(walkData)
	if err != nil {
		return nil, errors.Wrap(err, "walk data proto marshal error")
	}

	return b, nil
}

// Compare meets the checker.Comparer interface. It performs a fswalker Walk
// on the provided file path, unmarshals the comparison data as a fswalker Walk,
// and generates a fswalker report comparing the two Walks. If there are any differences
// an error is returned, and the full report will be written to the provided writer
// as JSON.
func (chk *WalkCompare) Compare(ctx context.Context, path string, data []byte, reportOut io.Writer) error {
	beforeWalk := &fspb.Walk{}

	err := proto.Unmarshal(data, beforeWalk)
	if err != nil {
		return errors.Wrap(err, "walk data unmarshal error")
	}

	afterWalk, err := walker.WalkPathHash(ctx, path)
	if err != nil {
		return errors.Wrap(err, "walk with hashing error during compare phase")
	}

	err = rerootWalkDataPaths(afterWalk, path)
	if err != nil {
		return errors.Wrap(err, "reroot walk paths error during compare phase")
	}

	report, err := reporter.Report(ctx, &fspb.ReportConfig{}, beforeWalk, afterWalk)
	if err != nil {
		return errors.Wrap(err, "report error")
	}

	chk.filterReportDiffs(report)

	err = validateReport(report)
	if err != nil && reportOut != nil {
		printReportSummary(report, reportOut)

		b, marshalErr := json.MarshalIndent(report, "", "   ")
		if marshalErr != nil {
			_, reportErr := reportOut.Write([]byte(marshalErr.Error()))
			if reportErr != nil {
				return errors.Wrapf(reportErr, "error while writing out marshal error (%v)", marshalErr.Error())
			}

			return errors.Wrap(marshalErr, "error JSON marshaling report")
		}

		if _, wrErr := reportOut.Write(b); wrErr != nil {
			return errors.Wrap(err, "error writing report to output")
		}

		return errors.Wrap(err, "validation error")
	}

	return nil
}

func printReportSummary(report *fswalker.Report, reportOut io.Writer) {
	rptr := &fswalker.Reporter{}
	rptr.PrintDiffSummary(reportOut, report)
	rptr.PrintReportSummary(reportOut, report)
	rptr.PrintRuleSummary(reportOut, report)
}

func (chk *WalkCompare) filterReportDiffs(report *fswalker.Report) {
	var newModList []fswalker.ActionData

	for _, mod := range report.Modified {
		var newDiffItemList []string

		diffItems := strings.Split(mod.Diff, "\n")

	DiffItemLoop:
		for _, diffItem := range diffItems {
			for _, filterFunc := range chk.GlobalFilterFuncs {
				if filterFunc(diffItem, mod) {
					continue DiffItemLoop
				}
			}

			newDiffItemList = append(newDiffItemList, diffItem)
		}

		if len(newDiffItemList) > 0 {
			log.Println("Not Filtering", newDiffItemList)
			mod.Diff = strings.Join(newDiffItemList, "\n")
			newModList = append(newModList, mod)
		}
	}

	report.Modified = newModList
}

func isRootDirectoryRename(diffItem string, mod fswalker.ActionData) bool {
	if !strings.HasPrefix(diffItem, "name: ") {
		return false
	}

	// The mod.Before.Path may be given from fswalker Report as "./", so
	// clean it before compare
	return mod.Before.Info.IsDir && filepath.Clean(mod.Before.Path) == "."
}

func dirSizeMightBeOffByBlockSizeMultiple(str string, mod fswalker.ActionData) bool {
	if !mod.Before.Info.IsDir {
		return false
	}

	if !strings.Contains(str, "size: ") {
		return false
	}

	const blockSize = 4096

	return (mod.Before.Stat.Size-mod.After.Stat.Size)%blockSize == 0
}

func filterFileTimeDiffs(str string, mod fswalker.ActionData) bool {
	return strings.Contains(str, "ctime:") || strings.Contains(str, "atime:") || strings.Contains(str, "mtime:")
}

func validateReport(report *fswalker.Report) error {
	if len(report.Modified) > 0 {
		return errors.New("files were modified")
	}

	if len(report.Added) > 0 {
		return errors.New("files were added")
	}

	if len(report.Deleted) > 0 {
		return errors.New("files were deleted")
	}

	if len(report.Errors) > 0 {
		return errors.New("errors were thrown in the walk")
	}

	return nil
}

func rerootWalkDataPaths(walk *fspb.Walk, newRoot string) error {
	for _, f := range walk.File {
		var err error

		f.Path, err = filepath.Rel(newRoot, f.Path)
		if err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>[Robustness] Fswalker should ignore host name (#531)<commit_after>// +build darwin,amd64 linux,amd64

// Package fswalker provides the checker.Comparer interface using FSWalker
// walker and reporter.
package fswalker

import (
	"context"
	"encoding/json"
	"io"
	"log"
	"path/filepath"
	"strings"

	// nolint:staticcheck
	"github.com/golang/protobuf/proto"
	"github.com/google/fswalker"
	fspb "github.com/google/fswalker/proto/fswalker"
	"github.com/pkg/errors"

	"github.com/kopia/kopia/tests/robustness/checker"
	"github.com/kopia/kopia/tests/tools/fswalker/reporter"
	"github.com/kopia/kopia/tests/tools/fswalker/walker"
)

var _ checker.Comparer = &WalkCompare{}

// WalkCompare is a checker.Comparer that utilizes the fswalker
// libraries to perform the data consistency check.
type WalkCompare struct {
	GlobalFilterFuncs []func(string, fswalker.ActionData) bool
}

// NewWalkCompare instantiates a new WalkCompare and returns its pointer.
func NewWalkCompare() *WalkCompare {
	return &WalkCompare{
		GlobalFilterFuncs: []func(string, fswalker.ActionData) bool{
			filterFileTimeDiffs,
			isRootDirectoryRename,
			dirSizeMightBeOffByBlockSizeMultiple,
		},
	}
}

// Gather meets the checker.Comparer interface. It performs a fswalker Walk
// and returns the resulting Walk as a protobuf Marshaled buffer.
func (chk *WalkCompare) Gather(ctx context.Context, path string) ([]byte, error) {
	walkData, err := performWalk(ctx, path)
	if err != nil {
		return nil, errors.Wrap(err, "walk with hashing error during gather phase")
	}

	// Store the walk data along with the snapshot ID
	b, err := proto.Marshal(walkData)

	return b, errors.Wrap(err, "walk data proto marshal error")
}

func performWalk(ctx context.Context, path string) (*fspb.Walk, error) {
	walkData, err := walker.WalkPathHash(ctx, path)
	if err != nil {
		return nil, errors.Wrap(err, "walk with hashing error during gather phase")
	}

	err = processWalk(walkData, path)

	return walkData, errors.Wrap(err, "error during gather phase")
}

func processWalk(walk *fspb.Walk, path string) error {
	// Hostname is cleared to allow reporting on walks done across
	// different hosts.
	clearHostname(walk)

	// Walk paths are rerooted to be relative to the root directory
	// so child entries of /source/rootDir/... won't be different than
	// child entries of /target/rootDir/...
	err := rerootWalkDataPaths(walk, path)

	return errors.Wrap(err, "reroot walk paths error")
}

func clearHostname(walk *fspb.Walk) {
	walk.Hostname = ""
}

// Compare meets the checker.Comparer interface. It performs a fswalker Walk
// on the provided file path, unmarshals the comparison data as a fswalker Walk,
// and generates a fswalker report comparing the two Walks. If there are any differences
// an error is returned, and the full report will be written to the provided writer
// as JSON.
func (chk *WalkCompare) Compare(ctx context.Context, path string, data []byte, reportOut io.Writer) error {
	beforeWalk := &fspb.Walk{}

	err := proto.Unmarshal(data, beforeWalk)
	if err != nil {
		return errors.Wrap(err, "walk data unmarshal error")
	}

	clearHostname(beforeWalk)

	afterWalk, err := performWalk(ctx, path)
	if err != nil {
		return errors.Wrap(err, "walk with hashing error during compare phase")
	}

	report, err := reporter.Report(ctx, &fspb.ReportConfig{}, beforeWalk, afterWalk)
	if err != nil {
		return errors.Wrap(err, "report error")
	}

	chk.filterReportDiffs(report)

	err = validateReport(report)
	if err != nil && reportOut != nil {
		printReportSummary(report, reportOut)

		// Don't spew these fields. They can be enormous and
		// are probably redundant
		report.WalkBefore = nil
		report.WalkAfter = nil

		b, marshalErr := json.MarshalIndent(report, "", "   ")
		if marshalErr != nil {
			_, reportErr := reportOut.Write([]byte(marshalErr.Error()))
			if reportErr != nil {
				return errors.Wrapf(reportErr, "error while writing out marshal error (%v)", marshalErr.Error())
			}

			return errors.Wrap(marshalErr, "error JSON marshaling report")
		}

		if _, wrErr := reportOut.Write(b); wrErr != nil {
			return errors.Wrap(err, "error writing report to output")
		}

		return errors.Wrap(err, "validation error")
	}

	return err
}

func printReportSummary(report *fswalker.Report, reportOut io.Writer) {
	rptr := &fswalker.Reporter{}
	rptr.PrintDiffSummary(reportOut, report)
	rptr.PrintReportSummary(reportOut, report)
	rptr.PrintRuleSummary(reportOut, report)
}

func (chk *WalkCompare) filterReportDiffs(report *fswalker.Report) {
	var newModList []fswalker.ActionData

	for _, mod := range report.Modified {
		var newDiffItemList []string

		diffItems := strings.Split(mod.Diff, "\n")

	DiffItemLoop:
		for _, diffItem := range diffItems {
			for _, filterFunc := range chk.GlobalFilterFuncs {
				if filterFunc(diffItem, mod) {
					continue DiffItemLoop
				}
			}

			newDiffItemList = append(newDiffItemList, diffItem)
		}

		if len(newDiffItemList) > 0 {
			log.Println("Not Filtering", newDiffItemList)
			mod.Diff = strings.Join(newDiffItemList, "\n")
			newModList = append(newModList, mod)
		}
	}

	report.Modified = newModList
}

func isRootDirectoryRename(diffItem string, mod fswalker.ActionData) bool {
	if !strings.HasPrefix(diffItem, "name: ") {
		return false
	}

	// The mod.Before.Path may be given from fswalker Report as "./", so
	// clean it before compare
	return mod.Before.Info.IsDir && filepath.Clean(mod.Before.Path) == "."
}

func dirSizeMightBeOffByBlockSizeMultiple(str string, mod fswalker.ActionData) bool {
	if !mod.Before.Info.IsDir {
		return false
	}

	if !strings.Contains(str, "size: ") {
		return false
	}

	const blockSize = 4096

	return (mod.Before.Stat.Size-mod.After.Stat.Size)%blockSize == 0
}

func filterFileTimeDiffs(str string, mod fswalker.ActionData) bool {
	return strings.Contains(str, "ctime:") || strings.Contains(str, "atime:") || strings.Contains(str, "mtime:")
}

func validateReport(report *fswalker.Report) error {
	if len(report.Modified) > 0 {
		return errors.New("files were modified")
	}

	if len(report.Added) > 0 {
		return errors.New("files were added")
	}

	if len(report.Deleted) > 0 {
		return errors.New("files were deleted")
	}

	if len(report.Errors) > 0 {
		return errors.New("errors were thrown in the walk")
	}

	return nil
}

func rerootWalkDataPaths(walk *fspb.Walk, newRoot string) error {
	for _, f := range walk.File {
		var err error

		f.Path, err = filepath.Rel(newRoot, f.Path)
		if err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/phayes/hookserve/hookserve"
	"os"
	"time"
)

func main() {

	app := cli.NewApp()
	app.Name = "hookserve"
	app.Usage = "A small little application that listens for commit / push webhook events from github and runs a specified command"

	app.Flags = []cli.Flag{
		cli.IntFlag{
			Name:  "port, p",
			Value: 80,
			Usage: "port on which to listen for github webhooks",
		},
		cli.StringFlag{
			Name:  "command, c",
			Value: "",
			Usage: "command to run when a webhook is received. The command will be called like so: <command> owner repo branch commit. If no command is specified, the webhook info will be printed to stdout.",
		},
	}

	app.Action = func(c *cli.Context) {
		server := hookserve.NewServer()
		server.GoListenAndServe()

		for {
			select {
			case commit := <-server.Events:
				fmt.Println(commit.Owner + " " + commit.Repo + " " + commit.Branch + " " + commit.Commit)
			default:
				time.Sleep(100)
			}
		}
	}

	app.Run(os.Args)
}
<commit_msg>Fixing path and a few other things<commit_after>package main

import (
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/phayes/hookserve/hookserve"
	"os"
	"time"
)

func main() {

	app := cli.NewApp()
	app.Name = "hookserve"
	app.Usage = "A small little application that listens for commit / push webhook events from github and runs a specified command"

	app.Flags = []cli.Flag{
		cli.IntFlag{
			Name:  "port, p",
			Value: 80,
			Usage: "port on which to listen for github webhooks",
		},
		cli.StringFlag{
			Name:  "command, c",
			Value: "",
			Usage: "command to run when a webhook is received. The command will be called like so: <command> owner repo branch commit. If no command is specified, the webhook info will be printed to stdout.",
		},
	}

	app.Action = func(c *cli.Context) {
		server := hookserve.NewServer()
		server.Port = c.Int("port")
		server.GoListenAndServe()

		for {
			select {
			case commit := <-server.Events:
				fmt.Println(commit.Owner + " " + commit.Repo + " " + commit.Branch + " " + commit.Commit)
			default:
				time.Sleep(100)
			}
		}
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package core

import (
	"net/http"
)

func SessionCookie(value string, valid bool) *http.Cookie {
	c := &http.Cookie{
		Name:  "shield7",
		Value: value,
		Path:  "/",
	}
	if !valid {
		c.MaxAge = 0
	}
	return c
}
<commit_msg>Remove unused core/session.go code<commit_after><|endoftext|>


<commit_before>package pilosa

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"sync"

	"github.com/gogo/protobuf/proto"
	"github.com/pilosa/pilosa/internal"
)

const (
	// FrameSuffixRank is the suffix used for rank-based frames.
	FrameSuffixRank = ".n"
)

// Frame represents a container for fragments.
type Frame struct {
	mu          sync.Mutex
	path        string
	db          string
	name        string
	timeQuantum TimeQuantum

	// Fragments by slice.
	fragments map[uint64]*Fragment

	// Bitmap attribute storage and cache
	bitmapAttrStore *AttrStore

	stats StatsClient
}

// NewFrame returns a new instance of frame.
func NewFrame(path, db, name string) *Frame {
	return &Frame{
		path: path,
		db:   db,
		name: name,

		fragments:       make(map[uint64]*Fragment),
		bitmapAttrStore: NewAttrStore(filepath.Join(path, "data")),

		stats: NopStatsClient,
	}
}

// Name returns the name the frame was initialized with.
func (f *Frame) Name() string { return f.name }

// DB returns the database name the frame was initialized with.
func (f *Frame) DB() string { return f.db }

// Path returns the path the frame was initialized with.
func (f *Frame) Path() string { return f.path }

// BitmapAttrStore returns the attribute storage.
func (f *Frame) BitmapAttrStore() *AttrStore { return f.bitmapAttrStore }

// MaxSlice returns the max slice in the frame.
func (f *Frame) MaxSlice() uint64 {
	f.mu.Lock()
	defer f.mu.Unlock()

	var max uint64
	for slice := range f.fragments {
		if slice > max {
			max = slice
		}
	}
	return max
}

// Open opens and initializes the frame.
func (f *Frame) Open() error {
	if err := func() error {
		// Ensure the frame's path exists.
		if err := os.MkdirAll(f.path, 0777); err != nil {
			return err
		}

		if err := f.loadMeta(); err != nil {
			return err
		}

		if err := f.openFragments(); err != nil {
			return err
		}

		if err := f.bitmapAttrStore.Open(); err != nil {
			return err
		}

		return nil
	}(); err != nil {
		f.Close()
		return err
	}

	return nil
}

// openFragments opens and initializes the fragments inside the frame.
func (f *Frame) openFragments() error {
	file, err := os.Open(f.path)
	if err != nil {
		return err
	}
	defer file.Close()

	fis, err := file.Readdir(0)
	if err != nil {
		return err
	}

	for _, fi := range fis {
		if fi.IsDir() {
			continue
		}

		// Parse filename into integer.
		slice, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64)
		if err != nil {
			continue
		}

		frag := f.newFragment(f.FragmentPath(slice), slice)
		if err := frag.Open(); err != nil {
			return fmt.Errorf("open fragment: slice=%s, err=%s", frag.Slice(), err)
		}
		frag.BitmapAttrStore = f.bitmapAttrStore
		f.fragments[frag.Slice()] = frag

		f.stats.Count("maxSlice", 1)
	}

	return nil
}

// loadMeta reads meta data for the frame, if any.
func (f *Frame) loadMeta() error {
	var pb internal.Frame

	// Read data from meta file.
	buf, err := ioutil.ReadFile(filepath.Join(f.path, "meta"))
	if os.IsNotExist(err) {
		f.timeQuantum = ""
		return nil
	} else if err != nil {
		return err
	} else {
		if err := proto.Unmarshal(buf, &pb); err != nil {
			return err
		}
	}

	// Copy metadata fields.
	f.timeQuantum = TimeQuantum(pb.TimeQuantum)

	return nil
}

// saveMeta writes meta data for the frame.
func (f *Frame) saveMeta() error {
	// Marshal metadata.
	buf, err := proto.Marshal(&internal.Frame{TimeQuantum: string(f.timeQuantum)})
	if err != nil {
		return err
	}

	// Write to meta file.
	if err := ioutil.WriteFile(filepath.Join(f.path, "meta"), buf, 0666); err != nil {
		return err
	}

	return nil
}

// Close closes the frame and its fragments.
func (f *Frame) Close() error {
	f.mu.Lock()
	defer f.mu.Unlock()

	// Close the attribute store.
	if f.bitmapAttrStore != nil {
		_ = f.bitmapAttrStore.Close()
	}

	// Close all fragments.
	for _, frag := range f.fragments {
		_ = frag.Close()
	}
	f.fragments = make(map[uint64]*Fragment)

	return nil
}

// TimeQuantum returns the time quantum for the frame.
func (f *Frame) TimeQuantum() TimeQuantum {
	f.mu.Lock()
	defer f.mu.Unlock()
	return f.timeQuantum
}

// SetTimeQuantum sets the time quantum for the frame.
func (f *Frame) SetTimeQuantum(q TimeQuantum) error {
	f.mu.Lock()
	defer f.mu.Unlock()

	// Validate input.
	if !q.Valid() {
		return ErrInvalidTimeQuantum
	}

	// Update value on frame.
	f.timeQuantum = q

	// Perist meta data to disk.
	if err := f.saveMeta(); err != nil {
		return err
	}

	return nil
}

// FragmentPath returns the path to a fragment in the frame.
func (f *Frame) FragmentPath(slice uint64) string {
	return filepath.Join(f.path, strconv.FormatUint(slice, 10))
}

// Fragment returns a fragment in the frame by slice.
func (f *Frame) Fragment(slice uint64) *Fragment {
	f.mu.Lock()
	defer f.mu.Unlock()
	return f.fragment(slice)
}

func (f *Frame) fragment(slice uint64) *Fragment { return f.fragments[slice] }

// Fragments returns a list of all fragments in the frame.
func (f *Frame) Fragments() []*Fragment {
	f.mu.Lock()
	defer f.mu.Unlock()

	other := make([]*Fragment, 0, len(f.fragments))
	for _, fragment := range f.fragments {
		other = append(other, fragment)
	}
	return other
}

// CreateFragmentIfNotExists returns a fragment in the frame by slice.
func (f *Frame) CreateFragmentIfNotExists(slice uint64) (*Fragment, error) {
	f.mu.Lock()
	defer f.mu.Unlock()
	return f.createFragmentIfNotExists(slice)
}

func (f *Frame) createFragmentIfNotExists(slice uint64) (*Fragment, error) {
	// Find fragment in cache first.
	if frag := f.fragments[slice]; frag != nil {
		return frag, nil
	}

	// Initialize and open fragment.
	frag := f.newFragment(f.FragmentPath(slice), slice)
	if err := frag.Open(); err != nil {
		return nil, err
	}
	frag.BitmapAttrStore = f.bitmapAttrStore

	// Save to lookup.
	f.fragments[slice] = frag

	f.stats.Count("maxSlice", 1)

	return frag, nil
}

func (f *Frame) newFragment(path string, slice uint64) *Fragment {
	frag := NewFragment(path, f.db, f.name, slice)
	frag.stats = f.stats.WithTags(fmt.Sprintf("slice:%d", slice))
	return frag
}

// SetBit sets a bit within the frame.
func (f *Frame) SetBit(bitmapID, profileID uint64) (changed bool, err error) {
	slice := bitmapID / SliceWidth
	frag, err := f.CreateFragmentIfNotExists(slice)
	if err != nil {
		return changed, err
	}
	return frag.SetBit(bitmapID, profileID)
}

type frameSlice []*Frame

func (p frameSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p frameSlice) Len() int           { return len(p) }
func (p frameSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }

// FrameInfo represents schema information for a frame.
type FrameInfo struct {
	Name string `json:"name"`
}

type frameInfoSlice []*FrameInfo

func (p frameInfoSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p frameInfoSlice) Len() int           { return len(p) }
func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
<commit_msg>Fix frame SetBit slice calculation.<commit_after>package pilosa

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"sync"

	"github.com/gogo/protobuf/proto"
	"github.com/pilosa/pilosa/internal"
)

const (
	// FrameSuffixRank is the suffix used for rank-based frames.
	FrameSuffixRank = ".n"
)

// Frame represents a container for fragments.
type Frame struct {
	mu          sync.Mutex
	path        string
	db          string
	name        string
	timeQuantum TimeQuantum

	// Fragments by slice.
	fragments map[uint64]*Fragment

	// Bitmap attribute storage and cache
	bitmapAttrStore *AttrStore

	stats StatsClient
}

// NewFrame returns a new instance of frame.
func NewFrame(path, db, name string) *Frame {
	return &Frame{
		path: path,
		db:   db,
		name: name,

		fragments:       make(map[uint64]*Fragment),
		bitmapAttrStore: NewAttrStore(filepath.Join(path, "data")),

		stats: NopStatsClient,
	}
}

// Name returns the name the frame was initialized with.
func (f *Frame) Name() string { return f.name }

// DB returns the database name the frame was initialized with.
func (f *Frame) DB() string { return f.db }

// Path returns the path the frame was initialized with.
func (f *Frame) Path() string { return f.path }

// BitmapAttrStore returns the attribute storage.
func (f *Frame) BitmapAttrStore() *AttrStore { return f.bitmapAttrStore }

// MaxSlice returns the max slice in the frame.
func (f *Frame) MaxSlice() uint64 {
	f.mu.Lock()
	defer f.mu.Unlock()

	var max uint64
	for slice := range f.fragments {
		if slice > max {
			max = slice
		}
	}
	return max
}

// Open opens and initializes the frame.
func (f *Frame) Open() error {
	if err := func() error {
		// Ensure the frame's path exists.
		if err := os.MkdirAll(f.path, 0777); err != nil {
			return err
		}

		if err := f.loadMeta(); err != nil {
			return err
		}

		if err := f.openFragments(); err != nil {
			return err
		}

		if err := f.bitmapAttrStore.Open(); err != nil {
			return err
		}

		return nil
	}(); err != nil {
		f.Close()
		return err
	}

	return nil
}

// openFragments opens and initializes the fragments inside the frame.
func (f *Frame) openFragments() error {
	file, err := os.Open(f.path)
	if err != nil {
		return err
	}
	defer file.Close()

	fis, err := file.Readdir(0)
	if err != nil {
		return err
	}

	for _, fi := range fis {
		if fi.IsDir() {
			continue
		}

		// Parse filename into integer.
		slice, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64)
		if err != nil {
			continue
		}

		frag := f.newFragment(f.FragmentPath(slice), slice)
		if err := frag.Open(); err != nil {
			return fmt.Errorf("open fragment: slice=%s, err=%s", frag.Slice(), err)
		}
		frag.BitmapAttrStore = f.bitmapAttrStore
		f.fragments[frag.Slice()] = frag

		f.stats.Count("maxSlice", 1)
	}

	return nil
}

// loadMeta reads meta data for the frame, if any.
func (f *Frame) loadMeta() error {
	var pb internal.Frame

	// Read data from meta file.
	buf, err := ioutil.ReadFile(filepath.Join(f.path, "meta"))
	if os.IsNotExist(err) {
		f.timeQuantum = ""
		return nil
	} else if err != nil {
		return err
	} else {
		if err := proto.Unmarshal(buf, &pb); err != nil {
			return err
		}
	}

	// Copy metadata fields.
	f.timeQuantum = TimeQuantum(pb.TimeQuantum)

	return nil
}

// saveMeta writes meta data for the frame.
func (f *Frame) saveMeta() error {
	// Marshal metadata.
	buf, err := proto.Marshal(&internal.Frame{TimeQuantum: string(f.timeQuantum)})
	if err != nil {
		return err
	}

	// Write to meta file.
	if err := ioutil.WriteFile(filepath.Join(f.path, "meta"), buf, 0666); err != nil {
		return err
	}

	return nil
}

// Close closes the frame and its fragments.
func (f *Frame) Close() error {
	f.mu.Lock()
	defer f.mu.Unlock()

	// Close the attribute store.
	if f.bitmapAttrStore != nil {
		_ = f.bitmapAttrStore.Close()
	}

	// Close all fragments.
	for _, frag := range f.fragments {
		_ = frag.Close()
	}
	f.fragments = make(map[uint64]*Fragment)

	return nil
}

// TimeQuantum returns the time quantum for the frame.
func (f *Frame) TimeQuantum() TimeQuantum {
	f.mu.Lock()
	defer f.mu.Unlock()
	return f.timeQuantum
}

// SetTimeQuantum sets the time quantum for the frame.
func (f *Frame) SetTimeQuantum(q TimeQuantum) error {
	f.mu.Lock()
	defer f.mu.Unlock()

	// Validate input.
	if !q.Valid() {
		return ErrInvalidTimeQuantum
	}

	// Update value on frame.
	f.timeQuantum = q

	// Perist meta data to disk.
	if err := f.saveMeta(); err != nil {
		return err
	}

	return nil
}

// FragmentPath returns the path to a fragment in the frame.
func (f *Frame) FragmentPath(slice uint64) string {
	return filepath.Join(f.path, strconv.FormatUint(slice, 10))
}

// Fragment returns a fragment in the frame by slice.
func (f *Frame) Fragment(slice uint64) *Fragment {
	f.mu.Lock()
	defer f.mu.Unlock()
	return f.fragment(slice)
}

func (f *Frame) fragment(slice uint64) *Fragment { return f.fragments[slice] }

// Fragments returns a list of all fragments in the frame.
func (f *Frame) Fragments() []*Fragment {
	f.mu.Lock()
	defer f.mu.Unlock()

	other := make([]*Fragment, 0, len(f.fragments))
	for _, fragment := range f.fragments {
		other = append(other, fragment)
	}
	return other
}

// CreateFragmentIfNotExists returns a fragment in the frame by slice.
func (f *Frame) CreateFragmentIfNotExists(slice uint64) (*Fragment, error) {
	f.mu.Lock()
	defer f.mu.Unlock()
	return f.createFragmentIfNotExists(slice)
}

func (f *Frame) createFragmentIfNotExists(slice uint64) (*Fragment, error) {
	// Find fragment in cache first.
	if frag := f.fragments[slice]; frag != nil {
		return frag, nil
	}

	// Initialize and open fragment.
	frag := f.newFragment(f.FragmentPath(slice), slice)
	if err := frag.Open(); err != nil {
		return nil, err
	}
	frag.BitmapAttrStore = f.bitmapAttrStore

	// Save to lookup.
	f.fragments[slice] = frag

	f.stats.Count("maxSlice", 1)

	return frag, nil
}

func (f *Frame) newFragment(path string, slice uint64) *Fragment {
	frag := NewFragment(path, f.db, f.name, slice)
	frag.stats = f.stats.WithTags(fmt.Sprintf("slice:%d", slice))
	return frag
}

// SetBit sets a bit within the frame.
func (f *Frame) SetBit(bitmapID, profileID uint64) (changed bool, err error) {
	slice := profileID / SliceWidth
	frag, err := f.CreateFragmentIfNotExists(slice)
	if err != nil {
		return changed, err
	}
	return frag.SetBit(bitmapID, profileID)
}

type frameSlice []*Frame

func (p frameSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p frameSlice) Len() int           { return len(p) }
func (p frameSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }

// FrameInfo represents schema information for a frame.
type FrameInfo struct {
	Name string `json:"name"`
}

type frameInfoSlice []*FrameInfo

func (p frameInfoSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p frameInfoSlice) Len() int           { return len(p) }
func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
<|endoftext|>


<commit_before>package fs

import (
	"encoding/binary"
	"sync"

	"bazil.org/bazil/cas/chunks"
	"bazil.org/bazil/fs/inodes"
	"bazil.org/bazil/fs/wire"
	"bazil.org/bazil/tokens"
	"bazil.org/fuse"
	"bazil.org/fuse/fs"
	"github.com/boltdb/bolt"
)

type Volume struct {
	mu         sync.Mutex
	db         *bolt.DB
	volID      VolumeID
	chunkStore chunks.Store
}

var _ = fs.FS(&Volume{})

var bucketVolume = []byte(tokens.BucketVolume)
var bucketDir = []byte("dir")
var bucketInode = []byte("inode")
var bucketSnap = []byte("snap")

func (v *Volume) bucket(tx *bolt.Tx) *bolt.Bucket {
	b := tx.Bucket(bucketVolume)
	b = b.Bucket(v.volID.Bytes())
	return b
}

// Open returns a FUSE filesystem instance serving content from the
// given database and chunk store. The result can be passed to
// bazil.org/fuse/fs#Serve to start serving file access requests from
// the kernel.
func Open(db *bolt.DB, chunkStore chunks.Store) (*Volume, error) {
	fs := &Volume{}
	copy(fs.volID[:], "defaultvol") // TODO
	fs.db = db
	fs.chunkStore = chunkStore
	return fs, nil
}

// TODO this needs to go away
func Init(tx *bolt.Tx) error {
	b := tx.Bucket(bucketVolume)
	var volID VolumeID
	copy(volID[:], "defaultvol") // TODO
	var err error
	b, err = b.CreateBucketIfNotExists(volID.Bytes())
	if err != nil {
		return err
	}
	if _, err := b.CreateBucketIfNotExists(bucketDir); err != nil {
		return err
	}
	if _, err := b.CreateBucketIfNotExists(bucketInode); err != nil {
		return err
	}
	if _, err := b.CreateBucketIfNotExists(bucketSnap); err != nil {
		return err
	}
	return nil
}

func (v *Volume) Root() (fs.Node, fuse.Error) {
	d := &dir{
		inode:  tokens.InodeRoot,
		parent: nil,
		fs:     v,
		active: make(map[string]node),
	}
	return d, nil
}

func (*Volume) GenerateInode(parent uint64, name string) uint64 {
	return inodes.Dynamic(parent, name)
}

var _ = fs.FSInodeGenerator(&Volume{})

func pathToKey(parentInode uint64, name string) []byte {
	buf := make([]byte, 8+len(name))
	binary.BigEndian.PutUint64(buf, parentInode)
	copy(buf[8:], name)
	return buf
}

type node interface {
	fs.Node

	marshal() (*wire.Dirent, error)
	getName() string
	setName(name string)
}
<commit_msg>fs: Allow readahead, let reads happen async<commit_after>package fs

import (
	"encoding/binary"
	"sync"

	"bazil.org/bazil/cas/chunks"
	"bazil.org/bazil/fs/inodes"
	"bazil.org/bazil/fs/wire"
	"bazil.org/bazil/tokens"
	"bazil.org/fuse"
	"bazil.org/fuse/fs"
	"github.com/boltdb/bolt"
)

type Volume struct {
	mu         sync.Mutex
	db         *bolt.DB
	volID      VolumeID
	chunkStore chunks.Store
}

var _ = fs.FS(&Volume{})
var _ = fs.FSIniter(&Volume{})

var bucketVolume = []byte(tokens.BucketVolume)
var bucketDir = []byte("dir")
var bucketInode = []byte("inode")
var bucketSnap = []byte("snap")

func (v *Volume) bucket(tx *bolt.Tx) *bolt.Bucket {
	b := tx.Bucket(bucketVolume)
	b = b.Bucket(v.volID.Bytes())
	return b
}

// Open returns a FUSE filesystem instance serving content from the
// given database and chunk store. The result can be passed to
// bazil.org/fuse/fs#Serve to start serving file access requests from
// the kernel.
func Open(db *bolt.DB, chunkStore chunks.Store) (*Volume, error) {
	fs := &Volume{}
	copy(fs.volID[:], "defaultvol") // TODO
	fs.db = db
	fs.chunkStore = chunkStore
	return fs, nil
}

// TODO this needs to go away
func Init(tx *bolt.Tx) error {
	b := tx.Bucket(bucketVolume)
	var volID VolumeID
	copy(volID[:], "defaultvol") // TODO
	var err error
	b, err = b.CreateBucketIfNotExists(volID.Bytes())
	if err != nil {
		return err
	}
	if _, err := b.CreateBucketIfNotExists(bucketDir); err != nil {
		return err
	}
	if _, err := b.CreateBucketIfNotExists(bucketInode); err != nil {
		return err
	}
	if _, err := b.CreateBucketIfNotExists(bucketSnap); err != nil {
		return err
	}
	return nil
}

func (f *Volume) Init(req *fuse.InitRequest, resp *fuse.InitResponse, intr fs.Intr) fuse.Error {
	resp.MaxReadahead = 32 * 1024 * 1024
	resp.Flags |= fuse.InitAsyncRead
	return nil
}

func (v *Volume) Root() (fs.Node, fuse.Error) {
	d := &dir{
		inode:  tokens.InodeRoot,
		parent: nil,
		fs:     v,
		active: make(map[string]node),
	}
	return d, nil
}

func (*Volume) GenerateInode(parent uint64, name string) uint64 {
	return inodes.Dynamic(parent, name)
}

var _ = fs.FSInodeGenerator(&Volume{})

func pathToKey(parentInode uint64, name string) []byte {
	buf := make([]byte, 8+len(name))
	binary.BigEndian.PutUint64(buf, parentInode)
	copy(buf[8:], name)
	return buf
}

type node interface {
	fs.Node

	marshal() (*wire.Dirent, error)
	getName() string
	setName(name string)
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fs

import (
	"fmt"
	"reflect"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/fs/inode"
	"github.com/jacobsa/gcsfuse/timeutil"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

type fileSystem struct {
	fuseutil.NotImplementedFileSystem

	/////////////////////////
	// Dependencies
	/////////////////////////

	clock  timeutil.Clock
	bucket gcs.Bucket

	/////////////////////////
	// Mutable state
	/////////////////////////

	// When acquiring this lock, the caller must hold no inode or dir handle
	// locks.
	mu syncutil.InvariantMutex

	// The user and group owning everything in the file system.
	//
	// GUARDED_BY(Mu)
	uid uint32
	gid uint32

	// The collection of live inodes, keyed by inode ID. No ID less than
	// fuse.RootInodeID is ever used.
	//
	// TODO(jacobsa): Implement ForgetInode support in the fuse package, then
	// implement the method here and clean up these maps.
	//
	// INVARIANT: All values are of type *inode.DirInode or *inode.FileInode
	// INVARIANT: For all keys k, k >= fuse.RootInodeID
	// INVARIANT: For all keys k, inodes[k].ID() == k
	// INVARIANT: inodes[fuse.RootInodeID] is of type *inode.DirInode
	//
	// GUARDED_BY(mu)
	inodes map[fuse.InodeID]inode.Inode

	// The next inode ID to hand out. We assume that this will never overflow,
	// since even if we were handing out inode IDs at 4 GHz, it would still take
	// over a century to do so.
	//
	// INVARIANT: For all keys k in inodes, k < nextInodeID
	//
	// GUARDED_BY(mu)
	nextInodeID fuse.InodeID

	// An index of all directory inodes by Name().
	//
	// INVARIANT: For each key k, isDirName(k)
	// INVARIANT: For each key k, dirIndex[k].Name() == k
	// INVARIANT: The values are all and only the values of the inodes map of
	// type *inode.DirInode.
	//
	// GUARDED_BY(mu)
	dirIndex map[string]*inode.DirInode

	// An index of all file inodes by (Name(), SourceGeneration()) pairs.
	//
	// INVARIANT: For each key k, !isDirName(k)
	// INVARIANT: For each key k, fileIndex[k].Name() == k.name
	// INVARIANT: For each key k, fileIndex[k].SourceGeneration() == k.gen
	// INVARIANT: The values are all and only the values of the inodes map of
	// type *inode.FileInode.
	//
	// GUARDED_BY(mu)
	fileIndex map[nameAndGen]*inode.FileInode

	// The collection of live handles, keyed by handle ID.
	//
	// INVARIANT: All values are of type *dirHandle
	//
	// GUARDED_BY(mu)
	handles map[fuse.HandleID]interface{}

	// The next handle ID to hand out. We assume that this will never overflow.
	//
	// INVARIANT: For all keys k in handles, k < nextHandleID
	//
	// GUARDED_BY(mu)
	nextHandleID fuse.HandleID
}

type nameAndGen struct {
	name string
	gen  int64
}

// Create a fuse file system whose root directory is the root of the supplied
// bucket. The supplied clock will be used for cache invalidation, modification
// times, etc.
func NewFileSystem(
	clock timeutil.Clock,
	bucket gcs.Bucket) (ffs fuse.FileSystem, err error) {
	// Set up the basic struct.
	fs := &fileSystem{
		clock:       clock,
		bucket:      bucket,
		inodes:      make(map[fuse.InodeID]inode.Inode),
		nextInodeID: fuse.RootInodeID + 1,
		dirIndex:    make(map[string]*inode.DirInode),
		fileIndex:   make(map[nameAndGen]*inode.FileInode),
		handles:     make(map[fuse.HandleID]interface{}),
	}

	// Set up the root inode.
	root := inode.NewDirInode(bucket, fuse.RootInodeID, "")
	fs.inodes[fuse.RootInodeID] = root
	fs.dirIndex[""] = root

	// Set up invariant checking.
	fs.mu = syncutil.NewInvariantMutex(fs.checkInvariants)

	ffs = fs
	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func isDirName(name string) bool {
	return name == "" || name[len(name)-1] == '/'
}

func (fs *fileSystem) checkInvariants() {
	// Check inode keys.
	for id, _ := range fs.inodes {
		if id < fuse.RootInodeID || id >= fs.nextInodeID {
			panic(fmt.Sprintf("Illegal inode ID: %v", id))
		}
	}

	// Check the root inode.
	_ = fs.inodes[fuse.RootInodeID].(*inode.DirInode)

	// Check each inode, and the indexes over them. Keep a count of each type
	// seen.
	dirsSeen := 0
	filesSeen := 0
	for id, in := range fs.inodes {
		// Check the ID.
		if in.ID() != id {
			panic(fmt.Sprintf("ID mismatch: %v vs. %v", in.ID(), id))
		}

		// Check type-specific stuff.
		switch typed := in.(type) {
		case *inode.DirInode:
			dirsSeen++

			if !isDirName(typed.Name()) {
				panic(fmt.Sprintf("Unexpected directory name: %s", typed.Name()))
			}

			if fs.dirIndex[typed.Name()] != typed {
				panic(fmt.Sprintf("dirIndex mismatch: %s", typed.Name()))
			}

		case *inode.FileInode:
			filesSeen++

			if isDirName(typed.Name()) {
				panic(fmt.Sprintf("Unexpected file name: %s", typed.Name()))
			}

			nandg := nameAndGen{typed.Name(), typed.SourceGeneration()}
			if fs.fileIndex[nandg] != typed {
				panic(
					fmt.Sprintf(
						"fileIndex mismatch: %s, %v",
						typed.Name(),
						typed.SourceGeneration()))
			}

		default:
			panic(fmt.Sprintf("Unexpected inode type: %v", reflect.TypeOf(in)))
		}
	}

	// Make sure that the indexes are exhaustive.
	if len(fs.dirIndex) != dirsSeen {
		panic(
			fmt.Sprintf(
				"dirIndex length mismatch: %v vs. %v",
				len(fs.dirIndex),
				dirsSeen))
	}

	if len(fs.fileIndex) != filesSeen {
		panic(
			fmt.Sprintf(
				"fileIndex length mismatch: %v vs. %v",
				len(fs.fileIndex),
				dirsSeen))
	}

	// Check handles.
	for id, h := range fs.handles {
		if id >= fs.nextHandleID {
			panic(fmt.Sprintf("Illegal handle ID: %v", id))
		}

		_ = h.(*dirHandle)
	}
}

// Find the given inode and return it with its lock held for reading. Panic if
// it doesn't exist or is the wrong type.
//
// SHARED_LOCKS_REQUIRED(fs.mu)
// SHARED_LOCK_FUNCTION(in.mu)
func (fs *fileSystem) getDirForReadingOrDie(
	id fuse.InodeID) (in *inode.DirInode) {
	in = fs.inodes[id].(*inode.DirInode)
	in.Mu.RLock()
	return
}

// Get attributes for the given directory inode, fixing up ownership information.
//
// SHARED_LOCKS_REQUIRED(fs.mu)
// SHARED_LOCK_FUNCTION(d.mu)
func (fs *fileSystem) getAttributes(
	ctx context.Context,
	d *inode.DirInode) (attrs fuse.InodeAttributes, err error) {
	attrs, err = d.Attributes(ctx)
	if err != nil {
		return
	}

	attrs.Uid = fs.uid
	attrs.Gid = fs.gid

	return
}

// Find a directory inode for the given object record. Create one if there
// isn't already one available.
//
// EXCLUSIVE_LOCKS_REQUIRED(fs.mu)
func (fs *fileSystem) lookUpOrCreateDirInode(
	ctx context.Context,
	o *storage.Object) (in *inode.DirInode, err error) {
	// Do we already have an inode for this name?
	if in = fs.dirIndex[o.Name]; in != nil {
		return
	}

	// Mint an ID.
	id := fs.nextInodeID
	fs.nextInodeID++

	// Create and index an inode.
	in = inode.NewDirInode(fs.bucket, id, o.Name)
	fs.inodes[id] = in
	fs.dirIndex[in.Name()] = in

	return
}

// Find a file inode for the given object record. Create one if there isn't
// already one available.
//
// EXCLUSIVE_LOCKS_REQUIRED(fs.mu)
func (fs *fileSystem) lookUpOrCreateFileInode(
	ctx context.Context,
	o *storage.Object) (in *inode.FileInode, err error) {
	nandg := nameAndGen{
		name: o.Name,
		gen:  o.Generation,
	}

	// Do we already have an inode for this (name, generation) pair?
	if in = fs.fileIndex[nandg]; in != nil {
		return
	}

	// Mint an ID.
	id := fs.nextInodeID
	fs.nextInodeID++

	// Create and index an inode.
	in = inode.NewFileInode(fs.bucket, id, o)
	fs.inodes[id] = in
	fs.fileIndex[nandg] = in

	return
}

////////////////////////////////////////////////////////////////////////
// fuse.FileSystem methods
////////////////////////////////////////////////////////////////////////

func (fs *fileSystem) Init(
	ctx context.Context,
	req *fuse.InitRequest) (resp *fuse.InitResponse, err error) {
	resp = &fuse.InitResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Store the mounting user's info for later.
	fs.uid = req.Header.Uid
	fs.gid = req.Header.Gid

	return
}

func (fs *fileSystem) LookUpInode(
	ctx context.Context,
	req *fuse.LookUpInodeRequest) (resp *fuse.LookUpInodeResponse, err error) {
	resp = &fuse.LookUpInodeResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Find the parent directory in question.
	parent := fs.inodes[req.Parent].(*inode.DirInode)

	// Find a record for the child with the given name.
	o, err := parent.LookUpChild(ctx, req.Name)
	if err != nil {
		return
	}

	// Is the child a directory or a file?
	var in inode.Inode
	if isDirName(o.Name) {
		in, err = fs.lookUpOrCreateDirInode(ctx, o)
	} else {
		in, err = fs.lookUpOrCreateFileInode(ctx, o)
	}

	if err != nil {
		return
	}

	// Fill out the response.
	resp.Entry.Child = in.ID()
	if resp.Entry.Attributes, err = in.Attributes(ctx); err != nil {
		return
	}

	return
}

func (fs *fileSystem) GetInodeAttributes(
	ctx context.Context,
	req *fuse.GetInodeAttributesRequest) (
	resp *fuse.GetInodeAttributesResponse, err error) {
	resp = &fuse.GetInodeAttributesResponse{}

	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// Find the inode.
	in := fs.inodes[req.Inode]

	// Grab its attributes.
	switch typed := in.(type) {
	case *inode.DirInode:
		resp.Attributes, err = fs.getAttributes(ctx, typed)
		if err != nil {
			err = fmt.Errorf("DirInode.Attributes: %v", err)
			return
		}

	default:
		panic(
			fmt.Sprintf(
				"Unknown inode type for ID %v: %v",
				req.Inode,
				reflect.TypeOf(in)))
	}

	return
}

func (fs *fileSystem) OpenDir(
	ctx context.Context,
	req *fuse.OpenDirRequest) (resp *fuse.OpenDirResponse, err error) {
	resp = &fuse.OpenDirResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Make sure the inode still exists and is a directory. If not, something has
	// screwed up because the VFS layer shouldn't have let us forget the inode
	// before opening it.
	in := fs.getDirForReadingOrDie(req.Inode)
	defer in.Mu.RUnlock()

	// Allocate a handle.
	handleID := fs.nextHandleID
	fs.nextHandleID++

	fs.handles[handleID] = newDirHandle(in)
	resp.Handle = handleID

	return
}

func (fs *fileSystem) ReadDir(
	ctx context.Context,
	req *fuse.ReadDirRequest) (resp *fuse.ReadDirResponse, err error) {
	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// Find the handle.
	dh := fs.handles[req.Handle].(*dirHandle)
	dh.Mu.Lock()
	defer dh.Mu.Unlock()

	// Serve the request.
	resp, err = dh.ReadDir(ctx, req)

	return
}

func (fs *fileSystem) ReleaseDirHandle(
	ctx context.Context,
	req *fuse.ReleaseDirHandleRequest) (
	resp *fuse.ReleaseDirHandleResponse, err error) {
	resp = &fuse.ReleaseDirHandleResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Sanity check that this handle exists and is of the correct type.
	_ = fs.handles[req.Handle].(*dirHandle)

	// Clear the entry from the map.
	delete(fs.handles, req.Handle)

	return
}
<commit_msg>Use getAttributes for lookup too.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fs

import (
	"fmt"
	"reflect"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/fs/inode"
	"github.com/jacobsa/gcsfuse/timeutil"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

type fileSystem struct {
	fuseutil.NotImplementedFileSystem

	/////////////////////////
	// Dependencies
	/////////////////////////

	clock  timeutil.Clock
	bucket gcs.Bucket

	/////////////////////////
	// Mutable state
	/////////////////////////

	// When acquiring this lock, the caller must hold no inode or dir handle
	// locks.
	mu syncutil.InvariantMutex

	// The user and group owning everything in the file system.
	//
	// GUARDED_BY(Mu)
	uid uint32
	gid uint32

	// The collection of live inodes, keyed by inode ID. No ID less than
	// fuse.RootInodeID is ever used.
	//
	// TODO(jacobsa): Implement ForgetInode support in the fuse package, then
	// implement the method here and clean up these maps.
	//
	// INVARIANT: All values are of type *inode.DirInode or *inode.FileInode
	// INVARIANT: For all keys k, k >= fuse.RootInodeID
	// INVARIANT: For all keys k, inodes[k].ID() == k
	// INVARIANT: inodes[fuse.RootInodeID] is of type *inode.DirInode
	//
	// GUARDED_BY(mu)
	inodes map[fuse.InodeID]inode.Inode

	// The next inode ID to hand out. We assume that this will never overflow,
	// since even if we were handing out inode IDs at 4 GHz, it would still take
	// over a century to do so.
	//
	// INVARIANT: For all keys k in inodes, k < nextInodeID
	//
	// GUARDED_BY(mu)
	nextInodeID fuse.InodeID

	// An index of all directory inodes by Name().
	//
	// INVARIANT: For each key k, isDirName(k)
	// INVARIANT: For each key k, dirIndex[k].Name() == k
	// INVARIANT: The values are all and only the values of the inodes map of
	// type *inode.DirInode.
	//
	// GUARDED_BY(mu)
	dirIndex map[string]*inode.DirInode

	// An index of all file inodes by (Name(), SourceGeneration()) pairs.
	//
	// INVARIANT: For each key k, !isDirName(k)
	// INVARIANT: For each key k, fileIndex[k].Name() == k.name
	// INVARIANT: For each key k, fileIndex[k].SourceGeneration() == k.gen
	// INVARIANT: The values are all and only the values of the inodes map of
	// type *inode.FileInode.
	//
	// GUARDED_BY(mu)
	fileIndex map[nameAndGen]*inode.FileInode

	// The collection of live handles, keyed by handle ID.
	//
	// INVARIANT: All values are of type *dirHandle
	//
	// GUARDED_BY(mu)
	handles map[fuse.HandleID]interface{}

	// The next handle ID to hand out. We assume that this will never overflow.
	//
	// INVARIANT: For all keys k in handles, k < nextHandleID
	//
	// GUARDED_BY(mu)
	nextHandleID fuse.HandleID
}

type nameAndGen struct {
	name string
	gen  int64
}

// Create a fuse file system whose root directory is the root of the supplied
// bucket. The supplied clock will be used for cache invalidation, modification
// times, etc.
func NewFileSystem(
	clock timeutil.Clock,
	bucket gcs.Bucket) (ffs fuse.FileSystem, err error) {
	// Set up the basic struct.
	fs := &fileSystem{
		clock:       clock,
		bucket:      bucket,
		inodes:      make(map[fuse.InodeID]inode.Inode),
		nextInodeID: fuse.RootInodeID + 1,
		dirIndex:    make(map[string]*inode.DirInode),
		fileIndex:   make(map[nameAndGen]*inode.FileInode),
		handles:     make(map[fuse.HandleID]interface{}),
	}

	// Set up the root inode.
	root := inode.NewDirInode(bucket, fuse.RootInodeID, "")
	fs.inodes[fuse.RootInodeID] = root
	fs.dirIndex[""] = root

	// Set up invariant checking.
	fs.mu = syncutil.NewInvariantMutex(fs.checkInvariants)

	ffs = fs
	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func isDirName(name string) bool {
	return name == "" || name[len(name)-1] == '/'
}

func (fs *fileSystem) checkInvariants() {
	// Check inode keys.
	for id, _ := range fs.inodes {
		if id < fuse.RootInodeID || id >= fs.nextInodeID {
			panic(fmt.Sprintf("Illegal inode ID: %v", id))
		}
	}

	// Check the root inode.
	_ = fs.inodes[fuse.RootInodeID].(*inode.DirInode)

	// Check each inode, and the indexes over them. Keep a count of each type
	// seen.
	dirsSeen := 0
	filesSeen := 0
	for id, in := range fs.inodes {
		// Check the ID.
		if in.ID() != id {
			panic(fmt.Sprintf("ID mismatch: %v vs. %v", in.ID(), id))
		}

		// Check type-specific stuff.
		switch typed := in.(type) {
		case *inode.DirInode:
			dirsSeen++

			if !isDirName(typed.Name()) {
				panic(fmt.Sprintf("Unexpected directory name: %s", typed.Name()))
			}

			if fs.dirIndex[typed.Name()] != typed {
				panic(fmt.Sprintf("dirIndex mismatch: %s", typed.Name()))
			}

		case *inode.FileInode:
			filesSeen++

			if isDirName(typed.Name()) {
				panic(fmt.Sprintf("Unexpected file name: %s", typed.Name()))
			}

			nandg := nameAndGen{typed.Name(), typed.SourceGeneration()}
			if fs.fileIndex[nandg] != typed {
				panic(
					fmt.Sprintf(
						"fileIndex mismatch: %s, %v",
						typed.Name(),
						typed.SourceGeneration()))
			}

		default:
			panic(fmt.Sprintf("Unexpected inode type: %v", reflect.TypeOf(in)))
		}
	}

	// Make sure that the indexes are exhaustive.
	if len(fs.dirIndex) != dirsSeen {
		panic(
			fmt.Sprintf(
				"dirIndex length mismatch: %v vs. %v",
				len(fs.dirIndex),
				dirsSeen))
	}

	if len(fs.fileIndex) != filesSeen {
		panic(
			fmt.Sprintf(
				"fileIndex length mismatch: %v vs. %v",
				len(fs.fileIndex),
				dirsSeen))
	}

	// Check handles.
	for id, h := range fs.handles {
		if id >= fs.nextHandleID {
			panic(fmt.Sprintf("Illegal handle ID: %v", id))
		}

		_ = h.(*dirHandle)
	}
}

// Find the given inode and return it with its lock held for reading. Panic if
// it doesn't exist or is the wrong type.
//
// SHARED_LOCKS_REQUIRED(fs.mu)
// SHARED_LOCK_FUNCTION(in.mu)
func (fs *fileSystem) getDirForReadingOrDie(
	id fuse.InodeID) (in *inode.DirInode) {
	in = fs.inodes[id].(*inode.DirInode)
	in.Mu.RLock()
	return
}

// Get attributes for the given directory inode, fixing up ownership information.
//
// SHARED_LOCKS_REQUIRED(fs.mu)
// SHARED_LOCK_FUNCTION(d.mu)
func (fs *fileSystem) getAttributes(
	ctx context.Context,
	d *inode.DirInode) (attrs fuse.InodeAttributes, err error) {
	attrs, err = d.Attributes(ctx)
	if err != nil {
		return
	}

	attrs.Uid = fs.uid
	attrs.Gid = fs.gid

	return
}

// Find a directory inode for the given object record. Create one if there
// isn't already one available.
//
// EXCLUSIVE_LOCKS_REQUIRED(fs.mu)
func (fs *fileSystem) lookUpOrCreateDirInode(
	ctx context.Context,
	o *storage.Object) (in *inode.DirInode, err error) {
	// Do we already have an inode for this name?
	if in = fs.dirIndex[o.Name]; in != nil {
		return
	}

	// Mint an ID.
	id := fs.nextInodeID
	fs.nextInodeID++

	// Create and index an inode.
	in = inode.NewDirInode(fs.bucket, id, o.Name)
	fs.inodes[id] = in
	fs.dirIndex[in.Name()] = in

	return
}

// Find a file inode for the given object record. Create one if there isn't
// already one available.
//
// EXCLUSIVE_LOCKS_REQUIRED(fs.mu)
func (fs *fileSystem) lookUpOrCreateFileInode(
	ctx context.Context,
	o *storage.Object) (in *inode.FileInode, err error) {
	nandg := nameAndGen{
		name: o.Name,
		gen:  o.Generation,
	}

	// Do we already have an inode for this (name, generation) pair?
	if in = fs.fileIndex[nandg]; in != nil {
		return
	}

	// Mint an ID.
	id := fs.nextInodeID
	fs.nextInodeID++

	// Create and index an inode.
	in = inode.NewFileInode(fs.bucket, id, o)
	fs.inodes[id] = in
	fs.fileIndex[nandg] = in

	return
}

////////////////////////////////////////////////////////////////////////
// fuse.FileSystem methods
////////////////////////////////////////////////////////////////////////

func (fs *fileSystem) Init(
	ctx context.Context,
	req *fuse.InitRequest) (resp *fuse.InitResponse, err error) {
	resp = &fuse.InitResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Store the mounting user's info for later.
	fs.uid = req.Header.Uid
	fs.gid = req.Header.Gid

	return
}

func (fs *fileSystem) LookUpInode(
	ctx context.Context,
	req *fuse.LookUpInodeRequest) (resp *fuse.LookUpInodeResponse, err error) {
	resp = &fuse.LookUpInodeResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Find the parent directory in question.
	parent := fs.inodes[req.Parent].(*inode.DirInode)

	// Find a record for the child with the given name.
	o, err := parent.LookUpChild(ctx, req.Name)
	if err != nil {
		return
	}

	// Is the child a directory or a file?
	var in inode.Inode
	if isDirName(o.Name) {
		in, err = fs.lookUpOrCreateDirInode(ctx, o)
	} else {
		in, err = fs.lookUpOrCreateFileInode(ctx, o)
	}

	if err != nil {
		return
	}

	// Fill out the response.
	resp.Entry.Child = in.ID()
	if resp.Entry.Attributes, err = fs.getAttributes(ctx, in); err != nil {
		return
	}

	return
}

func (fs *fileSystem) GetInodeAttributes(
	ctx context.Context,
	req *fuse.GetInodeAttributesRequest) (
	resp *fuse.GetInodeAttributesResponse, err error) {
	resp = &fuse.GetInodeAttributesResponse{}

	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// Find the inode.
	in := fs.inodes[req.Inode]

	// Grab its attributes.
	switch typed := in.(type) {
	case *inode.DirInode:
		resp.Attributes, err = fs.getAttributes(ctx, typed)
		if err != nil {
			err = fmt.Errorf("DirInode.Attributes: %v", err)
			return
		}

	default:
		panic(
			fmt.Sprintf(
				"Unknown inode type for ID %v: %v",
				req.Inode,
				reflect.TypeOf(in)))
	}

	return
}

func (fs *fileSystem) OpenDir(
	ctx context.Context,
	req *fuse.OpenDirRequest) (resp *fuse.OpenDirResponse, err error) {
	resp = &fuse.OpenDirResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Make sure the inode still exists and is a directory. If not, something has
	// screwed up because the VFS layer shouldn't have let us forget the inode
	// before opening it.
	in := fs.getDirForReadingOrDie(req.Inode)
	defer in.Mu.RUnlock()

	// Allocate a handle.
	handleID := fs.nextHandleID
	fs.nextHandleID++

	fs.handles[handleID] = newDirHandle(in)
	resp.Handle = handleID

	return
}

func (fs *fileSystem) ReadDir(
	ctx context.Context,
	req *fuse.ReadDirRequest) (resp *fuse.ReadDirResponse, err error) {
	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// Find the handle.
	dh := fs.handles[req.Handle].(*dirHandle)
	dh.Mu.Lock()
	defer dh.Mu.Unlock()

	// Serve the request.
	resp, err = dh.ReadDir(ctx, req)

	return
}

func (fs *fileSystem) ReleaseDirHandle(
	ctx context.Context,
	req *fuse.ReleaseDirHandleRequest) (
	resp *fuse.ReleaseDirHandleResponse, err error) {
	resp = &fuse.ReleaseDirHandleResponse{}

	fs.mu.Lock()
	defer fs.mu.Unlock()

	// Sanity check that this handle exists and is of the correct type.
	_ = fs.handles[req.Handle].(*dirHandle)

	// Clear the entry from the map.
	delete(fs.handles, req.Handle)

	return
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/hoisie/web"
	"github.com/xyproto/browserspeak"
	"github.com/xyproto/genericsite"
	"github.com/xyproto/siteengines"
	//"github.com/xyproto/personplan"
)

// TODO: Norwegian everywhere
// TODO: Different Redis database than the other sites

const JQUERY_VERSION = "2.0.0"

func notFound2(ctx *web.Context, val string) {
	ctx.ResponseWriter.WriteHeader(404)
	ctx.ResponseWriter.Write([]byte(browserspeak.NotFound(ctx, val)))
}

func ServeEngines(userState *genericsite.UserState, mainMenuEntries genericsite.MenuEntries) {
	// The user engine
	userEngine := siteengines.NewUserEngine(userState)
	userEngine.ServePages("ftls2.roboticoverlords.org")

	// The admin engine
	adminEngine := siteengines.NewAdminEngine(userState)
	adminEngine.ServePages(FTLSBaseCP, mainMenuEntries)

	// Wiki engine
	wikiEngine := siteengines.NewWikiEngine(userState)
	wikiEngine.ServePages(FTLSBaseCP, mainMenuEntries)

	// Timetable engine
	ftlsEngine := siteengines.NewTimeTableEngine(userState)
	ftlsEngine.ServePages(FTLSBaseCP, mainMenuEntries)
}

func main() {

	// UserState with a Redis Connection Pool, using database index 2
	userState := genericsite.NewUserState(2)
	defer userState.Close()

	// The archlinux.no webpage,
	mainMenuEntries := ServeFTLS(userState, "/js/jquery-"+JQUERY_VERSION+".min.js")

	ServeEngines(userState, mainMenuEntries)

	// Compilation errors, vim-compatible filename
	web.Get("/error", browserspeak.GenerateErrorHandle("errors.err"))
	web.Get("/errors", browserspeak.GenerateErrorHandle("errors.err"))

	// Various .php and .asp urls that showed up in the log
	genericsite.ServeForFun()

	// TODO: Incorporate this check into web.go, to only return
	// stuff in the header when the HEAD method is requested:
	// if ctx.Request.Method == "HEAD" { return }
	// See also: curl -I

	// Serve on port 3002 for the Nginx instance to use
	web.Run("0.0.0.0:3002")
}
<commit_msg>Introducing webhandle<commit_after>package main

import (
	"github.com/hoisie/web"
	"github.com/xyproto/genericsite"
	"github.com/xyproto/siteengines"
	"github.com/xyproto/webhandle"
	//"github.com/xyproto/personplan"
)

// TODO: Norwegian everywhere
// TODO: Different Redis database than the other sites

const JQUERY_VERSION = "2.0.0"

func notFound2(ctx *web.Context, val string) {
	ctx.ResponseWriter.WriteHeader(404)
	ctx.ResponseWriter.Write([]byte(webhandle.NotFound(ctx, val)))
}

func ServeEngines(userState *genericsite.UserState, mainMenuEntries genericsite.MenuEntries) {
	// The user engine
	userEngine := siteengines.NewUserEngine(userState)
	userEngine.ServePages("ftls2.roboticoverlords.org")

	// The admin engine
	adminEngine := siteengines.NewAdminEngine(userState)
	adminEngine.ServePages(FTLSBaseCP, mainMenuEntries)

	// Wiki engine
	wikiEngine := siteengines.NewWikiEngine(userState)
	wikiEngine.ServePages(FTLSBaseCP, mainMenuEntries)

	// Timetable engine
	ftlsEngine := siteengines.NewTimeTableEngine(userState)
	ftlsEngine.ServePages(FTLSBaseCP, mainMenuEntries)
}

func main() {

	// UserState with a Redis Connection Pool, using database index 2
	userState := genericsite.NewUserState(2)
	defer userState.Close()

	// The archlinux.no webpage,
	mainMenuEntries := ServeFTLS(userState, "/js/jquery-"+JQUERY_VERSION+".min.js")

	ServeEngines(userState, mainMenuEntries)

	// Compilation errors, vim-compatible filename
	web.Get("/error", webhandle.GenerateErrorHandle("errors.err"))
	web.Get("/errors", webhandle.GenerateErrorHandle("errors.err"))

	// Various .php and .asp urls that showed up in the log
	genericsite.ServeForFun()

	// TODO: Incorporate this check into web.go, to only return
	// stuff in the header when the HEAD method is requested:
	// if ctx.Request.Method == "HEAD" { return }
	// See also: curl -I

	// Serve on port 3002 for the Nginx instance to use
	web.Run("0.0.0.0:3002")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strings"
)

var defaultFuncs = map[string]interface{}{
	"str": String,
	"dec": Decimal,
	"eq":  Equal,
}

func String(v interface{}) string {
	switch x := v.(type) {
	case string:
		x = strings.Replace(x, `\`, `\\`, -1)
		x = strings.Replace(x, `"`, `\"`, -1)
		return fmt.Sprintf("\"%s\"", x)
	case float64:
		return fmt.Sprintf("\"%f\"", x)
	}
	return ""
}

func Decimal(dec int, v interface{}) string {
	if f, ok := v.(float64); ok {
		fmtstr := fmt.Sprintf("%%.%df", dec)
		return fmt.Sprintf(fmtstr, f)
	}
	return ""
}

func Equal(v1, v2 interface{}) interface{} {
	if v1 == v2 {
		return v1
	}
	return nil
}
<commit_msg>Add function for string compare ignoring case<commit_after>package main

import (
	"fmt"
	"strings"
)

var defaultFuncs = map[string]interface{}{
	"str":        String,
	"dec":        Decimal,
	"eq":         Equal,
	"eq_igncase": EqualIgnoreCase,
}

func String(v interface{}) string {
	switch x := v.(type) {
	case string:
		x = strings.Replace(x, `\`, `\\`, -1)
		x = strings.Replace(x, `"`, `\"`, -1)
		return fmt.Sprintf("\"%s\"", x)
	case float64:
		return fmt.Sprintf("\"%f\"", x)
	}
	return ""
}

func Decimal(dec int, v interface{}) string {
	if f, ok := v.(float64); ok {
		fmtstr := fmt.Sprintf("%%.%df", dec)
		return fmt.Sprintf(fmtstr, f)
	}
	return ""
}

func Equal(v1, v2 interface{}) interface{} {
	if v1 == v2 {
		return v1
	}
	return nil
}

func EqualIgnoreCase(s1, s2 string) string {
	if strings.ToLower(s1) == strings.ToLower(s2) {
		return s1
	}
	return ""
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
)

// A Steam game in a library. May or may not be installed.
type Game struct {
	// Official Steam id.
	Id string
	// Warning, may contain Unicode characters.
	Name string
	// Tags, including user-created category and Steam's "Favorite" tag.
	Tags []string
	// Path for the grid image.
	ImagePath string
	// Raw bytes of the encoded image (usually jpg).
	ImageBytes []byte
}

// Pattern of game declarations in the public profile. It's actually JSON
// inside Javascript, but this way is easier to extract.
const profileGamePattern = `\{"appid":\s*(\d+),\s*"name":\s*"(.+?)"`

// Fetches the list of games from the public user profile. This is better than
// looking locally because the profiles give the full game name, which can be
// used for image searches later on.
func addGamesFromProfile(user User, games map[string]*Game) (err error) {
	profile, err := GetProfile(user)
	if err != nil {
		return
	}

	// Fetch game list from public profile.
	pattern := regexp.MustCompile(profileGamePattern)
	for _, groups := range pattern.FindAllStringSubmatch(profile, -1) {
		gameId := groups[1]
		gameName := groups[2]
		tags := []string{""}
		imagePath := ""
		games[gameId] = &Game{gameId, gameName, tags, imagePath, nil}
	}

	return
}

func addNonSteamGaems(user User, game map[string]*Game) {
}

// Loads the categories list. This finds the categories for the games loaded
// from the profile and sometimes find new games, although without names.
func addUnknownGames(user User, games map[string]*Game) {
	// Fetch game categories from local file.
	sharedConfFile := filepath.Join(user.Dir, "7", "remote", "sharedconfig.vdf")
	if _, err := os.Stat(sharedConfFile); err != nil {
		// No categories file found, skipping this part.
		return
	}
	sharedConfBytes, err := ioutil.ReadFile(sharedConfFile)
	if err != nil {
		return
	}

	sharedConf := string(sharedConfBytes)
	// VDF pattern: "steamid" { "tags { "0" "category" } }
	gamePattern := regexp.MustCompile(`"([0-9]+)"\s*{[^}]+?"tags"\s*{([^}]+?)}`)
	tagsPattern := regexp.MustCompile(`"[0-9]+"\s*"(.+?)"`)
	for _, gameGroups := range gamePattern.FindAllStringSubmatch(sharedConf, -1) {
		gameId := gameGroups[1]
		tagsText := gameGroups[2]

		for _, tagGroups := range tagsPattern.FindAllStringSubmatch(tagsText, -1) {
			tag := tagGroups[1]

			game, ok := games[gameId]
			if ok {
				game.Tags = append(game.Tags, tag)
			} else {
				// If for some reason it wasn't included in the profile, create a new
				// entry for it now. Unfortunately we don't have a name.
				gameName := ""
				games[gameId] = &Game{gameId, gameName, []string{tag}, "", nil}
			}
		}
	}
}

// Adds non-Steam games that have been registered locally.
func addNonSteamGames(user User, games map[string]*Game) {
	// File that holds list of all non-Steam games.
	file := filepath.Join(user.Dir, "760", "screenshots.vdf")
	if _, err := os.Stat(file); err != nil {
		// No custom games, skip this part.
		return
	}
	bytes, err := ioutil.ReadFile(file)
	if err != nil {
		return
	}

	content := string(bytes)
	// VDF pattern: "customid"		"name"
	gamePattern := regexp.MustCompile(`"([0-9]+)"\s*"(.+?)"`)
	for _, gameGroups := range gamePattern.FindAllStringSubmatch(content, -1) {
		gameId := gameGroups[1]
		name := gameGroups[2]
		games[gameId] = &Game{gameId, name, []string{}, "", nil}
	}
}

// Returns all games from a given user, using both the public profile and local
// files to gather the data. Returns a map of game by ID.
func GetGames(user User) (games map[string]*Game, err error) {
	games = make(map[string]*Game, 0)

	addGamesFromProfile(user, games)
	addUnknownGames(user, games)
	addNonSteamGames(user, games)

	return
}
<commit_msg>Full non-Steam games support<commit_after>package main

import (
	"io/ioutil"
	"bytes"
	"os"
	"path/filepath"
	"regexp"
	"hash/crc32"
	"strconv"
)

// A Steam game in a library. May or may not be installed.
type Game struct {
	// Official Steam id.
	Id string
	// Warning, may contain Unicode characters.
	Name string
	// Tags, including user-created category and Steam's "Favorite" tag.
	Tags []string
	// Path for the grid image.
	ImagePath string
	// Raw bytes of the encoded image (usually jpg).
	ImageBytes []byte
}

// Pattern of game declarations in the public profile. It's actually JSON
// inside Javascript, but this way is easier to extract.
const profileGamePattern = `\{"appid":\s*(\d+),\s*"name":\s*"(.+?)"`

// Fetches the list of games from the public user profile. This is better than
// looking locally because the profiles give the full game name, which can be
// used for image searches later on.
func addGamesFromProfile(user User, games map[string]*Game) (err error) {
	profile, err := GetProfile(user)
	if err != nil {
		return
	}

	// Fetch game list from public profile.
	pattern := regexp.MustCompile(profileGamePattern)
	for _, groups := range pattern.FindAllStringSubmatch(profile, -1) {
		gameId := groups[1]
		gameName := groups[2]
		tags := []string{""}
		imagePath := ""
		games[gameId] = &Game{gameId, gameName, tags, imagePath, nil}
	}

	return
}

// Loads the categories list. This finds the categories for the games loaded
// from the profile and sometimes find new games, although without names.
func addUnknownGames(user User, games map[string]*Game) {
	// Fetch game categories from local file.
	sharedConfFile := filepath.Join(user.Dir, "7", "remote", "sharedconfig.vdf")
	if _, err := os.Stat(sharedConfFile); err != nil {
		// No categories file found, skipping this part.
		return
	}
	sharedConfBytes, err := ioutil.ReadFile(sharedConfFile)
	if err != nil {
		return
	}

	sharedConf := string(sharedConfBytes)
	// VDF pattern: "steamid" { "tags { "0" "category" } }
	gamePattern := regexp.MustCompile(`"([0-9]+)"\s*{[^}]+?"tags"\s*{([^}]+?)}`)
	tagsPattern := regexp.MustCompile(`"[0-9]+"\s*"(.+?)"`)
	for _, gameGroups := range gamePattern.FindAllStringSubmatch(sharedConf, -1) {
		gameId := gameGroups[1]
		tagsText := gameGroups[2]

		for _, tagGroups := range tagsPattern.FindAllStringSubmatch(tagsText, -1) {
			tag := tagGroups[1]

			game, ok := games[gameId]
			if ok {
				game.Tags = append(game.Tags, tag)
			} else {
				// If for some reason it wasn't included in the profile, create a new
				// entry for it now. Unfortunately we don't have a name.
				gameName := ""
				games[gameId] = &Game{gameId, gameName, []string{tag}, "", nil}
			}
		}
	}
}

// Adds non-Steam games that have been registered locally.
// This information is in the file config/shortcuts.vdf, in binary format.
// It contains the non-Steam games with names, target (exe location) and
// tags/categories. To create a grid image we must compute the Steam ID, which
// is just crc32(target + label) + "02000000", using IEEE standard polynomials.
func addNonSteamGames(user User, games map[string]*Game) {
	shortcutsVdf := filepath.Join(user.Dir, "config", "shortcuts.vdf")
	if _, err := os.Stat(shortcutsVdf); err != nil {
		return
	}
	shortcutBytes, err := ioutil.ReadFile(shortcutsVdf)
	if err != nil {
		return
	}

	// The actual binary format is known, but using regexes is way easier than
	// parsing the entire file. If I run into any problems I'll replace this.
	gamePattern := regexp.MustCompile("appname\x00(.+?)\x00\x01exe\x00(.+?)\x00\x01.+?\x00tags\x00(.*?)\x08\x08")
	tagsPattern := regexp.MustCompile("\\d\x00(.+?)\x00")
	for _, gameGroups := range gamePattern.FindAllSubmatch(shortcutBytes, -1) {
		gameName := gameGroups[1]
		target := gameGroups[2]
		uniqueName := bytes.Join([][]byte{target, gameName}, []byte(""))
		// Does IEEE CRC32 of target concatenated with gameName, then convert
		// to 64bit Steam ID. No idea why Steam chose this operation.
		gameId := strconv.FormatUint(uint64(crc32.ChecksumIEEE(uniqueName)) << 32 | 0x02000000, 10)
		game := Game{gameId, string(gameName), []string{}, "", nil}
		games[gameId] = &game

		// TODO: some problems when generating SteamID for games with changed
		// labels. For some reason they didn't match.

		tagsText := gameGroups[3]
		for _, tagGroups := range tagsPattern.FindAllSubmatch(tagsText, -1) {
			tag := tagGroups[1]
			game.Tags = append(game.Tags, string(tag))
		}
	}
}

// Returns all games from a given user, using both the public profile and local
// files to gather the data. Returns a map of game by ID.
func GetGames(user User) (games map[string]*Game, err error) {
	games = make(map[string]*Game, 0)

	addGamesFromProfile(user, games)
	addUnknownGames(user, games)
	addNonSteamGames(user, games)

	return
}
<|endoftext|>