<commit_before>/*
 * key.go - Cryptographic key management for fscrypt. Ensures that sensitive
 * material is properly handled throughout the program.
 *
 * Copyright 2017 Google Inc.
 * Author: Joe Richey (joerichey@google.com)
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package crypto

import (
	"bytes"
	"crypto/subtle"
	"encoding/base32"
	"io"
	"log"
	"os"
	"runtime"
	"unsafe"

	"github.com/pkg/errors"
	"golang.org/x/sys/unix"

	"github.com/google/fscrypt/metadata"
	"github.com/google/fscrypt/util"
)

const (
	// DefaultService is the service which should be used for all encryption
	// keys unless not possible for legacy reasons. For ext4 systems before
	// v4.8 and f2fs systems before v4.6, filesystem specific services must
	// be used (these legacy services will still work with later kernels).
	DefaultService = unix.FS_KEY_DESC_PREFIX
	// keyType is always logon as required by filesystem encryption
	keyType = "logon"
	// Keys need to readable and writable, but hidden from other processes.
	keyProtection = unix.PROT_READ | unix.PROT_WRITE
	keyMmapFlags  = unix.MAP_PRIVATE | unix.MAP_ANONYMOUS
)

/*
UseMlock determines whether we should use the mlock/munlock syscalls to
prevent sensitive data like keys and passphrases from being paged to disk.
UseMlock defaults to true, but can be set to false if the application calling
into this library has insufficient privileges to lock memory. Code using this
package could also bind this setting to a flag by using:

	flag.BoolVar(&crypto.UseMlock, "lock-memory", true, "lock keys in memory")
*/
var UseMlock = true

/*
Key protects some arbitrary buffer of cryptographic material. Its methods
ensure that the Key's data is locked in memory before being used (if
UseMlock is set to true), and is wiped and unlocked after use (via the Wipe()
method). This data is never accessed outside of the fscrypt/crypto package
(except for the UnsafeData method). If a key is successfully created, the
Wipe() method should be called after it's use. For example:

	func UseKeyFromStdin() error {
		key, err := NewKeyFromReader(os.Stdin)
		if err != nil {
			return err
		}
		defer key.Wipe()

		// Do stuff with key

		return nil
	}

The Wipe() method will also be called when a key is garbage collected; however,
it is best practice to clear the key as soon as possible, so it spends a minimal
amount of time in memory.

Note that Key is not thread safe, as a key could be wiped while another thread
is using it. Also, calling Wipe() from two threads could cause an error as
memory could be freed twice.
*/
type Key struct {
	data []byte
}

// newBlankKey constructs a blank key of a specified length and returns an error
// if we are unable to allocate or lock the necessary memory.
func newBlankKey(length int) (*Key, error) {
	if length == 0 {
		return &Key{data: nil}, nil
	} else if length < 0 {
		return nil, errors.Wrapf(ErrNegitiveLength, "length of %d requested", length)
	}

	flags := keyMmapFlags
	if UseMlock {
		flags |= unix.MAP_LOCKED
	}

	// See MAP_ANONYMOUS in http://man7.org/linux/man-pages/man2/mmap.2.html
	data, err := unix.Mmap(-1, 0, length, keyProtection, flags)
	if err != nil {
		log.Printf("unix.Mmap() with length=%d failed: %v", length, err)
		return nil, ErrKeyAlloc
	}

	key := &Key{data: data}

	// Backup finalizer in case user forgets to "defer key.Wipe()"
	runtime.SetFinalizer(key, (*Key).Wipe)
	return key, nil
}

// Wipe destroys a Key by zeroing and freeing the memory. The data is zeroed
// even if Wipe returns an error, which occurs if we are unable to unlock or
// free the key memory. Wipe does nothing if the key is already wiped or is nil.
func (key *Key) Wipe() error {
	// We do nothing if key or key.data is nil so that Wipe() is idempotent
	// and so Wipe() can be called on keys which have already been cleared.
	if key != nil && key.data != nil {
		data := key.data
		key.data = nil

		for i := range data {
			data[i] = 0
		}

		if err := unix.Munmap(data); err != nil {
			log.Printf("unix.Munmap() failed: %v", err)
			return ErrKeyFree
		}
	}
	return nil
}

// Len is the underlying data buffer's length.
func (key *Key) Len() int {
	return len(key.data)
}

// UnsafeData exposes the underlying protected slice. This is unsafe because the
// data can be paged to disk if the buffer is copied, or the slice may be
// wiped while being used.
func (key *Key) UnsafeData() []byte {
	return key.data
}

// Equals compares the contents of two keys, returning true if they have the same
// key data. This function runs in constant time.
func (key *Key) Equals(key2 *Key) bool {
	return subtle.ConstantTimeCompare(key.data, key2.data) == 1
}

// resize returns a new key with size requestedSize and the appropriate data
// copied over. The original data is wiped. This method does nothing and returns
// itself if the key's length equals requestedSize.
func (key *Key) resize(requestedSize int) (*Key, error) {
	if key.Len() == requestedSize {
		return key, nil
	}
	defer key.Wipe()

	resizedKey, err := newBlankKey(requestedSize)
	if err != nil {
		return nil, err
	}
	copy(resizedKey.data, key.data)
	return resizedKey, nil
}

// NewKeyFromReader constructs a key of abritary length by reading from reader
// until hitting EOF.
func NewKeyFromReader(reader io.Reader) (*Key, error) {
	// Use an initial key size of a page. As Mmap allocates a page anyway,
	// there isn't much additional overhead from starting with a whole page.
	key, err := newBlankKey(os.Getpagesize())
	if err != nil {
		return nil, err
	}

	totalBytesRead := 0
	for {
		bytesRead, err := reader.Read(key.data[totalBytesRead:])
		totalBytesRead += bytesRead

		switch err {
		case nil:
			// Need to continue reading. Grow key if necessary
			if key.Len() == totalBytesRead {
				if key, err = key.resize(2 * key.Len()); err != nil {
					return nil, err
				}
			}
		case io.EOF:
			// Getting the EOF error means we are done
			return key.resize(totalBytesRead)
		default:
			// Fail if Read() has a failure
			key.Wipe()
			return nil, err
		}
	}
}

// NewFixedLengthKeyFromReader constructs a key with a specified length by
// reading exactly length bytes from reader.
func NewFixedLengthKeyFromReader(reader io.Reader, length int) (*Key, error) {
	key, err := newBlankKey(length)
	if err != nil {
		return nil, err
	}
	if _, err := io.ReadFull(reader, key.data); err != nil {
		key.Wipe()
		return nil, err
	}
	return key, nil
}

// getKeyring returns the id of the session keyring, or the id of the user
// session keyring if session keyring does not exist. We cannot directly use
// KEY_SPEC_SESSION_KEYRING, as that will make a new session keyring if one does
// not exist, which will be garbage collected when the process terminates.
func getKeyring() (int, error) {
	keyringID, err := unix.KeyctlGetKeyringID(unix.KEY_SPEC_SESSION_KEYRING, false)
	log.Printf("unix.KeyctlGetKeyringID(KEY_SPEC_SESSION_KEYRING) = %d, %v", keyringID, err)
	if err != nil {
		return 0, errors.Wrap(ErrKeyringLocate, err.Error())
	}
	return keyringID, nil
}

// FindPolicyKey tries to locate a policy key in the kernel keyring with the
// provided description. The keyring and key ids are returned if we can find the
// key. An error is returned if the key does not exist.
func FindPolicyKey(description string) (keyringID, keyID int, err error) {
	keyringID, err = getKeyring()
	if err != nil {
		return
	}

	keyID, err = unix.KeyctlSearch(keyringID, keyType, description, 0)
	log.Printf("unix.KeyctlSearch(%d, %s, %s) = %d, %v", keyringID, keyType, description, keyID, err)
	if err != nil {
		err = errors.Wrap(ErrKeyringSearch, err.Error())
	}
	return
}

// RemovePolicyKey tries to remove a policy key from the kernel keyring with the
// provided description. An error is returned if the key does not exist.
func RemovePolicyKey(description string) error {
	keyringID, keyID, err := FindPolicyKey(description)
	if err != nil {
		return err
	}

	_, err = unix.KeyctlInt(unix.KEYCTL_UNLINK, keyID, keyringID, 0, 0)
	log.Printf("unix.KeyctlUnlink(%d, %d) = %v", keyID, keyringID, err)
	if err != nil {
		return errors.Wrap(ErrKeyringDelete, err.Error())
	}
	return nil
}

// InsertPolicyKey puts the provided policy key into the kernel keyring with the
// provided description, and type logon. The key must be a policy key.
func InsertPolicyKey(key *Key, description string) error {
	if err := util.CheckValidLength(metadata.PolicyKeyLen, key.Len()); err != nil {
		return errors.Wrap(err, "policy key")
	}

	// Create our payload (containing an FscryptKey)
	payload, err := newBlankKey(int(unsafe.Sizeof(unix.FscryptKey{})))
	if err != nil {
		return err
	}
	defer payload.Wipe()

	// Cast the payload to an FscryptKey so we can initialize the fields.
	fscryptKey := (*unix.FscryptKey)(util.Ptr(payload.data))
	// Mode is ignored by the kernel
	fscryptKey.Mode = 0
	fscryptKey.Size = metadata.PolicyKeyLen
	copy(fscryptKey.Raw[:], key.data)

	keyringID, err := getKeyring()
	if err != nil {
		return err
	}

	keyID, err := unix.AddKey(keyType, description, payload.data, keyringID)
	log.Printf("unix.AddKey(%s, %s, <payload>, %d) = %d, %v",
		keyType, description, keyringID, keyID, err)
	if err != nil {
		return errors.Wrap(ErrKeyringInsert, err.Error())
	}
	return nil
}

var (
	// The recovery code is base32 with a dash between each block of 8 characters.
	encoding      = base32.StdEncoding
	blockSize     = 8
	separator     = []byte("-")
	encodedLength = encoding.EncodedLen(metadata.PolicyKeyLen)
	decodedLength = encoding.DecodedLen(encodedLength)
	// RecoveryCodeLength is the number of bytes in every recovery code
	RecoveryCodeLength = (encodedLength/blockSize)*(blockSize+len(separator)) - len(separator)
)

// WriteRecoveryCode outputs key's recovery code to the provided writer.
// WARNING: This recovery key is enough to derive the original key, so it must
// be given the same level of protection as a raw cryptographic key.
func WriteRecoveryCode(key *Key, writer io.Writer) error {
	if err := util.CheckValidLength(metadata.PolicyKeyLen, key.Len()); err != nil {
		return errors.Wrap(err, "recovery key")
	}

	// We store the base32 encoded data (without separators) in a temp key
	encodedKey, err := newBlankKey(encodedLength)
	if err != nil {
		return err
	}
	defer encodedKey.Wipe()
	encoding.Encode(encodedKey.data, key.data)

	w := util.NewErrWriter(writer)

	// Write the blocks with separators between them
	w.Write(encodedKey.data[:blockSize])
	for blockStart := blockSize; blockStart < encodedLength; blockStart += blockSize {
		w.Write(separator)

		blockEnd := util.MinInt(blockStart+blockSize, encodedLength)
		w.Write(encodedKey.data[blockStart:blockEnd])
	}

	// If any writes have failed, return the error
	return w.Err()
}

// ReadRecoveryCode gets the recovery code from the provided writer and returns
// the corresponding cryptographic key.
// WARNING: This recovery key is enough to derive the original key, so it must
// be given the same level of protection as a raw cryptographic key.
func ReadRecoveryCode(reader io.Reader) (*Key, error) {
	// We store the base32 encoded data (without separators) in a temp key
	encodedKey, err := newBlankKey(encodedLength)
	if err != nil {
		return nil, err
	}
	defer encodedKey.Wipe()

	r := util.NewErrReader(reader)

	// Read the other blocks, checking the separators between them
	r.Read(encodedKey.data[:blockSize])
	inputSeparator := make([]byte, len(separator))

	for blockStart := blockSize; blockStart < encodedLength; blockStart += blockSize {
		r.Read(inputSeparator)
		if r.Err() == nil && !bytes.Equal(separator, inputSeparator) {
			err := errors.Wrapf(ErrRecoveryCode, "invalid seperator %q", inputSeparator)
			return nil, err
		}

		blockEnd := util.MinInt(blockStart+blockSize, encodedLength)
		r.Read(encodedKey.data[blockStart:blockEnd])
	}

	// If any reads have failed, return the error
	if r.Err() != nil {
		return nil, errors.Wrapf(ErrRecoveryCode, "read error %v", r.Err())
	}

	// Now we decode the key, resizing if necessary
	decodedKey, err := newBlankKey(decodedLength)
	if err != nil {
		return nil, err
	}
	if _, err = encoding.Decode(decodedKey.data, encodedKey.data); err != nil {
		return nil, errors.Wrap(ErrRecoveryCode, err.Error())
	}
	return decodedKey.resize(metadata.PolicyKeyLen)
}
<commit_msg>crypto: Conversion to/from C strings<commit_after>/*
 * key.go - Cryptographic key management for fscrypt. Ensures that sensitive
 * material is properly handled throughout the program.
 *
 * Copyright 2017 Google Inc.
 * Author: Joe Richey (joerichey@google.com)
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package crypto

/*
#include <stdlib.h>
#include <string.h>
*/
import "C"

import (
	"bytes"
	"crypto/subtle"
	"encoding/base32"
	"io"
	"log"
	"os"
	"runtime"
	"unsafe"

	"github.com/pkg/errors"
	"golang.org/x/sys/unix"

	"github.com/google/fscrypt/metadata"
	"github.com/google/fscrypt/util"
)

const (
	// DefaultService is the service which should be used for all encryption
	// keys unless not possible for legacy reasons. For ext4 systems before
	// v4.8 and f2fs systems before v4.6, filesystem specific services must
	// be used (these legacy services will still work with later kernels).
	DefaultService = unix.FS_KEY_DESC_PREFIX
	// keyType is always logon as required by filesystem encryption
	keyType = "logon"
	// Keys need to readable and writable, but hidden from other processes.
	keyProtection = unix.PROT_READ | unix.PROT_WRITE
	keyMmapFlags  = unix.MAP_PRIVATE | unix.MAP_ANONYMOUS
)

/*
UseMlock determines whether we should use the mlock/munlock syscalls to
prevent sensitive data like keys and passphrases from being paged to disk.
UseMlock defaults to true, but can be set to false if the application calling
into this library has insufficient privileges to lock memory. Code using this
package could also bind this setting to a flag by using:

	flag.BoolVar(&crypto.UseMlock, "lock-memory", true, "lock keys in memory")
*/
var UseMlock = true

/*
Key protects some arbitrary buffer of cryptographic material. Its methods
ensure that the Key's data is locked in memory before being used (if
UseMlock is set to true), and is wiped and unlocked after use (via the Wipe()
method). This data is never accessed outside of the fscrypt/crypto package
(except for the UnsafeData method). If a key is successfully created, the
Wipe() method should be called after it's use. For example:

	func UseKeyFromStdin() error {
		key, err := NewKeyFromReader(os.Stdin)
		if err != nil {
			return err
		}
		defer key.Wipe()

		// Do stuff with key

		return nil
	}

The Wipe() method will also be called when a key is garbage collected; however,
it is best practice to clear the key as soon as possible, so it spends a minimal
amount of time in memory.

Note that Key is not thread safe, as a key could be wiped while another thread
is using it. Also, calling Wipe() from two threads could cause an error as
memory could be freed twice.
*/
type Key struct {
	data []byte
}

// newBlankKey constructs a blank key of a specified length and returns an error
// if we are unable to allocate or lock the necessary memory.
func newBlankKey(length int) (*Key, error) {
	if length == 0 {
		return &Key{data: nil}, nil
	} else if length < 0 {
		return nil, errors.Wrapf(ErrNegitiveLength, "length of %d requested", length)
	}

	flags := keyMmapFlags
	if UseMlock {
		flags |= unix.MAP_LOCKED
	}

	// See MAP_ANONYMOUS in http://man7.org/linux/man-pages/man2/mmap.2.html
	data, err := unix.Mmap(-1, 0, length, keyProtection, flags)
	if err != nil {
		log.Printf("unix.Mmap() with length=%d failed: %v", length, err)
		return nil, ErrKeyAlloc
	}

	key := &Key{data: data}

	// Backup finalizer in case user forgets to "defer key.Wipe()"
	runtime.SetFinalizer(key, (*Key).Wipe)
	return key, nil
}

// Wipe destroys a Key by zeroing and freeing the memory. The data is zeroed
// even if Wipe returns an error, which occurs if we are unable to unlock or
// free the key memory. Wipe does nothing if the key is already wiped or is nil.
func (key *Key) Wipe() error {
	// We do nothing if key or key.data is nil so that Wipe() is idempotent
	// and so Wipe() can be called on keys which have already been cleared.
	if key != nil && key.data != nil {
		data := key.data
		key.data = nil

		for i := range data {
			data[i] = 0
		}

		if err := unix.Munmap(data); err != nil {
			log.Printf("unix.Munmap() failed: %v", err)
			return ErrKeyFree
		}
	}
	return nil
}

// Len is the underlying data buffer's length.
func (key *Key) Len() int {
	return len(key.data)
}

// Equals compares the contents of two keys, returning true if they have the same
// key data. This function runs in constant time.
func (key *Key) Equals(key2 *Key) bool {
	return subtle.ConstantTimeCompare(key.data, key2.data) == 1
}

// resize returns a new key with size requestedSize and the appropriate data
// copied over. The original data is wiped. This method does nothing and returns
// itself if the key's length equals requestedSize.
func (key *Key) resize(requestedSize int) (*Key, error) {
	if key.Len() == requestedSize {
		return key, nil
	}
	defer key.Wipe()

	resizedKey, err := newBlankKey(requestedSize)
	if err != nil {
		return nil, err
	}
	copy(resizedKey.data, key.data)
	return resizedKey, nil
}

// UnsafeToCString makes a copy of the string's data into a null-terminated C
// string allocated by C. Note that this method is unsafe as this C copy has no
// locking or wiping functionality. The key shouldn't contain any `\0` bytes.
func (key *Key) UnsafeToCString() unsafe.Pointer {
	// Memory for the key must be moved into a C string allocated by C.
	size := C.size_t(key.Len())
	data := C.calloc(size+1, 1)
	C.memcpy(data, util.Ptr(key.data), size)
	return data
}

// NewKeyFromCString creates of a copy of some C string's data in a key. Note
// that the original C string is not modified at all, so steps must be taken to
// ensure that this original copy is secured.
func NewKeyFromCString(str unsafe.Pointer) (*Key, error) {
	size := C.strlen((*C.char)(str))
	key, err := newBlankKey(int(size))
	if err != nil {
		return nil, err
	}
	C.memcpy(util.Ptr(key.data), str, size)
	return key, nil
}

// NewKeyFromReader constructs a key of abritary length by reading from reader
// until hitting EOF.
func NewKeyFromReader(reader io.Reader) (*Key, error) {
	// Use an initial key size of a page. As Mmap allocates a page anyway,
	// there isn't much additional overhead from starting with a whole page.
	key, err := newBlankKey(os.Getpagesize())
	if err != nil {
		return nil, err
	}

	totalBytesRead := 0
	for {
		bytesRead, err := reader.Read(key.data[totalBytesRead:])
		totalBytesRead += bytesRead

		switch err {
		case nil:
			// Need to continue reading. Grow key if necessary
			if key.Len() == totalBytesRead {
				if key, err = key.resize(2 * key.Len()); err != nil {
					return nil, err
				}
			}
		case io.EOF:
			// Getting the EOF error means we are done
			return key.resize(totalBytesRead)
		default:
			// Fail if Read() has a failure
			key.Wipe()
			return nil, err
		}
	}
}

// NewFixedLengthKeyFromReader constructs a key with a specified length by
// reading exactly length bytes from reader.
func NewFixedLengthKeyFromReader(reader io.Reader, length int) (*Key, error) {
	key, err := newBlankKey(length)
	if err != nil {
		return nil, err
	}
	if _, err := io.ReadFull(reader, key.data); err != nil {
		key.Wipe()
		return nil, err
	}
	return key, nil
}

// getKeyring returns the id of the session keyring, or the id of the user
// session keyring if session keyring does not exist. We cannot directly use
// KEY_SPEC_SESSION_KEYRING, as that will make a new session keyring if one does
// not exist, which will be garbage collected when the process terminates.
func getKeyring() (int, error) {
	keyringID, err := unix.KeyctlGetKeyringID(unix.KEY_SPEC_SESSION_KEYRING, false)
	log.Printf("unix.KeyctlGetKeyringID(KEY_SPEC_SESSION_KEYRING) = %d, %v", keyringID, err)
	if err != nil {
		return 0, errors.Wrap(ErrKeyringLocate, err.Error())
	}
	return keyringID, nil
}

// FindPolicyKey tries to locate a policy key in the kernel keyring with the
// provided description. The keyring and key ids are returned if we can find the
// key. An error is returned if the key does not exist.
func FindPolicyKey(description string) (keyringID, keyID int, err error) {
	keyringID, err = getKeyring()
	if err != nil {
		return
	}

	keyID, err = unix.KeyctlSearch(keyringID, keyType, description, 0)
	log.Printf("unix.KeyctlSearch(%d, %s, %s) = %d, %v", keyringID, keyType, description, keyID, err)
	if err != nil {
		err = errors.Wrap(ErrKeyringSearch, err.Error())
	}
	return
}

// RemovePolicyKey tries to remove a policy key from the kernel keyring with the
// provided description. An error is returned if the key does not exist.
func RemovePolicyKey(description string) error {
	keyringID, keyID, err := FindPolicyKey(description)
	if err != nil {
		return err
	}

	_, err = unix.KeyctlInt(unix.KEYCTL_UNLINK, keyID, keyringID, 0, 0)
	log.Printf("unix.KeyctlUnlink(%d, %d) = %v", keyID, keyringID, err)
	if err != nil {
		return errors.Wrap(ErrKeyringDelete, err.Error())
	}
	return nil
}

// InsertPolicyKey puts the provided policy key into the kernel keyring with the
// provided description, and type logon. The key must be a policy key.
func InsertPolicyKey(key *Key, description string) error {
	if err := util.CheckValidLength(metadata.PolicyKeyLen, key.Len()); err != nil {
		return errors.Wrap(err, "policy key")
	}

	// Create our payload (containing an FscryptKey)
	payload, err := newBlankKey(int(unsafe.Sizeof(unix.FscryptKey{})))
	if err != nil {
		return err
	}
	defer payload.Wipe()

	// Cast the payload to an FscryptKey so we can initialize the fields.
	fscryptKey := (*unix.FscryptKey)(util.Ptr(payload.data))
	// Mode is ignored by the kernel
	fscryptKey.Mode = 0
	fscryptKey.Size = metadata.PolicyKeyLen
	copy(fscryptKey.Raw[:], key.data)

	keyringID, err := getKeyring()
	if err != nil {
		return err
	}

	keyID, err := unix.AddKey(keyType, description, payload.data, keyringID)
	log.Printf("unix.AddKey(%s, %s, <payload>, %d) = %d, %v",
		keyType, description, keyringID, keyID, err)
	if err != nil {
		return errors.Wrap(ErrKeyringInsert, err.Error())
	}
	return nil
}

var (
	// The recovery code is base32 with a dash between each block of 8 characters.
	encoding      = base32.StdEncoding
	blockSize     = 8
	separator     = []byte("-")
	encodedLength = encoding.EncodedLen(metadata.PolicyKeyLen)
	decodedLength = encoding.DecodedLen(encodedLength)
	// RecoveryCodeLength is the number of bytes in every recovery code
	RecoveryCodeLength = (encodedLength/blockSize)*(blockSize+len(separator)) - len(separator)
)

// WriteRecoveryCode outputs key's recovery code to the provided writer.
// WARNING: This recovery key is enough to derive the original key, so it must
// be given the same level of protection as a raw cryptographic key.
func WriteRecoveryCode(key *Key, writer io.Writer) error {
	if err := util.CheckValidLength(metadata.PolicyKeyLen, key.Len()); err != nil {
		return errors.Wrap(err, "recovery key")
	}

	// We store the base32 encoded data (without separators) in a temp key
	encodedKey, err := newBlankKey(encodedLength)
	if err != nil {
		return err
	}
	defer encodedKey.Wipe()
	encoding.Encode(encodedKey.data, key.data)

	w := util.NewErrWriter(writer)

	// Write the blocks with separators between them
	w.Write(encodedKey.data[:blockSize])
	for blockStart := blockSize; blockStart < encodedLength; blockStart += blockSize {
		w.Write(separator)

		blockEnd := util.MinInt(blockStart+blockSize, encodedLength)
		w.Write(encodedKey.data[blockStart:blockEnd])
	}

	// If any writes have failed, return the error
	return w.Err()
}

// ReadRecoveryCode gets the recovery code from the provided writer and returns
// the corresponding cryptographic key.
// WARNING: This recovery key is enough to derive the original key, so it must
// be given the same level of protection as a raw cryptographic key.
func ReadRecoveryCode(reader io.Reader) (*Key, error) {
	// We store the base32 encoded data (without separators) in a temp key
	encodedKey, err := newBlankKey(encodedLength)
	if err != nil {
		return nil, err
	}
	defer encodedKey.Wipe()

	r := util.NewErrReader(reader)

	// Read the other blocks, checking the separators between them
	r.Read(encodedKey.data[:blockSize])
	inputSeparator := make([]byte, len(separator))

	for blockStart := blockSize; blockStart < encodedLength; blockStart += blockSize {
		r.Read(inputSeparator)
		if r.Err() == nil && !bytes.Equal(separator, inputSeparator) {
			err := errors.Wrapf(ErrRecoveryCode, "invalid seperator %q", inputSeparator)
			return nil, err
		}

		blockEnd := util.MinInt(blockStart+blockSize, encodedLength)
		r.Read(encodedKey.data[blockStart:blockEnd])
	}

	// If any reads have failed, return the error
	if r.Err() != nil {
		return nil, errors.Wrapf(ErrRecoveryCode, "read error %v", r.Err())
	}

	// Now we decode the key, resizing if necessary
	decodedKey, err := newBlankKey(decodedLength)
	if err != nil {
		return nil, err
	}
	if _, err = encoding.Decode(decodedKey.data, encodedKey.data); err != nil {
		return nil, errors.Wrap(ErrRecoveryCode, err.Error())
	}
	return decodedKey.resize(metadata.PolicyKeyLen)
}
<|endoftext|>


<commit_before>package cortexbot

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/http"
	"strings"
	"time"

	valid "github.com/asaskevich/govalidator"
	"github.com/go-telegram-bot-api/telegram-bot-api"
	"gopkg.in/ilyaglow/go-cortex.v2"
)

// ProcessCortex asks Cortex about data submitted by a user
func (c *Client) ProcessCortex(input *tgbotapi.Message) error {
	var j cortex.Observable
	var err error

	if input.Document != nil {
		link, err := c.Bot.GetFileDirectURL(input.Document.FileID)
		if err != nil {
			log.Println("Can't get direct link to file")
			return err
		}

		j, err = newFileArtifactFromURL(link, c.TLP, c.Bot.Client)
		if err != nil {
			log.Println(err.Error())
			return err
		}
	} else {
		j, err = newArtifact(input.Text, c.TLP)
		if err != nil {
			log.Println(err.Error())
			return err
		}
	}

	// Run all analyzers over it with 1 minute timeout
	mul := c.Cortex.Analyzers.NewMultiRun(context.Background(), 5*time.Minute)
	mul.OnReport = func(r *cortex.Report) {
		if r.Status == "Failure" {
			log.Printf("Analyzer %s failed with error message: %s", r.AnalyzerName, r.ReportBody.ErrorMessage)
			return
		}

		// Send JSON file with full report and taxonomies
		tr, _ := json.MarshalIndent(r, "", "  ")

		fb := tgbotapi.FileBytes{
			Name:  fmt.Sprintf("%s-%s.json", r.AnalyzerName, r.ID),
			Bytes: tr,
		}

		attachment := tgbotapi.NewDocumentUpload(input.Chat.ID, fb)
		attachment.ReplyToMessageID = input.MessageID
		attachment.Caption = buildTaxonomies(r.Taxonomies())
		c.Bot.Send(attachment)

	}

	mul.OnError = func(e error, o cortex.Observable, a *cortex.Analyzer) {
		log.Println(fmt.Sprintf("Cortex analyzer %s failed on data %s with an error: %s", a.Name, o.Description(), e.Error()))
	}

	err = mul.Do(j)
	if err != nil {
		return err
	}

	return nil
}

// buildTaxonomies joins taxonomies in one formatted string
// Every taxonomy is separated with two spaces from each other
func buildTaxonomies(txs []cortex.Taxonomy) string {
	var stats []string

	for _, t := range txs {
		stats = append(stats, fmt.Sprintf("%s:%s = %s", t.Namespace, t.Predicate, t.Value))

	}
	return strings.Join(stats, ", ")
}

// newArtifact makes an Artifact depends on its type
func newArtifact(s string, tlp int) (cortex.Observable, error) {
	var dataType string

	if valid.IsIP(s) {
		dataType = "ip"
	} else if IsDNSName(s) {
		dataType = "domain"
	} else if IsHash(s) {
		dataType = "hash"
	} else if valid.IsEmail(s) {
		dataType = "mail"
	} else if valid.IsURL(s) {
		dataType = "url"
	} else {
		dataType = "unknown"
		return nil, errors.New("Unknown data type")
	}

	j := &cortex.Task{
		Data:     s,
		DataType: dataType,
		TLP:      &tlp,
	}

	return j, nil
}

// newFileArtifactFromURL makes a FileArtifact from URL
func newFileArtifactFromURL(link string, tlp int, client *http.Client) (cortex.Observable, error) {
	req, err := http.NewRequest("GET", link, nil)
	if err != nil {
		return nil, err
	}

	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}

	return &cortex.FileTask{
		FileTaskMeta: cortex.FileTaskMeta{
			DataType: "file",
			TLP:      &tlp,
		},
		FileName: link,
		Reader:   resp.Body,
	}, nil
}
<commit_msg>Reflect uploaded file name with the analyzed file name<commit_after>package cortexbot

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/http"
	"strings"
	"time"

	valid "github.com/asaskevich/govalidator"
	"github.com/go-telegram-bot-api/telegram-bot-api"
	"gopkg.in/ilyaglow/go-cortex.v2"
)

// ProcessCortex asks Cortex about data submitted by a user
func (c *Client) ProcessCortex(input *tgbotapi.Message) error {
	var j cortex.Observable
	var err error

	if input.Document != nil {
		link, err := c.Bot.GetFileDirectURL(input.Document.FileID)
		if err != nil {
			log.Println("Can't get direct link to file")
			return err
		}

		j, err = newFileArtifactFromURL(link, input.Document.FileName, c.TLP, c.Bot.Client)
		if err != nil {
			log.Println(err.Error())
			return err
		}
	} else {
		j, err = newArtifact(input.Text, c.TLP)
		if err != nil {
			log.Println(err.Error())
			return err
		}
	}

	// Run all analyzers over it with 1 minute timeout
	mul := c.Cortex.Analyzers.NewMultiRun(context.Background(), 5*time.Minute)
	mul.OnReport = func(r *cortex.Report) {
		if r.Status == "Failure" {
			log.Printf("Analyzer %s failed with error message: %s", r.AnalyzerName, r.ReportBody.ErrorMessage)
			return
		}

		// Send JSON file with full report and taxonomies
		tr, _ := json.MarshalIndent(r, "", "  ")

		fb := tgbotapi.FileBytes{
			Name:  fmt.Sprintf("%s-%s.json", r.AnalyzerName, r.ID),
			Bytes: tr,
		}

		attachment := tgbotapi.NewDocumentUpload(input.Chat.ID, fb)
		attachment.ReplyToMessageID = input.MessageID
		attachment.Caption = buildTaxonomies(r.Taxonomies())
		c.Bot.Send(attachment)

	}

	mul.OnError = func(e error, o cortex.Observable, a *cortex.Analyzer) {
		log.Println(fmt.Sprintf("Cortex analyzer %s failed on data %s with an error: %s", a.Name, o.Description(), e.Error()))
	}

	err = mul.Do(j)
	if err != nil {
		return err
	}

	return nil
}

// buildTaxonomies joins taxonomies in one formatted string
// Every taxonomy is separated with two spaces from each other
func buildTaxonomies(txs []cortex.Taxonomy) string {
	var stats []string

	for _, t := range txs {
		stats = append(stats, fmt.Sprintf("%s:%s = %s", t.Namespace, t.Predicate, t.Value))

	}
	return strings.Join(stats, ", ")
}

// newArtifact makes an Artifact depends on its type
func newArtifact(s string, tlp int) (cortex.Observable, error) {
	var dataType string

	if valid.IsIP(s) {
		dataType = "ip"
	} else if IsDNSName(s) {
		dataType = "domain"
	} else if IsHash(s) {
		dataType = "hash"
	} else if valid.IsEmail(s) {
		dataType = "mail"
	} else if valid.IsURL(s) {
		dataType = "url"
	} else {
		dataType = "unknown"
		return nil, errors.New("Unknown data type")
	}

	j := &cortex.Task{
		Data:     s,
		DataType: dataType,
		TLP:      &tlp,
	}

	return j, nil
}

// newFileArtifactFromURL makes a FileArtifact from URL
func newFileArtifactFromURL(link string, fname string, tlp int, client *http.Client) (cortex.Observable, error) {
	req, err := http.NewRequest("GET", link, nil)
	if err != nil {
		return nil, err
	}

	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}

	return &cortex.FileTask{
		FileTaskMeta: cortex.FileTaskMeta{
			DataType: "file",
			TLP:      &tlp,
		},
		FileName: fname,
		Reader:   resp.Body,
	}, nil
}
<|endoftext|>


<commit_before>package dashboard

import (
	"log"
	"sync"
	"time"
)

var (
	defaultProjectMap map[string]*Project
	defaultProjects   = []*Project{
		newProject("jekyll", "jekyll/jekyll", "master", "jekyll"),
		newProject("jemoji", "jekyll/jemoji", "master", "jemoji"),
		newProject("mercenary", "jekyll/mercenary", "master", "mercenary"),
		newProject("jekyll-import", "jekyll/jekyll-import", "master", "jekyll-import"),
		newProject("jekyll-feed", "jekyll/jekyll-feed", "master", "jekyll-feed"),
		newProject("jekyll-seo-tag", "jekyll/jekyll-seo-tag", "master", "jekyll-seo-tag"),
		newProject("jekyll-sitemap", "jekyll/jekyll-sitemap", "master", "jekyll-sitemap"),
		newProject("jekyll-mentions", "jekyll/jekyll-mentions", "master", "jekyll-mentions"),
		newProject("jekyll-watch", "jekyll/jekyll-watch", "master", "jekyll-watch"),
		newProject("jekyll-compose", "jekyll/jekyll-compose", "master", "jekyll-compose"),
		newProject("jekyll-paginate", "jekyll/jekyll-paginate", "master", "jekyll-paginate"),
		newProject("jekyll-gist", "jekyll/jekyll-gist", "master", "jekyll-gist"),
		newProject("jekyll-coffeescript", "jekyll/jekyll-coffeescript", "master", "jekyll-coffeescript"),
		newProject("jekyll-opal", "jekyll/jekyll-opal", "master", "jekyll-opal"),
		newProject("classifier-reborn", "jekyll/classifier-reborn", "master", "classifier-reborn"),
		newProject("jekyll-sass-converter", "jekyll/jekyll-sass-converter", "master", "jekyll-sass-converter"),
		newProject("jekyll-textile-converter", "jekyll/jekyll-textile-converter", "master", "jekyll-textile-converter"),
		newProject("jekyll-redirect-from", "jekyll/jekyll-redirect-from", "master", "jekyll-redirect-from"),
		newProject("github-metadata", "jekyll/github-metadata", "master", "jekyll-github-metadata"),
		newProject("plugins website", "jekyll/plugins", "master", ""),
		newProject("jekyll-archives", "jekyll/jekyll-archives", "master", "jekyll-archives"),
		newProject("jekyll docker", "jekyll/docker", "master", ""),
	}
)

func init() {
	go resetProjectsPeriodically()
}

func resetProjectsPeriodically() {
	for range time.Tick(time.Hour / 2) {
		log.Println("resetting projects' cache")
		resetProjects()
	}
}

func resetProjects() {
	for _, p := range defaultProjects {
		p.reset()
	}
}

type Project struct {
	Name    string `json:"name"`
	Nwo     string `json:"nwo"`
	Branch  string `json:"branch"`
	GemName string `json:"gem_name"`

	Gem     *RubyGem      `json:"gem"`
	Travis  *TravisReport `json:"travis"`
	GitHub  *GitHub       `json:"github"`
	fetched bool
}

func (p *Project) fetch() {
	rubyGemChan := rubygem(p.GemName)
	travisChan := travis(p.Nwo, p.Branch)
	githubChan := github(p.Nwo)

	if p.Gem == nil {
		p.Gem = <-rubyGemChan
	}

	if p.Travis == nil {
		p.Travis = <-travisChan
	}

	if p.GitHub == nil {
		p.GitHub = <-githubChan
	}

	p.fetched = true
}

func (p *Project) reset() {
	p.fetched = false
	p.Gem = nil
	p.Travis = nil
	p.GitHub = nil
}

func buildProjectMap() {
	defaultProjectMap = map[string]*Project{}
	for _, p := range defaultProjects {
		defaultProjectMap[p.Name] = p
	}
}

func newProject(name, nwo, branch, rubygem string) *Project {
	return &Project{
		Name:    name,
		Nwo:     nwo,
		Branch:  branch,
		GemName: rubygem,
	}
}

func getProject(name string) *Project {
	if defaultProjectMap == nil {
		buildProjectMap()
	}

	if p, ok := defaultProjectMap[name]; ok {
		if !p.fetched {
			p.fetch()
		}
		return p
	}

	return nil
}

func getAllProjects() []*Project {
	var wg sync.WaitGroup
	for _, p := range defaultProjects {
		wg.Add(1)
		go func(project *Project) {
			project.fetch()
			wg.Done()
		}(p)
	}
	wg.Wait()
	return defaultProjects
}

func getProjects() []*Project {
	return defaultProjects
}
<commit_msg>Update projects<commit_after>package dashboard

import (
	"log"
	"sync"
	"time"
)

var (
	defaultProjectMap map[string]*Project
	defaultProjects   = []*Project{
		newProject("jekyll", "jekyll/jekyll", "master", "jekyll"),
    newProject("jekyll-coffeescript", "jekyll/jekyll-coffeescript", "master", "jekyll-coffeescript"),
    newProject("jekyll-feed", "jekyll/jekyll-feed", "master", "jekyll-feed"),
		newProject("jekyll-gist", "jekyll/jekyll-gist", "master", "jekyll-gist"),
    newProject("github-metadata", "jekyll/github-metadata", "master", "jekyll-github-metadata"),
    newProject("jekyll-mentions", "jekyll/jekyll-mentions", "master", "jekyll-mentions"),
    newProject("jekyll-paginate", "jekyll/jekyll-paginate", "master", "jekyll-paginate"),
    newProject("jekyll-redirect-from", "jekyll/jekyll-redirect-from", "master", "jekyll-redirect-from"),
    newProject("jekyll-sass-converter", "jekyll/jekyll-sass-converter", "master", "jekyll-sass-converter"),
    newProject("jekyll-seo-tag", "jekyll/jekyll-seo-tag", "master", "jekyll-seo-tag"),
    newProject("jekyll-sitemap", "jekyll/jekyll-sitemap", "master", "jekyll-sitemap"),
    newProject("jemoji", "jekyll/jemoji", "master", "jemoji"),
		newProject("minima", "jekyll/minima", "master", "minima"),
    newProject("jekyll-compose", "jekyll/jekyll-compose", "master", "jekyll-compose"),
		newProject("jekyll-import", "jekyll/jekyll-import", "master", "jekyll-import"),
		newProject("jekyll-archives", "jekyll/jekyll-archives", "master", "jekyll-archives"),
    newProject("jekyll-watch", "jekyll/jekyll-watch", "master", "jekyll-watch"),
		newProject("jekyll-opal", "jekyll/jekyll-opal", "master", "jekyll-opal"),
		newProject("jekyll-commonmark", "jekyll/jekyll-commonmark", "master", "jekyll-commonmark"),
		newProject("jekyll-textile-converter", "jekyll/jekyll-textile-converter", "master", "jekyll-textile-converter"),
    newProject("jekyll-admin", "jekyll/jekyll-admin", "master", "jekyll-admin"),
		newProject("classifier-reborn", "jekyll/classifier-reborn", "master", "classifier-reborn"),
		newProject("mercenary", "jekyll/mercenary", "master", "mercenary"),
		newProject("plugins website", "jekyll/plugins", "master", ""),
	}
)

func init() {
	go resetProjectsPeriodically()
}

func resetProjectsPeriodically() {
	for range time.Tick(time.Hour / 2) {
		log.Println("resetting projects' cache")
		resetProjects()
	}
}

func resetProjects() {
	for _, p := range defaultProjects {
		p.reset()
	}
}

type Project struct {
	Name    string `json:"name"`
	Nwo     string `json:"nwo"`
	Branch  string `json:"branch"`
	GemName string `json:"gem_name"`

	Gem     *RubyGem      `json:"gem"`
	Travis  *TravisReport `json:"travis"`
	GitHub  *GitHub       `json:"github"`
	fetched bool
}

func (p *Project) fetch() {
	rubyGemChan := rubygem(p.GemName)
	travisChan := travis(p.Nwo, p.Branch)
	githubChan := github(p.Nwo)

	if p.Gem == nil {
		p.Gem = <-rubyGemChan
	}

	if p.Travis == nil {
		p.Travis = <-travisChan
	}

	if p.GitHub == nil {
		p.GitHub = <-githubChan
	}

	p.fetched = true
}

func (p *Project) reset() {
	p.fetched = false
	p.Gem = nil
	p.Travis = nil
	p.GitHub = nil
}

func buildProjectMap() {
	defaultProjectMap = map[string]*Project{}
	for _, p := range defaultProjects {
		defaultProjectMap[p.Name] = p
	}
}

func newProject(name, nwo, branch, rubygem string) *Project {
	return &Project{
		Name:    name,
		Nwo:     nwo,
		Branch:  branch,
		GemName: rubygem,
	}
}

func getProject(name string) *Project {
	if defaultProjectMap == nil {
		buildProjectMap()
	}

	if p, ok := defaultProjectMap[name]; ok {
		if !p.fetched {
			p.fetch()
		}
		return p
	}

	return nil
}

func getAllProjects() []*Project {
	var wg sync.WaitGroup
	for _, p := range defaultProjects {
		wg.Add(1)
		go func(project *Project) {
			project.fetch()
			wg.Done()
		}(p)
	}
	wg.Wait()
	return defaultProjects
}

func getProjects() []*Project {
	return defaultProjects
}
<|endoftext|>


<commit_before>// Package pub is a very simple event store publisher, designed to send JSON
// events to the file system.
package pub

import (
	"crypto/rand"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
)

// Publisher is a simple event publisher, designed to publish events to the
// file system.
type Publisher struct {
	// The directory the event publisher will depost its events in. This is
	// composed of a base directory supplied upon creation of the publisher, and
	// the package-local name of the type used for the event.
	dir string

	// The type for the event that this publisher processes. Events passed to the
	// publisher need to match this type.
	eventType reflect.Type
}

// NewPublisher creates a publisher for the specific type. The events are
// published to a directory composed of the base directory specified in dir,
// and the name of the type.
//
// Any data in event is ignored - it just serves to infer the type of event
// this publisher is locked to.
func NewPublisher(dir string, event interface{}) (*Publisher, error) {
	if event == nil {
		return nil, errors.New("event cannot be nil")
	}
	p := &Publisher{
		dir:       filepath.Clean(dir) + "/" + reflect.TypeOf(event).Name(),
		eventType: reflect.TypeOf(event),
	}

	stat, err := os.Stat(p.dir)
	switch {
	case err == nil:
		if !stat.Mode().IsDir() {
			return nil, fmt.Errorf("%s exists and is not a directory", p.dir)
		}
	case err != nil && os.IsNotExist(err):
		if err := os.Mkdir(p.dir, 0777); err != nil {
			return nil, fmt.Errorf("cannot create directory %s: %s", p.dir, err)
		}
	case err != nil:
		return nil, fmt.Errorf("Could not stat dir %s: %s", p.dir, err)
	}

	return p, nil
}

// Publish publishes an event. The event is a single file in the directory,
// with a random 32-byte hex as the ID and filename. The ID is returned as a
// string.
func (p *Publisher) Publish(event interface{}) (string, error) {
	if reflect.TypeOf(event) != p.eventType {
		return "", fmt.Errorf("event of type %s does not match publisher type %s", reflect.TypeOf(event), p.eventType)
	}
	data, err := json.Marshal(event)
	if err != nil {
		return "", fmt.Errorf("could not marshal event data: %s", err)
	}
	idb := make([]byte, 32)
	if _, err := rand.Read(idb); err != nil {
		return "", fmt.Errorf("could not generate ID: %s", err)
	}
	id := fmt.Sprintf("%x", idb)

	path := p.dir + "/" + id

	if err := ioutil.WriteFile(path, data, 0666); err != nil {
		return "", fmt.Errorf("error writing event to file %s: %s", path, err)
	}

	return id, nil
}
<commit_msg>Switch ID generation to UUID, add collision error type<commit_after>// Package pub is a very simple event store publisher, designed to send JSON
// events to the file system.
package pub

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"

	"github.com/google/uuid"
)

// IDCollisionError is an error type that is returned on a UUID collision.
// This is a retryable error.
//
// While this error exists and you can check for it, the changes of it
// happening are extremely rare, as the uuid package so explains:
//
//   Randomly generated UUIDs have 122 random bits.  One's annual risk of being
//   hit by a meteorite is estimated to be one chance in 17 billion, that means
//   the probability is about 0.00000000006 (6 × 10−11), equivalent to the odds
//   of creating a few tens of trillions of UUIDs in a year and having one
//   duplicate.
type IDCollisionError struct {
	s string
}

func (e *IDCollisionError) Error() string {
	return e.s
}

// Publisher is a simple event publisher, designed to publish events to the
// file system.
type Publisher struct {
	// The directory the event publisher will depost its events in. This is
	// composed of a base directory supplied upon creation of the publisher, and
	// the package-local name of the type used for the event.
	dir string

	// The type for the event that this publisher processes. Events passed to the
	// publisher need to match this type.
	eventType reflect.Type
}

// NewPublisher creates a publisher for the specific type. The events are
// published to a directory composed of the base directory specified in dir,
// and the name of the type.
//
// Any data in event is ignored - it just serves to infer the type of event
// this publisher is locked to.
func NewPublisher(dir string, event interface{}) (*Publisher, error) {
	if event == nil {
		return nil, errors.New("event cannot be nil")
	}
	p := &Publisher{
		dir:       filepath.Clean(dir) + "/" + reflect.TypeOf(event).Name(),
		eventType: reflect.TypeOf(event),
	}

	stat, err := os.Stat(p.dir)
	switch {
	case err == nil:
		if !stat.Mode().IsDir() {
			return nil, fmt.Errorf("%s exists and is not a directory", p.dir)
		}
	case err != nil && os.IsNotExist(err):
		if err := os.Mkdir(p.dir, 0777); err != nil {
			return nil, fmt.Errorf("cannot create directory %s: %s", p.dir, err)
		}
	case err != nil:
		return nil, fmt.Errorf("Could not stat dir %s: %s", p.dir, err)
	}

	return p, nil
}

// Publish publishes an event. The event is a single file in the directory,
// with a v4 UUID as the ID and filename. The ID is returned as a string.
func (p *Publisher) Publish(event interface{}) (string, error) {
	if reflect.TypeOf(event) != p.eventType {
		return "", fmt.Errorf("event of type %s does not match publisher type %s", reflect.TypeOf(event), p.eventType)
	}
	data, err := json.Marshal(event)
	if err != nil {
		return "", fmt.Errorf("could not marshal event data: %s", err)
	}
	id, err := uuid.NewRandom()
	if err != nil {
		return "", fmt.Errorf("could not generate ID: %s", err)
	}

	path := p.dir + "/" + id.String()
	// If this path responds to stat, then the path exists in some way, shape, or
	// form, and is not valid for use. This is almost always due to a UUID
	// collision, so return IDCollisionError. If the stat failed and it is due to
	// some other error other than the file being missing, it will be caught when
	// we write to the file.
	if _, err := os.Stat(path); err == nil {
		return "", &IDCollisionError{s: fmt.Sprintf("id collision: %s", id)}
	}

	if err := ioutil.WriteFile(path, data, 0666); err != nil {
		return "", fmt.Errorf("error writing event to file %s: %s", path, err)
	}

	return id.String(), nil
}
<|endoftext|>


<commit_before>/*
 Copyright 2016 Padduck, LLC

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 	http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"github.com/pufferpanel/pufferd/environments"
	"io/ioutil"
	"os"

	"github.com/braintree/manners"
	"github.com/gin-gonic/gin"
	"github.com/pufferpanel/apufferi/config"
	"github.com/pufferpanel/apufferi/logging"
	"github.com/pufferpanel/pufferd/commands"
	"github.com/pufferpanel/pufferd/data"
	"github.com/pufferpanel/pufferd/install"
	"github.com/pufferpanel/pufferd/programs"
	"github.com/pufferpanel/pufferd/routing"
	"github.com/pufferpanel/pufferd/sftp"
	"github.com/pufferpanel/pufferd/shutdown"
	"net/http"
	"os/signal"
	"path/filepath"
	"runtime"
	"runtime/debug"
	"strings"
	"syscall"
)

var (
	VERSION      = "nightly"
	MAJORVERSION = "nightly"
	GITHASH      = "unknown"
)

var runService = true
var configPath string

func main() {
	var loggingLevel string
	var authRoot string
	var authToken string
	var runInstaller bool
	var version bool
	var license bool
	var shutdownPid int
	var runDaemon bool
	var reloadPid int
	flag.StringVar(&loggingLevel, "logging", "INFO", "Lowest logging level to display")
	flag.StringVar(&authRoot, "auth", "", "Base URL to the authorization server")
	flag.StringVar(&authToken, "token", "", "Authorization token")
	flag.BoolVar(&runInstaller, "install", false, "If installing instead of running")
	flag.BoolVar(&version, "version", false, "Get the version")
	flag.BoolVar(&license, "license", false, "View license")
	flag.StringVar(&configPath, "config", "config.json", "Path to config.json")
	flag.IntVar(&shutdownPid, "shutdown", 0, "PID to shut down")
	flag.IntVar(&reloadPid, "reload", 0, "PID to shut down")
	flag.BoolVar(&runDaemon, "run", false, "Runs the daemon")
	flag.Parse()

	versionString := fmt.Sprintf("pufferd %s (%s)", VERSION, GITHASH)

	if shutdownPid != 0 {
		logging.Info("Shutting down")
		commands.Shutdown(shutdownPid)
	}

	if reloadPid != 0 {
		logging.Info("Reloading")
		commands.Reload(reloadPid)
	}

	if version {
		fmt.Println(versionString)
	}

	if license {
		fmt.Println(data.LICENSE)
	}

	if license || version || shutdownPid != 0 || reloadPid != 0 {
		return
	}

	if _, err := os.Stat(configPath); os.IsNotExist(err) && !runInstaller {
		if _, err := os.Stat("/etc/pufferd/config.json"); err == nil {
			logging.Info("No config passed, defaulting to /etc/pufferd/config.json")
			configPath = "/etc/pufferd/config.json"
		} else {
			logging.Error("Cannot find a config file!")
			return
		}
	}

	config.Load(configPath)

	logging.SetLevelByString(loggingLevel)
	var defaultLogFolder = "logs"
	if runtime.GOOS == "linux" {
		defaultLogFolder = "/var/log/pufferd"
	}
	var logPath = config.GetStringOrDefault("logPath", defaultLogFolder)
	logging.SetLogFolder(logPath)
	logging.Init()
	gin.SetMode(gin.ReleaseMode)

	logging.Info(versionString)
	logging.Info("Logging set to " + loggingLevel)

	if runInstaller {
		install.Install(configPath, authRoot, authToken)
	}

	if runInstaller || !runDaemon {
		return
	}

	environments.LoadModules()
	programs.Initialize()

	if _, err := os.Stat(programs.TemplateFolder); os.IsNotExist(err) {
		logging.Info("No template directory found, creating")
		err = os.MkdirAll(programs.TemplateFolder, 0755)
		if err != nil {
			logging.Error("Error creating template folder", err)
		}

	}

	if _, err := os.Stat(programs.ServerFolder); os.IsNotExist(err) {
		logging.Info("No server directory found, creating")
		err = os.MkdirAll(programs.ServerFolder, 0755)
		if err != nil {
			logging.Error("Error creating server folder directory", err)
			return
		}
	}

	//check if there's an update
	go CheckForUpdate()

	programs.LoadFromFolder()

	programs.InitService()

	for _, element := range programs.GetAll() {
		if element.IsEnabled() {
			element.GetEnvironment().DisplayToConsole("Daemon has been started\n")
			if element.IsAutoStart() {
				logging.Info("Queued server " + element.Id())
				element.GetEnvironment().DisplayToConsole("Server has been queued to start\n")
				programs.StartViaService(element)
			}
		}
	}

	defer recoverPanic()

	CreateHook()

	for runService {
		runServices()
	}

	shutdown.Shutdown()
}

func runServices() {
	r := routing.ConfigureWeb()

	useHttps := false

	dataFolder := config.GetStringOrDefault("dataFolder", "data")
	httpsPem := filepath.Join(dataFolder, "https.pem")
	httpsKey := filepath.Join(dataFolder, "https.key")

	if _, err := os.Stat(httpsPem); os.IsNotExist(err) {
		logging.Warn("No HTTPS.PEM found in data folder, will use http instead")
	} else if _, err := os.Stat(httpsKey); os.IsNotExist(err) {
		logging.Warn("No HTTPS.KEY found in data folder, will use http instead")
	} else {
		useHttps = true
	}

	sftp.Run()

	web := config.GetStringOrDefault("web", config.GetStringOrDefault("webHost", "0.0.0.0")+":"+config.GetStringOrDefault("webPort", "5656"))

	logging.Infof("Starting web access on %s", web)
	var err error
	if useHttps {
		err = manners.ListenAndServeTLS(web, httpsPem, httpsKey, r)
	} else {
		err = manners.ListenAndServe(web, r)
	}
	if err != nil {
		logging.Error("Error starting web service", err)
	}
}

func CreateHook() {
	c := make(chan os.Signal, 1)
	signal.Notify(c, syscall.Signal(15), syscall.Signal(1))
	go func() {
		defer func() {
			if err := recover(); err != nil {
				logging.Errorf("Error: %+v\n%s", err, debug.Stack())
			}
		}()

		var sig os.Signal

		for sig != syscall.Signal(15) {
			sig = <-c
			switch sig {
			case syscall.Signal(1):
				manners.Close()
				sftp.Stop()
				config.Load(configPath)
			}
		}

		runService = false
		shutdown.CompleteShutdown()
	}()
}

func CheckForUpdate() {
	if config.GetBoolOrDefault("update-check", true) {
		url := "https://dl.pufferpanel.com/pufferd/" + MAJORVERSION + "/version.txt"
		logging.Debug("Checking for updates using " + url)
		resp, err := http.Get(url)
		if err != nil {
			return
		}
		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return
		}
		onlineVersion := strings.TrimSpace(string(body))
		if string(onlineVersion) != GITHASH {
			logging.Infof("DL server reports a different hash than this version, an update may be available")
			logging.Infof("Installed: %s", GITHASH)
			logging.Infof("Online: %s", onlineVersion)
		}
	}
}

func recoverPanic() {
	if rec := recover(); rec != nil {
		err := rec.(error)
		logging.Critical("Unhandled error", err)
	}
}<commit_msg>Remove update checker Since we switched to packages, this check is no longer required<commit_after>/*
 Copyright 2016 Padduck, LLC

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 	http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"github.com/pufferpanel/pufferd/environments"
	"os"

	"github.com/braintree/manners"
	"github.com/gin-gonic/gin"
	"github.com/pufferpanel/apufferi/config"
	"github.com/pufferpanel/apufferi/logging"
	"github.com/pufferpanel/pufferd/commands"
	"github.com/pufferpanel/pufferd/data"
	"github.com/pufferpanel/pufferd/install"
	"github.com/pufferpanel/pufferd/programs"
	"github.com/pufferpanel/pufferd/routing"
	"github.com/pufferpanel/pufferd/sftp"
	"github.com/pufferpanel/pufferd/shutdown"
	"os/signal"
	"path/filepath"
	"runtime"
	"runtime/debug"
	"syscall"
)

var (
	VERSION      = "nightly"
	MAJORVERSION = "nightly"
	GITHASH      = "unknown"
)

var runService = true
var configPath string

func main() {
	var loggingLevel string
	var authRoot string
	var authToken string
	var runInstaller bool
	var version bool
	var license bool
	var shutdownPid int
	var runDaemon bool
	var reloadPid int
	flag.StringVar(&loggingLevel, "logging", "INFO", "Lowest logging level to display")
	flag.StringVar(&authRoot, "auth", "", "Base URL to the authorization server")
	flag.StringVar(&authToken, "token", "", "Authorization token")
	flag.BoolVar(&runInstaller, "install", false, "If installing instead of running")
	flag.BoolVar(&version, "version", false, "Get the version")
	flag.BoolVar(&license, "license", false, "View license")
	flag.StringVar(&configPath, "config", "config.json", "Path to config.json")
	flag.IntVar(&shutdownPid, "shutdown", 0, "PID to shut down")
	flag.IntVar(&reloadPid, "reload", 0, "PID to shut down")
	flag.BoolVar(&runDaemon, "run", false, "Runs the daemon")
	flag.Parse()

	versionString := fmt.Sprintf("pufferd %s (%s)", VERSION, GITHASH)

	if shutdownPid != 0 {
		logging.Info("Shutting down")
		commands.Shutdown(shutdownPid)
	}

	if reloadPid != 0 {
		logging.Info("Reloading")
		commands.Reload(reloadPid)
	}

	if version {
		fmt.Println(versionString)
	}

	if license {
		fmt.Println(data.LICENSE)
	}

	if license || version || shutdownPid != 0 || reloadPid != 0 {
		return
	}

	if _, err := os.Stat(configPath); os.IsNotExist(err) && !runInstaller {
		if _, err := os.Stat("/etc/pufferd/config.json"); err == nil {
			logging.Info("No config passed, defaulting to /etc/pufferd/config.json")
			configPath = "/etc/pufferd/config.json"
		} else {
			logging.Error("Cannot find a config file!")
			return
		}
	}

	config.Load(configPath)

	logging.SetLevelByString(loggingLevel)
	var defaultLogFolder = "logs"
	if runtime.GOOS == "linux" {
		defaultLogFolder = "/var/log/pufferd"
	}
	var logPath = config.GetStringOrDefault("logPath", defaultLogFolder)
	logging.SetLogFolder(logPath)
	logging.Init()
	gin.SetMode(gin.ReleaseMode)

	logging.Info(versionString)
	logging.Info("Logging set to " + loggingLevel)

	if runInstaller {
		install.Install(configPath, authRoot, authToken)
	}

	if runInstaller || !runDaemon {
		return
	}

	environments.LoadModules()
	programs.Initialize()

	if _, err := os.Stat(programs.TemplateFolder); os.IsNotExist(err) {
		logging.Info("No template directory found, creating")
		err = os.MkdirAll(programs.TemplateFolder, 0755)
		if err != nil {
			logging.Error("Error creating template folder", err)
		}

	}

	if _, err := os.Stat(programs.ServerFolder); os.IsNotExist(err) {
		logging.Info("No server directory found, creating")
		err = os.MkdirAll(programs.ServerFolder, 0755)
		if err != nil {
			logging.Error("Error creating server folder directory", err)
			return
		}
	}

	programs.LoadFromFolder()

	programs.InitService()

	for _, element := range programs.GetAll() {
		if element.IsEnabled() {
			element.GetEnvironment().DisplayToConsole("Daemon has been started\n")
			if element.IsAutoStart() {
				logging.Info("Queued server " + element.Id())
				element.GetEnvironment().DisplayToConsole("Server has been queued to start\n")
				programs.StartViaService(element)
			}
		}
	}

	defer recoverPanic()

	CreateHook()

	for runService {
		runServices()
	}

	shutdown.Shutdown()
}

func runServices() {
	r := routing.ConfigureWeb()

	useHttps := false

	dataFolder := config.GetStringOrDefault("dataFolder", "data")
	httpsPem := filepath.Join(dataFolder, "https.pem")
	httpsKey := filepath.Join(dataFolder, "https.key")

	if _, err := os.Stat(httpsPem); os.IsNotExist(err) {
		logging.Warn("No HTTPS.PEM found in data folder, will use http instead")
	} else if _, err := os.Stat(httpsKey); os.IsNotExist(err) {
		logging.Warn("No HTTPS.KEY found in data folder, will use http instead")
	} else {
		useHttps = true
	}

	sftp.Run()

	web := config.GetStringOrDefault("web", config.GetStringOrDefault("webHost", "0.0.0.0")+":"+config.GetStringOrDefault("webPort", "5656"))

	logging.Infof("Starting web access on %s", web)
	var err error
	if useHttps {
		err = manners.ListenAndServeTLS(web, httpsPem, httpsKey, r)
	} else {
		err = manners.ListenAndServe(web, r)
	}
	if err != nil {
		logging.Error("Error starting web service", err)
	}
}

func CreateHook() {
	c := make(chan os.Signal, 1)
	signal.Notify(c, syscall.Signal(15), syscall.Signal(1))
	go func() {
		defer func() {
			if err := recover(); err != nil {
				logging.Errorf("Error: %+v\n%s", err, debug.Stack())
			}
		}()

		var sig os.Signal

		for sig != syscall.Signal(15) {
			sig = <-c
			switch sig {
			case syscall.Signal(1):
				manners.Close()
				sftp.Stop()
				config.Load(configPath)
			}
		}

		runService = false
		shutdown.CompleteShutdown()
	}()
}

func recoverPanic() {
	if rec := recover(); rec != nil {
		err := rec.(error)
		logging.Critical("Unhandled error", err)
	}
}<|endoftext|>


<commit_before>// Use cgo to interface with nflog
//
// Docs: http://www.netfilter.org/projects/libnetfilter_log/doxygen/index.html
//
// Debian packages needed:
//   apt-get install iptables-dev linux-libc-dev libnetfilter-log-dev

package main

import (
	"log"
	"net"
	"reflect"
	"syscall"
	"unsafe"
)

/*
#cgo LDFLAGS: -lnetfilter_log
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <stdlib.h>
#include <libnetfilter_log/libnetfilter_log.h>
#include <inttypes.h>

// A record of each packet
typedef struct {
	char *payload;
	int payload_len;
	u_int32_t seq;
} packet;

// Max number of packets to collect at once
#define MAX_PACKETS (16*1024)

// A load of packets with count
typedef struct {
	int index;
	packet pkt[MAX_PACKETS];
} packets;

// Process the incoming packet putting pointers to the data to be handled by Go
static int _processPacket(struct nflog_g_handle *gh, struct nfgenmsg *nfmsg, struct nflog_data *nfd, void *data) {
	packets *ps = (packets *)data;
	if (ps->index >= MAX_PACKETS) {
		return 1;
	}
	packet *p = &ps->pkt[ps->index++];
	p->payload = 0;
	p->payload_len = nflog_get_payload(nfd, &p->payload);
	p->seq = 0;
	nflog_get_seq(nfd, &p->seq);
	return 0;
 }

// Register the callback - can't be done from Go
//
// We have to register a C function _processPacket
static int _callback_register(struct nflog_g_handle *gh, packets *data) {
	return nflog_callback_register(gh, _processPacket, data);
}
*/
import "C"

const (
	RecvBufferSize  = 4 * 1024 * 1024
	NflogBufferSize = 128 * 1024 // Must be <= 128k (checked in kernel source)
	NflogTimeout    = 100        // Timeout before sending data in 1/100th second
	MaxQueueLogs    = C.MAX_PACKETS - 1
)

// NfLog
type NfLog struct {
	// Main nflog_handle
	h *C.struct_nflog_handle
	// File descriptor for socket operations
	fd C.int
	// Group handle
	gh *C.struct_nflog_g_handle
	// The multicast address
	McastGroup int
	// The next expected sequence number
	seq uint32
	// Errors
	errors int64
	// Flavour of IP we are expecting, 4 or 6
	IpVersion byte
	// Mask for the IP
	Mask net.IPMask
	// Whether to apply the mask or not
	UseMask bool
	// Are we account the source or the destination address
	Direction IpDirection
	// Flavour of IP packet we are decoding
	IpPacket *IpPacketInfo
	// Accounting
	a *Accounting
	// Quit the loop
	quit chan struct{}
	// Buffer for accumulated packets
	addPackets []AddPacket
	// Index of this in nflogs - uses to pass to C instead of a Go pointer
	index int
	// Pointer to the packets
	packets *C.packets
}

// An array of NfLog pointers
type NfLogs [16]*NfLog

// A global array of all NfLogs in use
var nflogs NfLogs

// Count all active NfLogs
func (ns NfLogs) Count() int {
	active := 0
	for _, nflog := range ns {
		if nflog != nil {
			active++
		}
	}
	return active
}

// Stop all active NfLogs
func (ns NfLogs) Stop() {
	for _, nflog := range ns {
		if nflog != nil {
			nflog.Close()
		}
	}
}

// Create a new NfLog
//
// McastGroup is that specified in ip[6]tables
// IPv6 is a flag to say if it is IPv6 or not
// Direction is to monitor the source address or the dest address
func NewNfLog(McastGroup int, IpVersion byte, Direction IpDirection, MaskBits int, a *Accounting) *NfLog {
	h := C.nflog_open()
	if h == nil {
		log.Fatalf("Failed to open NFLOG: %s", strerror())
	}
	if *Verbose {
		log.Println("Binding nfnetlink_log to AF_INET")
	}
	if C.nflog_bind_pf(h, C.AF_INET) < 0 {
		log.Fatalf("nflog_bind_pf failed: %s", strerror())
	}

	nflog := &NfLog{
		h:          h,
		fd:         C.nflog_fd(h),
		McastGroup: McastGroup,
		IpVersion:  IpVersion,
		Direction:  Direction,
		a:          a,
		quit:       make(chan struct{}),
		packets:    (*C.packets)(C.malloc(C.sizeof_packets)),
	}
	for i := range nflogs {
		if nflogs[i] == nil {
			nflog.index = i
			nflogs[i] = nflog
			goto found
		}
	}
	log.Fatal("Too many filters")
found:
	switch IpVersion {
	case 4:
		nflog.IpPacket = Ip4Packet
	case 6:
		nflog.IpPacket = Ip6Packet
	default:
		log.Fatalf("Bad IP version %d", IpVersion)
	}
	addrBits := 8 * nflog.IpPacket.AddrLen
	nflog.UseMask = MaskBits < addrBits
	nflog.Mask = net.CIDRMask(MaskBits, addrBits)
	nflog.makeGroup(McastGroup, nflog.IpPacket.HeaderSize)
	// Start the background process
	go nflog.Loop()
	return nflog
}

// Process a packet
func (nflog *NfLog) ProcessPacket(packet []byte, seq uint32) AddPacket {
	// log.Printf("Packet %#v", packet)
	// Peek the IP Version out of the header
	ip_version := packet[IpVersion] >> IpVersionShift & IpVersionMask
	// log.Printf("Received %d: size %d, IPv%d", seq, payload_len, ip_version)
	if seq != 0 && seq != nflog.seq {
		nflog.errors++
		log.Printf("%d missing packets detected, %d to %d", seq-nflog.seq, seq, nflog.seq)
	}
	nflog.seq = seq + 1
	if ip_version != nflog.IpVersion {
		nflog.errors++
		log.Printf("Bad IP version: %d", ip_version)
		return AddPacket{Length: -1}
	}
	i := nflog.IpPacket
	if len(packet) < i.HeaderSize {
		nflog.errors++
		log.Printf("Short IPv%d packet %d/%d bytes", ip_version, len(packet), i.HeaderSize)
		return AddPacket{Length: -1}
	}

	var addr net.IP
	if nflog.Direction {
		addr = i.Src(packet)
	} else {
		addr = i.Dst(packet)
	}

	// Mask the address
	if nflog.UseMask {
		addr = addr.Mask(nflog.Mask)
	}

	return AddPacket{
		Direction: nflog.Direction,
		Addr:      string(addr),
		Length:    i.Length(packet),
	}
}

// Receive data from nflog stored in nflog.packets
func (nflog *NfLog) processPackets(addPackets []AddPacket) []AddPacket {
	n := int(nflog.packets.index)
	if n >= C.MAX_PACKETS {
		log.Printf("Packets buffer overflowed")
	}
	if *Verbose {
		log.Printf("%d: Processing %d packets", nflog.McastGroup, n)
	}

	var packet []byte
	sliceHeader := (*reflect.SliceHeader)((unsafe.Pointer(&packet)))

	for i := 0; i < n; i++ {
		p := &nflog.packets.pkt[i]

		// Get the packet into a []byte
		// NB if the C data goes away then BAD things will happen!
		// So don't keep slices from this after returning from this function
		sliceHeader.Cap = int(p.payload_len)
		sliceHeader.Len = int(p.payload_len)
		sliceHeader.Data = uintptr(unsafe.Pointer(p.payload))

		// Process the packet
		newAddPacket := nflog.ProcessPacket(packet, uint32(p.seq))
		if newAddPacket.Length >= 0 {
			addPackets = append(addPackets, newAddPacket)
		}
	}
	sliceHeader = nil
	packet = nil
	return addPackets
}

// Current nflog error
func strerror() error {
	return syscall.Errno(C.nflog_errno)
}

// Connects to the group specified with the size
func (nflog *NfLog) makeGroup(group, size int) {
	if *Verbose {
		log.Printf("Binding this socket to group %d", group)
	}
	gh := C.nflog_bind_group(nflog.h, (C.u_int16_t)(group))
	if gh == nil {
		log.Fatalf("nflog_bind_group failed: %s", strerror())
	}
	nflog.gh = gh

	// Set the maximum amount of logs in buffer for this group
	if C.nflog_set_qthresh(gh, MaxQueueLogs) < 0 {
		log.Fatalf("nflog_set_qthresh failed: %s", strerror())
	}

	// Set local sequence numbering to detect missing packets
	if C.nflog_set_flags(gh, C.NFULNL_CFG_F_SEQ) < 0 {
		log.Fatalf("nflog_set_flags failed: %s", strerror())
	}

	// Set buffer size large
	if C.nflog_set_nlbufsiz(gh, NflogBufferSize) < 0 {
		log.Fatalf("nflog_set_nlbufsiz: %s", strerror())
	}

	// Set timeout
	if C.nflog_set_timeout(gh, NflogTimeout) < 0 {
		log.Fatalf("nflog_set_timeout: %s", strerror())
	}

	if *Verbose {
		log.Printf("Setting copy_packet mode to %d bytes", size)
	}
	if C.nflog_set_mode(gh, C.NFULNL_COPY_PACKET, (C.uint)(size)) < 0 {
		log.Fatalf("nflog_set_mode failed: %s", strerror())
	}

	// Register the callback now we are set up
	//
	// Note that we pass a block of memory allocated by C.malloc -
	// it isn't a good idea for C to hold pointers to go objects
	// which might move
	C._callback_register(gh, nflog.packets)
}

// Receive packets in a loop until quit
func (nflog *NfLog) Loop() {
	buflen := C.size_t(RecvBufferSize)
	pbuf := C.malloc(buflen)
	if pbuf == nil {
		log.Fatal("No memory for malloc")
	}
	defer C.free(pbuf)
	for {
		nr := C.recv(nflog.fd, pbuf, buflen, 0)
		select {
		case <-nflog.quit:
			return
		default:
		}
		if nr < 0 {
			log.Printf("Recvfrom failed: %s", strerror())
			nflog.errors++
		} else {
			// Handle messages in packet reusing memory
			ps := <-nflog.a.returnAddPackets
			nflog.packets.index = 0
			C.nflog_handle_packet(nflog.h, (*C.char)(pbuf), (C.int)(nr))
			nflog.a.processAddPackets <- nflog.processPackets(ps[:0])
		}
	}

}

// Close the NfLog down
func (nflog *NfLog) Close() {
	close(nflog.quit)
	// Sometimes hangs and doesn't seem to be necessary
	// if *Verbose {
	// 	log.Printf("Unbinding socket %d from group %d", nflog.fd, nflog.McastGroup)
	// }
	// if C.nflog_unbind_group(nflog.gh) < 0 {
	// 	log.Printf("nflog_unbind_group(%d) failed: %s", nflog.McastGroup, strerror())
	// }
	if *Verbose {
		log.Printf("Closing nflog socket %d group %d", nflog.fd, nflog.McastGroup)
	}
	if C.nflog_close(nflog.h) < 0 {
		log.Printf("nflog_close failed: %s", strerror())
	}
	// Mark this index as no longer in use
	nflogs[nflog.index] = nil
	C.free(unsafe.Pointer(nflog.packets))
}
<commit_msg>Improve error reporting from C<commit_after>// Use cgo to interface with nflog
//
// Docs: http://www.netfilter.org/projects/libnetfilter_log/doxygen/index.html
//
// Debian packages needed:
//   apt-get install iptables-dev linux-libc-dev libnetfilter-log-dev

package main

import (
	"log"
	"net"
	"reflect"
	"syscall"
	"unsafe"
)

/*
#cgo LDFLAGS: -lnetfilter_log
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <stdlib.h>
#include <libnetfilter_log/libnetfilter_log.h>
#include <inttypes.h>

// A record of each packet
typedef struct {
	char *payload;
	int payload_len;
	u_int32_t seq;
} packet;

// Max number of packets to collect at once
#define MAX_PACKETS (16*1024)

// A load of packets with count
typedef struct {
	int index;
	packet pkt[MAX_PACKETS];
} packets;

// Process the incoming packet putting pointers to the data to be handled by Go
static int _processPacket(struct nflog_g_handle *gh, struct nfgenmsg *nfmsg, struct nflog_data *nfd, void *data) {
	packets *ps = (packets *)data;
	if (ps->index >= MAX_PACKETS) {
		return 1;
	}
	packet *p = &ps->pkt[ps->index++];
	p->payload = 0;
	p->payload_len = nflog_get_payload(nfd, &p->payload);
	p->seq = 0;
	nflog_get_seq(nfd, &p->seq);
	return 0;
 }

// Register the callback - can't be done from Go
//
// We have to register a C function _processPacket
static int _callback_register(struct nflog_g_handle *gh, packets *data) {
	return nflog_callback_register(gh, _processPacket, data);
}
*/
import "C"

const (
	RecvBufferSize  = 4 * 1024 * 1024
	NflogBufferSize = 128 * 1024 // Must be <= 128k (checked in kernel source)
	NflogTimeout    = 100        // Timeout before sending data in 1/100th second
	MaxQueueLogs    = C.MAX_PACKETS - 1
)

// NfLog
type NfLog struct {
	// Main nflog_handle
	h *C.struct_nflog_handle
	// File descriptor for socket operations
	fd C.int
	// Group handle
	gh *C.struct_nflog_g_handle
	// The multicast address
	McastGroup int
	// The next expected sequence number
	seq uint32
	// Errors
	errors int64
	// Flavour of IP we are expecting, 4 or 6
	IpVersion byte
	// Mask for the IP
	Mask net.IPMask
	// Whether to apply the mask or not
	UseMask bool
	// Are we account the source or the destination address
	Direction IpDirection
	// Flavour of IP packet we are decoding
	IpPacket *IpPacketInfo
	// Accounting
	a *Accounting
	// Quit the loop
	quit chan struct{}
	// Buffer for accumulated packets
	addPackets []AddPacket
	// Index of this in nflogs - uses to pass to C instead of a Go pointer
	index int
	// Pointer to the packets
	packets *C.packets
}

// An array of NfLog pointers
type NfLogs [16]*NfLog

// A global array of all NfLogs in use
var nflogs NfLogs

// Count all active NfLogs
func (ns NfLogs) Count() int {
	active := 0
	for _, nflog := range ns {
		if nflog != nil {
			active++
		}
	}
	return active
}

// Stop all active NfLogs
func (ns NfLogs) Stop() {
	for _, nflog := range ns {
		if nflog != nil {
			nflog.Close()
		}
	}
}

// Create a new NfLog
//
// McastGroup is that specified in ip[6]tables
// IPv6 is a flag to say if it is IPv6 or not
// Direction is to monitor the source address or the dest address
func NewNfLog(McastGroup int, IpVersion byte, Direction IpDirection, MaskBits int, a *Accounting) *NfLog {
	h, err := C.nflog_open()
	if h == nil || err != nil {
		log.Fatalf("Failed to open NFLOG: %s", nflogError(err))
	}
	if *Verbose {
		log.Println("Binding nfnetlink_log to AF_INET")
	}
	if rc, err := C.nflog_bind_pf(h, C.AF_INET); rc < 0 || err != nil {
		log.Fatalf("nflog_bind_pf failed: %s", nflogError(err))
	}

	nflog := &NfLog{
		h:          h,
		fd:         C.nflog_fd(h),
		McastGroup: McastGroup,
		IpVersion:  IpVersion,
		Direction:  Direction,
		a:          a,
		quit:       make(chan struct{}),
		packets:    (*C.packets)(C.malloc(C.sizeof_packets)),
	}
	for i := range nflogs {
		if nflogs[i] == nil {
			nflog.index = i
			nflogs[i] = nflog
			goto found
		}
	}
	log.Fatal("Too many filters")
found:
	switch IpVersion {
	case 4:
		nflog.IpPacket = Ip4Packet
	case 6:
		nflog.IpPacket = Ip6Packet
	default:
		log.Fatalf("Bad IP version %d", IpVersion)
	}
	addrBits := 8 * nflog.IpPacket.AddrLen
	nflog.UseMask = MaskBits < addrBits
	nflog.Mask = net.CIDRMask(MaskBits, addrBits)
	nflog.makeGroup(McastGroup, nflog.IpPacket.HeaderSize)
	// Start the background process
	go nflog.Loop()
	return nflog
}

// Process a packet
func (nflog *NfLog) ProcessPacket(packet []byte, seq uint32) AddPacket {
	// log.Printf("Packet %#v", packet)
	// Peek the IP Version out of the header
	ip_version := packet[IpVersion] >> IpVersionShift & IpVersionMask
	// log.Printf("Received %d: size %d, IPv%d", seq, payload_len, ip_version)
	if seq != 0 && seq != nflog.seq {
		nflog.errors++
		log.Printf("%d missing packets detected, %d to %d", seq-nflog.seq, seq, nflog.seq)
	}
	nflog.seq = seq + 1
	if ip_version != nflog.IpVersion {
		nflog.errors++
		log.Printf("Bad IP version: %d", ip_version)
		return AddPacket{Length: -1}
	}
	i := nflog.IpPacket
	if len(packet) < i.HeaderSize {
		nflog.errors++
		log.Printf("Short IPv%d packet %d/%d bytes", ip_version, len(packet), i.HeaderSize)
		return AddPacket{Length: -1}
	}

	var addr net.IP
	if nflog.Direction {
		addr = i.Src(packet)
	} else {
		addr = i.Dst(packet)
	}

	// Mask the address
	if nflog.UseMask {
		addr = addr.Mask(nflog.Mask)
	}

	return AddPacket{
		Direction: nflog.Direction,
		Addr:      string(addr),
		Length:    i.Length(packet),
	}
}

// Receive data from nflog stored in nflog.packets
func (nflog *NfLog) processPackets(addPackets []AddPacket) []AddPacket {
	n := int(nflog.packets.index)
	if n >= C.MAX_PACKETS {
		log.Printf("Packets buffer overflowed")
	}
	if *Verbose {
		log.Printf("%d: Processing %d packets", nflog.McastGroup, n)
	}

	var packet []byte
	sliceHeader := (*reflect.SliceHeader)((unsafe.Pointer(&packet)))

	for i := 0; i < n; i++ {
		p := &nflog.packets.pkt[i]

		// Get the packet into a []byte
		// NB if the C data goes away then BAD things will happen!
		// So don't keep slices from this after returning from this function
		sliceHeader.Cap = int(p.payload_len)
		sliceHeader.Len = int(p.payload_len)
		sliceHeader.Data = uintptr(unsafe.Pointer(p.payload))

		// Process the packet
		newAddPacket := nflog.ProcessPacket(packet, uint32(p.seq))
		if newAddPacket.Length >= 0 {
			addPackets = append(addPackets, newAddPacket)
		}
	}
	sliceHeader = nil
	packet = nil
	return addPackets
}

// Current nflog error
func nflogError(err error) error {
	if C.nflog_errno != 0 {
		return syscall.Errno(C.nflog_errno)
	}
	return err
}

// Connects to the group specified with the size
func (nflog *NfLog) makeGroup(group, size int) {
	if *Verbose {
		log.Printf("Binding this socket to group %d", group)
	}
	gh, err := C.nflog_bind_group(nflog.h, (C.u_int16_t)(group))
	if gh == nil || err != nil {
		log.Fatalf("nflog_bind_group failed: %s", nflogError(err))
	}
	nflog.gh = gh

	// Set the maximum amount of logs in buffer for this group
	if rc, err := C.nflog_set_qthresh(gh, MaxQueueLogs); rc < 0 || err != nil {
		log.Fatalf("nflog_set_qthresh failed: %s", nflogError(err))
	}

	// Set local sequence numbering to detect missing packets
	if rc, err := C.nflog_set_flags(gh, C.NFULNL_CFG_F_SEQ); rc < 0 || err != nil {
		log.Fatalf("nflog_set_flags failed: %s", nflogError(err))
	}

	// Set buffer size large
	if rc, err := C.nflog_set_nlbufsiz(gh, NflogBufferSize); rc < 0 || err != nil {
		log.Fatalf("nflog_set_nlbufsiz: %s", nflogError(err))
	}

	// Set timeout
	if rc, err := C.nflog_set_timeout(gh, NflogTimeout); rc < 0 || err != nil {
		log.Fatalf("nflog_set_timeout: %s", nflogError(err))
	}

	if *Verbose {
		log.Printf("Setting copy_packet mode to %d bytes", size)
	}
	if rc, err := C.nflog_set_mode(gh, C.NFULNL_COPY_PACKET, (C.uint)(size)); rc < 0 || err != nil {
		log.Fatalf("nflog_set_mode failed: %s", nflogError(err))
	}

	// Register the callback now we are set up
	//
	// Note that we pass a block of memory allocated by C.malloc -
	// it isn't a good idea for C to hold pointers to go objects
	// which might move
	C._callback_register(gh, nflog.packets)
}

// Receive packets in a loop until quit
func (nflog *NfLog) Loop() {
	buflen := C.size_t(RecvBufferSize)
	pbuf := C.malloc(buflen)
	if pbuf == nil {
		log.Fatal("No memory for malloc")
	}
	defer C.free(pbuf)
	for {
		nr, err := C.recv(nflog.fd, pbuf, buflen, 0)
		select {
		case <-nflog.quit:
			return
		default:
		}
		if nr < 0 || err != nil {
			log.Printf("Recv failed: %s", err)
			nflog.errors++
		} else {
			// Handle messages in packet reusing memory
			ps := <-nflog.a.returnAddPackets
			nflog.packets.index = 0
			C.nflog_handle_packet(nflog.h, (*C.char)(pbuf), (C.int)(nr))
			nflog.a.processAddPackets <- nflog.processPackets(ps[:0])
		}
	}

}

// Close the NfLog down
func (nflog *NfLog) Close() {
	close(nflog.quit)
	// Sometimes hangs and doesn't seem to be necessary
	// if *Verbose {
	// 	log.Printf("Unbinding socket %d from group %d", nflog.fd, nflog.McastGroup)
	// }
	// if rc, err := C.nflog_unbind_group(nflog.gh); rc < 0 || err != nil {
	// 	log.Printf("nflog_unbind_group(%d) failed: %s", nflog.McastGroup, nflogError(err))
	// }
	if *Verbose {
		log.Printf("Closing nflog socket %d group %d", nflog.fd, nflog.McastGroup)
	}
	if rc, err := C.nflog_close(nflog.h); rc < 0 || err != nil {
		log.Printf("nflog_close failed: %s", nflogError(nil))
	}
	// Mark this index as no longer in use
	nflogs[nflog.index] = nil
	C.free(unsafe.Pointer(nflog.packets))
}
<|endoftext|>


<commit_before>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package restore

import (
	"fmt"
	"io/ioutil"
	"log"
	"testing"

	"golang.org/x/net/context"

	"github.com/jacobsa/comeback/internal/blob"
	"github.com/jacobsa/comeback/internal/dag"
	"github.com/jacobsa/comeback/internal/fs"
	"github.com/jacobsa/comeback/internal/repr"
	"github.com/jacobsa/comeback/internal/util"
	"github.com/jacobsa/comeback/internal/wiring"
	"github.com/jacobsa/gcloud/gcs/gcsfake"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"github.com/jacobsa/timeutil"
)

func TestDependencyResolver(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func convertNodes(dagNodes []dag.Node) (nodes []*node) {
	for _, n := range dagNodes {
		nodes = append(nodes, n.(*node))
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type DependencyResolverTest struct {
	ctx       context.Context
	blobStore blob.Store
	dr        dag.DependencyResolver
}

var _ SetUpInterface = &DependencyResolverTest{}

func init() { RegisterTestSuite(&DependencyResolverTest{}) }

func (t *DependencyResolverTest) SetUp(ti *TestInfo) {
	var err error
	t.ctx = ti.Ctx

	// Create the blob store.
	bucket := gcsfake.NewFakeBucket(timeutil.RealClock(), "some_bucket")

	_, crypter, err := wiring.MakeRegistryAndCrypter(t.ctx, "password", bucket)
	AssertEq(nil, err)

	t.blobStore, err = wiring.MakeBlobStore(bucket, crypter, util.NewStringSet())
	AssertEq(nil, err)

	// Create the dependency resolver.
	t.dr = newDependencyResolver(t.blobStore, log.New(ioutil.Discard, "", 0))
}

func (t *DependencyResolverTest) call(n *node) (deps []*node, err error) {
	untyped, err := t.dr.FindDependencies(t.ctx, n)
	if err != nil {
		err = fmt.Errorf("FindDependencies: %v", err)
		return
	}

	for _, u := range untyped {
		deps = append(deps, u.(*node))
	}

	return
}

func (t *DependencyResolverTest) store(b []byte) (s blob.Score, err error) {
	s, err = t.blobStore.Store(
		t.ctx,
		&blob.StoreRequest{Blob: b})

	return
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *DependencyResolverTest) File() {
	node := &node{
		Info: fs.DirectoryEntry{
			Type: fs.TypeFile,
		},
	}

	// Call
	deps, err := t.call(node)

	AssertEq(nil, err)
	ExpectThat(deps, ElementsAre())
	ExpectThat(node.Children, ElementsAre())
}

func (t *DependencyResolverTest) Symlink() {
	node := &node{
		Info: fs.DirectoryEntry{
			Type: fs.TypeSymlink,
		},
	}

	// Call
	deps, err := t.call(node)

	AssertEq(nil, err)
	ExpectThat(deps, ElementsAre())
	ExpectThat(node.Children, ElementsAre())
}

func (t *DependencyResolverTest) BlobMissing() {
	s := blob.ComputeScore([]byte(""))
	node := &node{
		Info: fs.DirectoryEntry{
			Type:   fs.TypeDirectory,
			Scores: []blob.Score{s},
		},
	}

	// Call
	_, err := t.call(node)

	ExpectThat(err, Error(HasSubstr("TODO")))
	ExpectThat(err, Error(HasSubstr(s.Hex())))
}

func (t *DependencyResolverTest) BlobCorrupted() {
	var err error

	// Store some junk and set up a node with the junk's score as its contents.
	junk, err := t.store([]byte("foobar"))
	AssertEq(nil, err)

	node := &node{
		Info: fs.DirectoryEntry{
			Type:   fs.TypeDirectory,
			Scores: []blob.Score{junk},
		},
	}

	// Call
	_, err = t.call(node)

	ExpectThat(err, Error(HasSubstr("UnmarshalDir")))
	ExpectThat(err, Error(HasSubstr(junk.Hex())))
}

func (t *DependencyResolverTest) NoChildren() {
	var err error

	// Set up an empty listing.
	listing := []*fs.DirectoryEntry{}

	serialized, err := repr.MarshalDir(listing)
	AssertEq(nil, err)

	score, err := t.store(serialized)
	AssertEq(nil, err)

	// Set up the node.
	node := &node{
		RelPath: "taco/burrito",
		Info: fs.DirectoryEntry{
			Type:   fs.TypeDirectory,
			Scores: []blob.Score{score},
		},
	}

	// Call
	deps, err := t.call(node)

	AssertEq(nil, err)
	ExpectThat(deps, ElementsAre())
	ExpectThat(node.Children, ElementsAre())
}

func (t *DependencyResolverTest) SomeChildren() {
	AssertTrue(false, "TODO")
}
<commit_msg>DependencyResolverTest.SomeChildren<commit_after>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package restore

import (
	"fmt"
	"io/ioutil"
	"log"
	"testing"

	"golang.org/x/net/context"

	"github.com/jacobsa/comeback/internal/blob"
	"github.com/jacobsa/comeback/internal/dag"
	"github.com/jacobsa/comeback/internal/fs"
	"github.com/jacobsa/comeback/internal/repr"
	"github.com/jacobsa/comeback/internal/util"
	"github.com/jacobsa/comeback/internal/wiring"
	"github.com/jacobsa/gcloud/gcs/gcsfake"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"github.com/jacobsa/timeutil"
)

func TestDependencyResolver(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func convertNodes(dagNodes []dag.Node) (nodes []*node) {
	for _, n := range dagNodes {
		nodes = append(nodes, n.(*node))
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type DependencyResolverTest struct {
	ctx       context.Context
	blobStore blob.Store
	dr        dag.DependencyResolver
}

var _ SetUpInterface = &DependencyResolverTest{}

func init() { RegisterTestSuite(&DependencyResolverTest{}) }

func (t *DependencyResolverTest) SetUp(ti *TestInfo) {
	var err error
	t.ctx = ti.Ctx

	// Create the blob store.
	bucket := gcsfake.NewFakeBucket(timeutil.RealClock(), "some_bucket")

	_, crypter, err := wiring.MakeRegistryAndCrypter(t.ctx, "password", bucket)
	AssertEq(nil, err)

	t.blobStore, err = wiring.MakeBlobStore(bucket, crypter, util.NewStringSet())
	AssertEq(nil, err)

	// Create the dependency resolver.
	t.dr = newDependencyResolver(t.blobStore, log.New(ioutil.Discard, "", 0))
}

func (t *DependencyResolverTest) call(n *node) (deps []*node, err error) {
	untyped, err := t.dr.FindDependencies(t.ctx, n)
	if err != nil {
		err = fmt.Errorf("FindDependencies: %v", err)
		return
	}

	for _, u := range untyped {
		deps = append(deps, u.(*node))
	}

	return
}

func (t *DependencyResolverTest) store(b []byte) (s blob.Score, err error) {
	s, err = t.blobStore.Store(
		t.ctx,
		&blob.StoreRequest{Blob: b})

	return
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *DependencyResolverTest) File() {
	n := &node{
		Info: fs.DirectoryEntry{
			Type: fs.TypeFile,
		},
	}

	// Call
	deps, err := t.call(n)

	AssertEq(nil, err)
	ExpectThat(deps, ElementsAre())
	ExpectThat(n.Children, ElementsAre())
}

func (t *DependencyResolverTest) Symlink() {
	n := &node{
		Info: fs.DirectoryEntry{
			Type: fs.TypeSymlink,
		},
	}

	// Call
	deps, err := t.call(n)

	AssertEq(nil, err)
	ExpectThat(deps, ElementsAre())
	ExpectThat(n.Children, ElementsAre())
}

func (t *DependencyResolverTest) BlobMissing() {
	s := blob.ComputeScore([]byte(""))
	n := &node{
		Info: fs.DirectoryEntry{
			Type:   fs.TypeDirectory,
			Scores: []blob.Score{s},
		},
	}

	// Call
	_, err := t.call(n)

	ExpectThat(err, Error(HasSubstr("TODO")))
	ExpectThat(err, Error(HasSubstr(s.Hex())))
}

func (t *DependencyResolverTest) BlobCorrupted() {
	var err error

	// Store some junk and set up a node with the junk's score as its contents.
	junk, err := t.store([]byte("foobar"))
	AssertEq(nil, err)

	n := &node{
		Info: fs.DirectoryEntry{
			Type:   fs.TypeDirectory,
			Scores: []blob.Score{junk},
		},
	}

	// Call
	_, err = t.call(n)

	ExpectThat(err, Error(HasSubstr("UnmarshalDir")))
	ExpectThat(err, Error(HasSubstr(junk.Hex())))
}

func (t *DependencyResolverTest) NoChildren() {
	var err error

	// Set up an empty listing.
	listing := []*fs.DirectoryEntry{}

	serialized, err := repr.MarshalDir(listing)
	AssertEq(nil, err)

	score, err := t.store(serialized)
	AssertEq(nil, err)

	// Set up the node.
	n := &node{
		RelPath: "taco/burrito",
		Info: fs.DirectoryEntry{
			Type:   fs.TypeDirectory,
			Scores: []blob.Score{score},
		},
	}

	// Call
	deps, err := t.call(n)

	AssertEq(nil, err)
	ExpectThat(deps, ElementsAre())
	ExpectThat(n.Children, ElementsAre())
}

func (t *DependencyResolverTest) SomeChildren() {
	var err error

	// Set up a listing.
	listing := []*fs.DirectoryEntry{
		&fs.DirectoryEntry{
			Type:        fs.TypeFile,
			Name:        "foo",
			Permissions: 0754,
		},
		&fs.DirectoryEntry{
			Type:   fs.TypeDirectory,
			Name:   "bar",
			Scores: []blob.Score{blob.ComputeScore([]byte(""))},
		},
	}

	serialized, err := repr.MarshalDir(listing)
	AssertEq(nil, err)

	score, err := t.store(serialized)
	AssertEq(nil, err)

	// Set up the node.
	n := &node{
		RelPath: "taco/burrito",
		Info: fs.DirectoryEntry{
			Type:   fs.TypeDirectory,
			Scores: []blob.Score{score},
		},
	}

	// Call
	deps, err := t.call(n)

	AssertEq(nil, err)
	AssertEq(2, len(deps))
	AssertThat(n.Children, DeepEquals(deps))
	var child *node

	child = n.Children[0]
	ExpectEq("taco/burrito/foo", child.RelPath)
	ExpectThat(child.Info, DeepEquals(*listing[0]))

	child = n.Children[1]
	ExpectEq("taco/burrito/bar", child.RelPath)
	ExpectThat(child.Info, DeepEquals(*listing[1]))
}
<|endoftext|>