<commit_before>package iot

import (
	. "github.com/influxdata/influxdb-comparisons/bulk_data_gen/common"
	"math/rand"
	"time"
)

var (
	CameraDetectionByteString = []byte("camera_detection") // heap optimization
)

var DetectionObjects = [][]byte{
	[]byte("animal"),
	[]byte("human"),
	[]byte("vehicle"),
	[]byte("unknown"),
}

var Animals = [][]byte{
	[]byte("cat"),
	[]byte("dog"),
	[]byte("bird"),
	[]byte("unknown"),
}

var Humans = [][]byte{
	[]byte("man"),
	[]byte("woman"),
	[]byte("child"),
	[]byte("unknown"),
}

var Vehicles = [][]byte{
	[]byte("car"),
	[]byte("lorry"),
	[]byte("truck"),
	[]byte("motorcycle"),
	[]byte("bicycle"),
	[]byte("unknown"),
}

var (
	// Field keys for 'air condition indoor' points.
	CameraDetectionFieldKeys = [][]byte{
		[]byte("object_type"),
		[]byte("object_kind"),
		[]byte("battery_voltage"),
	}
)

type CameraDetectionMeasurement struct {
	sensorId    []byte
	timestamp   time.Time
	batteryDist Distribution
	object      []byte
	kind        []byte
}

func NewCameraDetectionMeasurement(start time.Time, id []byte) *CameraDetectionMeasurement {

	//battery_voltage
	batteryDist := MUDWD(ND(1, 0.5), 1, 3.2, 3.2)

	return &CameraDetectionMeasurement{
		timestamp:   start,
		batteryDist: batteryDist,
		sensorId:    id,
	}
}

func (m *CameraDetectionMeasurement) Tick(d time.Duration) {
	m.timestamp = m.timestamp.Add(d)
	object := rand.Int63n(int64(len(DetectionObjects)))
	m.object = Animals[object]
	switch object {
	case 0: //animal
		m.kind = Animals[rand.Int63n(int64(len(Animals)))]
		break
	case 1: //human
		m.kind = Humans[rand.Int63n(int64(len(Humans)))]
		break
	case 2: //vehicle
		m.kind = Vehicles[rand.Int63n(int64(len(Vehicles)))]
		break
	case 3: //uknown
		m.kind = []byte("uknown")

	}
	m.batteryDist.Advance()
}

func (m *CameraDetectionMeasurement) ToPoint(p *Point) {
	p.SetMeasurementName(CameraDetectionByteString)
	p.SetTimestamp(&m.timestamp)
	p.AppendTag(SensorHomeTagKeys[0], m.sensorId)
	p.AppendField(CameraDetectionFieldKeys[0], m.object)
	p.AppendField(CameraDetectionFieldKeys[1], m.kind)
	p.AppendField(CameraDetectionFieldKeys[2], m.batteryDist.Get())
}
<commit_msg>Fixed object name<commit_after>package iot

import (
	. "github.com/influxdata/influxdb-comparisons/bulk_data_gen/common"
	"math/rand"
	"time"
)

var (
	CameraDetectionByteString = []byte("camera_detection") // heap optimization
)

var DetectionObjects = [][]byte{
	[]byte("animal"),
	[]byte("human"),
	[]byte("vehicle"),
	[]byte("unknown"),
}

var Animals = [][]byte{
	[]byte("cat"),
	[]byte("dog"),
	[]byte("bird"),
	[]byte("unknown"),
}

var Humans = [][]byte{
	[]byte("man"),
	[]byte("woman"),
	[]byte("child"),
	[]byte("unknown"),
}

var Vehicles = [][]byte{
	[]byte("car"),
	[]byte("lorry"),
	[]byte("truck"),
	[]byte("motorcycle"),
	[]byte("bicycle"),
	[]byte("unknown"),
}

var (
	// Field keys for 'air condition indoor' points.
	CameraDetectionFieldKeys = [][]byte{
		[]byte("object_type"),
		[]byte("object_kind"),
		[]byte("battery_voltage"),
	}
)

type CameraDetectionMeasurement struct {
	sensorId    []byte
	timestamp   time.Time
	batteryDist Distribution
	object      []byte
	kind        []byte
}

func NewCameraDetectionMeasurement(start time.Time, id []byte) *CameraDetectionMeasurement {

	//battery_voltage
	batteryDist := MUDWD(ND(1, 0.5), 1, 3.2, 3.2)

	return &CameraDetectionMeasurement{
		timestamp:   start,
		batteryDist: batteryDist,
		sensorId:    id,
	}
}

func (m *CameraDetectionMeasurement) Tick(d time.Duration) {
	m.timestamp = m.timestamp.Add(d)
	object := rand.Int63n(int64(len(DetectionObjects)))
	m.object = DetectionObjects[object]
	switch object {
	case 0: //animal
		m.kind = Animals[rand.Int63n(int64(len(Animals)))]
		break
	case 1: //human
		m.kind = Humans[rand.Int63n(int64(len(Humans)))]
		break
	case 2: //vehicle
		m.kind = Vehicles[rand.Int63n(int64(len(Vehicles)))]
		break
	case 3: //uknown
		m.kind = []byte("uknown")

	}
	m.batteryDist.Advance()
}

func (m *CameraDetectionMeasurement) ToPoint(p *Point) {
	p.SetMeasurementName(CameraDetectionByteString)
	p.SetTimestamp(&m.timestamp)
	p.AppendTag(SensorHomeTagKeys[0], m.sensorId)
	p.AppendField(CameraDetectionFieldKeys[0], m.object)
	p.AppendField(CameraDetectionFieldKeys[1], m.kind)
	p.AppendField(CameraDetectionFieldKeys[2], m.batteryDist.Get())
}
<|endoftext|>


<commit_before>package repository

import (
	"bytes"
	"crypto/hmac"
	"crypto/rand"
	"crypto/sha512"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	"code.google.com/p/go.crypto/nacl/secretbox"
)

const (
	authPubkeyFileName     = "auth.pub"
	encryptionKeyFileName  = "encryption.key"
	hashingKeyFileName     = "hashing.key"
	signingPrivkeyFileName = "signing.priv"
	managementDirName      = ".lara"
	objectsDirName         = "objects"
	nibsDirName            = "nibs"
	defaultFilePerms       = 0600
	defaultDirPerms        = 0700
	defaultChunkSize       = 1 * 1024 * 1024
	// EncryptionKeySize represents the size of the key used for
	// encrypting.
	EncryptionKeySize = 32
	// HashingKeySize represents the size of the key used for
	// generating content hashes (HMAC).
	HashingKeySize = 32
)

// Repository represents an on-disk repository and provides methods to
// access its sub-items.
type Repository struct {
	Path               string
	storage            ContentStorage
	nibStore           NIBStore
	authPubkeyPath     string
	encryptionKeyPath  string
	signingPrivkeyPath string
	hashingKeyPath     string
	objectsPath        string
	nibsPath           string
}

// New returns a new repository instance with the given base path
func New(path string) *Repository {
	r := &Repository{Path: path}
	r.setupPaths()
	return r
}

// setupPaths initializes several attributes referring to internal repository paths
// such as encryption key paths.
func (r *Repository) setupPaths() {
	base := filepath.Join(r.Path, managementDirName)
	r.authPubkeyPath = filepath.Join(base, authPubkeyFileName)
	r.encryptionKeyPath = filepath.Join(base, encryptionKeyFileName)
	r.signingPrivkeyPath = filepath.Join(base, signingPrivkeyFileName)
	r.hashingKeyPath = filepath.Join(base, hashingKeyFileName)
	r.objectsPath = filepath.Join(base, objectsDirName)
	r.nibsPath = filepath.Join(base, nibsDirName)
}

// getStorage returns the currently configured content storage backend
// for the repository.
func (r *Repository) getStorage() (ContentStorage, error) {
	if r.storage == nil {
		storage := FileContentStorage{
			StoragePath: filepath.Join(
				r.GetManagementDir(),
				objectsDirName)}
		err := storage.CreateDir()
		if err != nil {
			return nil, err
		}
		r.storage = storage
	}
	return r.storage, nil
}

// getNIBStore returns the currently configured nib store backend
// for the repository.
func (r *Repository) getNIBStore() (NIBStore, error) {
	if r.nibStore == nil {
		nibStorage := FileContentStorage{
			StoragePath: filepath.Join(
				r.GetManagementDir(),
				nibsDirName)}
		err := nibStorage.CreateDir()
		if err != nil {
			return nil, err
		}

		nibStore := newClientNibStore(nibStorage, *r)
		r.nibStore = nibStore
	}
	return r.nibStore, nil
}

// CreateManagementDir ensures that this repository's management
// directory exists.
func (r *Repository) CreateManagementDir() error {
	err := os.Mkdir(r.GetManagementDir(), defaultDirPerms)
	if err != nil && !os.IsExist(err) {
		return err
	}
	path := r.nibsPath
	err = os.Mkdir(path, defaultDirPerms)
	if err != nil && !os.IsExist(err) {
		return err
	}
	_, err = r.getStorage()
	if err != nil {
		return err
	}

	return nil
}

// GetManagementDir returns the path to the management directory.
func (r *Repository) GetManagementDir() string {
	return filepath.Join(r.Path, managementDirName)
}

// Create initially creates the repository directory structure.
func (r *Repository) Create() error {
	err := os.Mkdir(r.Path, defaultDirPerms)
	if err != nil {
		return err
	}
	err = r.CreateManagementDir()
	return err
}

// CreateEncryptionKey generates a random encryption key.
func (r *Repository) CreateEncryptionKey() error {
	key := make([]byte, EncryptionKeySize)
	var arrKey [EncryptionKeySize]byte
	_, err := rand.Read(key)
	if err != nil {
		return err
	}
	copy(arrKey[:], key)
	err = r.SetEncryptionKey(arrKey)
	return err
}

// CreateSigningKey generates a random signing key.
func (r *Repository) CreateSigningKey() error {
	key := make([]byte, PrivateKeySize)
	var arrKey [PrivateKeySize]byte
	_, err := rand.Read(key)
	if err != nil {
		return err
	}
	copy(arrKey[:], key)
	err = r.SetSigningPrivkey(arrKey)
	return err
}

// CreateHashingKey generates a random hashing key.
func (r *Repository) CreateHashingKey() error {
	key := make([]byte, HashingKeySize)
	var arrKey [HashingKeySize]byte
	_, err := rand.Read(key)
	if err != nil {
		return err
	}
	copy(arrKey[:], key)
	err = r.SetHashingKey(arrKey)
	return err
}

// GetAuthPubkey returns the repository auth key's public key.
func (r *Repository) GetAuthPubkey() ([]byte, error) {
	pubkey, err := ioutil.ReadFile(r.authPubkeyPath)
	return pubkey, err
}

// SetAuthPubkey sets the repository auth key's public key.
func (r *Repository) SetAuthPubkey(key []byte) error {
	return ioutil.WriteFile(r.authPubkeyPath, key, defaultFilePerms)
}

// SetEncryptionKey sets the repository encryption key
func (r *Repository) SetEncryptionKey(key [EncryptionKeySize]byte) error {
	return ioutil.WriteFile(r.encryptionKeyPath, key[:], defaultFilePerms)
}

// GetEncryptionKey returns the repository encryption key.
func (r *Repository) GetEncryptionKey() ([EncryptionKeySize]byte, error) {
	key, err := ioutil.ReadFile(r.encryptionKeyPath)
	if len(key) != EncryptionKeySize {
		return [EncryptionKeySize]byte{}, fmt.Errorf(
			"invalid key length (%d)", len(key))
	}
	var arrKey [EncryptionKeySize]byte
	copy(arrKey[:], key)
	return arrKey, err
}

// SetSigningPrivkey sets the repository signing private key
func (r *Repository) SetSigningPrivkey(key [PrivateKeySize]byte) error {
	return ioutil.WriteFile(r.signingPrivkeyPath, key[:], defaultFilePerms)
}

// GetSigningPrivkey returns the repository signing private key.
func (r *Repository) GetSigningPrivkey() ([PrivateKeySize]byte, error) {
	key, err := ioutil.ReadFile(r.signingPrivkeyPath)
	if len(key) != PrivateKeySize {
		return [PrivateKeySize]byte{}, fmt.Errorf(
			"invalid key length (%d)", len(key))
	}
	var arrKey [PrivateKeySize]byte
	copy(arrKey[:], key)
	return arrKey, err
}

// SetHashingKey sets the repository hashing key (content addressing)
func (r *Repository) SetHashingKey(key [HashingKeySize]byte) error {
	return ioutil.WriteFile(r.hashingKeyPath, key[:], defaultFilePerms)
}

// GetHashingKey returns the repository signing private key.
func (r *Repository) GetHashingKey() ([HashingKeySize]byte, error) {
	key, err := ioutil.ReadFile(r.hashingKeyPath)
	if len(key) != HashingKeySize {
		return [HashingKeySize]byte{}, fmt.Errorf(
			"invalid key length (%d)", len(key))
	}
	var arrKey [HashingKeySize]byte
	copy(arrKey[:], key)
	return arrKey, err
}

// AddItem adds a new file or directory to the repository.
func (r *Repository) AddItem(absPath string) error {
	metadataID, err := r.writeMetadata(absPath)
	if err != nil {
		return err
	}

	contentIDs, err := r.writeFileToChunks(absPath)
	if err != nil {
		return err
	}

	//FIXME this only works for new files / non-existing NIBs
	rev := &Revision{}
	rev.MetadataID = metadataID
	rev.ContentIDs = contentIDs
	nib := NIB{}
	if err != nil {
		return err
	}
	// Setting an empty UUID to trigger the generation of a new one.
	nib.UUID = ""
	nib.AppendRevision(rev)
	//FIXME: timestamp, deviceID etc.
	nibStore, err := r.getNIBStore()
	if err != nil {
		return err
	}

	return nibStore.Add(&nib)
}

// AddObject adds an object into the storage with the given
// id and adds the data in the reader to it.
func (r *Repository) AddObject(objectID string, data io.Reader) error {
	storage, err := r.getStorage()
	if err != nil {
		return err
	}
	return storage.Set(objectID, data)
}

// GetObjectData returns the data stored for the given objectID in this
// repository.
func (r *Repository) GetObjectData(objectID string) (io.Reader, error) {
	storage, err := r.getStorage()
	if err != nil {
		return nil, err
	}
	return storage.Get(objectID)
}

// HasObject returns if the given objectID exists in this repository.
func (r *Repository) HasObject(objectID string) bool {
	storage, err := r.getStorage()
	if err != nil {
		return false
	}

	return storage.Exists(objectID)
}

// hasUUID checks if the given UUID is already in use in this repository;
// this is a local-only check.
func (r *Repository) hasUUID(hash []byte) (bool, error) {
	UUID := formatUUID(hash)
	nibStore, err := r.getNIBStore()
	if err != nil {
		return false, err
	}
	return nibStore.Exists(UUID), nil
}

// getRepoRelativePath turns the given path into a path relative to the
// repository root and returns it.
func (r *Repository) getRepoRelativePath(absPath string) (string, error) {
	if len(absPath) < len(r.Path)+1 {
		return "", errors.New("unable to resolve path: path too short")
	}
	rel := absPath[len(r.Path)+1:]
	return rel, nil
}

// writeMetadata writes the metadata object for the given path
// to disk and returns its id.
func (r *Repository) writeMetadata(absPath string) (string, error) {
	relPath, err := r.getRepoRelativePath(absPath)
	if err != nil {
		return "", err
	}
	m := Metadata{
		RepoRelativePath: relPath,
		Type:             MetadataTypeFile, //FIXME
	}
	raw := &bytes.Buffer{}
	_, err = m.WriteTo(raw)
	if err != nil {
		return "", err
	}
	cid, err := r.writeContentAddressedCryptoContainer(raw.Bytes())
	if err != nil {
		return "", err
	}
	return cid, nil
}

// writeContentAddressedCryptoContainer takes a piece of raw data and
// streams it to disk in one content-addressed chunk while encrypting the
// data in the process.
func (r *Repository) writeContentAddressedCryptoContainer(data []byte) (string, error) {
	// hash for content-addressing
	hexHash, err := r.hashChunk(data)
	if err != nil {
		return "", err
	}

	var enc []byte
	enc, err = r.encryptWithRandomKey(data)
	if err != nil {
		return "", err
	}

	err = r.AddObject(hexHash, bytes.NewReader(enc))
	if err != nil {
		return "", err
	}

	return hexHash, nil
}

// encryptWithRandomKey takes a piece of data, encrypts it with a random
// key and returns the result, prefixed by the random key encrypted by
// the repository encryption key.
func (r *Repository) encryptWithRandomKey(data []byte) ([]byte, error) {
	var enc []byte

	// first generate and encrypt the per-file key and append it to
	// the result buffer:
	var nonce1 [24]byte
	_, err := rand.Read(nonce1[:])
	if err != nil {
		return nil, err
	}

	var fileKey [32]byte
	_, err = rand.Read(fileKey[:])
	if err != nil {
		return nil, err
	}
	repoKey, err := r.GetEncryptionKey()
	if err != nil {
		return nil, err
	}
	out := secretbox.Seal(enc, fileKey[:], &nonce1, &repoKey)

	// then append the actual encrypted contents
	var nonce2 [24]byte
	_, err = rand.Read(nonce2[:])
	if err != nil {
		return nil, err
	}
	out = secretbox.Seal(out, data, &nonce2, &fileKey)
	return out, nil
}

// hashChunk takes a chunk of data and constructs its content-addressing
// hash.
func (r *Repository) hashChunk(chunk []byte) (string, error) {
	key, err := r.GetHashingKey()
	if err != nil {
		return "", err
	}
	hasher := hmac.New(sha512.New, key[:])
	hasher.Write(chunk)
	hash := hasher.Sum(nil)
	hexHash := hex.EncodeToString(hash)
	return hexHash, nil
}

// writeFileToChunks takes a file path and saves its contents to the
// storage in encrypted form with a content-addressing id.
func (r *Repository) writeFileToChunks(path string) ([]string, error) {
	chunker, err := NewChunker(path, defaultChunkSize)
	if err != nil {
		return nil, err
	}
	var ids []string
	for chunker.HasNext() {
		chunk, err := chunker.Next()
		if err != nil {
			return nil, err
		}
		id, err := r.writeContentAddressedCryptoContainer(chunk)
		if err != nil {
			return nil, err
		}
		ids = append(ids, id)
	}
	return ids, nil
}
<commit_msg>repository: added GetSigningPubkey function to the repository struct.<commit_after>package repository

import (
	"bytes"
	"crypto/hmac"
	"crypto/rand"
	"crypto/sha512"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	"code.google.com/p/go.crypto/nacl/secretbox"

	edhelpers "github.com/hoffie/larasync/helpers/ed25519"
)

const (
	authPubkeyFileName     = "auth.pub"
	encryptionKeyFileName  = "encryption.key"
	hashingKeyFileName     = "hashing.key"
	signingPrivkeyFileName = "signing.priv"
	managementDirName      = ".lara"
	objectsDirName         = "objects"
	nibsDirName            = "nibs"
	defaultFilePerms       = 0600
	defaultDirPerms        = 0700
	defaultChunkSize       = 1 * 1024 * 1024
	// EncryptionKeySize represents the size of the key used for
	// encrypting.
	EncryptionKeySize = 32
	// HashingKeySize represents the size of the key used for
	// generating content hashes (HMAC).
	HashingKeySize = 32
)

// Repository represents an on-disk repository and provides methods to
// access its sub-items.
type Repository struct {
	Path               string
	storage            ContentStorage
	nibStore           NIBStore
	authPubkeyPath     string
	encryptionKeyPath  string
	signingPrivkeyPath string
	hashingKeyPath     string
	objectsPath        string
	nibsPath           string
}

// New returns a new repository instance with the given base path
func New(path string) *Repository {
	r := &Repository{Path: path}
	r.setupPaths()
	return r
}

// setupPaths initializes several attributes referring to internal repository paths
// such as encryption key paths.
func (r *Repository) setupPaths() {
	base := filepath.Join(r.Path, managementDirName)
	r.authPubkeyPath = filepath.Join(base, authPubkeyFileName)
	r.encryptionKeyPath = filepath.Join(base, encryptionKeyFileName)
	r.signingPrivkeyPath = filepath.Join(base, signingPrivkeyFileName)
	r.hashingKeyPath = filepath.Join(base, hashingKeyFileName)
	r.objectsPath = filepath.Join(base, objectsDirName)
	r.nibsPath = filepath.Join(base, nibsDirName)
}

// getStorage returns the currently configured content storage backend
// for the repository.
func (r *Repository) getStorage() (ContentStorage, error) {
	if r.storage == nil {
		storage := FileContentStorage{
			StoragePath: filepath.Join(
				r.GetManagementDir(),
				objectsDirName)}
		err := storage.CreateDir()
		if err != nil {
			return nil, err
		}
		r.storage = storage
	}
	return r.storage, nil
}

// getNIBStore returns the currently configured nib store backend
// for the repository.
func (r *Repository) getNIBStore() (NIBStore, error) {
	if r.nibStore == nil {
		nibStorage := FileContentStorage{
			StoragePath: filepath.Join(
				r.GetManagementDir(),
				nibsDirName)}
		err := nibStorage.CreateDir()
		if err != nil {
			return nil, err
		}

		storage := ContentStorage(nibStorage)
		clientNIBStore := newClientNIBStore(&storage, r)

		nibStore := NIBStore(clientNIBStore)
		r.nibStore = nibStore
	}
	return r.nibStore, nil
}

// CreateManagementDir ensures that this repository's management
// directory exists.
func (r *Repository) CreateManagementDir() error {
	err := os.Mkdir(r.GetManagementDir(), defaultDirPerms)
	if err != nil && !os.IsExist(err) {
		return err
	}
	path := r.nibsPath
	err = os.Mkdir(path, defaultDirPerms)
	if err != nil && !os.IsExist(err) {
		return err
	}
	_, err = r.getStorage()
	if err != nil {
		return err
	}

	return nil
}

// GetManagementDir returns the path to the management directory.
func (r *Repository) GetManagementDir() string {
	return filepath.Join(r.Path, managementDirName)
}

// Create initially creates the repository directory structure.
func (r *Repository) Create() error {
	err := os.Mkdir(r.Path, defaultDirPerms)
	if err != nil {
		return err
	}
	err = r.CreateManagementDir()
	return err
}

// CreateEncryptionKey generates a random encryption key.
func (r *Repository) CreateEncryptionKey() error {
	key := make([]byte, EncryptionKeySize)
	var arrKey [EncryptionKeySize]byte
	_, err := rand.Read(key)
	if err != nil {
		return err
	}
	copy(arrKey[:], key)
	err = r.SetEncryptionKey(arrKey)
	return err
}

// CreateSigningKey generates a random signing key.
func (r *Repository) CreateSigningKey() error {
	key := make([]byte, PrivateKeySize)
	var arrKey [PrivateKeySize]byte
	_, err := rand.Read(key)
	if err != nil {
		return err
	}
	copy(arrKey[:], key)
	err = r.SetSigningPrivkey(arrKey)
	return err
}

// CreateHashingKey generates a random hashing key.
func (r *Repository) CreateHashingKey() error {
	key := make([]byte, HashingKeySize)
	var arrKey [HashingKeySize]byte
	_, err := rand.Read(key)
	if err != nil {
		return err
	}
	copy(arrKey[:], key)
	err = r.SetHashingKey(arrKey)
	return err
}

// GetAuthPubkey returns the repository auth key's public key.
func (r *Repository) GetAuthPubkey() ([]byte, error) {
	pubkey, err := ioutil.ReadFile(r.authPubkeyPath)
	return pubkey, err
}

// SetAuthPubkey sets the repository auth key's public key.
func (r *Repository) SetAuthPubkey(key []byte) error {
	return ioutil.WriteFile(r.authPubkeyPath, key, defaultFilePerms)
}

// SetEncryptionKey sets the repository encryption key
func (r *Repository) SetEncryptionKey(key [EncryptionKeySize]byte) error {
	return ioutil.WriteFile(r.encryptionKeyPath, key[:], defaultFilePerms)
}

// GetEncryptionKey returns the repository encryption key.
func (r *Repository) GetEncryptionKey() ([EncryptionKeySize]byte, error) {
	key, err := ioutil.ReadFile(r.encryptionKeyPath)
	if len(key) != EncryptionKeySize {
		return [EncryptionKeySize]byte{}, fmt.Errorf(
			"invalid key length (%d)", len(key))
	}
	var arrKey [EncryptionKeySize]byte
	copy(arrKey[:], key)
	return arrKey, err
}

// SetSigningPrivkey sets the repository signing private key
func (r *Repository) SetSigningPrivkey(key [PrivateKeySize]byte) error {
	return ioutil.WriteFile(r.signingPrivkeyPath, key[:], defaultFilePerms)
}

// GetSigningPrivkey returns the repository signing private key.
func (r *Repository) GetSigningPrivkey() ([PrivateKeySize]byte, error) {
	key, err := ioutil.ReadFile(r.signingPrivkeyPath)
	if len(key) != PrivateKeySize {
		return [PrivateKeySize]byte{}, fmt.Errorf(
			"invalid key length (%d)", len(key))
	}
	var arrKey [PrivateKeySize]byte
	copy(arrKey[:], key)
	return arrKey, err
}

// GetSigningPubkey returns the repository signing public key.
func (r *Repository) GetSigningPubkey() ([PubkeySize]byte, error) {
	privKey, err := r.GetSigningPrivkey()
	if err != nil {
		return [PubkeySize]byte{}, err
	}
	return edhelpers.GetPublicKeyFromPrivate(privKey), nil
}

// SetHashingKey sets the repository hashing key (content addressing)
func (r *Repository) SetHashingKey(key [HashingKeySize]byte) error {
	return ioutil.WriteFile(r.hashingKeyPath, key[:], defaultFilePerms)
}

// GetHashingKey returns the repository signing private key.
func (r *Repository) GetHashingKey() ([HashingKeySize]byte, error) {
	key, err := ioutil.ReadFile(r.hashingKeyPath)
	if len(key) != HashingKeySize {
		return [HashingKeySize]byte{}, fmt.Errorf(
			"invalid key length (%d)", len(key))
	}
	var arrKey [HashingKeySize]byte
	copy(arrKey[:], key)
	return arrKey, err
}

// AddItem adds a new file or directory to the repository.
func (r *Repository) AddItem(absPath string) error {
	metadataID, err := r.writeMetadata(absPath)
	if err != nil {
		return err
	}

	contentIDs, err := r.writeFileToChunks(absPath)
	if err != nil {
		return err
	}

	//FIXME this only works for new files / non-existing NIBs
	rev := &Revision{}
	rev.MetadataID = metadataID
	rev.ContentIDs = contentIDs
	nib := NIB{}
	if err != nil {
		return err
	}
	// Setting an empty UUID to trigger the generation of a new one.
	nib.UUID = ""
	nib.AppendRevision(rev)
	//FIXME: timestamp, deviceID etc.
	nibStore, err := r.getNIBStore()
	if err != nil {
		return err
	}

	return nibStore.Add(&nib)
}

// AddObject adds an object into the storage with the given
// id and adds the data in the reader to it.
func (r *Repository) AddObject(objectID string, data io.Reader) error {
	storage, err := r.getStorage()
	if err != nil {
		return err
	}
	return storage.Set(objectID, data)
}

// GetObjectData returns the data stored for the given objectID in this
// repository.
func (r *Repository) GetObjectData(objectID string) (io.Reader, error) {
	storage, err := r.getStorage()
	if err != nil {
		return nil, err
	}
	return storage.Get(objectID)
}

// HasObject returns if the given objectID exists in this repository.
func (r *Repository) HasObject(objectID string) bool {
	storage, err := r.getStorage()
	if err != nil {
		return false
	}

	return storage.Exists(objectID)
}

// hasUUID checks if the given UUID is already in use in this repository;
// this is a local-only check.
func (r *Repository) hasUUID(hash []byte) (bool, error) {
	UUID := formatUUID(hash)
	nibStore, err := r.getNIBStore()
	if err != nil {
		return false, err
	}
	return nibStore.Exists(UUID), nil
}

// getRepoRelativePath turns the given path into a path relative to the
// repository root and returns it.
func (r *Repository) getRepoRelativePath(absPath string) (string, error) {
	if len(absPath) < len(r.Path)+1 {
		return "", errors.New("unable to resolve path: path too short")
	}
	rel := absPath[len(r.Path)+1:]
	return rel, nil
}

// writeMetadata writes the metadata object for the given path
// to disk and returns its id.
func (r *Repository) writeMetadata(absPath string) (string, error) {
	relPath, err := r.getRepoRelativePath(absPath)
	if err != nil {
		return "", err
	}
	m := Metadata{
		RepoRelativePath: relPath,
		Type:             MetadataTypeFile, //FIXME
	}
	raw := &bytes.Buffer{}
	_, err = m.WriteTo(raw)
	if err != nil {
		return "", err
	}
	cid, err := r.writeContentAddressedCryptoContainer(raw.Bytes())
	if err != nil {
		return "", err
	}
	return cid, nil
}

// writeContentAddressedCryptoContainer takes a piece of raw data and
// streams it to disk in one content-addressed chunk while encrypting the
// data in the process.
func (r *Repository) writeContentAddressedCryptoContainer(data []byte) (string, error) {
	// hash for content-addressing
	hexHash, err := r.hashChunk(data)
	if err != nil {
		return "", err
	}

	var enc []byte
	enc, err = r.encryptWithRandomKey(data)
	if err != nil {
		return "", err
	}

	err = r.AddObject(hexHash, bytes.NewReader(enc))
	if err != nil {
		return "", err
	}

	return hexHash, nil
}

// encryptWithRandomKey takes a piece of data, encrypts it with a random
// key and returns the result, prefixed by the random key encrypted by
// the repository encryption key.
func (r *Repository) encryptWithRandomKey(data []byte) ([]byte, error) {
	var enc []byte

	// first generate and encrypt the per-file key and append it to
	// the result buffer:
	var nonce1 [24]byte
	_, err := rand.Read(nonce1[:])
	if err != nil {
		return nil, err
	}

	var fileKey [32]byte
	_, err = rand.Read(fileKey[:])
	if err != nil {
		return nil, err
	}
	repoKey, err := r.GetEncryptionKey()
	if err != nil {
		return nil, err
	}
	out := secretbox.Seal(enc, fileKey[:], &nonce1, &repoKey)

	// then append the actual encrypted contents
	var nonce2 [24]byte
	_, err = rand.Read(nonce2[:])
	if err != nil {
		return nil, err
	}
	out = secretbox.Seal(out, data, &nonce2, &fileKey)
	return out, nil
}

// hashChunk takes a chunk of data and constructs its content-addressing
// hash.
func (r *Repository) hashChunk(chunk []byte) (string, error) {
	key, err := r.GetHashingKey()
	if err != nil {
		return "", err
	}
	hasher := hmac.New(sha512.New, key[:])
	hasher.Write(chunk)
	hash := hasher.Sum(nil)
	hexHash := hex.EncodeToString(hash)
	return hexHash, nil
}

// writeFileToChunks takes a file path and saves its contents to the
// storage in encrypted form with a content-addressing id.
func (r *Repository) writeFileToChunks(path string) ([]string, error) {
	chunker, err := NewChunker(path, defaultChunkSize)
	if err != nil {
		return nil, err
	}
	var ids []string
	for chunker.HasNext() {
		chunk, err := chunker.Next()
		if err != nil {
			return nil, err
		}
		id, err := r.writeContentAddressedCryptoContainer(chunk)
		if err != nil {
			return nil, err
		}
		ids = append(ids, id)
	}
	return ids, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"

	"github.com/ClusterHQ/dvol/pkg/api"
)

const PLUGINS_DIR = "/run/docker/plugins"
const DVOL_SOCKET = PLUGINS_DIR + "/dvol.sock"
const VOL_DIR = "/var/lib/dvol/volumes"

type ResponseImplements struct {
	// A response to the Plugin.Activate request
	Implements []string
}

type RequestCreate struct {
	// A request to create a volume for Docker
	Name string
	Opts map[string]string
}

type RequestMount struct {
	// A request to mount a volume for Docker
	Name string
}

type RequestRemove struct {
	// A request to remove a volume for Docker
	Name string
}

type ResponseSimple struct {
	// A response which only indicates if there was an error or not
	Err string
}

type ResponseMount struct {
	// A response to the VolumeDriver.Mount request
	Mountpoint string
	Err        string
}

func main() {
	log.Print("Starting dvol plugin")

	if _, err := os.Stat(PLUGINS_DIR); err != nil {
		if err := os.MkdirAll(PLUGINS_DIR, 0700); err != nil {
			log.Fatalf("Could not make plugin directory %s: %v", PLUGINS_DIR, err)
		}
	}
	if _, err := os.Stat(DVOL_SOCKET); err == nil {
		if err = os.Remove(DVOL_SOCKET); err != nil {
			log.Fatalf("Could not clean up existing socket at %s: %v", DVOL_SOCKET, err)
		}
	}
	if _, err := os.Stat(VOL_DIR); err != nil {
		if err := os.MkdirAll(VOL_DIR, 0700); err != nil {
			log.Fatalf("Could not make volumes directory %s: %v", VOL_DIR, err)
		}
	}

	http.HandleFunc("/Plugin.Activate", func(w http.ResponseWriter, r *http.Request) {
		log.Print("<= /Plugin.Activate")
		responseJSON, _ := json.Marshal(&ResponseImplements{
			Implements: []string{"VolumeDriver"},
		})
		w.Write(responseJSON)
	})

	http.HandleFunc("/VolumeDriver.Create", func(w http.ResponseWriter, r *http.Request) {
		log.Print("<= /VolumeDriver.Create")
		requestJSON, err := ioutil.ReadAll(r.Body)
		if err != nil {
			log.Fatalf("Unable to read response body %s", err)
		}
		request := new(RequestCreate)
		json.Unmarshal(requestJSON, request)
		name := request.Name
		dvol := api.NewDvolAPI(VOL_DIR)
		if dvol.VolumeExists(name) {
			log.Print("Volume already exists", name)
		} else {
			err := dvol.CreateVolume(name)
			if err != nil {
				writeResponseErr(fmt.Errorf("Could not create volume %s: %v", name, err), w)
				return
			}
		}
		writeResponseOK(w)
	})

	http.HandleFunc("/VolumeDriver.Remove", func(w http.ResponseWriter, r *http.Request) {
		writeResponseOK(w)
	})

	http.HandleFunc("/VolumeDriver.Path", func(w http.ResponseWriter, r *http.Request) {
	})

	http.HandleFunc("/VolumeDriver.Mount", func(w http.ResponseWriter, r *http.Request) {
		log.Print("<= /VolumeDriver.Mount")
		requestJSON, err := ioutil.ReadAll(r.Body)
		if err != nil {
			log.Fatalf("Unable to read response body %s", err)
		}
		request := new(RequestMount)
		json.Unmarshal(requestJSON, request)
		name := request.Name

		dvol := api.NewDvolAPI(VOL_DIR)

		if dvol.VolumeExists(name) {
			err := dvol.SwitchVolume(name)
			if err != nil {
				writeResponseErr(err, w)
			}
			_, err = dvol.ActiveVolume()
			if err != nil {
				writeResponseErr(err, w)
			}
			// mountpoint should be:
			// /var/lib/docker/volumes/<volumename>/running_point
			responseJSON, _ := json.Marshal(&ResponseMount{
				Mountpoint: "/tmp", // TODO: Get the real path
				Err:        "",
			})
			w.Write(responseJSON)
		} else {
			writeResponseErr(fmt.Errorf("Requested to mount unknown volume %s", name), w)
		}
	})

	http.HandleFunc("/VolumeDriver.Unmount", func(w http.ResponseWriter, r *http.Request) {
		writeResponseOK(w)
	})

	http.HandleFunc("/VolumeDriver.List", func(w http.ResponseWriter, r *http.Request) {
	})

	listener, err := net.Listen("unix", DVOL_SOCKET)
	if err != nil {
		log.Fatalf("Could not listen on %s: %v", DVOL_SOCKET, err)
	}

	http.Serve(listener, nil)
}

func writeResponseOK(w http.ResponseWriter) {
	// A shortcut to writing a ResponseOK to w
	responseJSON, _ := json.Marshal(&ResponseSimple{Err: ""})
	w.Write(responseJSON)
}

func writeResponseErr(err error, w http.ResponseWriter) {
	// A shortcut to responding with an error, and then log the error
	errString := fmt.Sprintln(err)
	log.Printf("Error: %v", err)
	responseJSON, _ := json.Marshal(&ResponseSimple{Err: errString})
	w.Write(responseJSON)
}
<commit_msg>Add a note about VolumeDriver.Remove.<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"

	"github.com/ClusterHQ/dvol/pkg/api"
)

const PLUGINS_DIR = "/run/docker/plugins"
const DVOL_SOCKET = PLUGINS_DIR + "/dvol.sock"
const VOL_DIR = "/var/lib/dvol/volumes"

type ResponseImplements struct {
	// A response to the Plugin.Activate request
	Implements []string
}

type RequestCreate struct {
	// A request to create a volume for Docker
	Name string
	Opts map[string]string
}

type RequestMount struct {
	// A request to mount a volume for Docker
	Name string
}

type RequestRemove struct {
	// A request to remove a volume for Docker
	Name string
}

type ResponseSimple struct {
	// A response which only indicates if there was an error or not
	Err string
}

type ResponseMount struct {
	// A response to the VolumeDriver.Mount request
	Mountpoint string
	Err        string
}

func main() {
	log.Print("Starting dvol plugin")

	if _, err := os.Stat(PLUGINS_DIR); err != nil {
		if err := os.MkdirAll(PLUGINS_DIR, 0700); err != nil {
			log.Fatalf("Could not make plugin directory %s: %v", PLUGINS_DIR, err)
		}
	}
	if _, err := os.Stat(DVOL_SOCKET); err == nil {
		if err = os.Remove(DVOL_SOCKET); err != nil {
			log.Fatalf("Could not clean up existing socket at %s: %v", DVOL_SOCKET, err)
		}
	}
	if _, err := os.Stat(VOL_DIR); err != nil {
		if err := os.MkdirAll(VOL_DIR, 0700); err != nil {
			log.Fatalf("Could not make volumes directory %s: %v", VOL_DIR, err)
		}
	}

	http.HandleFunc("/Plugin.Activate", func(w http.ResponseWriter, r *http.Request) {
		log.Print("<= /Plugin.Activate")
		responseJSON, _ := json.Marshal(&ResponseImplements{
			Implements: []string{"VolumeDriver"},
		})
		w.Write(responseJSON)
	})

	http.HandleFunc("/VolumeDriver.Create", func(w http.ResponseWriter, r *http.Request) {
		log.Print("<= /VolumeDriver.Create")
		requestJSON, err := ioutil.ReadAll(r.Body)
		if err != nil {
			log.Fatalf("Unable to read response body %s", err)
		}
		request := new(RequestCreate)
		json.Unmarshal(requestJSON, request)
		name := request.Name
		dvol := api.NewDvolAPI(VOL_DIR)
		if dvol.VolumeExists(name) {
			log.Print("Volume already exists", name)
		} else {
			err := dvol.CreateVolume(name)
			if err != nil {
				writeResponseErr(fmt.Errorf("Could not create volume %s: %v", name, err), w)
				return
			}
		}
		writeResponseOK(w)
	})

	http.HandleFunc("/VolumeDriver.Remove", func(w http.ResponseWriter, r *http.Request) {
		/*
		We do not actually want to remove the dvol volume when Docker references to them are removed.

		This is a no-op.
		*/
		writeResponseOK(w)
	})

	http.HandleFunc("/VolumeDriver.Path", func(w http.ResponseWriter, r *http.Request) {
	})

	http.HandleFunc("/VolumeDriver.Mount", func(w http.ResponseWriter, r *http.Request) {
		log.Print("<= /VolumeDriver.Mount")
		requestJSON, err := ioutil.ReadAll(r.Body)
		if err != nil {
			log.Fatalf("Unable to read response body %s", err)
		}
		request := new(RequestMount)
		json.Unmarshal(requestJSON, request)
		name := request.Name

		dvol := api.NewDvolAPI(VOL_DIR)

		if dvol.VolumeExists(name) {
			err := dvol.SwitchVolume(name)
			if err != nil {
				writeResponseErr(err, w)
			}
			_, err = dvol.ActiveVolume()
			if err != nil {
				writeResponseErr(err, w)
			}
			// mountpoint should be:
			// /var/lib/docker/volumes/<volumename>/running_point
			responseJSON, _ := json.Marshal(&ResponseMount{
				Mountpoint: "/tmp", // TODO: Get the real path
				Err:        "",
			})
			w.Write(responseJSON)
		} else {
			writeResponseErr(fmt.Errorf("Requested to mount unknown volume %s", name), w)
		}
	})

	http.HandleFunc("/VolumeDriver.Unmount", func(w http.ResponseWriter, r *http.Request) {
		writeResponseOK(w)
	})

	http.HandleFunc("/VolumeDriver.List", func(w http.ResponseWriter, r *http.Request) {
	})

	listener, err := net.Listen("unix", DVOL_SOCKET)
	if err != nil {
		log.Fatalf("Could not listen on %s: %v", DVOL_SOCKET, err)
	}

	http.Serve(listener, nil)
}

func writeResponseOK(w http.ResponseWriter) {
	// A shortcut to writing a ResponseOK to w
	responseJSON, _ := json.Marshal(&ResponseSimple{Err: ""})
	w.Write(responseJSON)
}

func writeResponseErr(err error, w http.ResponseWriter) {
	// A shortcut to responding with an error, and then log the error
	errString := fmt.Sprintln(err)
	log.Printf("Error: %v", err)
	responseJSON, _ := json.Marshal(&ResponseSimple{Err: errString})
	w.Write(responseJSON)
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/kisielk/whisper-go/whisper"
	"github.com/raintank/metrictank/mdata"
)

type conversion struct {
	archives []whisper.ArchiveInfo
	points   map[int][]whisper.Point
	method   string
}

func newConversion(arch []whisper.ArchiveInfo, points map[int][]whisper.Point, method string) *conversion {
	return &conversion{archives: arch, points: points, method: method}
}

func (c *conversion) findSmallestLargestArchive(spp, nop uint32) (int, int) {
	// find smallest archive that still contains enough data to satisfy requested range
	largestArchiveIdx := len(c.archives) - 1
	for i := largestArchiveIdx; i >= 0; i-- {
		arch := c.archives[i]
		if arch.Points*arch.SecondsPerPoint < nop*spp {
			break
		}
		largestArchiveIdx = i
	}

	// find largest archive that still has a higher or equal resolution than requested
	smallestArchiveIdx := 0
	for i := 0; i < len(c.archives); i++ {
		arch := c.archives[i]
		if arch.SecondsPerPoint > spp {
			break
		}
		smallestArchiveIdx = i
	}

	return smallestArchiveIdx, largestArchiveIdx
}

// generates points according to specified parameters by finding and using the best archives as input
func (c *conversion) getPoints(retIdx int, spp, nop uint32) map[string][]whisper.Point {
	res := make(map[string][]whisper.Point)

	if len(c.points) == 0 {
		return res
	}

	// figure out the range of archives that make sense to use for the requested specs
	smallestArchiveIdx, largestArchiveIdx := c.findSmallestLargestArchive(spp, nop)
	rawRes := c.archives[0].SecondsPerPoint

	adjustedPoints := make(map[string]map[uint32]float64)
	if retIdx > 0 && c.method == "avg" {
		adjustedPoints["cnt"] = make(map[uint32]float64)
		adjustedPoints["sum"] = make(map[uint32]float64)
	} else {
		adjustedPoints[c.method] = make(map[uint32]float64)
	}

	// Out of the input archives that we'll use, start with the lowest resolution one by converting
	// it to the requested resolution and filling the resulting points into adjustedPoints.
	// Then continue with archives of increasing resolutions while overwriting the generated points
	// of previous ones.
	for i := largestArchiveIdx; i >= smallestArchiveIdx; i-- {
		in := c.points[i]
		arch := c.archives[i]
		if arch.SecondsPerPoint == spp {
			if retIdx == 0 || c.method != "avg" {
				for _, p := range in {
					if p.Timestamp > uint32(*importUpTo) {
						continue
					}
					adjustedPoints[c.method][p.Timestamp] = p.Value
				}
			} else {
				for _, p := range in {
					if p.Timestamp > uint32(*importUpTo) {
						continue
					}
					adjustedPoints["sum"][p.Timestamp] = p.Value
					adjustedPoints["cnt"][p.Timestamp] = 1
				}
			}
		} else if arch.SecondsPerPoint > spp {
			method := c.method
			if method == "avg" && retIdx > 0 {
				method = "fakeavg"
			}
			for m, points := range incResolution(in, method, arch.SecondsPerPoint, spp, rawRes) {
				for _, p := range points {
					adjustedPoints[m][p.Timestamp] = p.Value
				}
			}
		} else {
			methods := []string{c.method}
			if c.method == "avg" && retIdx > 0 {
				methods = []string{"sum", "cnt"}
			}
			for m, points := range decResolution(in, methods, arch.SecondsPerPoint, spp) {
				for _, p := range points {
					adjustedPoints[m][p.Timestamp] = p.Value
				}
			}
		}

	}

	// merge the results that are keyed by timestamp into a slice of points
	for m, p := range adjustedPoints {
		for t, v := range p {
			res[m] = append(res[m], whisper.Point{Timestamp: t, Value: v})
		}
		res[m] = sortPoints(res[m])
	}

	return res
}

// increase resolution of given points according to defined specs by generating
// additional datapoints to bridge the gaps between the given points. depending
// on what aggregation method is specified, those datapoints may be generated in
// slightly different ways.
func incResolution(points []whisper.Point, method string, inRes, outRes, rawRes uint32) map[string][]whisper.Point {
	out := make(map[string][]whisper.Point)
	aggFactor := float64(outRes) / float64(rawRes)
	for _, inPoint := range points {
		if inPoint.Timestamp == 0 {
			continue
		}

		rangeEnd := inPoint.Timestamp - (inPoint.Timestamp % outRes)

		var outPoints []whisper.Point
		for ts := rangeEnd; ts > inPoint.Timestamp-inRes; ts = ts - outRes {
			if ts > uint32(*importUpTo) {
				continue
			}
			outPoints = append(outPoints, whisper.Point{Timestamp: ts})
		}

		for _, outPoint := range outPoints {
			if method == "cnt" || method == "sum" {
				outPoint.Value = inPoint.Value / float64(len(outPoints))
			} else if method == "fakeavg" {
				outPoint.Value = inPoint.Value * aggFactor
			} else {
				outPoint.Value = inPoint.Value
			}

			if method == "fakeavg" {
				out["sum"] = append(out["sum"], outPoint)
				out["cnt"] = append(out["cnt"], whisper.Point{Timestamp: outPoint.Timestamp, Value: aggFactor})
			} else {
				out[method] = append(out[method], outPoint)
			}
		}
	}
	for m := range out {
		out[m] = sortPoints(out[m])
	}
	return out
}

// decreases the resolution of given points by using the aggregation method specified
// in the second argument. emulates the way metrictank aggregates data when it generates
// rollups of the raw data.
func decResolution(points []whisper.Point, methods []string, inRes, outRes uint32) map[string][]whisper.Point {
	out := make(map[string][]whisper.Point)
	agg := mdata.NewAggregation()
	currentBoundary := uint32(0)

	flush := func() {
		if agg.Cnt == 0 {
			return
		}

		for _, method := range methods {
			var value float64
			switch method {
			case "min":
				value = agg.Min
			case "max":
				value = agg.Max
			case "sum":
				value = agg.Sum
			case "cnt":
				value = agg.Cnt
			case "lst":
				value = agg.Lst
			case "avg":
				value = agg.Sum / agg.Cnt
			default:
				return
			}
			out[method] = append(out[method], whisper.Point{
				Timestamp: currentBoundary,
				Value:     value,
			})
		}
		agg.Reset()
	}

	for _, inPoint := range sortPoints(points) {
		if inPoint.Timestamp == 0 {
			continue
		}
		boundary := mdata.AggBoundary(inPoint.Timestamp, outRes)
		if boundary > uint32(*importUpTo) {
			break
		}

		if boundary == currentBoundary {
			agg.Add(inPoint.Value)
			if inPoint.Timestamp == boundary {
				flush()
			}
		} else {
			flush()
			currentBoundary = boundary
			agg.Add(inPoint.Value)
		}
	}

	return out
}
<commit_msg>add comments<commit_after>package main

import (
	"github.com/kisielk/whisper-go/whisper"
	"github.com/raintank/metrictank/mdata"
)

type conversion struct {
	archives []whisper.ArchiveInfo
	points   map[int][]whisper.Point
	method   string
}

func newConversion(arch []whisper.ArchiveInfo, points map[int][]whisper.Point, method string) *conversion {
	return &conversion{archives: arch, points: points, method: method}
}

func (c *conversion) findSmallestLargestArchive(spp, nop uint32) (int, int) {
	// find smallest archive that still contains enough data to satisfy requested range
	largestArchiveIdx := len(c.archives) - 1
	for i := largestArchiveIdx; i >= 0; i-- {
		arch := c.archives[i]
		if arch.Points*arch.SecondsPerPoint < nop*spp {
			break
		}
		largestArchiveIdx = i
	}

	// find largest archive that still has a higher or equal resolution than requested
	smallestArchiveIdx := 0
	for i := 0; i < len(c.archives); i++ {
		arch := c.archives[i]
		if arch.SecondsPerPoint > spp {
			break
		}
		smallestArchiveIdx = i
	}

	return smallestArchiveIdx, largestArchiveIdx
}

// generates points according to specified parameters by finding and using the best archives as input
func (c *conversion) getPoints(retIdx int, spp, nop uint32) map[string][]whisper.Point {
	res := make(map[string][]whisper.Point)

	if len(c.points) == 0 {
		return res
	}

	// figure out the range of archives that make sense to use for the requested specs
	smallestArchiveIdx, largestArchiveIdx := c.findSmallestLargestArchive(spp, nop)
	rawRes := c.archives[0].SecondsPerPoint

	adjustedPoints := make(map[string]map[uint32]float64)
	if retIdx > 0 && c.method == "avg" {
		adjustedPoints["cnt"] = make(map[uint32]float64)
		adjustedPoints["sum"] = make(map[uint32]float64)
	} else {
		adjustedPoints[c.method] = make(map[uint32]float64)
	}

	// Out of the input archives that we'll use, start with the lowest resolution one by converting
	// it to the requested resolution and filling the resulting points into adjustedPoints.
	// Then continue with archives of increasing resolutions while overwriting the generated points
	// of previous ones.
	for i := largestArchiveIdx; i >= smallestArchiveIdx; i-- {
		in := c.points[i]
		arch := c.archives[i]
		if arch.SecondsPerPoint == spp {
			if retIdx == 0 || c.method != "avg" {
				for _, p := range in {
					if p.Timestamp > uint32(*importUpTo) {
						continue
					}
					adjustedPoints[c.method][p.Timestamp] = p.Value
				}
			} else {
				for _, p := range in {
					if p.Timestamp > uint32(*importUpTo) {
						continue
					}
					adjustedPoints["sum"][p.Timestamp] = p.Value
					adjustedPoints["cnt"][p.Timestamp] = 1
				}
			}
		} else if arch.SecondsPerPoint > spp {
			method := c.method
			if method == "avg" && retIdx > 0 {
				method = "fakeavg"
			}
			for m, points := range incResolution(in, method, arch.SecondsPerPoint, spp, rawRes) {
				for _, p := range points {
					adjustedPoints[m][p.Timestamp] = p.Value
				}
			}
		} else {
			methods := []string{c.method}
			if c.method == "avg" && retIdx > 0 {
				methods = []string{"sum", "cnt"}
			}
			for m, points := range decResolution(in, methods, arch.SecondsPerPoint, spp) {
				for _, p := range points {
					adjustedPoints[m][p.Timestamp] = p.Value
				}
			}
		}

	}

	// merge the results that are keyed by timestamp into a slice of points
	for m, p := range adjustedPoints {
		for t, v := range p {
			res[m] = append(res[m], whisper.Point{Timestamp: t, Value: v})
		}
		res[m] = sortPoints(res[m])
	}

	return res
}

// increase resolution of given points according to defined specs by generating
// additional datapoints to bridge the gaps between the given points. depending
// on what aggregation method is specified, those datapoints may be generated in
// slightly different ways.
func incResolution(points []whisper.Point, method string, inRes, outRes, rawRes uint32) map[string][]whisper.Point {
	out := make(map[string][]whisper.Point)
	aggFactor := float64(outRes) / float64(rawRes)
	for _, inPoint := range points {
		if inPoint.Timestamp == 0 {
			continue
		}

		// inPoints are guaranteed to be quantized by whisper
		// outRes is > inRes, otherwise this function should never be called
		// rangeEnd is the the TS of the last datapoint that will be generated based on inPoint
		rangeEnd := inPoint.Timestamp - (inPoint.Timestamp % outRes)

		// generate datapoints based on inPoint in reverse order
		var outPoints []whisper.Point
		for ts := rangeEnd; ts > inPoint.Timestamp-inRes; ts = ts - outRes {
			if ts > uint32(*importUpTo) {
				continue
			}
			outPoints = append(outPoints, whisper.Point{Timestamp: ts})
		}

		for _, outPoint := range outPoints {
			if method == "cnt" || method == "sum" {
				outPoint.Value = inPoint.Value / float64(len(outPoints))
			} else if method == "fakeavg" {
				outPoint.Value = inPoint.Value * aggFactor
			} else {
				outPoint.Value = inPoint.Value
			}

			if method == "fakeavg" {
				out["sum"] = append(out["sum"], outPoint)
				out["cnt"] = append(out["cnt"], whisper.Point{Timestamp: outPoint.Timestamp, Value: aggFactor})
			} else {
				out[method] = append(out[method], outPoint)
			}
		}
	}
	for m := range out {
		out[m] = sortPoints(out[m])
	}
	return out
}

// decreases the resolution of given points by using the aggregation method specified
// in the second argument. emulates the way metrictank aggregates data when it generates
// rollups of the raw data.
func decResolution(points []whisper.Point, methods []string, inRes, outRes uint32) map[string][]whisper.Point {
	out := make(map[string][]whisper.Point)
	agg := mdata.NewAggregation()
	currentBoundary := uint32(0)

	flush := func() {
		if agg.Cnt == 0 {
			return
		}

		for _, method := range methods {
			var value float64
			switch method {
			case "min":
				value = agg.Min
			case "max":
				value = agg.Max
			case "sum":
				value = agg.Sum
			case "cnt":
				value = agg.Cnt
			case "lst":
				value = agg.Lst
			case "avg":
				value = agg.Sum / agg.Cnt
			default:
				return
			}
			out[method] = append(out[method], whisper.Point{
				Timestamp: currentBoundary,
				Value:     value,
			})
		}
		agg.Reset()
	}

	for _, inPoint := range sortPoints(points) {
		if inPoint.Timestamp == 0 {
			continue
		}
		boundary := mdata.AggBoundary(inPoint.Timestamp, outRes)
		if boundary > uint32(*importUpTo) {
			break
		}

		if boundary == currentBoundary {
			agg.Add(inPoint.Value)
			if inPoint.Timestamp == boundary {
				flush()
			}
		} else {
			flush()
			currentBoundary = boundary
			agg.Add(inPoint.Value)
		}
	}

	return out
}
<|endoftext|>


<commit_before>package discordgo

import (
	"net/http"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

// customRateLimit holds information for defining a custom rate limit
type customRateLimit struct {
	suffix   string
	requests int
	reset    time.Duration
}

// RateLimiter holds all ratelimit buckets
type RateLimiter struct {
	sync.Mutex
	global           *int64
	buckets          map[string]*Bucket
	globalRateLimit  time.Duration
	customRateLimits []*customRateLimit

	MaxConcurrentRequests int
	numConcurrentLocks    *int32
}

// NewRatelimiter returns a new RateLimiter
func NewRatelimiter() *RateLimiter {

	return &RateLimiter{
		buckets:            make(map[string]*Bucket),
		global:             new(int64),
		numConcurrentLocks: new(int32),
		customRateLimits: []*customRateLimit{
			&customRateLimit{
				suffix:   "/reactions//",
				requests: 1,
				reset:    250 * time.Millisecond,
			},
		},
	}
}

func (r *RateLimiter) CurrentConcurrentLocks() int {
	return int(atomic.LoadInt32(r.numConcurrentLocks))
}

// GetBucket retrieves or creates a bucket
func (r *RateLimiter) GetBucket(key string) *Bucket {
	r.Lock()
	defer r.Unlock()

	if bucket, ok := r.buckets[key]; ok {
		return bucket
	}

	b := &Bucket{
		Remaining: 1,
		Key:       key,
		global:    r.global,
	}

	if r.MaxConcurrentRequests > 0 {
		b.numConcurrentLocks = r.numConcurrentLocks
	}

	// Check if there is a custom ratelimit set for this bucket ID.
	for _, rl := range r.customRateLimits {
		if strings.HasSuffix(b.Key, rl.suffix) {
			b.customRateLimit = rl
			break
		}
	}

	r.buckets[key] = b
	return b
}

// GetWaitTime returns the duration you should wait for a Bucket
func (r *RateLimiter) GetWaitTime(b *Bucket, minRemaining int) time.Duration {
	// If we ran out of calls and the reset time is still ahead of us
	// then we need to take it easy and relax a little

	wait := time.Duration(0)
	if b.Remaining < minRemaining && b.reset.After(time.Now()) {
		wait = b.reset.Sub(time.Now())
	}

	// Check for global ratelimits
	sleepTo := time.Unix(0, atomic.LoadInt64(r.global))
	if now := time.Now(); now.Before(sleepTo) {

		// time until the global ratelimit is over
		globalWait := sleepTo.Sub(now)

		if globalWait < wait {
			// if the per route wait time is greater than the global wait time
			// return the per route wait time
			return wait
		}

		// otherwise return the global wait
		return globalWait
	}

	// either 0 or the per route wait time
	return wait
}

// LockBucket Locks until a request can be made
func (r *RateLimiter) LockBucket(bucketID string) *Bucket {
	return r.LockBucketObject(r.GetBucket(bucketID))
}

// LockBucketObject Locks an already resolved bucket until a request can be made
func (r *RateLimiter) LockBucketObject(b *Bucket) *Bucket {
	b.Lock()

	if wait := r.GetWaitTime(b, 1); wait > 0 {
		time.Sleep(wait)
	}

	if r.MaxConcurrentRequests > 0 {
		// sleep until were below the maximum
		for {
			numNow := atomic.AddInt32(r.numConcurrentLocks, 1)
			if int(numNow) >= r.MaxConcurrentRequests {
				atomic.AddInt32(r.numConcurrentLocks, -1)
				time.Sleep(time.Millisecond * 25)
			} else {
				break
			}
		}
	}

	b.Remaining--
	return b
}

func (r *RateLimiter) SetGlobalTriggered(to time.Time) {
	atomic.StoreInt64(r.global, to.UnixNano())
}

// Bucket represents a ratelimit bucket, each bucket gets ratelimited individually (-global ratelimits)
type Bucket struct {
	sync.Mutex
	Key                string
	Remaining          int
	limit              int
	reset              time.Time
	global             *int64
	numConcurrentLocks *int32

	lastReset       time.Time
	customRateLimit *customRateLimit
	Userdata        interface{}
}

// Release unlocks the bucket and reads the headers to update the buckets ratelimit info
// and locks up the whole thing in case if there's a global ratelimit.
func (b *Bucket) Release(headers http.Header) error {
	defer b.Unlock()

	if b.numConcurrentLocks != nil {
		atomic.AddInt32(b.numConcurrentLocks, -1)
	}

	// Check if the bucket uses a custom ratelimiter
	if rl := b.customRateLimit; rl != nil {
		if time.Now().Sub(b.lastReset) >= rl.reset {
			b.Remaining = rl.requests - 1
			b.lastReset = time.Now()
		}
		if b.Remaining < 1 {
			b.reset = time.Now().Add(rl.reset)
		}
		return nil
	}

	if headers == nil {
		return nil
	}

	remaining := headers.Get("X-RateLimit-Remaining")
	reset := headers.Get("X-RateLimit-Reset")
	global := headers.Get("X-RateLimit-Global")
	retryAfter := headers.Get("Retry-After")

	// Update global and per bucket reset time if the proper headers are available
	// If global is set, then it will block all buckets until after Retry-After
	// If Retry-After without global is provided it will use that for the new reset
	// time since it's more accurate than X-RateLimit-Reset.
	// If Retry-After after is not proided, it will update the reset time from X-RateLimit-Reset
	if retryAfter != "" {
		parsedAfter, err := strconv.ParseInt(retryAfter, 10, 64)
		if err != nil {
			return err
		}

		resetAt := time.Now().Add(time.Duration(parsedAfter) * time.Millisecond)

		// Lock either this single bucket or all buckets
		if global != "" {
			atomic.StoreInt64(b.global, resetAt.UnixNano())
		} else {
			b.reset = resetAt
		}
	} else if reset != "" {
		// Calculate the reset time by using the date header returned from discord
		discordTime, err := http.ParseTime(headers.Get("Date"))
		if err != nil {
			return err
		}

		unix, err := strconv.ParseInt(reset, 10, 64)
		if err != nil {
			return err
		}

		// Calculate the time until reset and add it to the current local time
		// some extra time is added because without it i still encountered 429's.
		// The added amount is the lowest amount that gave no 429's
		// in 1k requests
		delta := time.Unix(unix, 0).Sub(discordTime) + time.Millisecond*250
		b.reset = time.Now().Add(delta)
	}

	// Udpate remaining if header is present
	if remaining != "" {
		parsedRemaining, err := strconv.ParseInt(remaining, 10, 32)
		if err != nil {
			return err
		}
		b.Remaining = int(parsedRemaining)
	}

	return nil
}
<commit_msg>ratelimit: check if a global is triggered while in the ccr queue<commit_after>package discordgo

import (
	"net/http"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

// customRateLimit holds information for defining a custom rate limit
type customRateLimit struct {
	suffix   string
	requests int
	reset    time.Duration
}

// RateLimiter holds all ratelimit buckets
type RateLimiter struct {
	sync.Mutex
	global           *int64
	buckets          map[string]*Bucket
	globalRateLimit  time.Duration
	customRateLimits []*customRateLimit

	MaxConcurrentRequests int
	numConcurrentLocks    *int32
}

// NewRatelimiter returns a new RateLimiter
func NewRatelimiter() *RateLimiter {

	return &RateLimiter{
		buckets:            make(map[string]*Bucket),
		global:             new(int64),
		numConcurrentLocks: new(int32),
		customRateLimits: []*customRateLimit{
			&customRateLimit{
				suffix:   "/reactions//",
				requests: 1,
				reset:    250 * time.Millisecond,
			},
		},
	}
}

func (r *RateLimiter) CurrentConcurrentLocks() int {
	return int(atomic.LoadInt32(r.numConcurrentLocks))
}

// GetBucket retrieves or creates a bucket
func (r *RateLimiter) GetBucket(key string) *Bucket {
	r.Lock()
	defer r.Unlock()

	if bucket, ok := r.buckets[key]; ok {
		return bucket
	}

	b := &Bucket{
		Remaining: 1,
		Key:       key,
		global:    r.global,
	}

	if r.MaxConcurrentRequests > 0 {
		b.numConcurrentLocks = r.numConcurrentLocks
	}

	// Check if there is a custom ratelimit set for this bucket ID.
	for _, rl := range r.customRateLimits {
		if strings.HasSuffix(b.Key, rl.suffix) {
			b.customRateLimit = rl
			break
		}
	}

	r.buckets[key] = b
	return b
}

// GetWaitTime returns the duration you should wait for a Bucket
func (r *RateLimiter) GetWaitTime(b *Bucket, minRemaining int) time.Duration {
	// If we ran out of calls and the reset time is still ahead of us
	// then we need to take it easy and relax a little

	wait := time.Duration(0)
	if b.Remaining < minRemaining && b.reset.After(time.Now()) {
		wait = b.reset.Sub(time.Now())
	}

	// Check for global ratelimits
	sleepTo := time.Unix(0, atomic.LoadInt64(r.global))
	if now := time.Now(); now.Before(sleepTo) {

		// time until the global ratelimit is over
		globalWait := sleepTo.Sub(now)

		if globalWait < wait {
			// if the per route wait time is greater than the global wait time
			// return the per route wait time
			return wait
		}

		// otherwise return the global wait
		return globalWait
	}

	// either 0 or the per route wait time
	return wait
}

// LockBucket Locks until a request can be made
func (r *RateLimiter) LockBucket(bucketID string) *Bucket {
	return r.LockBucketObject(r.GetBucket(bucketID))
}

// LockBucketObject Locks an already resolved bucket until a request can be made
func (r *RateLimiter) LockBucketObject(b *Bucket) *Bucket {
	b.Lock()

	if wait := r.GetWaitTime(b, 1); wait > 0 {
		time.Sleep(wait)
	}

	didWaitForMaxCCR := false
	if r.MaxConcurrentRequests > 0 {
		// sleep until were below the maximum
		for {
			numNow := atomic.AddInt32(r.numConcurrentLocks, 1)
			if int(numNow) >= r.MaxConcurrentRequests {
				atomic.AddInt32(r.numConcurrentLocks, -1)
				didWaitForMaxCCR = true
				time.Sleep(time.Millisecond * 25)
			} else {
				break
			}
		}
	}

	if didWaitForMaxCCR {
		// If things changed while waiting for max ccr (like a global ratelimit)
		if wait := r.GetWaitTime(b, 1); wait > 0 {
			time.Sleep(wait)
		}
	}

	b.Remaining--
	return b
}

func (r *RateLimiter) SetGlobalTriggered(to time.Time) {
	atomic.StoreInt64(r.global, to.UnixNano())
}

// Bucket represents a ratelimit bucket, each bucket gets ratelimited individually (-global ratelimits)
type Bucket struct {
	sync.Mutex
	Key                string
	Remaining          int
	limit              int
	reset              time.Time
	global             *int64
	numConcurrentLocks *int32

	lastReset       time.Time
	customRateLimit *customRateLimit
	Userdata        interface{}
}

// Release unlocks the bucket and reads the headers to update the buckets ratelimit info
// and locks up the whole thing in case if there's a global ratelimit.
func (b *Bucket) Release(headers http.Header) error {
	defer b.Unlock()

	if b.numConcurrentLocks != nil {
		atomic.AddInt32(b.numConcurrentLocks, -1)
	}

	// Check if the bucket uses a custom ratelimiter
	if rl := b.customRateLimit; rl != nil {
		if time.Now().Sub(b.lastReset) >= rl.reset {
			b.Remaining = rl.requests - 1
			b.lastReset = time.Now()
		}
		if b.Remaining < 1 {
			b.reset = time.Now().Add(rl.reset)
		}
		return nil
	}

	if headers == nil {
		return nil
	}

	remaining := headers.Get("X-RateLimit-Remaining")
	reset := headers.Get("X-RateLimit-Reset")
	global := headers.Get("X-RateLimit-Global")
	retryAfter := headers.Get("Retry-After")

	// Update global and per bucket reset time if the proper headers are available
	// If global is set, then it will block all buckets until after Retry-After
	// If Retry-After without global is provided it will use that for the new reset
	// time since it's more accurate than X-RateLimit-Reset.
	// If Retry-After after is not proided, it will update the reset time from X-RateLimit-Reset
	if retryAfter != "" {
		parsedAfter, err := strconv.ParseInt(retryAfter, 10, 64)
		if err != nil {
			return err
		}

		resetAt := time.Now().Add(time.Duration(parsedAfter) * time.Millisecond)

		// Lock either this single bucket or all buckets
		if global != "" {
			atomic.StoreInt64(b.global, resetAt.UnixNano())
		} else {
			b.reset = resetAt
		}
	} else if reset != "" {
		// Calculate the reset time by using the date header returned from discord
		discordTime, err := http.ParseTime(headers.Get("Date"))
		if err != nil {
			return err
		}

		unix, err := strconv.ParseInt(reset, 10, 64)
		if err != nil {
			return err
		}

		// Calculate the time until reset and add it to the current local time
		// some extra time is added because without it i still encountered 429's.
		// The added amount is the lowest amount that gave no 429's
		// in 1k requests
		delta := time.Unix(unix, 0).Sub(discordTime) + time.Millisecond*250
		b.reset = time.Now().Add(delta)
	}

	// Udpate remaining if header is present
	if remaining != "" {
		parsedRemaining, err := strconv.ParseInt(remaining, 10, 32)
		if err != nil {
			return err
		}
		b.Remaining = int(parsedRemaining)
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"html/template"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"time"

	"github.com/cenkalti/backoff"
	"github.com/facebookgo/stackerr"
	"github.com/fsouza/go-dockerclient"
	"github.com/gorilla/mux"
)

// If |useTLS| is enabled, look for the cert files here.
var defaultCertPath = os.Getenv("DOCKER_CERT_PATH")

var (
	staticPath  = flag.String("static", "./static", "serve static files located in this directory")
	dockerHost  = flag.String("docker", "tcp://192.168.59.103:2376", "address of docker daemon host")
	tlsCertPath = flag.String("tls-cert-path", defaultCertPath, "path to docker host TLS certificate")
	host        = flag.String("host", ":8080", "address:port for HTTP listener")
	useTLS      = flag.Bool("tls", true, "use Docker TLS")
)

func main() {
	flag.Parse()
	if err := run(); err != nil {
		log.Fatal(err)
	}
}

func run() error {
	var client *docker.Client
	var err error
	if *useTLS {
		client, err = docker.NewTLSClient(
			*dockerHost,
			path.Join(*tlsCertPath, "cert.pem"),
			path.Join(*tlsCertPath, "key.pem"),
			path.Join(*tlsCertPath, "ca.pem"),
		)
		if err != nil {
			return stackerr.Wrap(err)
		}
	} else {
		client, err = docker.NewClient(*dockerHost)
		if err != nil {
			return err
		}
	}
	s := &AgentSupervisor{Client: client}
	log.Fatal(http.ListenAndServe(*host, Router(s)))
	return nil
}

// Router defines URL routes
func Router(s *AgentSupervisor) *mux.Router {
	r := mux.NewRouter()
	WireAPIRoutes(r, s)
	WireHTMLRoutes(r, s) // after API routes

	// Finally, if none of the above routes match, delegate to the single-page
	// app's client-side router. Rewrite the path in order to load the
	// single-page app's root HTML entrypoint. The app will handle the route.
	r.NotFoundHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		r.URL.Path = "/"
		http.FileServer(http.Dir(*staticPath)).ServeHTTP(w, r)
	})
	return r
}

var templateHome = `

fxx >>= ai

	<form action=/api/v0/validate method=GET>

	validate your AI... Enter the name of the Docker image:
		<input type=text name=image>
		</input>
	</form>

`

func WireHTMLRoutes(r *mux.Router, s *AgentSupervisor) {

	r.HandleFunc("/html", func(w http.ResponseWriter, r *http.Request) {
		tmpl, err := template.New("name").Parse(templateHome)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		// Error checking elided
		if err := tmpl.Execute(w, nil); err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
	})
}

func WireAPIRoutes(r *mux.Router, s *AgentSupervisor) {

	apiV0 := r.PathPrefix("/api/v0").Subrouter()

	// TODO Build an agent from a GitHub repo
	// TODO Run Game between two running agents

	agents := apiV0.Path("/agents").Subrouter()
	agents.Methods("GET").HandlerFunc(ListAgentsHandler(s)) // list all available agents

	// pull an agent from Dockerhub
	dockerhub := apiV0.Path("/pull/dockerhub").Subrouter()
	dockerhub.Methods("GET").HandlerFunc(PullDockerHubAgentHandler(s)) // FIXME make this a POST

	// TODO List running agents with parameter
	apiV0.Path("/running").HandlerFunc(ListActiveAgentsHandler(s))

	// checks that the agent implements the right protocol
	apiV0.Path("/validate").HandlerFunc(ValidateAgentHandler(s))

	apiV0.Path("/image").HandlerFunc(ShowAgentInfoHandler(s))
}

// PullDockerHubAgentHandler pulls the Docker image named |image| from
// DockerHub.
func PullDockerHubAgentHandler(s *AgentSupervisor) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if r.FormValue("image") == "" {
			w.WriteHeader(http.StatusBadRequest)
			return
		}
		if err := s.Client.PullImage(docker.PullImageOptions{
			OutputStream: w,
			Registry:     "https://index.docker.io",
			Repository:   r.FormValue("image"),
			Tag:          "latest",
		}, docker.AuthConfiguration{}); err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintln(w, err)
		}
	}
}

// ListAgentsHandler lists AI agents.
func ListAgentsHandler(s *AgentSupervisor) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		images, err := s.Client.ListImages(docker.ListImagesOptions{
			All: false,
		})
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			_, err := io.WriteString(w, err.Error())
			if err != nil {
				log.Println("error writing err: %s", err)
			}
			return
		}
		for _, img := range images {
			_, err := fmt.Fprintln(w, fmt.Sprintln(append([]string{img.ID, "\t"}, img.RepoTags...)))
			if err != nil {
				w.WriteHeader(http.StatusInternalServerError)
				_, err := io.WriteString(w, err.Error())
				if err != nil {
					log.Println("error writing err: %s", err)
				}
			}
		}
	}
}

// ListActiveAgentsHandler lists agents that are currently running.
func ListActiveAgentsHandler(s *AgentSupervisor) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		containers, err := s.Client.ListContainers(docker.ListContainersOptions{})
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		for _, ps := range containers {
			fmt.Fprintln(w, fmt.Sprintf("%+v\n", ps))
		}
	}
}

// ValidateAgentHandler ensures that the Agent image responds to the PING
// command.
func ValidateAgentHandler(s *AgentSupervisor) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		defer log.Println("returned")

		// client gives the |image| as a URL parameter
		if r.FormValue("image") == "" {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintln(w, "`image` parameter is required")
			return
		}
		image := r.FormValue("image")

		if err := s.ValidateImage(image); err != nil {
			fmt.Fprintf(w, "image %s is not valid. error: %s", image, err)
			return
		}
		fmt.Fprintf(w, "image %s is valid", image)
	}
}

type AgentInfo struct {
	Owner  string
	Taunts []string
}

// ShowImageHandler shows information about an AI Agent
func ShowAgentInfoHandler(s *AgentSupervisor) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		image, err := s.Client.InspectImage("jbenet/go-ipfs")
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		json.NewEncoder(w).Encode(image.Config.ExposedPorts)
	}
}

// AgentSupervisor manages AI agents running in Docker containers
type AgentSupervisor struct {
	Client *docker.Client
}

func (s *AgentSupervisor) ValidateImage(image string) error {

	// run the container in a two-phase process. First, create the
	// container.
	container, err := s.Client.CreateContainer(docker.CreateContainerOptions{
		Config: &docker.Config{
			Image: image, // the only required argument
		},
	})
	if err != nil {
		return fmt.Errorf("error creating container:", err.Error())
	}

	hc := &docker.HostConfig{
		PublishAllPorts: true,
	}

	// run the created container
	if err := s.Client.StartContainer(container.ID, hc); err != nil {
		return fmt.Errorf("error starting container: %s", err.Error())
	}

	// ensure the proper port is exposed
	info, err := s.Client.InspectContainer(container.ID)
	if err != nil {
		return fmt.Errorf("error inspecting container: %s", err.Error())
	}

	mappings, ok := info.NetworkSettings.Ports[docker.Port("3423/tcp")]
	if !ok {
		return fmt.Errorf(
			"container must expose port 3423/tcp. Found: %+v",
			info.NetworkSettings.Ports)
	}
	if len(mappings) != 1 {
		return fmt.Errorf(
			"error. expected one port mapping. found: %+v",
			info.NetworkSettings.Ports)
	}
	ip, port := mappings[0].HostIP, mappings[0].HostPort

	backoffConfig := backoff.NewExponentialBackOff()
	backoffConfig.InitialInterval = time.Second
	backoffConfig.MaxInterval = 10
	backoffConfig.MaxElapsedTime = 10 * time.Second
	err = backoff.Retry(func() error {
		resp, err := http.Get(fmt.Sprintf("http://%s:%s/ping", ip, port))
		if err != nil {
			log.Println("error pinging agent. found:", err)
			// TODO handle err
			return err
		}
		defer resp.Body.Close()
		var agentInfo AgentInfo
		if err := json.NewDecoder(resp.Body).Decode(&agentInfo); err != nil {
			return err
		}
		if agentInfo.Owner == "btc" {
			log.Println("yay!")
		}
		return nil
	}, backoffConfig)

	const kStopContainerTimeout = 5 // seconds
	if err := s.Client.StopContainer(container.ID, kStopContainerTimeout); err != nil {
		return err
	}
	return nil
}
<commit_msg>refactor(httpd) pass path as argument<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"html/template"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"time"

	"github.com/cenkalti/backoff"
	"github.com/facebookgo/stackerr"
	"github.com/fsouza/go-dockerclient"
	"github.com/gorilla/mux"
)

// If |useTLS| is enabled, look for the cert files here.
var defaultCertPath = os.Getenv("DOCKER_CERT_PATH")

var (
	staticPath  = flag.String("static", "./static", "serve static files located in this directory")
	dockerHost  = flag.String("docker", "tcp://192.168.59.103:2376", "address of docker daemon host")
	tlsCertPath = flag.String("tls-cert-path", defaultCertPath, "path to docker host TLS certificate")
	host        = flag.String("host", ":8080", "address:port for HTTP listener")
	useTLS      = flag.Bool("tls", true, "use Docker TLS")
)

func main() {
	flag.Parse()
	if err := run(); err != nil {
		log.Fatal(err)
	}
}

func run() error {
	var client *docker.Client
	var err error
	if *useTLS {
		client, err = docker.NewTLSClient(
			*dockerHost,
			path.Join(*tlsCertPath, "cert.pem"),
			path.Join(*tlsCertPath, "key.pem"),
			path.Join(*tlsCertPath, "ca.pem"),
		)
		if err != nil {
			return stackerr.Wrap(err)
		}
	} else {
		client, err = docker.NewClient(*dockerHost)
		if err != nil {
			return err
		}
	}
	s := &AgentSupervisor{Client: client}
	log.Fatal(http.ListenAndServe(*host, Router(s, *staticPath)))
	return nil
}

// Router defines URL routes
func Router(s *AgentSupervisor, path string) *mux.Router {
	r := mux.NewRouter()
	WireAPIRoutes(r, s)
	WireHTMLRoutes(r, s) // after API routes

	// Finally, if none of the above routes match, delegate to the single-page
	// app's client-side router. Rewrite the path in order to load the
	// single-page app's root HTML entrypoint. The app will handle the route.
	r.NotFoundHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		r.URL.Path = "/"
		http.FileServer(http.Dir(path)).ServeHTTP(w, r)
	})
	return r
}

var templateHome = `

fxx >>= ai

	<form action=/api/v0/validate method=GET>

	validate your AI... Enter the name of the Docker image:
		<input type=text name=image>
		</input>
	</form>

`

func WireHTMLRoutes(r *mux.Router, s *AgentSupervisor) {

	r.HandleFunc("/html", func(w http.ResponseWriter, r *http.Request) {
		tmpl, err := template.New("name").Parse(templateHome)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		// Error checking elided
		if err := tmpl.Execute(w, nil); err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
	})
}

func WireAPIRoutes(r *mux.Router, s *AgentSupervisor) {

	apiV0 := r.PathPrefix("/api/v0").Subrouter()

	// TODO Build an agent from a GitHub repo
	// TODO Run Game between two running agents

	agents := apiV0.Path("/agents").Subrouter()
	agents.Methods("GET").HandlerFunc(ListAgentsHandler(s)) // list all available agents

	// pull an agent from Dockerhub
	dockerhub := apiV0.Path("/pull/dockerhub").Subrouter()
	dockerhub.Methods("GET").HandlerFunc(PullDockerHubAgentHandler(s)) // FIXME make this a POST

	// TODO List running agents with parameter
	apiV0.Path("/running").HandlerFunc(ListActiveAgentsHandler(s))

	// checks that the agent implements the right protocol
	apiV0.Path("/validate").HandlerFunc(ValidateAgentHandler(s))

	apiV0.Path("/image").HandlerFunc(ShowAgentInfoHandler(s))
}

// PullDockerHubAgentHandler pulls the Docker image named |image| from
// DockerHub.
func PullDockerHubAgentHandler(s *AgentSupervisor) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if r.FormValue("image") == "" {
			w.WriteHeader(http.StatusBadRequest)
			return
		}
		if err := s.Client.PullImage(docker.PullImageOptions{
			OutputStream: w,
			Registry:     "https://index.docker.io",
			Repository:   r.FormValue("image"),
			Tag:          "latest",
		}, docker.AuthConfiguration{}); err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintln(w, err)
		}
	}
}

// ListAgentsHandler lists AI agents.
func ListAgentsHandler(s *AgentSupervisor) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		images, err := s.Client.ListImages(docker.ListImagesOptions{
			All: false,
		})
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			_, err := io.WriteString(w, err.Error())
			if err != nil {
				log.Println("error writing err: %s", err)
			}
			return
		}
		for _, img := range images {
			_, err := fmt.Fprintln(w, fmt.Sprintln(append([]string{img.ID, "\t"}, img.RepoTags...)))
			if err != nil {
				w.WriteHeader(http.StatusInternalServerError)
				_, err := io.WriteString(w, err.Error())
				if err != nil {
					log.Println("error writing err: %s", err)
				}
			}
		}
	}
}

// ListActiveAgentsHandler lists agents that are currently running.
func ListActiveAgentsHandler(s *AgentSupervisor) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		containers, err := s.Client.ListContainers(docker.ListContainersOptions{})
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		for _, ps := range containers {
			fmt.Fprintln(w, fmt.Sprintf("%+v\n", ps))
		}
	}
}

// ValidateAgentHandler ensures that the Agent image responds to the PING
// command.
func ValidateAgentHandler(s *AgentSupervisor) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		defer log.Println("returned")

		// client gives the |image| as a URL parameter
		if r.FormValue("image") == "" {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintln(w, "`image` parameter is required")
			return
		}
		image := r.FormValue("image")

		if err := s.ValidateImage(image); err != nil {
			fmt.Fprintf(w, "image %s is not valid. error: %s", image, err)
			return
		}
		fmt.Fprintf(w, "image %s is valid", image)
	}
}

type AgentInfo struct {
	Owner  string
	Taunts []string
}

// ShowImageHandler shows information about an AI Agent
func ShowAgentInfoHandler(s *AgentSupervisor) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		image, err := s.Client.InspectImage("jbenet/go-ipfs")
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		json.NewEncoder(w).Encode(image.Config.ExposedPorts)
	}
}

// AgentSupervisor manages AI agents running in Docker containers
type AgentSupervisor struct {
	Client *docker.Client
}

func (s *AgentSupervisor) ValidateImage(image string) error {

	// run the container in a two-phase process. First, create the
	// container.
	container, err := s.Client.CreateContainer(docker.CreateContainerOptions{
		Config: &docker.Config{
			Image: image, // the only required argument
		},
	})
	if err != nil {
		return fmt.Errorf("error creating container:", err.Error())
	}

	hc := &docker.HostConfig{
		PublishAllPorts: true,
	}

	// run the created container
	if err := s.Client.StartContainer(container.ID, hc); err != nil {
		return fmt.Errorf("error starting container: %s", err.Error())
	}

	// ensure the proper port is exposed
	info, err := s.Client.InspectContainer(container.ID)
	if err != nil {
		return fmt.Errorf("error inspecting container: %s", err.Error())
	}

	mappings, ok := info.NetworkSettings.Ports[docker.Port("3423/tcp")]
	if !ok {
		return fmt.Errorf(
			"container must expose port 3423/tcp. Found: %+v",
			info.NetworkSettings.Ports)
	}
	if len(mappings) != 1 {
		return fmt.Errorf(
			"error. expected one port mapping. found: %+v",
			info.NetworkSettings.Ports)
	}
	ip, port := mappings[0].HostIP, mappings[0].HostPort

	backoffConfig := backoff.NewExponentialBackOff()
	backoffConfig.InitialInterval = time.Second
	backoffConfig.MaxInterval = 10
	backoffConfig.MaxElapsedTime = 10 * time.Second
	err = backoff.Retry(func() error {
		resp, err := http.Get(fmt.Sprintf("http://%s:%s/ping", ip, port))
		if err != nil {
			log.Println("error pinging agent. found:", err)
			// TODO handle err
			return err
		}
		defer resp.Body.Close()
		var agentInfo AgentInfo
		if err := json.NewDecoder(resp.Body).Decode(&agentInfo); err != nil {
			return err
		}
		if agentInfo.Owner == "btc" {
			log.Println("yay!")
		}
		return nil
	}, backoffConfig)

	const kStopContainerTimeout = 5 // seconds
	if err := s.Client.StopContainer(container.ID, kStopContainerTimeout); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015-2016 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libfs

import (
	"bytes"
	"context"
	"io"
	"os"
	"regexp"
	"runtime/pprof"
	"runtime/trace"
	"strings"
	"time"

	"github.com/keybase/client/go/kbfs/libkbfs"
	"github.com/pkg/errors"
	billy "gopkg.in/src-d/go-billy.v4"
)

const (
	// CPUProfilePrefix is the prefix to a CPU profile file (a
	// duration should follow the prefix in the actual file name).
	CPUProfilePrefix = "profile."

	// TraceProfilePrefix is the prefix to a trace profile file (a
	// duration should follow the prefix in the actual file name).
	TraceProfilePrefix = "trace."
)

type timedProfile interface {
	Start(w io.Writer) error
	Stop()
}

type cpuProfile struct{}

func (p cpuProfile) Start(w io.Writer) error {
	return pprof.StartCPUProfile(w)
}

func (p cpuProfile) Stop() {
	pprof.StopCPUProfile()
}

type traceProfile struct{}

func (p traceProfile) Start(w io.Writer) error {
	return trace.Start(w)
}

func (p traceProfile) Stop() {
	trace.Stop()
}

// ProfileGet gets the relevant read function for the profile or nil if it doesn't exist.
func ProfileGet(name string) func(context.Context) ([]byte, time.Time, error) {
	p := pprof.Lookup(name)
	if p == nil {
		return nil
	}

	// See https://golang.org/pkg/runtime/pprof/#Profile.WriteTo
	// for the meaning of debug.
	debug := 1
	if name == "goroutine" {
		debug = 2
	}
	return profileRead(p, debug)
}

// profileRead reads from a Profile.
func profileRead(p *pprof.Profile, debug int) func(context.Context) ([]byte, time.Time, error) {
	return func(_ context.Context) ([]byte, time.Time, error) {
		var b bytes.Buffer
		err := p.WriteTo(&b, debug)
		if err != nil {
			return nil, time.Time{}, err
		}

		return b.Bytes(), time.Now(), nil
	}
}

var profileNameRE = regexp.MustCompile("^[a-zA-Z0-9_]*$")

// IsSupportedProfileName matches a string against allowed profile names.
func IsSupportedProfileName(name string) bool {
	// https://golang.org/pkg/runtime/pprof/#NewProfile recommends
	// using an import path for profile names. But supporting that
	// would require faking out sub-directories, too. For now,
	// just support alphanumeric filenames.
	return profileNameRE.MatchString(name)
}

// ProfileFS provides an easy way to browse the go profiles.
type ProfileFS struct {
	config libkbfs.Config
}

// NewProfileFS returns a read-only filesystem for browsing profiles.
func NewProfileFS(config libkbfs.Config) ProfileFS {
	return ProfileFS{config}
}

var _ libkbfs.NodeFSReadOnly = ProfileFS{}

// Lstat implements the libkbfs.NodeFSReadOnly interface.
func (pfs ProfileFS) Lstat(filename string) (os.FileInfo, error) {
	if strings.HasPrefix(filename, CPUProfilePrefix) ||
		strings.HasPrefix(filename, TraceProfilePrefix) {
		// Set profile sizes to 0 because it's too expensive to read them
		// ahead of time.
		return &wrappedReadFileInfo{filename, 0, time.Now(), false}, nil
	}

	if !IsSupportedProfileName(filename) {
		return nil, errors.Errorf("Unsupported profile %s", filename)
	}

	f := ProfileGet(filename)
	// Get the data, just for the size.
	b, t, err := f(context.Background())
	if err != nil {
		return nil, err
	}
	return &wrappedReadFileInfo{filename, int64(len(b)), t, false}, nil
}

// ListProfileNames returns the name of all profiles to list.
func ListProfileNames() (res []string) {
	profiles := pprof.Profiles()
	res = make([]string, 0, len(profiles)+2)
	for _, p := range profiles {
		name := p.Name()
		if !IsSupportedProfileName(name) {
			continue
		}
		res = append(res, name)
	}
	res = append(res, CPUProfilePrefix+"30s")
	res = append(res, TraceProfilePrefix+"30s")
	return res
}

// ReadDir implements the libkbfs.NodeFSReadOnly interface.
func (pfs ProfileFS) ReadDir(path string) ([]os.FileInfo, error) {
	if path != "" && path != "." {
		return nil, errors.New("Can't read subdirectories in profile list")
	}

	profiles := ListProfileNames()
	res := make([]os.FileInfo, 0, len(profiles))
	for _, p := range profiles {
		fi, err := pfs.Lstat(p)
		if err != nil {
			return nil, err
		}
		res = append(res, fi)
	}
	return res, nil
}

// Readlink implements the libkbfs.NodeFSReadOnly interface.
func (pfs ProfileFS) Readlink(_ string) (string, error) {
	return "", errors.New("Readlink not supported")
}

func (pfs ProfileFS) openTimedProfile(
	ctx context.Context, durationStr string, prof timedProfile) (
	[]byte, error) {
	duration, err := time.ParseDuration(durationStr)
	if err != nil {
		return nil, err
	}

	// TODO: Blocking here until the profile is done is
	// weird. Blocking on read is better.
	//
	// TODO: Maybe keep around a special last_profile file to be
	// able to start capturing a profile and then interrupt when
	// done, which would also be useful in general, since you be
	// able to save a profile even if you open it up with a tool.
	var buf bytes.Buffer
	err = prof.Start(&buf)
	if err != nil {
		return nil, err
	}

	defer prof.Stop()

	select {
	case <-time.After(duration):
	case <-ctx.Done():
		return nil, ctx.Err()
	}

	prof.Stop()

	return buf.Bytes(), nil
}

// OpenWithContext opens a profile, with a custom context.
func (pfs ProfileFS) OpenWithContext(
	ctx context.Context, filename string) (billy.File, error) {
	var durationStr string
	var prof timedProfile
	if strings.HasPrefix(filename, CPUProfilePrefix) {
		durationStr = strings.TrimPrefix(filename, CPUProfilePrefix)
		prof = cpuProfile{}
	} else if strings.HasPrefix(filename, TraceProfilePrefix) {
		durationStr = strings.TrimPrefix(filename, TraceProfilePrefix)
		prof = traceProfile{}
	}
	if durationStr != "" {
		// Read and cache the timed profiles contents on open, so we
		// don't re-do it on every read.
		b, err := pfs.openTimedProfile(ctx, durationStr, prof)
		if err != nil {
			return nil, err
		}
		now := pfs.config.Clock().Now()
		return &wrappedReadFile{
			filename,
			func(_ context.Context) ([]byte, time.Time, error) {
				return b, now, nil
			},
			pfs.config.MakeLogger(""),
			0}, nil
	}

	if !IsSupportedProfileName(filename) {
		return nil, errors.Errorf("Unsupported profile %s", filename)
	}

	f := ProfileGet(filename)
	return &wrappedReadFile{filename, f, pfs.config.MakeLogger(""), 0}, nil
}

// Open implements the libkbfs.NodeFSReadOnly interface.
func (pfs ProfileFS) Open(filename string) (billy.File, error) {
	// TODO: we should figure out some way to route the actual
	// request context here if at all possible, so we can end
	// early if it's canceled.
	return pfs.OpenWithContext(context.TODO(), filename)
}

// OpenFile implements the libkbfs.NodeFSReadOnly interface.
func (pfs ProfileFS) OpenFile(
	filename string, flag int, _ os.FileMode) (billy.File, error) {
	if flag&os.O_CREATE != 0 {
		return nil, errors.New("read-only filesystem")
	}

	return pfs.Open(filename)
}
<commit_msg>libfs: fix profilelist comment typo<commit_after>// Copyright 2015-2016 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libfs

import (
	"bytes"
	"context"
	"io"
	"os"
	"regexp"
	"runtime/pprof"
	"runtime/trace"
	"strings"
	"time"

	"github.com/keybase/client/go/kbfs/libkbfs"
	"github.com/pkg/errors"
	billy "gopkg.in/src-d/go-billy.v4"
)

const (
	// CPUProfilePrefix is the prefix to a CPU profile file (a
	// duration should follow the prefix in the actual file name).
	CPUProfilePrefix = "profile."

	// TraceProfilePrefix is the prefix to a trace profile file (a
	// duration should follow the prefix in the actual file name).
	TraceProfilePrefix = "trace."
)

type timedProfile interface {
	Start(w io.Writer) error
	Stop()
}

type cpuProfile struct{}

func (p cpuProfile) Start(w io.Writer) error {
	return pprof.StartCPUProfile(w)
}

func (p cpuProfile) Stop() {
	pprof.StopCPUProfile()
}

type traceProfile struct{}

func (p traceProfile) Start(w io.Writer) error {
	return trace.Start(w)
}

func (p traceProfile) Stop() {
	trace.Stop()
}

// ProfileGet gets the relevant read function for the profile or nil if it doesn't exist.
func ProfileGet(name string) func(context.Context) ([]byte, time.Time, error) {
	p := pprof.Lookup(name)
	if p == nil {
		return nil
	}

	// See https://golang.org/pkg/runtime/pprof/#Profile.WriteTo
	// for the meaning of debug.
	debug := 1
	if name == "goroutine" {
		debug = 2
	}
	return profileRead(p, debug)
}

// profileRead reads from a Profile.
func profileRead(p *pprof.Profile, debug int) func(context.Context) ([]byte, time.Time, error) {
	return func(_ context.Context) ([]byte, time.Time, error) {
		var b bytes.Buffer
		err := p.WriteTo(&b, debug)
		if err != nil {
			return nil, time.Time{}, err
		}

		return b.Bytes(), time.Now(), nil
	}
}

var profileNameRE = regexp.MustCompile("^[a-zA-Z0-9_]*$")

// IsSupportedProfileName matches a string against allowed profile names.
func IsSupportedProfileName(name string) bool {
	// https://golang.org/pkg/runtime/pprof/#NewProfile recommends
	// using an import path for profile names. But supporting that
	// would require faking out sub-directories, too. For now,
	// just support alphanumeric filenames.
	return profileNameRE.MatchString(name)
}

// ProfileFS provides an easy way to browse the go profiles.
type ProfileFS struct {
	config libkbfs.Config
}

// NewProfileFS returns a read-only filesystem for browsing profiles.
func NewProfileFS(config libkbfs.Config) ProfileFS {
	return ProfileFS{config}
}

var _ libkbfs.NodeFSReadOnly = ProfileFS{}

// Lstat implements the libkbfs.NodeFSReadOnly interface.
func (pfs ProfileFS) Lstat(filename string) (os.FileInfo, error) {
	if strings.HasPrefix(filename, CPUProfilePrefix) ||
		strings.HasPrefix(filename, TraceProfilePrefix) {
		// Set profile sizes to 0 because it's too expensive to read them
		// ahead of time.
		return &wrappedReadFileInfo{filename, 0, time.Now(), false}, nil
	}

	if !IsSupportedProfileName(filename) {
		return nil, errors.Errorf("Unsupported profile %s", filename)
	}

	f := ProfileGet(filename)
	// Get the data, just for the size.
	b, t, err := f(context.Background())
	if err != nil {
		return nil, err
	}
	return &wrappedReadFileInfo{filename, int64(len(b)), t, false}, nil
}

// ListProfileNames returns the name of all profiles to list.
func ListProfileNames() (res []string) {
	profiles := pprof.Profiles()
	res = make([]string, 0, len(profiles)+2)
	for _, p := range profiles {
		name := p.Name()
		if !IsSupportedProfileName(name) {
			continue
		}
		res = append(res, name)
	}
	res = append(res, CPUProfilePrefix+"30s")
	res = append(res, TraceProfilePrefix+"30s")
	return res
}

// ReadDir implements the libkbfs.NodeFSReadOnly interface.
func (pfs ProfileFS) ReadDir(path string) ([]os.FileInfo, error) {
	if path != "" && path != "." {
		return nil, errors.New("Can't read subdirectories in profile list")
	}

	profiles := ListProfileNames()
	res := make([]os.FileInfo, 0, len(profiles))
	for _, p := range profiles {
		fi, err := pfs.Lstat(p)
		if err != nil {
			return nil, err
		}
		res = append(res, fi)
	}
	return res, nil
}

// Readlink implements the libkbfs.NodeFSReadOnly interface.
func (pfs ProfileFS) Readlink(_ string) (string, error) {
	return "", errors.New("Readlink not supported")
}

func (pfs ProfileFS) openTimedProfile(
	ctx context.Context, durationStr string, prof timedProfile) (
	[]byte, error) {
	duration, err := time.ParseDuration(durationStr)
	if err != nil {
		return nil, err
	}

	// TODO: Blocking here until the profile is done is
	// weird. Blocking on read is better.
	//
	// TODO: Maybe keep around a special last_profile file to be able
	// to start capturing a profile and then interrupt when done,
	// which would also be useful in general, since you'd be able to
	// save a profile even if you open it up with a tool.
	var buf bytes.Buffer
	err = prof.Start(&buf)
	if err != nil {
		return nil, err
	}

	defer prof.Stop()

	select {
	case <-time.After(duration):
	case <-ctx.Done():
		return nil, ctx.Err()
	}

	prof.Stop()

	return buf.Bytes(), nil
}

// OpenWithContext opens a profile, with a custom context.
func (pfs ProfileFS) OpenWithContext(
	ctx context.Context, filename string) (billy.File, error) {
	var durationStr string
	var prof timedProfile
	if strings.HasPrefix(filename, CPUProfilePrefix) {
		durationStr = strings.TrimPrefix(filename, CPUProfilePrefix)
		prof = cpuProfile{}
	} else if strings.HasPrefix(filename, TraceProfilePrefix) {
		durationStr = strings.TrimPrefix(filename, TraceProfilePrefix)
		prof = traceProfile{}
	}
	if durationStr != "" {
		// Read and cache the timed profiles contents on open, so we
		// don't re-do it on every read.
		b, err := pfs.openTimedProfile(ctx, durationStr, prof)
		if err != nil {
			return nil, err
		}
		now := pfs.config.Clock().Now()
		return &wrappedReadFile{
			filename,
			func(_ context.Context) ([]byte, time.Time, error) {
				return b, now, nil
			},
			pfs.config.MakeLogger(""),
			0}, nil
	}

	if !IsSupportedProfileName(filename) {
		return nil, errors.Errorf("Unsupported profile %s", filename)
	}

	f := ProfileGet(filename)
	return &wrappedReadFile{filename, f, pfs.config.MakeLogger(""), 0}, nil
}

// Open implements the libkbfs.NodeFSReadOnly interface.
func (pfs ProfileFS) Open(filename string) (billy.File, error) {
	// TODO: we should figure out some way to route the actual
	// request context here if at all possible, so we can end
	// early if it's canceled.
	return pfs.OpenWithContext(context.TODO(), filename)
}

// OpenFile implements the libkbfs.NodeFSReadOnly interface.
func (pfs ProfileFS) OpenFile(
	filename string, flag int, _ os.FileMode) (billy.File, error) {
	if flag&os.O_CREATE != 0 {
		return nil, errors.New("read-only filesystem")
	}

	return pfs.Open(filename)
}
<|endoftext|>


<commit_before>package armor

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/base64"
	"os"
)

func init() {
	os.Setenv("GODEBUG", os.Getenv("GODEBUG")+",tls13=1")
}

// setupTLSConfig builds the TLS configuration
func (a *Armor) setupTLSConfig() *tls.Config {
	ret := new(tls.Config)
	ret.GetConfigForClient = a.GetConfigForClient

	if a.TLS.Secured {
		ret.MinVersion = tls.VersionTLS12
		ret.CipherSuites = []uint16{
			tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,

			tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
		}
	}

	return ret
}

// GetConfigForClient implements the Config.GetClientCertificate callback
func (a *Armor) GetConfigForClient(clientHelloInfo *tls.ClientHelloInfo) (*tls.Config, error) {
	// Get the host from the hello info
	host := a.Hosts[clientHelloInfo.ServerName]
	// If the host or the clientCAs are not configured the function
	// returns the default TLS configuration
	if host == nil || len(host.ClientCAs) == 0 {
		return nil, nil
	}

	// Use existing host config if exist
	if host.TLSConfig != nil {
		return host.TLSConfig, nil
	}

	// Build and save the host config
	host.TLSConfig = a.buildTLSConfig(clientHelloInfo, host)

	return host.TLSConfig, nil
}

func (a *Armor) buildTLSConfig(clientHelloInfo *tls.ClientHelloInfo, host *Host) *tls.Config {
	// Copy the configurations from the regular server
	tlsConfig := new(tls.Config)
	*tlsConfig = *a.Echo.TLSServer.TLSConfig

	// Set the client validation and the certification pool
	tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert
	tlsConfig.ClientCAs = a.buildClientCertPool(host)

	return tlsConfig
}

func (a *Armor) buildClientCertPool(host *Host) (certPool *x509.CertPool) {
	certPool = x509.NewCertPool()

	// Loop every CA certs given as base64 DER encoding
	for _, clientCAString := range host.ClientCAs {
		// Decode base64
		derBytes, err := base64.StdEncoding.DecodeString(clientCAString)
		if err != nil {
			continue
		}
		if len(derBytes) == 0 {
			continue
		}

		// Parse the DER encoded certificate
		var caCert *x509.Certificate
		caCert, err = x509.ParseCertificate(derBytes)
		if err != nil {
			continue
		}

		// Add the certificate to CertPool
		certPool.AddCert(caCert)
	}

	return certPool
}
<commit_msg>Renamed variable as askeck from @vishr<commit_after>package armor

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/base64"
	"os"
)

func init() {
	os.Setenv("GODEBUG", os.Getenv("GODEBUG")+",tls13=1")
}

// setupTLSConfig builds the TLS configuration
func (a *Armor) setupTLSConfig() *tls.Config {
	cfg := new(tls.Config)
	cfg.GetConfigForClient = a.GetConfigForClient

	if a.TLS.Secured {
		cfg.MinVersion = tls.VersionTLS12
		cfg.CipherSuites = []uint16{
			tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,

			tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
		}
	}

	return cfg
}

// GetConfigForClient implements the Config.GetClientCertificate callback
func (a *Armor) GetConfigForClient(clientHelloInfo *tls.ClientHelloInfo) (*tls.Config, error) {
	// Get the host from the hello info
	host := a.Hosts[clientHelloInfo.ServerName]
	// If the host or the clientCAs are not configured the function
	// returns the default TLS configuration
	if host == nil || len(host.ClientCAs) == 0 {
		return nil, nil
	}

	// Use existing host config if exist
	if host.TLSConfig != nil {
		return host.TLSConfig, nil
	}

	// Build and save the host config
	host.TLSConfig = a.buildTLSConfig(clientHelloInfo, host)

	return host.TLSConfig, nil
}

func (a *Armor) buildTLSConfig(clientHelloInfo *tls.ClientHelloInfo, host *Host) *tls.Config {
	// Copy the configurations from the regular server
	tlsConfig := new(tls.Config)
	*tlsConfig = *a.Echo.TLSServer.TLSConfig

	// Set the client validation and the certification pool
	tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert
	tlsConfig.ClientCAs = a.buildClientCertPool(host)

	return tlsConfig
}

func (a *Armor) buildClientCertPool(host *Host) (certPool *x509.CertPool) {
	certPool = x509.NewCertPool()

	// Loop every CA certs given as base64 DER encoding
	for _, clientCAString := range host.ClientCAs {
		// Decode base64
		derBytes, err := base64.StdEncoding.DecodeString(clientCAString)
		if err != nil {
			continue
		}
		if len(derBytes) == 0 {
			continue
		}

		// Parse the DER encoded certificate
		var caCert *x509.Certificate
		caCert, err = x509.ParseCertificate(derBytes)
		if err != nil {
			continue
		}

		// Add the certificate to CertPool
		certPool.AddCert(caCert)
	}

	return certPool
}
<|endoftext|>