<commit_before>package units

type Unit struct {
	dim  Dimensionality
	base baseUnits

	// For scaled units (millis, kilos, megas, etc) this stores the scale
	// factor. If positive then a value must be divided by it to recover
	// the unscaled unit, while if it's negative a value must be multiplied
	// by its absolute value. 0 means "unscaled"
	//
	// Scaling is only used for derived units. Units of base dimensions are
	// just represented directly.
	scale int
}

var dimless = &Unit{Dimensionality{}, baseUnits{}, 0}

var unitByName map[string]*Unit = map[string]*Unit{
	// Dimensionless
	"": dimless,

	// Mass Units
	"kg": &Unit{Dimensionality{Mass: 1}, baseUnits{Mass: kilogram}, 0},
	"g":  &Unit{Dimensionality{Mass: 1}, baseUnits{Mass: gram}, 0},
	"lb": &Unit{Dimensionality{Mass: 1}, baseUnits{Mass: pound}, 0},
	"st": &Unit{Dimensionality{Mass: 1}, baseUnits{Mass: stone}, 0},

	// Length Units
	"m":   &Unit{Dimensionality{Length: 1}, baseUnits{Length: meter}, 0},
	"mm":  &Unit{Dimensionality{Length: 1}, baseUnits{Length: millimeter}, 0},
	"cm":  &Unit{Dimensionality{Length: 1}, baseUnits{Length: centimeter}, 0},
	"km":  &Unit{Dimensionality{Length: 1}, baseUnits{Length: kilometer}, 0},
	"mil": &Unit{Dimensionality{Length: 1}, baseUnits{Length: mil}, 0},
	"in":  &Unit{Dimensionality{Length: 1}, baseUnits{Length: inch}, 0},
	"ft":  &Unit{Dimensionality{Length: 1}, baseUnits{Length: foot}, 0},
	"yd":  &Unit{Dimensionality{Length: 1}, baseUnits{Length: yard}, 0},

	// Angle Units
	"deg":  &Unit{Dimensionality{Angle: 1}, baseUnits{Angle: degree}, 0},
	"rad":  &Unit{Dimensionality{Angle: 1}, baseUnits{Angle: radian}, 0},
	"turn": &Unit{Dimensionality{Angle: 1}, baseUnits{Angle: turn}, 0},

	// Time Units
	"s":  &Unit{Dimensionality{Time: 1}, baseUnits{Time: second}, 0}, // There is no secs in physics.
	"ms": &Unit{Dimensionality{Time: 1}, baseUnits{Time: millisecond}, 0},
	"us": &Unit{Dimensionality{Time: 1}, baseUnits{Time: microsecond}, 0},

	// Electric Current Units
	"A": &Unit{Dimensionality{ElectricCurrent: 1}, baseUnits{ElectricCurrent: ampere}, 0},

	// Luminous Intensity Units
	"cd": &Unit{Dimensionality{LuminousIntensity: 1}, baseUnits{LuminousIntensity: candela}, 0},

	// Electic Resistance Units
	"ohm": &Unit{
		Dimensionality{Mass: 1, Length: 2, Time: -3, ElectricCurrent: -2},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		0,
	},
	"kohm": &Unit{
		Dimensionality{Mass: 1, Length: 2, Time: -3, ElectricCurrent: -2},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		1000,
	},
	"Mohm": &Unit{
		Dimensionality{Mass: 1, Length: 2, Time: -3, ElectricCurrent: -2},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		1000000,
	},

	// Electric Voltage Units
	"V": &Unit{
		Dimensionality{Mass: 1, Length: 2, Time: -3, ElectricCurrent: -1},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		0,
	},
	"mV": &Unit{
		Dimensionality{Mass: 1, Length: 2, Time: -3, ElectricCurrent: -1},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		-1000,
	},
	"kV": &Unit{
		Dimensionality{Mass: 1, Length: 2, Time: -3, ElectricCurrent: -1},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		1000,
	},

	// Frequency Units
	"Hz": &Unit{
		Dimensionality{Time: -1},
		baseUnits{Time: second},
		0,
	},
	"kHz": &Unit{
		Dimensionality{Time: -1},
		baseUnits{Time: second},
		1000,
	},
	"MHz": &Unit{
		Dimensionality{Time: -1},
		baseUnits{Time: second},
		1000000,
	},

	// Force Units
	"N": &Unit{
		Dimensionality{Mass: 1, Length: 1, Time: -2},
		baseUnits{Mass: kilogram, Length: meter, Time: second},
		0,
	},

	// Power Units
	"W": &Unit{
		Dimensionality{Mass: 1, Length: 2, Time: -3},
		baseUnits{Mass: kilogram, Length: meter, Time: second},
		0,
	},
	"mW": &Unit{
		Dimensionality{Time: -1},
		baseUnits{Time: second},
		-1000,
	},
	"kW": &Unit{
		Dimensionality{Time: -1},
		baseUnits{Time: second},
		1000,
	},
	"MW": &Unit{
		Dimensionality{Time: -1},
		baseUnits{Time: second},
		1000000,
	},
	"GW": &Unit{
		Dimensionality{Time: -1},
		baseUnits{Time: second},
		1000000000,
	},

	// Electrical Capacitance Units
	"F": &Unit{
		Dimensionality{Mass: -1, Length: -2, Time: 4, ElectricCurrent: 2},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		0,
	},
	"mF": &Unit{
		Dimensionality{Mass: -1, Length: -2, Time: 4, ElectricCurrent: 2},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		-1000,
	},
	"uF": &Unit{
		Dimensionality{Mass: -1, Length: -2, Time: 4, ElectricCurrent: 2},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		-1000000,
	},

	// Electrical Inductance Units
	"H": &Unit{
		Dimensionality{Mass: 1, Length: 2, Time: -2, ElectricCurrent: -2},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		0,
	},
	"uH": &Unit{
		Dimensionality{Mass: 1, Length: 2, Time: -2, ElectricCurrent: -2},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		-1000000,
	},

	// Illuminance units
	"lx": &Unit{
		Dimensionality{Length: -2, LuminousIntensity: 1},
		baseUnits{Length: meter, LuminousIntensity: candela},
		0,
	},
}

var unitName map[Unit]string
var units map[Unit]*Unit

func init() {
	unitName = make(map[Unit]string, len(unitByName))
	units = make(map[Unit]*Unit, len(unitByName))
	for name, unit := range unitByName {
		unitName[*unit] = name
		units[*unit] = unit
	}
}

// CommensurableWith returns true if the receiver and the given unit
// have the same dimensionality.
//
// For example, two length units are commensurable but a mass unit is not
// commensurable with a length unit.
func (u *Unit) CommensurableWith(other *Unit) bool {
	if u == nil && other == nil {
		return true
	}
	if u == nil || other == nil {
		return false
	}

	return u.dim == other.dim
}
<commit_msg>unit: unit normalization<commit_after>package units

type Unit struct {
	dim  Dimensionality
	base baseUnits

	// For scaled units (millis, kilos, megas, etc) this stores the scale
	// factor. If positive then a value must be divided by it to recover
	// the unscaled unit, while if it's negative a value must be multiplied
	// by its absolute value. 0 means "unscaled"
	//
	// Scaling is only used for derived units. Units of base dimensions are
	// just represented directly.
	scale int
}

var dimless = &Unit{Dimensionality{}, baseUnits{}, 0}

var unitByName map[string]*Unit = map[string]*Unit{
	// Dimensionless
	"": dimless,

	// Mass Units
	"kg": &Unit{Dimensionality{Mass: 1}, baseUnits{Mass: kilogram}, 0},
	"g":  &Unit{Dimensionality{Mass: 1}, baseUnits{Mass: gram}, 0},
	"lb": &Unit{Dimensionality{Mass: 1}, baseUnits{Mass: pound}, 0},
	"st": &Unit{Dimensionality{Mass: 1}, baseUnits{Mass: stone}, 0},

	// Length Units
	"m":   &Unit{Dimensionality{Length: 1}, baseUnits{Length: meter}, 0},
	"mm":  &Unit{Dimensionality{Length: 1}, baseUnits{Length: millimeter}, 0},
	"cm":  &Unit{Dimensionality{Length: 1}, baseUnits{Length: centimeter}, 0},
	"km":  &Unit{Dimensionality{Length: 1}, baseUnits{Length: kilometer}, 0},
	"mil": &Unit{Dimensionality{Length: 1}, baseUnits{Length: mil}, 0},
	"in":  &Unit{Dimensionality{Length: 1}, baseUnits{Length: inch}, 0},
	"ft":  &Unit{Dimensionality{Length: 1}, baseUnits{Length: foot}, 0},
	"yd":  &Unit{Dimensionality{Length: 1}, baseUnits{Length: yard}, 0},

	// Angle Units
	"deg":  &Unit{Dimensionality{Angle: 1}, baseUnits{Angle: degree}, 0},
	"rad":  &Unit{Dimensionality{Angle: 1}, baseUnits{Angle: radian}, 0},
	"turn": &Unit{Dimensionality{Angle: 1}, baseUnits{Angle: turn}, 0},

	// Time Units
	"s":  &Unit{Dimensionality{Time: 1}, baseUnits{Time: second}, 0}, // There is no secs in physics.
	"ms": &Unit{Dimensionality{Time: 1}, baseUnits{Time: millisecond}, 0},
	"us": &Unit{Dimensionality{Time: 1}, baseUnits{Time: microsecond}, 0},

	// Electric Current Units
	"A": &Unit{Dimensionality{ElectricCurrent: 1}, baseUnits{ElectricCurrent: ampere}, 0},

	// Luminous Intensity Units
	"cd": &Unit{Dimensionality{LuminousIntensity: 1}, baseUnits{LuminousIntensity: candela}, 0},

	// Electic Resistance Units
	"ohm": &Unit{
		Dimensionality{Mass: 1, Length: 2, Time: -3, ElectricCurrent: -2},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		0,
	},
	"kohm": &Unit{
		Dimensionality{Mass: 1, Length: 2, Time: -3, ElectricCurrent: -2},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		1000,
	},
	"Mohm": &Unit{
		Dimensionality{Mass: 1, Length: 2, Time: -3, ElectricCurrent: -2},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		1000000,
	},

	// Electric Voltage Units
	"V": &Unit{
		Dimensionality{Mass: 1, Length: 2, Time: -3, ElectricCurrent: -1},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		0,
	},
	"mV": &Unit{
		Dimensionality{Mass: 1, Length: 2, Time: -3, ElectricCurrent: -1},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		-1000,
	},
	"kV": &Unit{
		Dimensionality{Mass: 1, Length: 2, Time: -3, ElectricCurrent: -1},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		1000,
	},

	// Frequency Units
	"Hz": &Unit{
		Dimensionality{Time: -1},
		baseUnits{Time: second},
		0,
	},
	"kHz": &Unit{
		Dimensionality{Time: -1},
		baseUnits{Time: second},
		1000,
	},
	"MHz": &Unit{
		Dimensionality{Time: -1},
		baseUnits{Time: second},
		1000000,
	},

	// Force Units
	"N": &Unit{
		Dimensionality{Mass: 1, Length: 1, Time: -2},
		baseUnits{Mass: kilogram, Length: meter, Time: second},
		0,
	},

	// Power Units
	"W": &Unit{
		Dimensionality{Mass: 1, Length: 2, Time: -3},
		baseUnits{Mass: kilogram, Length: meter, Time: second},
		0,
	},
	"mW": &Unit{
		Dimensionality{Time: -1},
		baseUnits{Time: second},
		-1000,
	},
	"kW": &Unit{
		Dimensionality{Time: -1},
		baseUnits{Time: second},
		1000,
	},
	"MW": &Unit{
		Dimensionality{Time: -1},
		baseUnits{Time: second},
		1000000,
	},
	"GW": &Unit{
		Dimensionality{Time: -1},
		baseUnits{Time: second},
		1000000000,
	},

	// Electrical Capacitance Units
	"F": &Unit{
		Dimensionality{Mass: -1, Length: -2, Time: 4, ElectricCurrent: 2},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		0,
	},
	"mF": &Unit{
		Dimensionality{Mass: -1, Length: -2, Time: 4, ElectricCurrent: 2},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		-1000,
	},
	"uF": &Unit{
		Dimensionality{Mass: -1, Length: -2, Time: 4, ElectricCurrent: 2},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		-1000000,
	},

	// Electrical Inductance Units
	"H": &Unit{
		Dimensionality{Mass: 1, Length: 2, Time: -2, ElectricCurrent: -2},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		0,
	},
	"uH": &Unit{
		Dimensionality{Mass: 1, Length: 2, Time: -2, ElectricCurrent: -2},
		baseUnits{Mass: kilogram, Length: meter, Time: second, ElectricCurrent: ampere},
		-1000000,
	},

	// Illuminance units
	"lx": &Unit{
		Dimensionality{Length: -2, LuminousIntensity: 1},
		baseUnits{Length: meter, LuminousIntensity: candela},
		0,
	},
}

var units map[Unit]*Unit
var unitName map[*Unit]string

func init() {
	units = make(map[Unit]*Unit, len(unitByName))
	unitName = make(map[*Unit]string, len(unitByName))
	for name, unit := range unitByName {
		units[*unit] = unit
		unitName[unit] = name
	}
}

// CommensurableWith returns true if the receiver and the given unit
// have the same dimensionality.
//
// For example, two length units are commensurable but a mass unit is not
// commensurable with a length unit.
func (u *Unit) CommensurableWith(other *Unit) bool {
	if u == nil && other == nil {
		return true
	}
	if u == nil || other == nil {
		return false
	}

	return u.dim == other.dim
}

// normalize checks if the receiver is one of the named units, and if so
// returns the pointer to the canonical instance of that unit, which can
// then in turn by looked up in the unitName table to recover the name.
func (u *Unit) normalize() *Unit {
	if nu := units[*u]; nu != nil {
		return nu
	}
	return u
}
<|endoftext|>


<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package saltpack

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha512"
	"fmt"
	"io"

	"golang.org/x/crypto/ed25519"
	"golang.org/x/crypto/nacl/secretbox"
)

type signcryptSealStream struct {
	version         Version
	output          io.Writer
	encoder         encoder
	encryptionKey   SymmetricKey
	signingKey      SigningSecretKey
	senderAnonymous bool
	keyring         Keyring
	buffer          bytes.Buffer
	headerHash      headerHash

	numBlocks encryptionBlockNumber // the lower 64 bits of the nonce

	didHeader bool
	eof       bool
	err       error
}

func (sss *signcryptSealStream) Write(plaintext []byte) (int, error) {

	if sss.err != nil {
		return 0, sss.err
	}

	var ret int
	if ret, sss.err = sss.buffer.Write(plaintext); sss.err != nil {
		return 0, sss.err
	}
	for sss.buffer.Len() > encryptionBlockSize {
		sss.err = sss.signcryptBlock(false)
		if sss.err != nil {
			return 0, sss.err
		}
	}
	return ret, nil
}

func (sss *signcryptSealStream) signcryptBlock(isFinal bool) error {
	// NOTE: plaintext is a slice into sss.buffer's buffer, so
	// make sure not to stash it anywhere.
	plaintext := sss.buffer.Next(encryptionBlockSize)
	if isFinal && (sss.buffer.Len() != 0) {
		panic(fmt.Sprintf("isFinal=true and (sss.buffer.Len()=%d != 0)", sss.buffer.Len()))
	}

	if err := sss.numBlocks.check(); err != nil {
		return err
	}

	nonce := nonceForChunkSigncryption(sss.headerHash, isFinal, sss.numBlocks)

	// Handle regular signing mode and anonymous mode (where we don't actually
	// sign anything).
	var detachedSig []byte
	if sss.signingKey == nil {
		detachedSig = make([]byte, ed25519.SignatureSize)
	} else {
		signatureInput := computeSigncryptionSignatureInput(sss.headerHash, nonce, isFinal, plaintext)

		var err error
		detachedSig, err = sss.signingKey.Sign(signatureInput)
		if err != nil {
			return err
		}
	}

	attachedSig := append(detachedSig, plaintext...)

	ciphertext := secretbox.Seal([]byte{}, attachedSig, (*[24]byte)(&nonce), (*[32]byte)(&sss.encryptionKey))

	assertEncodedChunkState(sss.version, ciphertext, secretbox.Overhead, uint64(sss.numBlocks), isFinal)

	block := signcryptionBlock{
		PayloadCiphertext: ciphertext,
		IsFinal:           isFinal,
	}

	if err := sss.encoder.Encode(block); err != nil {
		return err
	}

	sss.numBlocks++
	return nil
}

// Similar to the encryption format, we derive a symmetric key from our DH keys
// (one of which is ephemeral) by encrypting 32 bytes of zeros. We could have
// used crypto_box_beforenm directly instead, but that would be a slight abuse
// of that function, and also we don't expect all NaCl/libsodium wrapper libs
// to expose it. This key does *not* mix in the recipient index -- it will be
// the same for two different recipients if they claim the same public key.
func derivedEphemeralKeyFromBoxKeys(public BoxPublicKey, private BoxSecretKey) *SymmetricKey {
	sharedSecretBox := private.Box(public, nonceForDerivedSharedKey(), make([]byte, 32))
	derivedKey, err := symmetricKeyFromSlice(sharedSecretBox[len(sharedSecretBox)-32 : len(sharedSecretBox)])
	if err != nil {
		panic(err) // should be statically impossible, if the slice above is the right length
	}
	return derivedKey
}

// Compute the visible identifier that the recipient will use to find the right
// recipient entry. Include the entry index, so that this identifier is unique
// even if two recipients claim the same public key (though unfortunately that
// means that recipients will need to recompute the identifier for each entry
// in the recipients list). This identifier is somewhat redundant, because a
// recipient could instead just attempt to decrypt the payload key secretbox
// and see if it works, but including them adds a bit to anonymity by making
// box key recipients indistinguishable from symmetric key recipients.
func keyIdentifierFromDerivedKey(derivedKey *SymmetricKey, recipientIndex uint64) []byte {
	keyIdentifierDigest := hmac.New(sha512.New, []byte(signcryptionBoxKeyIdentifierContext))
	keyIdentifierDigest.Write(derivedKey[:])
	nonce := nonceForPayloadKeyBoxV2(recipientIndex)
	keyIdentifierDigest.Write(nonce[:])
	return keyIdentifierDigest.Sum(nil)[0:32]
}

// A receiverKeysMaker is either a (wrapped) BoxPublicKey or a
// ReceiverSymmetricKey.
type receiverKeysMaker interface {
	makeReceiverKeys(ephemeralPriv BoxSecretKey, payloadKey SymmetricKey, index uint64) receiverKeys
}

type receiverBoxKey struct {
	pk BoxPublicKey
}

func (r receiverBoxKey) makeReceiverKeys(ephemeralPriv BoxSecretKey, payloadKey SymmetricKey, index uint64) receiverKeys {
	derivedKey := derivedEphemeralKeyFromBoxKeys(r.pk, ephemeralPriv)
	identifier := keyIdentifierFromDerivedKey(derivedKey, index)

	nonce := nonceForPayloadKeyBoxV2(index)
	payloadKeyBox := secretbox.Seal(
		nil,
		payloadKey[:],
		(*[24]byte)(&nonce),
		(*[32]byte)(derivedKey))

	return receiverKeys{
		ReceiverKID:   identifier,
		PayloadKeyBox: payloadKeyBox,
	}
}

// ReceiverSymmetricKey is a symmetric key paired with an identifier.
type ReceiverSymmetricKey struct {
	// In practice these identifiers will be KBFS TLF keys.
	Key SymmetricKey
	// In practice these identifiers will be KBFS TLF pseudonyms.
	Identifier []byte
}

func (r ReceiverSymmetricKey) makeReceiverKeys(ephemeralPriv BoxSecretKey, payloadKey SymmetricKey, index uint64) receiverKeys {
	// Derive a message-specific shared secret by hashing the symmetric key and
	// the ephemeral public key together. This lets us use nonces that are
	// simple counters.
	derivedKeyDigest := hmac.New(sha512.New, []byte(signcryptionSymmetricKeyContext))
	derivedKeyDigest.Write(ephemeralPriv.GetPublicKey().ToKID())
	derivedKeyDigest.Write(r.Key[:])
	derivedKey, err := rawBoxKeyFromSlice(derivedKeyDigest.Sum(nil)[0:32])
	if err != nil {
		panic(err) // should be statically impossible, if the slice above is the right length
	}

	nonce := nonceForPayloadKeyBoxV2(index)
	payloadKeyBox := secretbox.Seal(
		nil,
		payloadKey[:],
		(*[24]byte)(&nonce),
		(*[32]byte)(derivedKey))

	// Unlike the box key case, the identifier is supplied by the caller rather
	// than computed. (These will be KBFS TLF pseudonyms.)
	return receiverKeys{
		ReceiverKID:   r.Identifier,
		PayloadKeyBox: payloadKeyBox,
	}
}

func shuffleSigncryptionReceivers(receiverBoxKeys []BoxPublicKey, receiverSymmetricKeys []ReceiverSymmetricKey) []receiverKeysMaker {
	totalLen := len(receiverBoxKeys) + len(receiverSymmetricKeys)
	order := randomPerm(totalLen)
	receivers := make([]receiverKeysMaker, totalLen)
	for i, r := range receiverBoxKeys {
		receivers[order[i]] = receiverBoxKey{r}
	}

	for i, r := range receiverSymmetricKeys {
		receivers[order[len(receiverBoxKeys)+i]] = r
	}
	return receivers
}

// This generates the payload key, and encrypts it for all the different
// recipients of the two different types. Symmetric key recipients and DH key
// recipients use different types of identifiers, but they are the same length,
// and should both be indistinguishable from random noise.
func (sss *signcryptSealStream) init(receivers []receiverKeysMaker) error {
	ephemeralKey, err := sss.keyring.CreateEphemeralKey()
	if err != nil {
		return err
	}

	eh := SigncryptionHeader{
		FormatName: FormatName,
		Version:    sss.version,
		Type:       MessageTypeSigncryption,
		Ephemeral:  ephemeralKey.GetPublicKey().ToKID(),
	}
	if err := randomFill(sss.encryptionKey[:]); err != nil {
		return err
	}

	// Prepare the secretbox that contains the sender's public key. If the
	// sender is anonymous, use an all-zeros key, so that the anonymity bit
	// doesn't leak out.
	nonce := nonceForSenderKeySecretBox()
	if sss.signingKey == nil {
		// anonymous sender mode, all zeros
		eh.SenderSecretbox = secretbox.Seal([]byte{}, make([]byte, ed25519.PublicKeySize), (*[24]byte)(&nonce), (*[32]byte)(&sss.encryptionKey))
	} else {
		// regular sender mode, an actual key
		signingPublicKeyBytes := sss.signingKey.GetPublicKey().ToKID()
		if len(signingPublicKeyBytes) != ed25519.PublicKeySize {
			panic("unexpected signing key length, anonymity bit will leak")
		}
		eh.SenderSecretbox = secretbox.Seal([]byte{}, sss.signingKey.GetPublicKey().ToKID(), (*[24]byte)(&nonce), (*[32]byte)(&sss.encryptionKey))
	}

	// Collect all the recipient identifiers, and encrypt the payload key for
	// all of them.
	for i, r := range receivers {
		eh.Receivers = append(eh.Receivers, r.makeReceiverKeys(ephemeralKey, sss.encryptionKey, uint64(i)))
	}

	// Encode the header to bytes, hash it, then double encode it.
	headerBytes, err := encodeToBytes(eh)
	if err != nil {
		return err
	}
	sss.headerHash = sha512.Sum512(headerBytes)
	err = sss.encoder.Encode(headerBytes)
	if err != nil {
		return err
	}

	return nil
}

func (sss *signcryptSealStream) Close() error {
	err := sss.signcryptBlock(true)
	if err != nil {
		return err
	}

	if sss.buffer.Len() > 0 {
		panic(fmt.Sprintf("sss.buffer.Len()=%d > 0", sss.buffer.Len()))
	}

	return nil
}

// NewSigncryptSealStream creates a stream that consumes plaintext data. It
// will write out signed and encrypted data to the io.Writer passed in as
// ciphertext. The encryption is from the specified sender, and is encrypted
// for the given receivers.
//
// Returns an io.WriteClose that accepts plaintext data to be signcrypted; and
// also returns an error if initialization failed.
func NewSigncryptSealStream(ciphertext io.Writer, keyring Keyring, sender SigningSecretKey, receiverBoxKeys []BoxPublicKey, receiverSymmetricKeys []ReceiverSymmetricKey) (io.WriteCloser, error) {
	sss := &signcryptSealStream{
		version:    Version2(),
		output:     ciphertext,
		encoder:    newEncoder(ciphertext),
		signingKey: sender,
		keyring:    keyring,
	}
	receivers := shuffleSigncryptionReceivers(receiverBoxKeys, receiverSymmetricKeys)
	err := sss.init(receivers)
	return sss, err
}

// SigncryptSeal seals a plaintext from the given sender, for the
// specified receiver groups.
//
// Returns a ciphertext, or an error if something bad happened.
func SigncryptSeal(plaintext []byte, keyring Keyring, sender SigningSecretKey, receiverBoxKeys []BoxPublicKey, receiverSymmetricKeys []ReceiverSymmetricKey) (out []byte, err error) {
	var buf bytes.Buffer
	sss, err := NewSigncryptSealStream(&buf, keyring, sender, receiverBoxKeys, receiverSymmetricKeys)
	if err != nil {
		return nil, err
	}
	if _, err := sss.Write(plaintext); err != nil {
		return nil, err
	}
	if err := sss.Close(); err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}
<commit_msg>Fix gofmt (#56)<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package saltpack

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha512"
	"fmt"
	"io"

	"golang.org/x/crypto/ed25519"
	"golang.org/x/crypto/nacl/secretbox"
)

type signcryptSealStream struct {
	version         Version
	output          io.Writer
	encoder         encoder
	encryptionKey   SymmetricKey
	signingKey      SigningSecretKey
	senderAnonymous bool
	keyring         Keyring
	buffer          bytes.Buffer
	headerHash      headerHash

	numBlocks encryptionBlockNumber // the lower 64 bits of the nonce

	didHeader bool
	eof       bool
	err       error
}

func (sss *signcryptSealStream) Write(plaintext []byte) (int, error) {

	if sss.err != nil {
		return 0, sss.err
	}

	var ret int
	if ret, sss.err = sss.buffer.Write(plaintext); sss.err != nil {
		return 0, sss.err
	}
	for sss.buffer.Len() > encryptionBlockSize {
		sss.err = sss.signcryptBlock(false)
		if sss.err != nil {
			return 0, sss.err
		}
	}
	return ret, nil
}

func (sss *signcryptSealStream) signcryptBlock(isFinal bool) error {
	// NOTE: plaintext is a slice into sss.buffer's buffer, so
	// make sure not to stash it anywhere.
	plaintext := sss.buffer.Next(encryptionBlockSize)
	if isFinal && (sss.buffer.Len() != 0) {
		panic(fmt.Sprintf("isFinal=true and (sss.buffer.Len()=%d != 0)", sss.buffer.Len()))
	}

	if err := sss.numBlocks.check(); err != nil {
		return err
	}

	nonce := nonceForChunkSigncryption(sss.headerHash, isFinal, sss.numBlocks)

	// Handle regular signing mode and anonymous mode (where we don't actually
	// sign anything).
	var detachedSig []byte
	if sss.signingKey == nil {
		detachedSig = make([]byte, ed25519.SignatureSize)
	} else {
		signatureInput := computeSigncryptionSignatureInput(sss.headerHash, nonce, isFinal, plaintext)

		var err error
		detachedSig, err = sss.signingKey.Sign(signatureInput)
		if err != nil {
			return err
		}
	}

	attachedSig := append(detachedSig, plaintext...)

	ciphertext := secretbox.Seal([]byte{}, attachedSig, (*[24]byte)(&nonce), (*[32]byte)(&sss.encryptionKey))

	assertEncodedChunkState(sss.version, ciphertext, secretbox.Overhead, uint64(sss.numBlocks), isFinal)

	block := signcryptionBlock{
		PayloadCiphertext: ciphertext,
		IsFinal:           isFinal,
	}

	if err := sss.encoder.Encode(block); err != nil {
		return err
	}

	sss.numBlocks++
	return nil
}

// Similar to the encryption format, we derive a symmetric key from our DH keys
// (one of which is ephemeral) by encrypting 32 bytes of zeros. We could have
// used crypto_box_beforenm directly instead, but that would be a slight abuse
// of that function, and also we don't expect all NaCl/libsodium wrapper libs
// to expose it. This key does *not* mix in the recipient index -- it will be
// the same for two different recipients if they claim the same public key.
func derivedEphemeralKeyFromBoxKeys(public BoxPublicKey, private BoxSecretKey) *SymmetricKey {
	sharedSecretBox := private.Box(public, nonceForDerivedSharedKey(), make([]byte, 32))
	derivedKey, err := symmetricKeyFromSlice(sharedSecretBox[len(sharedSecretBox)-32:])
	if err != nil {
		panic(err) // should be statically impossible, if the slice above is the right length
	}
	return derivedKey
}

// Compute the visible identifier that the recipient will use to find the right
// recipient entry. Include the entry index, so that this identifier is unique
// even if two recipients claim the same public key (though unfortunately that
// means that recipients will need to recompute the identifier for each entry
// in the recipients list). This identifier is somewhat redundant, because a
// recipient could instead just attempt to decrypt the payload key secretbox
// and see if it works, but including them adds a bit to anonymity by making
// box key recipients indistinguishable from symmetric key recipients.
func keyIdentifierFromDerivedKey(derivedKey *SymmetricKey, recipientIndex uint64) []byte {
	keyIdentifierDigest := hmac.New(sha512.New, []byte(signcryptionBoxKeyIdentifierContext))
	keyIdentifierDigest.Write(derivedKey[:])
	nonce := nonceForPayloadKeyBoxV2(recipientIndex)
	keyIdentifierDigest.Write(nonce[:])
	return keyIdentifierDigest.Sum(nil)[0:32]
}

// A receiverKeysMaker is either a (wrapped) BoxPublicKey or a
// ReceiverSymmetricKey.
type receiverKeysMaker interface {
	makeReceiverKeys(ephemeralPriv BoxSecretKey, payloadKey SymmetricKey, index uint64) receiverKeys
}

type receiverBoxKey struct {
	pk BoxPublicKey
}

func (r receiverBoxKey) makeReceiverKeys(ephemeralPriv BoxSecretKey, payloadKey SymmetricKey, index uint64) receiverKeys {
	derivedKey := derivedEphemeralKeyFromBoxKeys(r.pk, ephemeralPriv)
	identifier := keyIdentifierFromDerivedKey(derivedKey, index)

	nonce := nonceForPayloadKeyBoxV2(index)
	payloadKeyBox := secretbox.Seal(
		nil,
		payloadKey[:],
		(*[24]byte)(&nonce),
		(*[32]byte)(derivedKey))

	return receiverKeys{
		ReceiverKID:   identifier,
		PayloadKeyBox: payloadKeyBox,
	}
}

// ReceiverSymmetricKey is a symmetric key paired with an identifier.
type ReceiverSymmetricKey struct {
	// In practice these identifiers will be KBFS TLF keys.
	Key SymmetricKey
	// In practice these identifiers will be KBFS TLF pseudonyms.
	Identifier []byte
}

func (r ReceiverSymmetricKey) makeReceiverKeys(ephemeralPriv BoxSecretKey, payloadKey SymmetricKey, index uint64) receiverKeys {
	// Derive a message-specific shared secret by hashing the symmetric key and
	// the ephemeral public key together. This lets us use nonces that are
	// simple counters.
	derivedKeyDigest := hmac.New(sha512.New, []byte(signcryptionSymmetricKeyContext))
	derivedKeyDigest.Write(ephemeralPriv.GetPublicKey().ToKID())
	derivedKeyDigest.Write(r.Key[:])
	derivedKey, err := rawBoxKeyFromSlice(derivedKeyDigest.Sum(nil)[0:32])
	if err != nil {
		panic(err) // should be statically impossible, if the slice above is the right length
	}

	nonce := nonceForPayloadKeyBoxV2(index)
	payloadKeyBox := secretbox.Seal(
		nil,
		payloadKey[:],
		(*[24]byte)(&nonce),
		(*[32]byte)(derivedKey))

	// Unlike the box key case, the identifier is supplied by the caller rather
	// than computed. (These will be KBFS TLF pseudonyms.)
	return receiverKeys{
		ReceiverKID:   r.Identifier,
		PayloadKeyBox: payloadKeyBox,
	}
}

func shuffleSigncryptionReceivers(receiverBoxKeys []BoxPublicKey, receiverSymmetricKeys []ReceiverSymmetricKey) []receiverKeysMaker {
	totalLen := len(receiverBoxKeys) + len(receiverSymmetricKeys)
	order := randomPerm(totalLen)
	receivers := make([]receiverKeysMaker, totalLen)
	for i, r := range receiverBoxKeys {
		receivers[order[i]] = receiverBoxKey{r}
	}

	for i, r := range receiverSymmetricKeys {
		receivers[order[len(receiverBoxKeys)+i]] = r
	}
	return receivers
}

// This generates the payload key, and encrypts it for all the different
// recipients of the two different types. Symmetric key recipients and DH key
// recipients use different types of identifiers, but they are the same length,
// and should both be indistinguishable from random noise.
func (sss *signcryptSealStream) init(receivers []receiverKeysMaker) error {
	ephemeralKey, err := sss.keyring.CreateEphemeralKey()
	if err != nil {
		return err
	}

	eh := SigncryptionHeader{
		FormatName: FormatName,
		Version:    sss.version,
		Type:       MessageTypeSigncryption,
		Ephemeral:  ephemeralKey.GetPublicKey().ToKID(),
	}
	if err := randomFill(sss.encryptionKey[:]); err != nil {
		return err
	}

	// Prepare the secretbox that contains the sender's public key. If the
	// sender is anonymous, use an all-zeros key, so that the anonymity bit
	// doesn't leak out.
	nonce := nonceForSenderKeySecretBox()
	if sss.signingKey == nil {
		// anonymous sender mode, all zeros
		eh.SenderSecretbox = secretbox.Seal([]byte{}, make([]byte, ed25519.PublicKeySize), (*[24]byte)(&nonce), (*[32]byte)(&sss.encryptionKey))
	} else {
		// regular sender mode, an actual key
		signingPublicKeyBytes := sss.signingKey.GetPublicKey().ToKID()
		if len(signingPublicKeyBytes) != ed25519.PublicKeySize {
			panic("unexpected signing key length, anonymity bit will leak")
		}
		eh.SenderSecretbox = secretbox.Seal([]byte{}, sss.signingKey.GetPublicKey().ToKID(), (*[24]byte)(&nonce), (*[32]byte)(&sss.encryptionKey))
	}

	// Collect all the recipient identifiers, and encrypt the payload key for
	// all of them.
	for i, r := range receivers {
		eh.Receivers = append(eh.Receivers, r.makeReceiverKeys(ephemeralKey, sss.encryptionKey, uint64(i)))
	}

	// Encode the header to bytes, hash it, then double encode it.
	headerBytes, err := encodeToBytes(eh)
	if err != nil {
		return err
	}
	sss.headerHash = sha512.Sum512(headerBytes)
	err = sss.encoder.Encode(headerBytes)
	if err != nil {
		return err
	}

	return nil
}

func (sss *signcryptSealStream) Close() error {
	err := sss.signcryptBlock(true)
	if err != nil {
		return err
	}

	if sss.buffer.Len() > 0 {
		panic(fmt.Sprintf("sss.buffer.Len()=%d > 0", sss.buffer.Len()))
	}

	return nil
}

// NewSigncryptSealStream creates a stream that consumes plaintext data. It
// will write out signed and encrypted data to the io.Writer passed in as
// ciphertext. The encryption is from the specified sender, and is encrypted
// for the given receivers.
//
// Returns an io.WriteClose that accepts plaintext data to be signcrypted; and
// also returns an error if initialization failed.
func NewSigncryptSealStream(ciphertext io.Writer, keyring Keyring, sender SigningSecretKey, receiverBoxKeys []BoxPublicKey, receiverSymmetricKeys []ReceiverSymmetricKey) (io.WriteCloser, error) {
	sss := &signcryptSealStream{
		version:    Version2(),
		output:     ciphertext,
		encoder:    newEncoder(ciphertext),
		signingKey: sender,
		keyring:    keyring,
	}
	receivers := shuffleSigncryptionReceivers(receiverBoxKeys, receiverSymmetricKeys)
	err := sss.init(receivers)
	return sss, err
}

// SigncryptSeal seals a plaintext from the given sender, for the
// specified receiver groups.
//
// Returns a ciphertext, or an error if something bad happened.
func SigncryptSeal(plaintext []byte, keyring Keyring, sender SigningSecretKey, receiverBoxKeys []BoxPublicKey, receiverSymmetricKeys []ReceiverSymmetricKey) (out []byte, err error) {
	var buf bytes.Buffer
	sss, err := NewSigncryptSealStream(&buf, keyring, sender, receiverBoxKeys, receiverSymmetricKeys)
	if err != nil {
		return nil, err
	}
	if _, err := sss.Write(plaintext); err != nil {
		return nil, err
	}
	if err := sss.Close(); err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}
<|endoftext|>


<commit_before>// Copyright 2018 Red Hat, Inc.
// Copyright (c) 2018 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"net"
	"os"
	"os/signal"
	"path"
	"syscall"
	"time"

	"github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc"
	"github.com/networkservicemesh/networkservicemesh/controlplane/pkg/apis/nsmdapi"
	"github.com/networkservicemesh/networkservicemesh/controlplane/pkg/nsmd"
	"github.com/networkservicemesh/networkservicemesh/controlplane/pkg/serviceregistry"
	"github.com/networkservicemesh/networkservicemesh/pkg/tools"
	opentracing "github.com/opentracing/opentracing-go"
	"github.com/sirupsen/logrus"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1"
)

const (
	// SocketBaseDir defines the location of NSM client socket
	resourceName = "networkservicemesh.io/socket"
	// ServerSock defines the name of NSM client socket
	ServerSock = "networkservicemesh.io.sock"
)

type nsmClientEndpoints struct {
	serviceRegistry serviceregistry.ServiceRegistry
}

func (n *nsmClientEndpoints) Allocate(ctx context.Context, reqs *pluginapi.AllocateRequest) (*pluginapi.AllocateResponse, error) {
	logrus.Info("Client request for nsmdp resource...")
	responses := &pluginapi.AllocateResponse{}
	for _, req := range reqs.ContainerRequests {
		id := req.DevicesIDs[0]
		logrus.Infof("Requesting Workspace, device ID: %s", id)
		workspace, err := nsmd.RequestWorkspace(n.serviceRegistry, id)
		logrus.Infof("Received Workspace %v", workspace)
		if err != nil {
			logrus.Errorf("error talking to nsmd: %v", err)
		} else {
			mount := &pluginapi.Mount{
				ContainerPath: workspace.ClientBaseDir,
				HostPath:      workspace.HostBasedir + workspace.Workspace,
				ReadOnly:      false,
			}
			responses.ContainerResponses = append(responses.ContainerResponses, &pluginapi.ContainerAllocateResponse{
				Mounts: []*pluginapi.Mount{mount},
				Envs: map[string]string{
					nsmd.NsmDevicePluginEnv: "true",
					nsmd.NsmServerSocketEnv: mount.ContainerPath + workspace.NsmServerSocket,
					nsmd.NsmClientSocketEnv: mount.ContainerPath + workspace.NsmClientSocket,
					nsmd.WorkspaceEnv:       workspace.ClientBaseDir,
				},
			})
		}
	}
	logrus.Infof("AllocateResponse: %v", responses)
	return responses, nil
}

// Register registers
func Register(kubeletEndpoint string) error {
	tracer := opentracing.GlobalTracer()
	conn, err := grpc.Dial(kubeletEndpoint, grpc.WithInsecure(),
		grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
			return net.DialTimeout("unix", addr, timeout)
		}),
		grpc.WithUnaryInterceptor(
			otgrpc.OpenTracingClientInterceptor(tracer, otgrpc.LogPayloads())),
		grpc.WithStreamInterceptor(
			otgrpc.OpenTracingStreamClientInterceptor(tracer)))
	defer conn.Close()
	if err != nil {
		return fmt.Errorf("device-plugin: cannot connect to kubelet service: %v", err)
	}
	client := pluginapi.NewRegistrationClient(conn)
	reqt := &pluginapi.RegisterRequest{
		Version:      pluginapi.Version,
		Endpoint:     ServerSock,
		ResourceName: resourceName,
	}

	_, err = client.Register(context.Background(), reqt)
	if err != nil {
		return fmt.Errorf("device-plugin: cannot register to kubelet service: %v", err)
	}
	return nil
}

// Define functions needed to meet the Kubernetes DevicePlugin API
func (n *nsmClientEndpoints) GetDevicePluginOptions(context.Context, *pluginapi.Empty) (*pluginapi.DevicePluginOptions, error) {
	return &pluginapi.DevicePluginOptions{}, nil
}

func (n *nsmClientEndpoints) PreStartContainer(context.Context, *pluginapi.PreStartContainerRequest) (*pluginapi.PreStartContainerResponse, error) {
	return &pluginapi.PreStartContainerResponse{}, nil
}

func enumWorkspaces(serviceRegistry serviceregistry.ServiceRegistry) (*nsmdapi.EnumConnectionReply, error) {
	client, con, err := serviceRegistry.NSMDApiClient()
	if err != nil {
		logrus.Fatalf("Failed to connect to NSMD: %+v...", err)
	}
	defer con.Close()
	reply, err := client.EnumConnection(context.Background(), &nsmdapi.EnumConnectionRequest{})
	if err != nil {
		return nil, err
	}
	logrus.Infof("nsmd allocated workspace %+v for client operations...", reply)
	return reply, nil
}

func indexOf(slice []string, value string) int {
	for i, v := range slice {
		if v == value {
			return i
		}
	}
	return -1
}

func (n *nsmClientEndpoints) ListAndWatch(e *pluginapi.Empty, s pluginapi.DevicePlugin_ListAndWatchServer) error {
	logrus.Infof("ListAndWatch was called with s: %+v", s)
	for {
		resp := new(pluginapi.ListAndWatchResponse)
		enumWS, err := enumWorkspaces(n.serviceRegistry)
		if err != nil {
			logrus.Errorf("error retrieving workspaces from nsmd: %v", err)
		}
		workspaces := enumWS.Workspace
		id := 1
		for pool := 0; pool < 10; id++ {
			workspace := fmt.Sprintf("nsm-%d", id)
			if indexOf(workspaces, workspace) != -1 {
				continue
			}
			pool++
			resp.Devices = append(resp.Devices, &pluginapi.Device{
				ID:     workspace,
				Health: pluginapi.Healthy,
			})
		}
		for _, w := range workspaces {
			resp.Devices = append(resp.Devices, &pluginapi.Device{
				ID:     w,
				Health: pluginapi.Healthy,
			})
		}
		if err := s.Send(resp); err != nil {
			logrus.Errorf("Failed to send response to kubelet: %v\n", err)
		}
		time.Sleep(30 * time.Second)
	}
}

func startDeviceServer(nsm *nsmClientEndpoints) error {
	listenEndpoint := path.Join(pluginapi.DevicePluginPath, ServerSock)
	if err := tools.SocketCleanup(listenEndpoint); err != nil {
		return err
	}
	sock, err := net.Listen("unix", listenEndpoint)
	if err != nil {
		return err
	}
	tracer := opentracing.GlobalTracer()
	grpcServer := grpc.NewServer(
		grpc.UnaryInterceptor(
			otgrpc.OpenTracingServerInterceptor(tracer, otgrpc.LogPayloads())),
		grpc.StreamInterceptor(
			otgrpc.OpenTracingStreamServerInterceptor(tracer)))

	pluginapi.RegisterDevicePluginServer(grpcServer, nsm)

	logrus.Infof("Starting Device Plugin's gRPC server listening on socket: %s", ServerSock)
	go func() {
		if err := grpcServer.Serve(sock); err != nil {
			logrus.Error("failed to start device plugin grpc server", listenEndpoint, err)
		}
	}()
	// Check if the socket of device plugin server is operation
	conn, err := tools.SocketOperationCheck(listenEndpoint)
	if err != nil {
		return err
	}
	conn.Close()

	return nil
}

func waitForNsmdAvailable() {
	for {
		if tools.WaitForPortAvailable(context.Background(), "unix", nsmd.ServerSock, 100*time.Millisecond) == nil {
			break
		}
	}
}

// NewNSMDeviceServer registers and starts Kubelet's device plugin
func NewNSMDeviceServer(serviceRegistry serviceregistry.ServiceRegistry) error {
	waitForNsmdAvailable()
	nsm := &nsmClientEndpoints{
		serviceRegistry: serviceRegistry,
	}

	if err := startDeviceServer(nsm); err != nil {
		return err
	}
	// Registers with Kubelet.
	err := Register(pluginapi.KubeletSocket)

	return err
}

func main() {
	// Capture signals to cleanup before exiting
	c := make(chan os.Signal, 1)
	signal.Notify(c,
		syscall.SIGHUP,
		syscall.SIGINT,
		syscall.SIGTERM,
		syscall.SIGQUIT)

	tracer, closer := tools.InitJaeger("nsmdp")
	defer closer.Close()
	opentracing.SetGlobalTracer(tracer)

	serviceRegistry := nsmd.NewServiceRegistry()
	err := NewNSMDeviceServer(serviceRegistry)

	if err != nil {
		logrus.Errorf("failed to start server: %v", err)
		os.Exit(1)
	}

	<-c
}
<commit_msg>#801 Fixes NSMDp (#823)<commit_after>// Copyright 2018 Red Hat, Inc.
// Copyright (c) 2018 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"github.com/networkservicemesh/networkservicemesh/controlplane/pkg/apis/nsmdapi"
	"net"
	"os"
	"os/signal"
	"path"
	"sync"
	"syscall"
	"time"

	"github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc"
	"github.com/networkservicemesh/networkservicemesh/controlplane/pkg/nsmd"
	"github.com/networkservicemesh/networkservicemesh/controlplane/pkg/serviceregistry"
	"github.com/networkservicemesh/networkservicemesh/pkg/tools"
	"github.com/opentracing/opentracing-go"
	"github.com/sirupsen/logrus"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1"
)

const (
	// SocketBaseDir defines the location of NSM client socket
	resourceName = "networkservicemesh.io/socket"
	// ServerSock defines the name of NSM client socket
	ServerSock = "networkservicemesh.io.sock"

	// A number of devices we have in buffer for use, so we hold extra DeviceBuffer count of deviceids send to kubelet.
	DeviceBuffer = 30

	// Send device ids to kubelet every N seconds.
	KubeletNotifyDelay = 30 * time.Second
)

type nsmClientEndpoints struct {
	serviceRegistry serviceregistry.ServiceRegistry
	resp            *pluginapi.ListAndWatchResponse
	devs            map[string]*pluginapi.Device
	pluginApi       *pluginapi.DevicePlugin_ListAndWatchServer
	mutext          sync.Mutex
	clientId        int
}

func (n *nsmClientEndpoints) Allocate(ctx context.Context, reqs *pluginapi.AllocateRequest) (*pluginapi.AllocateResponse, error) {
	logrus.Info("Client request for nsmdp resource...")
	responses := &pluginapi.AllocateResponse{}
	for _, req := range reqs.ContainerRequests {
		id := req.DevicesIDs[0]
		logrus.Infof("Requesting Workspace, device ID: %s", id)
		workspace, err := nsmd.RequestWorkspace(n.serviceRegistry, id)
		logrus.Infof("Received Workspace %v", workspace)
		if err != nil {
			logrus.Errorf("error talking to nsmd: %v", err)
		} else {
			mount := &pluginapi.Mount{
				ContainerPath: workspace.ClientBaseDir,
				HostPath:      workspace.HostBasedir + workspace.Workspace,
				ReadOnly:      false,
			}
			responses.ContainerResponses = append(responses.ContainerResponses, &pluginapi.ContainerAllocateResponse{
				Mounts: []*pluginapi.Mount{mount},
				Envs: map[string]string{
					nsmd.NsmDevicePluginEnv: "true",
					nsmd.NsmServerSocketEnv: mount.ContainerPath + workspace.NsmServerSocket,
					nsmd.NsmClientSocketEnv: mount.ContainerPath + workspace.NsmClientSocket,
					nsmd.WorkspaceEnv:       workspace.ClientBaseDir,
				},
			})
		}
	}
	logrus.Infof("AllocateResponse: %v", responses)
	return responses, nil
}

// Register registers
func Register(kubeletEndpoint string) error {
	tracer := opentracing.GlobalTracer()
	conn, err := grpc.Dial(kubeletEndpoint, grpc.WithInsecure(),
		grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
			return net.DialTimeout("unix", addr, timeout)
		}),
		grpc.WithUnaryInterceptor(
			otgrpc.OpenTracingClientInterceptor(tracer, otgrpc.LogPayloads())),
		grpc.WithStreamInterceptor(
			otgrpc.OpenTracingStreamClientInterceptor(tracer)))
	defer conn.Close()
	if err != nil {
		return fmt.Errorf("device-plugin: cannot connect to kubelet service: %v", err)
	}
	client := pluginapi.NewRegistrationClient(conn)
	reqt := &pluginapi.RegisterRequest{
		Version:      pluginapi.Version,
		Endpoint:     ServerSock,
		ResourceName: resourceName,
	}

	_, err = client.Register(context.Background(), reqt)
	if err != nil {
		return fmt.Errorf("device-plugin: cannot register to kubelet service: %v", err)
	}
	return nil
}

// Define functions needed to meet the Kubernetes DevicePlugin API
func (n *nsmClientEndpoints) GetDevicePluginOptions(context.Context, *pluginapi.Empty) (*pluginapi.DevicePluginOptions, error) {
	return &pluginapi.DevicePluginOptions{}, nil
}

func (n *nsmClientEndpoints) PreStartContainer(ctx context.Context, info *pluginapi.PreStartContainerRequest) (*pluginapi.PreStartContainerResponse, error) {
	logrus.Infof("Pre start container called... %v ", info)
	return &pluginapi.PreStartContainerResponse{}, nil
}

func enumWorkspaces(serviceRegistry serviceregistry.ServiceRegistry) (*nsmdapi.EnumConnectionReply, error) {
	client, con, err := serviceRegistry.NSMDApiClient()
	if err != nil {
		logrus.Fatalf("Failed to connect to NSMD: %+v...", err)
	}
	defer con.Close()
	reply, err := client.EnumConnection(context.Background(), &nsmdapi.EnumConnectionRequest{})
	if err != nil {
		return nil, err
	}
	return reply, nil
}

func indexOf(slice []string, value string) int {
	for i, v := range slice {
		if v == value {
			return i
		}
	}
	return -1
}

func (n *nsmClientEndpoints) ListAndWatch(e *pluginapi.Empty, s pluginapi.DevicePlugin_ListAndWatchServer) error {
	logrus.Infof("ListAndWatch was called with s: %+v", s)
	n.pluginApi = &s

	// Restore state from NSMD
	for {
		n.receiveWorkspaces()
		n.sendDeviceUpdate()

		// Sleep before next notification.
		time.Sleep(KubeletNotifyDelay)
	}
}

func startDeviceServer(nsm *nsmClientEndpoints) error {
	listenEndpoint := path.Join(pluginapi.DevicePluginPath, ServerSock)
	if err := tools.SocketCleanup(listenEndpoint); err != nil {
		return err
	}
	sock, err := net.Listen("unix", listenEndpoint)
	if err != nil {
		return err
	}
	tracer := opentracing.GlobalTracer()
	grpcServer := grpc.NewServer(
		grpc.UnaryInterceptor(
			otgrpc.OpenTracingServerInterceptor(tracer, otgrpc.LogPayloads())),
		grpc.StreamInterceptor(
			otgrpc.OpenTracingStreamServerInterceptor(tracer)))

	pluginapi.RegisterDevicePluginServer(grpcServer, nsm)

	logrus.Infof("Starting Device Plugin's gRPC server listening on socket: %s", ServerSock)
	go func() {
		if err := grpcServer.Serve(sock); err != nil {
			logrus.Error("failed to start device plugin grpc server", listenEndpoint, err)
		}
	}()
	// Check if the socket of device plugin server is operation
	conn, err := tools.SocketOperationCheck(listenEndpoint)
	if err != nil {
		return err
	}
	conn.Close()

	return nil
}

func waitForNsmdAvailable() {
	for {
		if tools.WaitForPortAvailable(context.Background(), "unix", nsmd.ServerSock, 100*time.Millisecond) == nil {
			break
		}
	}
}

// NewNSMDeviceServer registers and starts Kubelet's device plugin
func NewNSMDeviceServer(serviceRegistry serviceregistry.ServiceRegistry) error {
	waitForNsmdAvailable()
	nsm := &nsmClientEndpoints{
		serviceRegistry: serviceRegistry,
		resp:            new(pluginapi.ListAndWatchResponse),
		devs:            map[string]*pluginapi.Device{},
	}

	for i := 0; i < DeviceBuffer; i++ {
		nsm.addClientDevice()
	}

	if err := startDeviceServer(nsm); err != nil {
		return err
	}
	// Registers with Kubelet.
	err := Register(pluginapi.KubeletSocket)

	return err
}

func (nsm *nsmClientEndpoints) addClientDevice() {
	nsm.mutext.Lock()
	defer nsm.mutext.Unlock()

	for {
		nsm.clientId++

		id := fmt.Sprintf("nsm-%d", nsm.clientId)
		if _, ok := nsm.devs[id]; ok {
			// Item already exists, increment to new one.
			continue
		}
		// Add a new device Id
		dev := &pluginapi.Device{
			ID:     id,
			Health: pluginapi.Healthy,
		}
		nsm.devs[id] = dev
		nsm.resp.Devices = append(nsm.resp.Devices, dev)
		break
	}

}

func (n *nsmClientEndpoints) sendDeviceUpdate() {
	n.mutext.Lock()
	defer n.mutext.Unlock()
	if n.pluginApi != nil {
		api := *n.pluginApi
		logrus.Infof("Send device plugins intfo %v", n.resp)
		if err := api.Send(n.resp); err != nil {
			logrus.Errorf("Failed to send response to kubelet: %v\n", err)
		}
	}
}

func (n *nsmClientEndpoints) receiveWorkspaces() {
	for {
		reply, err := enumWorkspaces(n.serviceRegistry)
		if err != nil {
			logrus.Errorf("Error receive devices from NSM. %v", err)
			// Make a fast delay to faster startup of NSMD.
			<-time.Tick(100 * time.Millisecond)
			continue
		}
		n.mutext.Lock()

		// Check we had all workspaces in our update list
		// This list could be changed in case both NSMDp and NSMD restart.
		for _, w := range reply.GetWorkspace() {
			if len(w) > 0 {
				if _, ok := n.devs[w]; !ok {
					// We do not have this one in list of our devices
					dev := &pluginapi.Device{
						ID:     w,
						Health: pluginapi.Healthy,
					}
					n.devs[w] = dev
					n.resp.Devices = append(n.resp.Devices, dev)
				}
			}
		}
		n.mutext.Unlock()
		// Be sure we have enought deviced ids allocated.
		for len(n.resp.Devices) < len(reply.GetWorkspace())+DeviceBuffer {
			n.addClientDevice()
		}

		break
	}
}

func main() {
	// Capture signals to cleanup before exiting
	c := make(chan os.Signal, 1)
	signal.Notify(c,
		syscall.SIGHUP,
		syscall.SIGINT,
		syscall.SIGTERM,
		syscall.SIGQUIT)

	tracer, closer := tools.InitJaeger("nsmdp")
	defer closer.Close()
	opentracing.SetGlobalTracer(tracer)

	serviceRegistry := nsmd.NewServiceRegistry()
	err := NewNSMDeviceServer(serviceRegistry)

	if err != nil {
		logrus.Errorf("failed to start server: %v", err)
		os.Exit(1)
	}

	<-c
}
<|endoftext|>


<commit_before>// Derived from Inferno utils/6l/l.h and related files.
// http://code.google.com/p/inferno-os/source/browse/utils/6l/l.h
//
//	Copyright © 1994-1999 Lucent Technologies Inc.  All rights reserved.
//	Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net)
//	Portions Copyright © 1997-1999 Vita Nuova Limited
//	Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com)
//	Portions Copyright © 2004,2006 Bruce Ellis
//	Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net)
//	Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others
//	Portions Copyright © 2009 The Go Authors.  All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package obj

// An As denotes an assembler opcode.
// There are some portable opcodes, declared here in package obj,
// that are common to all architectures.
// However, the majority of opcodes are arch-specific
// and are declared in their respective architecture's subpackage.
type As int16

// Prog.as opcodes.
// These are the portable opcodes, common to all architectures.
// Each architecture defines many more arch-specific opcodes,
// with values starting at A_ARCHSPECIFIC.
// Each architecture adds an offset to this so each machine has
// distinct space for its instructions. The offset is a power of
// two so it can be masked to return to origin zero.
// See the definitions of ABase386 etc.
const (
	AXXX = 0 + iota
	ACALL
	ACHECKNIL
	ADATA
	ADUFFCOPY
	ADUFFZERO
	AEND
	AFUNCDATA
	AGLOBL
	AJMP
	ANOP
	APCDATA
	ARET
	ATEXT
	ATYPE
	AUNDEF
	AUSEFIELD
	AVARDEF
	AVARKILL
	A_ARCHSPECIFIC
)

// An LSym is the sort of symbol that is written to an object file.
type LSym struct {
	Name    string
	Version int16
	Size    int
}

// The compiler needs LSym to satisfy fmt.Stringer, because it stores
// an LSym in ssa.ExternSymbol.
func (s *LSym) String() string {
	return s.Name
}

// Link holds the context for writing object code from a compiler
// to be linker input or for reading that input into the linker.
type Link struct {
	Arch                 *LinkArch
	Pathname             string
	Flag_shared          bool
	Flag_dynlink         bool
	Framepointer_enabled bool
	DiagFunc             func(string, ...interface{})
	Errors               int
}

// LinkArch is a stubbed struct
type LinkArch struct {
}

// The smallest possible offset from the hardware stack pointer to a local
// variable on the stack. Architectures that use a link register save its value
// on the stack in the function prologue and so always have a pointer between
// the hardware stack pointer and the local variable area.
func (ctxt *Link) FixedFrameSize() int64 {
	// TODO: for now, assume amd64
	return 0
}

func (ctxt *Link) Diag(format string, args ...interface{}) {
	ctxt.Errors++
	ctxt.DiagFunc(format, args...)
}
<commit_msg>Add LinkArch.IntSize<commit_after>// Derived from Inferno utils/6l/l.h and related files.
// http://code.google.com/p/inferno-os/source/browse/utils/6l/l.h
//
//	Copyright © 1994-1999 Lucent Technologies Inc.  All rights reserved.
//	Portions Copyright © 1995-1997 C H Forsyth (forsyth@terzarima.net)
//	Portions Copyright © 1997-1999 Vita Nuova Limited
//	Portions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com)
//	Portions Copyright © 2004,2006 Bruce Ellis
//	Portions Copyright © 2005-2007 C H Forsyth (forsyth@terzarima.net)
//	Revisions Copyright © 2000-2007 Lucent Technologies Inc. and others
//	Portions Copyright © 2009 The Go Authors.  All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package obj

// An As denotes an assembler opcode.
// There are some portable opcodes, declared here in package obj,
// that are common to all architectures.
// However, the majority of opcodes are arch-specific
// and are declared in their respective architecture's subpackage.
type As int16

// Prog.as opcodes.
// These are the portable opcodes, common to all architectures.
// Each architecture defines many more arch-specific opcodes,
// with values starting at A_ARCHSPECIFIC.
// Each architecture adds an offset to this so each machine has
// distinct space for its instructions. The offset is a power of
// two so it can be masked to return to origin zero.
// See the definitions of ABase386 etc.
const (
	AXXX = 0 + iota
	ACALL
	ACHECKNIL
	ADATA
	ADUFFCOPY
	ADUFFZERO
	AEND
	AFUNCDATA
	AGLOBL
	AJMP
	ANOP
	APCDATA
	ARET
	ATEXT
	ATYPE
	AUNDEF
	AUSEFIELD
	AVARDEF
	AVARKILL
	A_ARCHSPECIFIC
)

// An LSym is the sort of symbol that is written to an object file.
type LSym struct {
	Name    string
	Version int16
	Size    int
}

// The compiler needs LSym to satisfy fmt.Stringer, because it stores
// an LSym in ssa.ExternSymbol.
func (s *LSym) String() string {
	return s.Name
}

// Link holds the context for writing object code from a compiler
// to be linker input or for reading that input into the linker.
type Link struct {
	Arch                 *LinkArch
	Pathname             string
	Flag_shared          bool
	Flag_dynlink         bool
	Framepointer_enabled bool
	DiagFunc             func(string, ...interface{})
	Errors               int
}

// LinkArch is a stubbed struct
type LinkArch struct {
	IntSize int
}

// The smallest possible offset from the hardware stack pointer to a local
// variable on the stack. Architectures that use a link register save its value
// on the stack in the function prologue and so always have a pointer between
// the hardware stack pointer and the local variable area.
func (ctxt *Link) FixedFrameSize() int64 {
	// TODO: for now, assume amd64
	return 0
}

func (ctxt *Link) Diag(format string, args ...interface{}) {
	ctxt.Errors++
	ctxt.DiagFunc(format, args...)
}
<|endoftext|>


<commit_before>// Copyright 2017 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package kbfsgit

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"testing"

	"github.com/keybase/kbfs/libfs"
	"github.com/keybase/kbfs/libkbfs"
	"github.com/keybase/kbfs/tlf"
	"github.com/stretchr/testify/require"
	gogitcfg "gopkg.in/src-d/go-git.v4/config"
)

type testErrput struct {
	t *testing.T
}

func (te testErrput) Write(buf []byte) (int, error) {
	te.t.Log(string(buf))
	return 0, nil
}

func TestRunnerCapabilities(t *testing.T) {
	ctx := libkbfs.BackgroundContextWithCancellationDelayer()
	config := libkbfs.MakeTestConfigOrBust(t, "user1")
	defer libkbfs.CheckConfigAndShutdown(ctx, t, config)

	inputReader, inputWriter := io.Pipe()
	defer inputWriter.Close()
	go func() {
		inputWriter.Write([]byte("capabilities\n\n"))
	}()

	var output bytes.Buffer
	r, err := newRunner(ctx, config, "origin", "keybase://private/user1/test",
		"", inputReader, &output, testErrput{t})
	require.NoError(t, err)
	err = r.processCommands(ctx)
	require.NoError(t, err)
	require.Equal(t, "fetch\npush\noption\n\n", output.String())
}

func initConfigForRunner(t *testing.T) (
	ctx context.Context, config *libkbfs.ConfigLocal, tempDir string) {
	ctx = libkbfs.BackgroundContextWithCancellationDelayer()
	config = libkbfs.MakeTestConfigOrBustLoggedInWithMode(
		t, 0, libkbfs.InitSingleOp, "user1")
	success := false

	var err error
	tempdir, err := ioutil.TempDir(os.TempDir(), "journal_server")
	require.NoError(t, err)
	defer func() {
		if !success {
			os.RemoveAll(tempdir)
		}
	}()

	err = config.EnableDiskLimiter(tempdir)
	require.NoError(t, err)
	err = config.EnableJournaling(
		ctx, tempdir, libkbfs.TLFJournalSingleOpBackgroundWorkEnabled)
	require.NoError(t, err)

	return ctx, config, tempDir
}

func TestRunnerInitRepo(t *testing.T) {
	ctx, config, tempdir := initConfigForRunner(t)
	defer os.RemoveAll(tempdir)

	inputReader, inputWriter := io.Pipe()
	defer inputWriter.Close()
	go func() {
		inputWriter.Write([]byte("list\n\n"))
	}()

	var output bytes.Buffer
	r, err := newRunner(ctx, config, "origin", "keybase://private/user1/test",
		"", inputReader, &output, testErrput{t})
	require.NoError(t, err)
	err = r.processCommands(ctx)
	require.NoError(t, err)
	// Just one symref, from HEAD to master (and master has no commits yet).
	require.Equal(t, output.String(), "@refs/heads/master HEAD\n\n")

	// Now there should be a valid git repo stored in KBFS.  Check the
	// existence of the HEAD file to be sure.
	h, err := libkbfs.ParseTlfHandle(ctx, config.KBPKI(), "user1", tlf.Private)
	require.NoError(t, err)
	fs, err := libfs.NewFS(ctx, config, h, ".kbfs_git/test", "")
	require.NoError(t, err)
	head, err := fs.Open("HEAD")
	require.NoError(t, err)
	buf, err := ioutil.ReadAll(head)
	require.NoError(t, err)
	require.Equal(t, "ref: refs/heads/master\n", string(buf))
}

func makeLocalRepoWithOneFile(t *testing.T,
	gitDir, filename, contents string) {
	t.Logf("Make a new repo in %s with one file", gitDir)
	err := ioutil.WriteFile(
		filepath.Join(gitDir, filename), []byte(contents), 0600)
	require.NoError(t, err)
	dotgit := filepath.Join(gitDir, ".git")
	cmd := exec.Command(
		"git", "--git-dir", dotgit, "--work-tree", gitDir, "init")
	err = cmd.Run()
	require.NoError(t, err)

	cmd = exec.Command(
		"git", "--git-dir", dotgit, "--work-tree", gitDir, "add", "foo")
	err = cmd.Run()
	require.NoError(t, err)

	cmd = exec.Command(
		"git", "--git-dir", dotgit, "--work-tree", gitDir,
		"-c", "user.name=Foo", "-c", "user.email=foo@foo.com",
		"commit", "-a", "-m", "foo")
	err = cmd.Run()
	require.NoError(t, err)
}

func testPush(t *testing.T, ctx context.Context, config libkbfs.Config,
	gitDir, refspec string) {
	// Use the runner to push the local data into the KBFS repo.
	inputReader, inputWriter := io.Pipe()
	defer inputWriter.Close()
	go func() {
		inputWriter.Write([]byte(fmt.Sprintf("push %s\n\n\n", refspec)))
	}()

	var output bytes.Buffer
	r, err := newRunner(ctx, config, "origin", "keybase://private/user1/test",
		filepath.Join(gitDir, ".git"), inputReader, &output, testErrput{t})
	require.NoError(t, err)
	err = r.processCommands(ctx)
	require.NoError(t, err)
	// Just one symref, from HEAD to master (and master has no commits yet).
	dst := gogitcfg.RefSpec(refspec).Dst("")
	require.Equal(t, fmt.Sprintf("ok %s\n\n", dst), output.String())
}

func testListAndGetHeads(t *testing.T, ctx context.Context,
	config libkbfs.Config, gitDir string, expectedRefs []string) (
	heads []string) {
	inputReader, inputWriter := io.Pipe()
	defer inputWriter.Close()
	go func() {
		inputWriter.Write([]byte("list\n\n"))
	}()

	var output bytes.Buffer
	r, err := newRunner(ctx, config, "origin", "keybase://private/user1/test",
		filepath.Join(gitDir, ".git"), inputReader, &output, testErrput{t})
	require.NoError(t, err)
	err = r.processCommands(ctx)
	require.NoError(t, err)
	listLines := strings.Split(output.String(), "\n")
	t.Log(listLines)
	require.Len(t, listLines, len(expectedRefs)+2 /* extra blank line */)
	refs := make(map[string]string, len(expectedRefs))
	for _, line := range listLines {
		if line == "" {
			continue
		}
		refParts := strings.Split(line, " ")
		require.Len(t, refParts, 2)
		refs[refParts[1]] = refParts[0]
	}

	for _, expectedRef := range expectedRefs {
		head, ok := refs[expectedRef]
		require.True(t, ok)
		heads = append(heads, head)
	}
	return heads
}

// This tests pushing code to a bare repo stored in KBFS, and pulling
// code from that bare repo into a new working tree.  This is a simple
// version of how the full KBFS Git system will work.  Specifically,
// this test does the following:
//
// 1) Initializes a new repo on the local file system with one file.
// 2) Initializes a new bare repo in KBFS.
// 3) User pushes from that repo into the remote KBFS repo.
// 4) Initializes a second new repo on the local file system.
// 5) User pulls from the remote KBFS repo into the second repo.
func testRunnerPushFetch(t *testing.T, cloning bool) {
	ctx, config, tempdir := initConfigForRunner(t)
	defer os.RemoveAll(tempdir)

	git1, err := ioutil.TempDir(os.TempDir(), "kbfsgittest")
	require.NoError(t, err)
	defer os.RemoveAll(git1)

	makeLocalRepoWithOneFile(t, git1, "foo", "hello")

	testPush(t, ctx, config, git1, "refs/heads/master:refs/heads/master")

	git2, err := ioutil.TempDir(os.TempDir(), "kbfsgittest")
	require.NoError(t, err)
	defer os.RemoveAll(git2)

	t.Logf("Make a new repo in %s to clone from the KBFS repo", git2)
	dotgit2 := filepath.Join(git2, ".git")
	cmd := exec.Command(
		"git", "--git-dir", dotgit2, "--work-tree", git2, "init")
	err = cmd.Run()
	require.NoError(t, err)

	// Find out the head hash.
	heads := testListAndGetHeads(t, ctx, config, git2,
		[]string{"refs/heads/master", "HEAD"})

	cloningStr := ""
	cloningRetStr := ""
	if cloning {
		cloningStr = "option cloning true\n"
		cloningRetStr = "ok\n"
	}

	// Use the runner to fetch the KBFS data into the new git repo.
	inputReader, inputWriter := io.Pipe()
	defer inputWriter.Close()
	go func() {
		inputWriter.Write([]byte(fmt.Sprintf(
			"%sfetch %s refs/heads/master\n\n\n", cloningStr, heads[0])))
	}()

	var output3 bytes.Buffer
	r, err := newRunner(ctx, config, "origin", "keybase://private/user1/test",
		dotgit2, inputReader, &output3, testErrput{t})
	require.NoError(t, err)
	err = r.processCommands(ctx)
	require.NoError(t, err)
	// Just one symref, from HEAD to master (and master has no commits yet).
	require.Equal(t, cloningRetStr+"\n", output3.String())

	// Checkout the head directly (fetching directly via the runner
	// doesn't leave any refs, those would normally be created by the
	// `git` process that invokes the runner).
	cmd = exec.Command(
		"git", "--git-dir", dotgit2, "--work-tree", git2, "checkout", heads[0])
	err = cmd.Run()
	require.NoError(t, err)

	data, err := ioutil.ReadFile(filepath.Join(git2, "foo"))
	require.NoError(t, err)
	require.Equal(t, "hello", string(data))
}

func TestRunnerPushFetch(t *testing.T) {
	testRunnerPushFetch(t, false)
}

func TestRunnerPushClone(t *testing.T) {
	testRunnerPushFetch(t, true)
}

func TestRunnerDeleteBranch(t *testing.T) {
	ctx, config, tempdir := initConfigForRunner(t)
	defer os.RemoveAll(tempdir)

	git, err := ioutil.TempDir(os.TempDir(), "kbfsgittest")
	require.NoError(t, err)
	defer os.RemoveAll(git)

	makeLocalRepoWithOneFile(t, git, "foo", "hello")

	testPush(t, ctx, config, git, "refs/heads/master:refs/heads/master")
	testPush(t, ctx, config, git, "refs/heads/master:refs/heads/test")

	// Make sure there are 2 remote branches.
	testListAndGetHeads(t, ctx, config, git,
		[]string{"refs/heads/master", "refs/heads/test", "HEAD"})

	// Delete the test branch and make sure it goes away.
	testPush(t, ctx, config, git, ":refs/heads/test")
	testListAndGetHeads(t, ctx, config, git,
		[]string{"refs/heads/master", "HEAD"})
}

func TestRunnerExitEarlyOnEOF(t *testing.T) {
	ctx, config, tempdir := initConfigForRunner(t)
	defer os.RemoveAll(tempdir)

	git, err := ioutil.TempDir(os.TempDir(), "kbfsgittest")
	require.NoError(t, err)
	defer os.RemoveAll(git)

	makeLocalRepoWithOneFile(t, git, "foo", "hello")

	h, err := libkbfs.ParseTlfHandle(ctx, config.KBPKI(), "user1", tlf.Private)
	require.NoError(t, err)
	rootNode, _, err := config.KBFSOps().GetOrCreateRootNode(
		ctx, h, libkbfs.MasterBranch)
	require.NoError(t, err)

	// Pause journal to force the processing to pause.
	jServer, err := libkbfs.GetJournalServer(config)
	require.NoError(t, err)
	jServer.PauseBackgroundWork(ctx, rootNode.GetFolderBranch().Tlf)

	// Input a full push batch, but let the reader EOF without giving
	// the final \n.
	input := bytes.NewBufferString(
		"push refs/heads/master:refs/heads/master\n\n")
	var output bytes.Buffer
	r, err := newRunner(ctx, config, "origin", "keybase://private/user1/test",
		filepath.Join(git, ".git"), input, &output, testErrput{t})
	require.NoError(t, err)

	// Make sure we don't hang when EOF comes early.
	err = r.processCommands(ctx)
	require.NoError(t, err)
}
<commit_msg>small fix to set success flag in kbgit test (#1187)<commit_after>// Copyright 2017 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package kbfsgit

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"testing"

	"github.com/keybase/kbfs/libfs"
	"github.com/keybase/kbfs/libkbfs"
	"github.com/keybase/kbfs/tlf"
	"github.com/stretchr/testify/require"
	gogitcfg "gopkg.in/src-d/go-git.v4/config"
)

type testErrput struct {
	t *testing.T
}

func (te testErrput) Write(buf []byte) (int, error) {
	te.t.Log(string(buf))
	return 0, nil
}

func TestRunnerCapabilities(t *testing.T) {
	ctx := libkbfs.BackgroundContextWithCancellationDelayer()
	config := libkbfs.MakeTestConfigOrBust(t, "user1")
	defer libkbfs.CheckConfigAndShutdown(ctx, t, config)

	inputReader, inputWriter := io.Pipe()
	defer inputWriter.Close()
	go func() {
		inputWriter.Write([]byte("capabilities\n\n"))
	}()

	var output bytes.Buffer
	r, err := newRunner(ctx, config, "origin", "keybase://private/user1/test",
		"", inputReader, &output, testErrput{t})
	require.NoError(t, err)
	err = r.processCommands(ctx)
	require.NoError(t, err)
	require.Equal(t, "fetch\npush\noption\n\n", output.String())
}

func initConfigForRunner(t *testing.T) (
	ctx context.Context, config *libkbfs.ConfigLocal, tempDir string) {
	ctx = libkbfs.BackgroundContextWithCancellationDelayer()
	config = libkbfs.MakeTestConfigOrBustLoggedInWithMode(
		t, 0, libkbfs.InitSingleOp, "user1")
	success := false

	var err error
	tempdir, err := ioutil.TempDir(os.TempDir(), "journal_server")
	require.NoError(t, err)
	defer func() {
		if !success {
			os.RemoveAll(tempdir)
		}
	}()

	err = config.EnableDiskLimiter(tempdir)
	require.NoError(t, err)
	err = config.EnableJournaling(
		ctx, tempdir, libkbfs.TLFJournalSingleOpBackgroundWorkEnabled)
	require.NoError(t, err)

	success = true
	return ctx, config, tempDir
}

func TestRunnerInitRepo(t *testing.T) {
	ctx, config, tempdir := initConfigForRunner(t)
	defer os.RemoveAll(tempdir)

	inputReader, inputWriter := io.Pipe()
	defer inputWriter.Close()
	go func() {
		inputWriter.Write([]byte("list\n\n"))
	}()

	var output bytes.Buffer
	r, err := newRunner(ctx, config, "origin", "keybase://private/user1/test",
		"", inputReader, &output, testErrput{t})
	require.NoError(t, err)
	err = r.processCommands(ctx)
	require.NoError(t, err)
	// Just one symref, from HEAD to master (and master has no commits yet).
	require.Equal(t, output.String(), "@refs/heads/master HEAD\n\n")

	// Now there should be a valid git repo stored in KBFS.  Check the
	// existence of the HEAD file to be sure.
	h, err := libkbfs.ParseTlfHandle(ctx, config.KBPKI(), "user1", tlf.Private)
	require.NoError(t, err)
	fs, err := libfs.NewFS(ctx, config, h, ".kbfs_git/test", "")
	require.NoError(t, err)
	head, err := fs.Open("HEAD")
	require.NoError(t, err)
	buf, err := ioutil.ReadAll(head)
	require.NoError(t, err)
	require.Equal(t, "ref: refs/heads/master\n", string(buf))
}

func makeLocalRepoWithOneFile(t *testing.T,
	gitDir, filename, contents string) {
	t.Logf("Make a new repo in %s with one file", gitDir)
	err := ioutil.WriteFile(
		filepath.Join(gitDir, filename), []byte(contents), 0600)
	require.NoError(t, err)
	dotgit := filepath.Join(gitDir, ".git")
	cmd := exec.Command(
		"git", "--git-dir", dotgit, "--work-tree", gitDir, "init")
	err = cmd.Run()
	require.NoError(t, err)

	cmd = exec.Command(
		"git", "--git-dir", dotgit, "--work-tree", gitDir, "add", "foo")
	err = cmd.Run()
	require.NoError(t, err)

	cmd = exec.Command(
		"git", "--git-dir", dotgit, "--work-tree", gitDir,
		"-c", "user.name=Foo", "-c", "user.email=foo@foo.com",
		"commit", "-a", "-m", "foo")
	err = cmd.Run()
	require.NoError(t, err)
}

func testPush(t *testing.T, ctx context.Context, config libkbfs.Config,
	gitDir, refspec string) {
	// Use the runner to push the local data into the KBFS repo.
	inputReader, inputWriter := io.Pipe()
	defer inputWriter.Close()
	go func() {
		inputWriter.Write([]byte(fmt.Sprintf("push %s\n\n\n", refspec)))
	}()

	var output bytes.Buffer
	r, err := newRunner(ctx, config, "origin", "keybase://private/user1/test",
		filepath.Join(gitDir, ".git"), inputReader, &output, testErrput{t})
	require.NoError(t, err)
	err = r.processCommands(ctx)
	require.NoError(t, err)
	// Just one symref, from HEAD to master (and master has no commits yet).
	dst := gogitcfg.RefSpec(refspec).Dst("")
	require.Equal(t, fmt.Sprintf("ok %s\n\n", dst), output.String())
}

func testListAndGetHeads(t *testing.T, ctx context.Context,
	config libkbfs.Config, gitDir string, expectedRefs []string) (
	heads []string) {
	inputReader, inputWriter := io.Pipe()
	defer inputWriter.Close()
	go func() {
		inputWriter.Write([]byte("list\n\n"))
	}()

	var output bytes.Buffer
	r, err := newRunner(ctx, config, "origin", "keybase://private/user1/test",
		filepath.Join(gitDir, ".git"), inputReader, &output, testErrput{t})
	require.NoError(t, err)
	err = r.processCommands(ctx)
	require.NoError(t, err)
	listLines := strings.Split(output.String(), "\n")
	t.Log(listLines)
	require.Len(t, listLines, len(expectedRefs)+2 /* extra blank line */)
	refs := make(map[string]string, len(expectedRefs))
	for _, line := range listLines {
		if line == "" {
			continue
		}
		refParts := strings.Split(line, " ")
		require.Len(t, refParts, 2)
		refs[refParts[1]] = refParts[0]
	}

	for _, expectedRef := range expectedRefs {
		head, ok := refs[expectedRef]
		require.True(t, ok)
		heads = append(heads, head)
	}
	return heads
}

// This tests pushing code to a bare repo stored in KBFS, and pulling
// code from that bare repo into a new working tree.  This is a simple
// version of how the full KBFS Git system will work.  Specifically,
// this test does the following:
//
// 1) Initializes a new repo on the local file system with one file.
// 2) Initializes a new bare repo in KBFS.
// 3) User pushes from that repo into the remote KBFS repo.
// 4) Initializes a second new repo on the local file system.
// 5) User pulls from the remote KBFS repo into the second repo.
func testRunnerPushFetch(t *testing.T, cloning bool) {
	ctx, config, tempdir := initConfigForRunner(t)
	defer os.RemoveAll(tempdir)

	git1, err := ioutil.TempDir(os.TempDir(), "kbfsgittest")
	require.NoError(t, err)
	defer os.RemoveAll(git1)

	makeLocalRepoWithOneFile(t, git1, "foo", "hello")

	testPush(t, ctx, config, git1, "refs/heads/master:refs/heads/master")

	git2, err := ioutil.TempDir(os.TempDir(), "kbfsgittest")
	require.NoError(t, err)
	defer os.RemoveAll(git2)

	t.Logf("Make a new repo in %s to clone from the KBFS repo", git2)
	dotgit2 := filepath.Join(git2, ".git")
	cmd := exec.Command(
		"git", "--git-dir", dotgit2, "--work-tree", git2, "init")
	err = cmd.Run()
	require.NoError(t, err)

	// Find out the head hash.
	heads := testListAndGetHeads(t, ctx, config, git2,
		[]string{"refs/heads/master", "HEAD"})

	cloningStr := ""
	cloningRetStr := ""
	if cloning {
		cloningStr = "option cloning true\n"
		cloningRetStr = "ok\n"
	}

	// Use the runner to fetch the KBFS data into the new git repo.
	inputReader, inputWriter := io.Pipe()
	defer inputWriter.Close()
	go func() {
		inputWriter.Write([]byte(fmt.Sprintf(
			"%sfetch %s refs/heads/master\n\n\n", cloningStr, heads[0])))
	}()

	var output3 bytes.Buffer
	r, err := newRunner(ctx, config, "origin", "keybase://private/user1/test",
		dotgit2, inputReader, &output3, testErrput{t})
	require.NoError(t, err)
	err = r.processCommands(ctx)
	require.NoError(t, err)
	// Just one symref, from HEAD to master (and master has no commits yet).
	require.Equal(t, cloningRetStr+"\n", output3.String())

	// Checkout the head directly (fetching directly via the runner
	// doesn't leave any refs, those would normally be created by the
	// `git` process that invokes the runner).
	cmd = exec.Command(
		"git", "--git-dir", dotgit2, "--work-tree", git2, "checkout", heads[0])
	err = cmd.Run()
	require.NoError(t, err)

	data, err := ioutil.ReadFile(filepath.Join(git2, "foo"))
	require.NoError(t, err)
	require.Equal(t, "hello", string(data))
}

func TestRunnerPushFetch(t *testing.T) {
	testRunnerPushFetch(t, false)
}

func TestRunnerPushClone(t *testing.T) {
	testRunnerPushFetch(t, true)
}

func TestRunnerDeleteBranch(t *testing.T) {
	ctx, config, tempdir := initConfigForRunner(t)
	defer os.RemoveAll(tempdir)

	git, err := ioutil.TempDir(os.TempDir(), "kbfsgittest")
	require.NoError(t, err)
	defer os.RemoveAll(git)

	makeLocalRepoWithOneFile(t, git, "foo", "hello")

	testPush(t, ctx, config, git, "refs/heads/master:refs/heads/master")
	testPush(t, ctx, config, git, "refs/heads/master:refs/heads/test")

	// Make sure there are 2 remote branches.
	testListAndGetHeads(t, ctx, config, git,
		[]string{"refs/heads/master", "refs/heads/test", "HEAD"})

	// Delete the test branch and make sure it goes away.
	testPush(t, ctx, config, git, ":refs/heads/test")
	testListAndGetHeads(t, ctx, config, git,
		[]string{"refs/heads/master", "HEAD"})
}

func TestRunnerExitEarlyOnEOF(t *testing.T) {
	ctx, config, tempdir := initConfigForRunner(t)
	defer os.RemoveAll(tempdir)

	git, err := ioutil.TempDir(os.TempDir(), "kbfsgittest")
	require.NoError(t, err)
	defer os.RemoveAll(git)

	makeLocalRepoWithOneFile(t, git, "foo", "hello")

	h, err := libkbfs.ParseTlfHandle(ctx, config.KBPKI(), "user1", tlf.Private)
	require.NoError(t, err)
	rootNode, _, err := config.KBFSOps().GetOrCreateRootNode(
		ctx, h, libkbfs.MasterBranch)
	require.NoError(t, err)

	// Pause journal to force the processing to pause.
	jServer, err := libkbfs.GetJournalServer(config)
	require.NoError(t, err)
	jServer.PauseBackgroundWork(ctx, rootNode.GetFolderBranch().Tlf)

	// Input a full push batch, but let the reader EOF without giving
	// the final \n.
	input := bytes.NewBufferString(
		"push refs/heads/master:refs/heads/master\n\n")
	var output bytes.Buffer
	r, err := newRunner(ctx, config, "origin", "keybase://private/user1/test",
		filepath.Join(git, ".git"), input, &output, testErrput{t})
	require.NoError(t, err)

	// Make sure we don't hang when EOF comes early.
	err = r.processCommands(ctx)
	require.NoError(t, err)
}
<|endoftext|>


<commit_before>package op

import (
	"bytes"
	"fmt"
	"math"

	"github.com/emicklei/melrose/core"
)

type Merge struct {
	Target []core.Sequenceable
}

func (m Merge) Storex() string {
	var b bytes.Buffer
	fmt.Fprintf(&b, "join(")
	core.AppendStorexList(&b, true, m.Target)
	fmt.Fprintf(&b, ")")
	return b.String()
}

// Replaced is part of Replaceable
func (m Merge) Replaced(from, to core.Sequenceable) core.Sequenceable {
	if core.IsIdenticalTo(m, from) {
		return to
	}
	return Join{Target: replacedAll(m.Target, from, to)}
}

func (m Merge) S() core.Sequence {
	readers := []*sequenceReader{}
	for _, each := range m.Target {
		// ungroup each
		for _, other := range each.S().Split() {
			readers = append(readers, &sequenceReader{sequence: other})
		}
	}
	groups := [][]core.Note{}
	done := false
	duration := float32(0.0)
	for !done {
		shortest := float32(math.MaxFloat32)
		// first collect the shortest duration to advance
		emptyReaders := true
		for _, each := range readers {
			dur, ok := each.durationUntilNextNote(duration)
			if ok {
				// e.g. pedals do not have a duration
				hasDuration := dur > 0
				if !hasDuration {
					// consume and add it immediately ,reader could become empty
					ns := each.take()
					groups = append(groups, ns)
				} else {
					emptyReaders = false
					// update shortest
					if dur < shortest {
						shortest = dur
					}
				}
			}
		}
		done = true
		if !emptyReaders {
			group := []core.Note{}
			// collect notes from each reader that fit into shortest; could be a rest as filler
			for _, each := range readers {
				ns, ok := each.noteUpto(duration, shortest)
				if ok {
					done = false
					group = append(group, ns...)
				}
			}
			duration += shortest
			c := compactGroup(group)
			// do not add empty ones
			if len(c) > 0 {
				groups = append(groups, c)
			}
		}
	}
	return core.Sequence{Notes: groups}
}

// compactGroup returns a group without superfluous rest notes. Does not have pedals
func compactGroup(g []core.Note) (compacted []core.Note) {
	if len(g) <= 1 {
		return g
	}
	durationShortest := float32(math.MaxFloat32)
	shortestIsRest := false
	for _, each := range g {
		f := each.DurationFactor()
		if f < durationShortest {
			durationShortest = f
			shortestIsRest = each.IsRest()
		} else {
			// non-rest notes has prio
			if f == durationShortest && shortestIsRest {
				shortestIsRest = each.IsRest()
			}
		}
	}
	hasIncludedRest := false
	for _, each := range g {
		if each.IsRest() {
			if each.DurationFactor() > durationShortest {
				continue
			}
			if shortestIsRest && !hasIncludedRest {
				hasIncludedRest = true
			} else {
				continue
			}
		}
		compacted = append(compacted, each)
	}
	return
}

// sequenceReader is to read notes and keeping the absolute duration for each note read
type sequenceReader struct {
	index              int           // zero based
	sequence           core.Sequence // must be a single note group sequence
	durationAtLastNote float32       // total duration factor at the end of the note at the index
}

func (r *sequenceReader) noteUpto(duration float32, shortest float32) (list []core.Note, ok bool) {
	// first check duration, could be last note
	// too soon? then no note
	if duration < r.durationAtLastNote {
		diff := r.durationAtLastNote - duration
		if shortest < diff {
			diff = shortest
		}
		var rest core.Note
		switch diff {
		case 1.5:
			rest = core.MustParseNote("1.=")
		case 1.0:
			rest = core.MustParseNote("1=")
		case 0.75:
			rest = core.MustParseNote("2.=")
		case 0.5:
			rest = core.MustParseNote("2=")
		case 0.375:
			rest = core.MustParseNote(".=")
		case 0.25:
			rest = core.MustParseNote("=")
		case 0.1875:
			rest = core.MustParseNote("8.=")
		case 0.125:
			rest = core.MustParseNote("8=")
		case 0.09375:
			rest = core.MustParseNote("16.=")
		case 0.0625:
			rest = core.MustParseNote("16=")
		default:
			return list, false
		}
		return append(list, rest), true
	}
	// any notes left?
	if r.index == len(r.sequence.Notes) {
		return list, false
	}
	r.index++
	n := r.sequence.At(r.index - 1)
	// n is a single note group
	r.durationAtLastNote += n[0].DurationFactor()
	return n, true
}

func (r *sequenceReader) durationUntilNextNote(duration float32) (float32, bool) {
	// first check duration, could be last note
	if duration < r.durationAtLastNote {
		return r.durationAtLastNote - duration, true
	}
	// any notes left?
	if r.index == len(r.sequence.Notes) {
		return 0.0, false
	}
	n := r.sequence.At(r.index)
	// n is a single note group
	return n[0].DurationFactor(), true
}

func (r *sequenceReader) take() []core.Note {
	r.index++
	n := r.sequence.At(r.index - 1)
	// n is a single note group
	r.durationAtLastNote += n[0].DurationFactor()
	return n
}
<commit_msg>fix merge storex<commit_after>package op

import (
	"bytes"
	"fmt"
	"math"

	"github.com/emicklei/melrose/core"
)

type Merge struct {
	Target []core.Sequenceable
}

func (m Merge) Storex() string {
	var b bytes.Buffer
	fmt.Fprintf(&b, "merge(")
	core.AppendStorexList(&b, true, m.Target)
	fmt.Fprintf(&b, ")")
	return b.String()
}

// Replaced is part of Replaceable
func (m Merge) Replaced(from, to core.Sequenceable) core.Sequenceable {
	if core.IsIdenticalTo(m, from) {
		return to
	}
	return Join{Target: replacedAll(m.Target, from, to)}
}

func (m Merge) S() core.Sequence {
	readers := []*sequenceReader{}
	for _, each := range m.Target {
		// ungroup each
		for _, other := range each.S().Split() {
			readers = append(readers, &sequenceReader{sequence: other})
		}
	}
	groups := [][]core.Note{}
	done := false
	duration := float32(0.0)
	for !done {
		shortest := float32(math.MaxFloat32)
		// first collect the shortest duration to advance
		emptyReaders := true
		for _, each := range readers {
			dur, ok := each.durationUntilNextNote(duration)
			if ok {
				// e.g. pedals do not have a duration
				hasDuration := dur > 0
				if !hasDuration {
					// consume and add it immediately ,reader could become empty
					ns := each.take()
					groups = append(groups, ns)
				} else {
					emptyReaders = false
					// update shortest
					if dur < shortest {
						shortest = dur
					}
				}
			}
		}
		done = true
		if !emptyReaders {
			group := []core.Note{}
			// collect notes from each reader that fit into shortest; could be a rest as filler
			for _, each := range readers {
				ns, ok := each.noteUpto(duration, shortest)
				if ok {
					done = false
					group = append(group, ns...)
				}
			}
			duration += shortest
			c := compactGroup(group)
			// do not add empty ones
			if len(c) > 0 {
				groups = append(groups, c)
			}
		}
	}
	return core.Sequence{Notes: groups}
}

// compactGroup returns a group without superfluous rest notes. Does not have pedals
func compactGroup(g []core.Note) (compacted []core.Note) {
	if len(g) <= 1 {
		return g
	}
	durationShortest := float32(math.MaxFloat32)
	shortestIsRest := false
	for _, each := range g {
		f := each.DurationFactor()
		if f < durationShortest {
			durationShortest = f
			shortestIsRest = each.IsRest()
		} else {
			// non-rest notes has prio
			if f == durationShortest && shortestIsRest {
				shortestIsRest = each.IsRest()
			}
		}
	}
	hasIncludedRest := false
	for _, each := range g {
		if each.IsRest() {
			if each.DurationFactor() > durationShortest {
				continue
			}
			if shortestIsRest && !hasIncludedRest {
				hasIncludedRest = true
			} else {
				continue
			}
		}
		compacted = append(compacted, each)
	}
	return
}

// sequenceReader is to read notes and keeping the absolute duration for each note read
type sequenceReader struct {
	index              int           // zero based
	sequence           core.Sequence // must be a single note group sequence
	durationAtLastNote float32       // total duration factor at the end of the note at the index
}

func (r *sequenceReader) noteUpto(duration float32, shortest float32) (list []core.Note, ok bool) {
	// first check duration, could be last note
	// too soon? then no note
	if duration < r.durationAtLastNote {
		diff := r.durationAtLastNote - duration
		if shortest < diff {
			diff = shortest
		}
		var rest core.Note
		switch diff {
		case 1.5:
			rest = core.MustParseNote("1.=")
		case 1.0:
			rest = core.MustParseNote("1=")
		case 0.75:
			rest = core.MustParseNote("2.=")
		case 0.5:
			rest = core.MustParseNote("2=")
		case 0.375:
			rest = core.MustParseNote(".=")
		case 0.25:
			rest = core.MustParseNote("=")
		case 0.1875:
			rest = core.MustParseNote("8.=")
		case 0.125:
			rest = core.MustParseNote("8=")
		case 0.09375:
			rest = core.MustParseNote("16.=")
		case 0.0625:
			rest = core.MustParseNote("16=")
		default:
			return list, false
		}
		return append(list, rest), true
	}
	// any notes left?
	if r.index == len(r.sequence.Notes) {
		return list, false
	}
	r.index++
	n := r.sequence.At(r.index - 1)
	// n is a single note group
	r.durationAtLastNote += n[0].DurationFactor()
	return n, true
}

func (r *sequenceReader) durationUntilNextNote(duration float32) (float32, bool) {
	// first check duration, could be last note
	if duration < r.durationAtLastNote {
		return r.durationAtLastNote - duration, true
	}
	// any notes left?
	if r.index == len(r.sequence.Notes) {
		return 0.0, false
	}
	n := r.sequence.At(r.index)
	// n is a single note group
	return n[0].DurationFactor(), true
}

func (r *sequenceReader) take() []core.Note {
	r.index++
	n := r.sequence.At(r.index - 1)
	// n is a single note group
	r.durationAtLastNote += n[0].DurationFactor()
	return n
}
<|endoftext|>


<commit_before>package toolbox

import (
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"path"
)

//FileSchema file://
var FileSchema = "file://"

//ExtractMimeType extracts mime type by extension
func ExtractMimeType(file string) string {
	extension := path.Ext(file)
	if len(extension) > 1 {
		extension = extension[1:]
	}

	if mimeType, ok := FileExtensionMimeType[extension]; ok {
		return mimeType
	}
	return "text/plain"
}

//OpenReaderFromURL opens a reader from URL
func OpenReaderFromURL(rawURL string) (io.ReadCloser, string, error) {
	var url, err = url.Parse(rawURL)
	if err != nil {
		return nil, "", err
	}
	switch url.Scheme {
	case "http", "https":
		response, err := http.Get(rawURL)
		if err != nil {
			return nil, "", err
		}
		mimeType := response.Header.Get("Content-Type")
		return response.Body, mimeType, nil
	case "file":
		file, err := os.Open(url.Path)
		if err != nil {
			return nil, "", fmt.Errorf("Failed to open file: %v due to %v", rawURL, err.Error())
		}
		return file, ExtractMimeType(url.Path), nil
	}
	return nil, "", fmt.Errorf("Unsupprted url.Scheme: %v on %v", url.Scheme, rawURL)
}

//FileFromURL returns file path from passed in URL.
func FileFromURL(fileURL string) (string, error) {
	var url, err = url.Parse(fileURL)
	if err != nil {
		return "", err
	}
	switch url.Scheme {
	case "file":
		return url.Path, nil
	}
	return "", fmt.Errorf("Unsupprted url.Scheme: %v on %v", url.Scheme, fileURL)
}

//OpenURL opens passed in url as file, or error. Only file:// scheme is supported
func OpenURL(fileURL string, flag int, permissions os.FileMode) (*os.File, error) {
	filePath, err := FileFromURL(fileURL)
	if err != nil {
		return nil, err
	}
	file, err := os.OpenFile(filePath, flag, permissions)
	if err != nil {
		return nil, fmt.Errorf("Failed to create file: %v due to %v", fileURL, err.Error())
	}
	return file, nil
}

//QueryIntValue returns query value for passed in url's name or default value
func QueryIntValue(u *url.URL, name string, defaultValue int) int {
	value := u.Query().Get(name)
	if value == "" {
		return defaultValue
	}

	result := AsInt(value)
	if result != 0 {
		return result
	}
	return defaultValue
}

//QueryBoolValue returns query value for passed in url's name or default value
func QueryBoolValue(u *url.URL, name string, defaultValue bool) bool {
	value := u.Query().Get(name)
	if value == "" {
		return defaultValue
	}
	return AsBoolean(value)
}

//QueryValue returns query value for passed in url's name or default value
func QueryValue(u *url.URL, name, defaultValue string) string {
	value := u.Query().Get(name)
	if value == "" {
		return defaultValue
	}
	return value
}
<commit_msg>Update uri_helper.go<commit_after>package toolbox

import (
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"path"
	"strings"
)

//FileSchema file://
var FileSchema = "file://"

//ExtractMimeType extracts mime type by extension
func ExtractMimeType(file string) string {
	extension := path.Ext(file)
	if len(extension) > 1 {
		extension = extension[1:]
	}

	if mimeType, ok := FileExtensionMimeType[extension]; ok {
		return mimeType
	}
	return "text/plain"
}

//OpenReaderFromURL opens a reader from URL
// to indicate a relative path from the working directory
// the schema must start with "file://.../". "..." will be replaced by
// the working directory. Note that you can not have a relative path
// in the middle of the uri. for instance, "file:///local/../path
// is not valid
func OpenReaderFromURL(rawURL string) (io.ReadCloser, string, error) {
	//this indicates a relative path
	if strings.HasPrefix(rawURL, "file://.../") {
		baseDirectory, _ := os.Getwd()
		rawURL = strings.Replace(rawURL, "...", baseDirectory, 1)
	}

	var url, err = url.Parse(rawURL)
	if err != nil {
		return nil, "", err
	}
	switch url.Scheme {
	case "http", "https":
		response, err := http.Get(rawURL)
		if err != nil {
			return nil, "", err
		}
		mimeType := response.Header.Get("Content-Type")
		return response.Body, mimeType, nil
	case "file":
		file, err := os.Open(url.Path)
		if err != nil {
			return nil, "", fmt.Errorf("Failed to open file: %v due to %v", rawURL, err.Error())
		}
		return file, ExtractMimeType(url.Path), nil
	}
	return nil, "", fmt.Errorf("Unsupprted url.Scheme: %v on %v", url.Scheme, rawURL)
}

//FileFromURL returns file path from passed in URL.
func FileFromURL(fileURL string) (string, error) {
	var url, err = url.Parse(fileURL)
	if err != nil {
		return "", err
	}
	switch url.Scheme {
	case "file":
		return url.Path, nil
	}
	return "", fmt.Errorf("Unsupprted url.Scheme: %v on %v", url.Scheme, fileURL)
}

//OpenURL opens passed in url as file, or error. Only file:// scheme is supported
func OpenURL(fileURL string, flag int, permissions os.FileMode) (*os.File, error) {
	filePath, err := FileFromURL(fileURL)
	if err != nil {
		return nil, err
	}
	file, err := os.OpenFile(filePath, flag, permissions)
	if err != nil {
		return nil, fmt.Errorf("Failed to create file: %v due to %v", fileURL, err.Error())
	}
	return file, nil
}

//QueryIntValue returns query value for passed in url's name or default value
func QueryIntValue(u *url.URL, name string, defaultValue int) int {
	value := u.Query().Get(name)
	if value == "" {
		return defaultValue
	}

	result := AsInt(value)
	if result != 0 {
		return result
	}
	return defaultValue
}

//QueryBoolValue returns query value for passed in url's name or default value
func QueryBoolValue(u *url.URL, name string, defaultValue bool) bool {
	value := u.Query().Get(name)
	if value == "" {
		return defaultValue
	}
	return AsBoolean(value)
}

//QueryValue returns query value for passed in url's name or default value
func QueryValue(u *url.URL, name, defaultValue string) string {
	value := u.Query().Get(name)
	if value == "" {
		return defaultValue
	}
	return value
}
<|endoftext|>


<commit_before>//
// openzwave provides a thin Go wrapper around the openzwave library.
//
// The following shows a simple use of the API which will log every notification received.
//
//        var loop = func(api openzwave.API) {
//                fmt.Printf("event loop starts\n")
//                for {
//                        select {
//                        case notification := <-api.Notifications():
//                                fmt.Printf("notification received - %v", notification)
//                                api.FreeNotification(notification)
//                        case quitNow := <-api.QuitSignal():
//                                _ = quitNow
//                                fmt.Printf("event loop ends\n")
//                                return
//                        }
//                }
//        }
//
//        os.Exit(openzwave.
//                BuildAPI("../go-openzwave/openzwave/config", "", "").
//                AddIntOption("SaveLogLevel", LOG_LEVEL.NONE).
//                AddIntOption("QueueLogLevel", LOG_LEVEL.NONE).
//                AddIntOption("DumpTrigger", LOG_LEVEL.NONE).
//                AddIntOption("PollInterval", 500).
//                AddBoolOption("IntervalBetweenPolls", true).
//                AddBoolOption("ValidateValueChanges", true).
//                Run(loop))
package openzwave
<commit_msg>Remove misleading example - api.Notifications() was removed from the API in favour of a callback.<commit_after>//
// openzwave provides a thin Go wrapper around the openzwave library.
//
// The following shows a simple use of the API which will log every notification received.
//
//        var loop = func(api openzwave.API) {
//                fmt.Printf("event loop starts\n")
//                for {
//                        select {
//                        case quitNow := <-api.QuitSignal():
//                                _ = quitNow
//                                fmt.Printf("event loop ends\n")
//                                return
//                        }
//                }
//        }
//
//        os.Exit(openzwave.
//                BuildAPI("../go-openzwave/openzwave/config", "", "").
//                AddIntOption("SaveLogLevel", LOG_LEVEL.NONE).
//                AddIntOption("QueueLogLevel", LOG_LEVEL.NONE).
//                AddIntOption("DumpTrigger", LOG_LEVEL.NONE).
//                AddIntOption("PollInterval", 500).
//                AddBoolOption("IntervalBetweenPolls", true).
//                AddBoolOption("ValidateValueChanges", true).
//                Run(loop))
package openzwave
<|endoftext|>


<commit_before>package main

import "runtime"
import "sync"
import "fmt"

const TOTAL int = 100

func Dine(id int, l_fork, r_fork chan int) {
	//Get the left fork
	left := <-l_fork
	runtime.Gosched()
	right := <-r_fork

	fmt.Println(id, "is eating with forks", left, "and", right)
	runtime.Gosched()
	fmt.Println(id, "is finished eating")

	//Give back the forks
	l_fork <- left
	r_fork <- right
	runtime.Gosched()
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	var wg sync.WaitGroup
	forks := make([]chan int, TOTAL)
	for i := 0; i < TOTAL; i++ {
		wg.Add(1)
		go func(k int, wg *sync.WaitGroup) {
			defer wg.Done()

			fork := make(chan int, 1)
			forks[k] = fork
			fork <- k
		}(i, &wg)
	}
	wg.Wait()
	var wg2 sync.WaitGroup
	for i := 0; i < TOTAL; i++ {
		wg2.Add(1)
		go func(id int, wg *sync.WaitGroup) {
			defer wg.Done()

			Dine(id, forks[id], forks[(id+1)%TOTAL])
		}(i, &wg2)
	}
	wg2.Wait()
	for i := 0; i < TOTAL; i++ {
		close(forks[i])
		<-forks[i]
	}
}
<commit_msg>Reset count to 5<commit_after>package main

import "runtime"
import "sync"
import "fmt"

const TOTAL int = 5

func Dine(id int, l_fork, r_fork chan int) {
	//Get the left fork
	left := <-l_fork
	runtime.Gosched()
	right := <-r_fork

	fmt.Println(id, "is eating with forks", left, "and", right)
	runtime.Gosched()
	fmt.Println(id, "is finished eating")

	//Give back the forks
	l_fork <- left
	r_fork <- right
	runtime.Gosched()
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	var wg sync.WaitGroup
	forks := make([]chan int, TOTAL)
	for i := 0; i < TOTAL; i++ {
		wg.Add(1)
		go func(k int, wg *sync.WaitGroup) {
			defer wg.Done()

			fork := make(chan int, 1)
			forks[k] = fork
			fork <- k
		}(i, &wg)
	}
	wg.Wait()
	var wg2 sync.WaitGroup
	for i := 0; i < TOTAL; i++ {
		wg2.Add(1)
		go func(id int, wg *sync.WaitGroup) {
			defer wg.Done()

			Dine(id, forks[id], forks[(id+1)%TOTAL])
		}(i, &wg2)
	}
	wg2.Wait()
	for i := 0; i < TOTAL; i++ {
		close(forks[i])
		<-forks[i]
	}
}
<|endoftext|>