<commit_before>package compress

import (
	"fmt"
	"io"
	"os"
	"sort"

	"github.com/golang/snappy"
)

// TODO: Tests schreiben (leere dateien, chunkgröße -1, +0, +1 etc.)
// TODO: linter durchlaufen lassen.

type chunkBuffer struct {
	buf      [MaxChunkSize]byte
	readOff  int64
	writeOff int64
	size     int64
}

func (c *chunkBuffer) Write(p []byte) (int, error) {
	n := copy(c.buf[c.writeOff:MaxChunkSize], p)
	c.writeOff += int64(n)
	if c.writeOff > c.size {
		c.size = c.writeOff
	}
	return n, nil
}

func (c *chunkBuffer) Reset() {
	c.readOff = 0
	c.writeOff = 0
	c.size = 0
}

func (c *chunkBuffer) Len() int {
	return int(c.size - c.readOff)
}

func (c *chunkBuffer) Read(p []byte) (int, error) {
	n := copy(p, c.buf[c.readOff:c.size])
	c.readOff += int64(n)
	if n == 0 {
		return n, io.EOF
	}
	return n, nil
}

func (c *chunkBuffer) Seek(offset int64, whence int) (int64, error) {
	switch whence {
	case os.SEEK_CUR:
		c.readOff += offset
	case os.SEEK_END:
		c.readOff = c.size + offset
	case os.SEEK_SET:
		c.readOff = offset
	}

	if c.readOff > c.size {
		c.readOff = c.size
	}
	c.writeOff = c.readOff
	return c.readOff, nil
}

func newChunkBuffer() chunkBuffer {
	return chunkBuffer{}
}

type reader struct {
	// Underlying raw, compressed datastream.
	rawR io.ReadSeeker

	// Decompression layer, reader is based on chosen algorithm.
	zipR io.Reader

	// Index with records which contain chunk offsets.
	index []record

	// Buffer holds currently read data; MaxChunkSize.
	chunkBuf chunkBuffer

	// Structure with parsed trailer.
	trailer *trailer
}

func (r *reader) Seek(rawOff int64, whence int) (int64, error) {
	if err := r.parseHeaderIfNeeded(); err != nil {
		return 0, err
	}

	if whence == os.SEEK_END {
		if rawOff > 0 {
			return 0, io.EOF
		}
		return r.Seek(r.index[len(r.index)-1].rawOff+rawOff, os.SEEK_SET)
	}

	if whence == os.SEEK_CUR {
		currPos, err := r.rawR.Seek(0, os.SEEK_CUR)
		if err != nil {
			return currPos, err
		}
		return r.Seek(currPos+rawOff, os.SEEK_SET)
	}

	// Check if given raw offset equals current offset.
	currRawOff, err := r.rawR.Seek(0, os.SEEK_CUR)
	if err != nil || currRawOff == rawOff {
		return currRawOff, err
	}

	currRecord, _ := r.chunkLookup(currRawOff)
	prevRecord, _ := r.chunkLookup(rawOff)
	if _, err := r.rawR.Seek(prevRecord.zipOff, os.SEEK_SET); err != nil {
		return 0, err
	}

	// Don't re-read if offset is in current chunk.
	if currRecord.rawOff == prevRecord.rawOff {
		if _, err := r.readChunk(); err != nil {
			return 0, err
		}
	}

	toRead := rawOff - prevRecord.rawOff
	if _, err := r.chunkBuf.Seek(toRead, os.SEEK_SET); err != nil {
		return 0, err
	}

	return rawOff, nil
}

// Return start (prev offset) and end (curr offset) of the chunk currOff is
// located in. If currOff is 0, the startoffset of the first and second record is
// returned. If currOff is at the end of file the end offset of the last chunk
// is returned twice.  The difference between prev record and curr chunk is then
// equal to 0.
func (r *reader) chunkLookup(currOff int64) (*record, *record) {
	i := sort.Search(len(r.index), func(i int) bool {
		return r.index[i].zipOff > currOff
	})

	// Beginning of the file, first chunk: prev offset is 0, curr offset is 1
	if i == 0 {
		fmt.Println("INDEX:", r.index, i, currOff)
		return &r.index[i], &r.index[i+1]
	}

	// End of the file, last chunk: prev and curr offset is the last index.
	if i == len(r.index) {
		return &r.index[i-1], &r.index[i-1]
	}
	return &r.index[i-1], &r.index[i]
}

func (r *reader) parseHeaderIfNeeded() error {
	if r.trailer != nil {
		return nil
	}

	// Goto end of file and read trailer buffer.
	if _, err := r.rawR.Seek(-TrailerSize, os.SEEK_END); err != nil {
		return err
	}

	buf := [TrailerSize]byte{}
	if n, err := r.rawR.Read(buf[:]); err != nil || n != TrailerSize {
		return err
	}
	r.trailer = &trailer{}
	r.trailer.unmarshal(buf[:])

	// Handle uncompressed stream.
	if r.trailer.algo == AlgoNone {
		if _, err := r.rawR.Seek(0, os.SEEK_SET); err != nil {
			return err
		}
		// No need to go further.
		return nil
	}

	// Handle compressed stream.
	// Seek and read index into buffer.
	seekIdx := -(int64(r.trailer.indexSize) + TrailerSize)
	if _, err := r.rawR.Seek(seekIdx, os.SEEK_END); err != nil {
		return err
	}
	indexBuf := make([]byte, r.trailer.indexSize)
	if _, err := r.rawR.Read(indexBuf); err != nil {
		return err
	}

	// Build index with records. A record encapsulates a raw offset and the
	// compressed offset it is mapped to.
	prevRecord := record{-1, -1}
	for i := uint64(0); i < (r.trailer.indexSize / IndexChunkSize); i++ {
		currRecord := record{}
		currRecord.unmarshal(indexBuf)

		if prevRecord.rawOff >= currRecord.rawOff {
			return ErrBadIndex
		}

		if prevRecord.zipOff >= currRecord.zipOff {
			return ErrBadIndex
		}
		fmt.Println(i, currRecord)
		r.index = append(r.index, currRecord)
		indexBuf = indexBuf[IndexChunkSize:]
	}

	// Set reader to beginning of file
	if _, err := r.rawR.Seek(0, os.SEEK_SET); err != nil {
		return err
	}
	return nil
}

// Read reads len(p) bytes from the compressed stream into p.
func (r *reader) Read(p []byte) (int, error) {
	if err := r.parseHeaderIfNeeded(); err != nil {
		return 0, err
	}

	read := 0
	for {
		if r.chunkBuf.Len() != 0 {
			n, err := r.chunkBuf.Read(p)
			if err != nil {
				return n, err
			}
			read += n
			p = p[n:]
		}

		if len(p) == 0 {
			break
		}

		if _, err := r.readChunk(); err != nil {
			return read, err
		}
	}

	return read, nil
}

//TODO: Save maxOffset in trailer and read from trailer instead of SEEK.
func (r *reader) maxOff(pSize int64) (int64, error) {
	// get current position.
	currOff, err := r.rawR.Seek(0, os.SEEK_CUR)
	if err != nil {
		return currOff, err
	}

	// get max offset (possition without trailer).
	maxOff, err := r.rawR.Seek(-TrailerSize, os.SEEK_END)
	if err != nil {
		return maxOff, err
	}

	// go back to current offset.
	_, err = r.rawR.Seek(currOff, os.SEEK_SET)
	if err != nil {
		return 0, err
	}

	// determinate max offset using.
	if pSize+currOff > maxOff {
		return maxOff - currOff, io.EOF
	}
	return pSize, nil
}

func (r *reader) readZipChunk() (int64, error) {
	// Get current position of the reader; offset of the compressed file.
	r.chunkBuf.Reset()
	currOff, err := r.rawR.Seek(0, os.SEEK_CUR)
	if err != nil {
		return 0, err
	}

	// Get the start and end record of the chunk currOff is located in.
	prevRecord, currRecord := r.chunkLookup(currOff)
	if currRecord == nil || prevRecord == nil {
		return 0, ErrBadIndex
	}

	// Determinate uncompressed chunksize; should only be 0 on empty file or at the end of file.
	chunkSize := currRecord.rawOff - prevRecord.rawOff
	if chunkSize == 0 {
		return 0, io.EOF
	}

	// Set reader to compressed offset.
	if _, err = r.rawR.Seek(prevRecord.zipOff, os.SEEK_SET); err != nil {
		return 0, err
	}

	n, err := io.CopyN(&r.chunkBuf, r.zipR, chunkSize)
	return n, err
}

// Return a new ReadSeeker with compression support. As random access is the
// purpose of this layer, a ReadSeeker is required as parameter. The used
// compression algorithm is chosen based on trailer information.
func NewReader(r io.ReadSeeker) io.ReadSeeker {
	return &reader{
		rawR:     r,
		zipR:     snappy.NewReader(r),
		chunkBuf: newChunkBuffer(),
	}
}
<commit_msg>compress/reader.go: Handle AlgoNone in Read.<commit_after>package compress

import (
	"fmt"
	"io"
	"os"
	"sort"

	"github.com/golang/snappy"
)

// TODO: Tests schreiben (leere dateien, chunkgröße -1, +0, +1 etc.)
// TODO: linter durchlaufen lassen.

type chunkBuffer struct {
	buf      [MaxChunkSize]byte
	readOff  int64
	writeOff int64
	size     int64
}

func (c *chunkBuffer) Write(p []byte) (int, error) {
	n := copy(c.buf[c.writeOff:MaxChunkSize], p)
	c.writeOff += int64(n)
	if c.writeOff > c.size {
		c.size = c.writeOff
	}
	return n, nil
}

func (c *chunkBuffer) Reset() {
	c.readOff = 0
	c.writeOff = 0
	c.size = 0
}

func (c *chunkBuffer) Len() int {
	return int(c.size - c.readOff)
}

func (c *chunkBuffer) Read(p []byte) (int, error) {
	n := copy(p, c.buf[c.readOff:c.size])
	c.readOff += int64(n)
	if n == 0 {
		return n, io.EOF
	}
	return n, nil
}

func (c *chunkBuffer) Seek(offset int64, whence int) (int64, error) {
	switch whence {
	case os.SEEK_CUR:
		c.readOff += offset
	case os.SEEK_END:
		c.readOff = c.size + offset
	case os.SEEK_SET:
		c.readOff = offset
	}

	if c.readOff > c.size {
		c.readOff = c.size
	}
	c.writeOff = c.readOff
	return c.readOff, nil
}

func newChunkBuffer() chunkBuffer {
	return chunkBuffer{}
}

type reader struct {
	// Underlying raw, compressed datastream.
	rawR io.ReadSeeker

	// Decompression layer, reader is based on chosen algorithm.
	zipR io.Reader

	// Index with records which contain chunk offsets.
	index []record

	// Buffer holds currently read data; MaxChunkSize.
	chunkBuf chunkBuffer

	// Structure with parsed trailer.
	trailer *trailer
}

func (r *reader) Seek(rawOff int64, whence int) (int64, error) {
	if err := r.parseHeaderIfNeeded(); err != nil {
		return 0, err
	}

	if whence == os.SEEK_END {
		if rawOff > 0 {
			return 0, io.EOF
		}
		return r.Seek(r.index[len(r.index)-1].rawOff+rawOff, os.SEEK_SET)
	}

	if whence == os.SEEK_CUR {
		currPos, err := r.rawR.Seek(0, os.SEEK_CUR)
		if err != nil {
			return currPos, err
		}
		return r.Seek(currPos+rawOff, os.SEEK_SET)
	}

	// Check if given raw offset equals current offset.
	currRawOff, err := r.rawR.Seek(0, os.SEEK_CUR)
	if err != nil || currRawOff == rawOff {
		return currRawOff, err
	}

	currRecord, _ := r.chunkLookup(currRawOff)
	prevRecord, _ := r.chunkLookup(rawOff)
	if _, err := r.rawR.Seek(prevRecord.zipOff, os.SEEK_SET); err != nil {
		return 0, err
	}

	// Don't re-read if offset is in current chunk.
	if currRecord.rawOff == prevRecord.rawOff {
		if _, err := r.readZipChunk(); err != nil {
			return 0, err
		}
	}

	toRead := rawOff - prevRecord.rawOff
	if _, err := r.chunkBuf.Seek(toRead, os.SEEK_SET); err != nil {
		return 0, err
	}

	return rawOff, nil
}

// Return start (prev offset) and end (curr offset) of the chunk currOff is
// located in. If currOff is 0, the startoffset of the first and second record is
// returned. If currOff is at the end of file the end offset of the last chunk
// is returned twice.  The difference between prev record and curr chunk is then
// equal to 0.
func (r *reader) chunkLookup(currOff int64) (*record, *record) {
	i := sort.Search(len(r.index), func(i int) bool {
		return r.index[i].zipOff > currOff
	})

	// Beginning of the file, first chunk: prev offset is 0, curr offset is 1
	if i == 0 {
		fmt.Println("INDEX:", r.index, i, currOff)
		return &r.index[i], &r.index[i+1]
	}

	// End of the file, last chunk: prev and curr offset is the last index.
	if i == len(r.index) {
		return &r.index[i-1], &r.index[i-1]
	}
	return &r.index[i-1], &r.index[i]
}

func (r *reader) parseHeaderIfNeeded() error {
	if r.trailer != nil {
		return nil
	}

	// Goto end of file and read trailer buffer.
	if _, err := r.rawR.Seek(-TrailerSize, os.SEEK_END); err != nil {
		return err
	}

	buf := [TrailerSize]byte{}
	if n, err := r.rawR.Read(buf[:]); err != nil || n != TrailerSize {
		return err
	}
	r.trailer = &trailer{}
	r.trailer.unmarshal(buf[:])

	// Handle uncompressed stream.
	if r.trailer.algo == AlgoNone {
		if _, err := r.rawR.Seek(0, os.SEEK_SET); err != nil {
			return err
		}
		// No need to go further.
		return nil
	}

	// Handle compressed stream.
	// Seek and read index into buffer.
	seekIdx := -(int64(r.trailer.indexSize) + TrailerSize)
	if _, err := r.rawR.Seek(seekIdx, os.SEEK_END); err != nil {
		return err
	}
	indexBuf := make([]byte, r.trailer.indexSize)
	if _, err := r.rawR.Read(indexBuf); err != nil {
		return err
	}

	// Build index with records. A record encapsulates a raw offset and the
	// compressed offset it is mapped to.
	prevRecord := record{-1, -1}
	for i := uint64(0); i < (r.trailer.indexSize / IndexChunkSize); i++ {
		currRecord := record{}
		currRecord.unmarshal(indexBuf)

		if prevRecord.rawOff >= currRecord.rawOff {
			return ErrBadIndex
		}

		if prevRecord.zipOff >= currRecord.zipOff {
			return ErrBadIndex
		}
		fmt.Println(i, currRecord)
		r.index = append(r.index, currRecord)
		indexBuf = indexBuf[IndexChunkSize:]
	}

	// Set reader to beginning of file
	if _, err := r.rawR.Seek(0, os.SEEK_SET); err != nil {
		return err
	}
	return nil
}

// Read reads len(p) bytes from the compressed stream into p.
func (r *reader) Read(p []byte) (int, error) {
	if err := r.parseHeaderIfNeeded(); err != nil {
		return 0, err
	}

	// Handle uncompressed stream.
	if r.trailer.algo == AlgoNone {
		maxOff, errMax := r.maxOff(int64(len(p)))
		n, err := r.rawR.Read(p[:maxOff])
		if err != nil {
			return n, err
		}
		return n, errMax
	}

	// Handle stream using compression.
	read := 0
	for {
		if r.chunkBuf.Len() != 0 {
			n, err := r.chunkBuf.Read(p)
			if err != nil {
				return n, err
			}
			read += n
			p = p[n:]
		}

		if len(p) == 0 {
			break
		}

		if _, err := r.readZipChunk(); err != nil {
			return read, err
		}
	}

	return read, nil
}

//TODO: Save maxOffset in trailer and read from trailer instead of SEEK.
func (r *reader) maxOff(pSize int64) (int64, error) {
	// get current position.
	currOff, err := r.rawR.Seek(0, os.SEEK_CUR)
	if err != nil {
		return currOff, err
	}

	// get max offset (possition without trailer).
	maxOff, err := r.rawR.Seek(-TrailerSize, os.SEEK_END)
	if err != nil {
		return maxOff, err
	}

	// go back to current offset.
	_, err = r.rawR.Seek(currOff, os.SEEK_SET)
	if err != nil {
		return 0, err
	}

	// determinate max offset using.
	if pSize+currOff > maxOff {
		return maxOff - currOff, io.EOF
	}
	return pSize, nil
}

func (r *reader) readZipChunk() (int64, error) {
	// Get current position of the reader; offset of the compressed file.
	r.chunkBuf.Reset()
	currOff, err := r.rawR.Seek(0, os.SEEK_CUR)
	if err != nil {
		return 0, err
	}

	// Get the start and end record of the chunk currOff is located in.
	prevRecord, currRecord := r.chunkLookup(currOff)
	if currRecord == nil || prevRecord == nil {
		return 0, ErrBadIndex
	}

	// Determinate uncompressed chunksize; should only be 0 on empty file or at the end of file.
	chunkSize := currRecord.rawOff - prevRecord.rawOff
	if chunkSize == 0 {
		return 0, io.EOF
	}

	// Set reader to compressed offset.
	if _, err = r.rawR.Seek(prevRecord.zipOff, os.SEEK_SET); err != nil {
		return 0, err
	}

	n, err := io.CopyN(&r.chunkBuf, r.zipR, chunkSize)
	return n, err
}

// Return a new ReadSeeker with compression support. As random access is the
// purpose of this layer, a ReadSeeker is required as parameter. The used
// compression algorithm is chosen based on trailer information.
func NewReader(r io.ReadSeeker) io.ReadSeeker {
	return &reader{
		rawR:     r,
		zipR:     snappy.NewReader(r),
		chunkBuf: newChunkBuffer(),
	}
}
<|endoftext|>


<commit_before>package wallet

import (
	"sort"

	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/types"
)

const (
	// defragThreshold is the number of outputs a wallet is allowed before it is
	// defragmented.
	defragThreshold = 50

	// defragBatchSize defines how many outputs are combined during one defrag.
	defragBatchSize = 40
)

// defragWallet computes the sum of the 15 largest outputs in the wallet and
// sends that sum to itself, effectively defragmenting the wallet. This defrag
// operation is only performed if the wallet has greater than defragThreshold
// outputs.
func (w *Wallet) defragWallet() {
	if len(w.siacoinOutputs) < defragThreshold {
		return
	}

	// grab all the outputs from the wallet and sort them from
	// largest -> smallest
	var so sortedOutputs
	for scoid, sco := range w.siacoinOutputs {
		so.ids = append(so.ids, scoid)
		so.outputs = append(so.outputs, sco)
	}
	sort.Sort(sort.Reverse(so))

	// choose the defragBatchSizeth largest outputs from the wallet and sum them
	defragOutputs := so.outputs[:defragBatchSize]
	var totalOutputValue types.Currency
	for _, output := range defragOutputs {
		totalOutputValue = totalOutputValue.Add(output.Value)
	}

	w.log.Printf("performing wallet defragmentation transaction: %v outputs, %v total value\n", len(defragOutputs), totalOutputValue)

	// grab a new address from the wallet
	addr, err := w.nextPrimarySeedAddress()
	if err != nil {
		w.log.Println("error getting an address for defrag: ", err)
		return
	}

	// send the sum of the outputs to this wallet. This operation is done in a
	// goroutine since defragWallet() is called under lock.
	go func() {
		txns, err := w.SendSiacoins(totalOutputValue, addr.UnlockHash())
		if err != nil {
			w.log.Printf("error sending defrag transaction with value %v: %v\n ", totalOutputValue, err)
			return
		}

		w.log.Println("defrag transaction size: ", len(encoding.Marshal(txns)))
	}()
}
<commit_msg>full puncuation in logging statements<commit_after>package wallet

import (
	"sort"

	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/types"
)

const (
	// defragThreshold is the number of outputs a wallet is allowed before it is
	// defragmented.
	defragThreshold = 50

	// defragBatchSize defines how many outputs are combined during one defrag.
	defragBatchSize = 40
)

// defragWallet computes the sum of the 15 largest outputs in the wallet and
// sends that sum to itself, effectively defragmenting the wallet. This defrag
// operation is only performed if the wallet has greater than defragThreshold
// outputs.
func (w *Wallet) defragWallet() {
	if len(w.siacoinOutputs) < defragThreshold {
		return
	}

	// grab all the outputs from the wallet and sort them from
	// largest -> smallest
	var so sortedOutputs
	for scoid, sco := range w.siacoinOutputs {
		so.ids = append(so.ids, scoid)
		so.outputs = append(so.outputs, sco)
	}
	sort.Sort(sort.Reverse(so))

	// choose the defragBatchSizeth largest outputs from the wallet and sum them
	defragOutputs := so.outputs[:defragBatchSize]
	var totalOutputValue types.Currency
	for _, output := range defragOutputs {
		totalOutputValue = totalOutputValue.Add(output.Value)
	}

	// grab a new address from the wallet
	addr, err := w.nextPrimarySeedAddress()
	if err != nil {
		w.log.Println("Error getting an address for defragmentation: ", err)
		return
	}

	// send the sum of the outputs to this wallet. This operation is done in a
	// goroutine since defragWallet() is called under lock.
	go func() {
		txns, err := w.SendSiacoins(totalOutputValue, addr.UnlockHash())
		if err != nil {
			w.log.Printf("Attempted to defragment wallet but failed. %v outputs used, %vH total coins. Error: %v.\n ", len(defragOutputs), totalOutputValue, err)
			return
		}

		w.log.Printf("Successfully defragmented wallet. %v outputs used, %vH coins defragmented. Defragment transaction size: %vB.\n", len(defragOutputs), totalOutputValue, len(encoding.Marshal(txns)))
	}()
}
<|endoftext|>


<commit_before>package wallet

import (
	"errors"
	"fmt"
	"os"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/sync"
	"github.com/NebulousLabs/Sia/types"
)

const (
	// AgeDelay indicates how long the wallet will wait before allowing the
	// user to double-spend a transaction under standard circumstances. The
	// rationale is that most transactions are meant to be submitted to the
	// blockchain immediately, and ones that take more than AgeDelay blocks
	// have probably failed in some way.
	AgeDelay = 80
)

// A Wallet uses the state and transaction pool to track the unconfirmed
// balance of a user. All of the keys are stored in 'saveDir'/wallet.dat.
//
// One feature of the wallet is preventing accidental double spends. The wallet
// will block an output from being spent if it has been spent in the last
// 'AgeDelay' blocks. This is managed by tracking a global age for the wallet
// and then an age for each output, set to the age of the wallet that the
// output was most recently spent. If the wallet is 'AgeDelay' blocks older
// than an output, then the output can be spent again.
//
// A second feature of the wallet is the transaction builder, which is a series
// of functions that can be used to build independent transactions for use with
// untrusted parties. The transactions can be cobbled together piece by piece
// and then signed. When using the transaction builder, the wallet will always
// have exact outputs (by creating another transaction first if needed) and
// thus the transaction does not need to be spent for the transaction builder
// to be able to use any refunds.
type Wallet struct {
	state            modules.ConsensusSet
	tpool            modules.TransactionPool
	unconfirmedDiffs []modules.SiacoinOutputDiff

	// Location of the wallet directory, for saving and loading keys.
	saveDir string

	// A key contains all the information necessary to spend a particular
	// address, as well as all the known outputs that use the address.
	//
	// age is a tool to determine whether or not an output can be spent. When
	// an output is spent by the wallet, the age of the output is marked equal
	// to the age of the wallet. It will not be spent again until the age is
	// `AgeDelay` less than the wallet. The wallet ages by 1 every block. The
	// wallet can also be manually aged, which is a convenient and efficient
	// way of resetting spent outputs. Transactions are not intended to be
	// broadcast for a while can be given an age that is much greater than the
	// wallet.
	//
	// Timelocked keys is a list of addresses found in `keys` that can't be
	// spent until a certain height. The wallet will use `timelockedKeys` to
	// mark keys as unspendable until the timelock has lifted.
	//
	// Visible keys will be displayed to the user.
	consensusHeight  types.BlockHeight
	age              int
	keys             map[types.UnlockHash]*key
	timelockedKeys   map[types.BlockHeight][]types.UnlockHash
	visibleAddresses map[types.UnlockHash]struct{}

	// transactions is a list of transactions that are currently being built by
	// the wallet. Each transaction has a unique id, which is enforced by the
	// transactionCounter.
	transactionCounter int
	transactions       map[string]*openTransaction

	subscribers []chan struct{}

	mu *sync.RWMutex
}

// New creates a new wallet, loading any known addresses from the input file
// name and then using the file to save in the future.
func New(cs modules.ConsensusSet, tpool modules.TransactionPool, saveDir string) (w *Wallet, err error) {
	if cs == nil {
		err = errors.New("wallet cannot use a nil state")
		return
	}
	if tpool == nil {
		err = errors.New("wallet cannot use a nil transaction pool")
		return
	}

	w = &Wallet{
		state: cs,
		tpool: tpool,

		saveDir: saveDir,

		age:              AgeDelay + 100,
		keys:             make(map[types.UnlockHash]*key),
		timelockedKeys:   make(map[types.BlockHeight][]types.UnlockHash),
		visibleAddresses: make(map[types.UnlockHash]struct{}),

		transactions: make(map[string]*openTransaction),

		mu: sync.New(modules.SafeMutexDelay, 1),
	}

	// Create the wallet folder.
	err = os.MkdirAll(saveDir, 0700)
	if err != nil {
		return
	}

	// Try to load a previously saved wallet file. If it doesn't exist, assume
	// that we're creating a new wallet file.
	// TODO: log warning if no file found?
	err = w.load()
	if os.IsNotExist(err) {
		err = nil
		// No wallet file exists... make a visible address for the user.
		_, _, err = w.coinAddress(true)
		if err != nil {
			return nil, err
		}
	}
	if err != nil {
		err = fmt.Errorf("couldn't load wallet file %s: %v", saveDir, err)
		// TODO: try to recover from wallet.backup?
		return
	}

	w.tpool.TransactionPoolSubscribe(w)

	return
}

func (w *Wallet) Close() error {
	id := w.mu.RLock()
	defer w.mu.RUnlock(id)
	return w.save()
}

// SpendCoins creates a transaction sending 'amount' to 'dest'. The transaction
// is submitted to the transaction pool and is also returned.
func (w *Wallet) SpendCoins(amount types.Currency, dest types.UnlockHash) (t types.Transaction, err error) {
	// Create and send the transaction.
	output := types.SiacoinOutput{
		Value:      amount,
		UnlockHash: dest,
	}
	id, err := w.RegisterTransaction(t)
	if err != nil {
		return
	}
	_, err = w.FundTransaction(id, amount.Sub(10*types.CoinbaseAugment))
	if err != nil {
		return
	}
	_, _, err = w.AddOutput(id, output)
	if err != nil {
		return
	}
	t, err = w.SignTransaction(id, true)
	if err != nil {
		return
	}
	err = w.tpool.AcceptTransaction(t)
	if err != nil {
		return
	}

	return
}
<commit_msg>one instance of coinBaseAugment slipped through<commit_after>package wallet

import (
	"errors"
	"fmt"
	"os"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/sync"
	"github.com/NebulousLabs/Sia/types"
)

const (
	// AgeDelay indicates how long the wallet will wait before allowing the
	// user to double-spend a transaction under standard circumstances. The
	// rationale is that most transactions are meant to be submitted to the
	// blockchain immediately, and ones that take more than AgeDelay blocks
	// have probably failed in some way.
	AgeDelay = 80
)

// A Wallet uses the state and transaction pool to track the unconfirmed
// balance of a user. All of the keys are stored in 'saveDir'/wallet.dat.
//
// One feature of the wallet is preventing accidental double spends. The wallet
// will block an output from being spent if it has been spent in the last
// 'AgeDelay' blocks. This is managed by tracking a global age for the wallet
// and then an age for each output, set to the age of the wallet that the
// output was most recently spent. If the wallet is 'AgeDelay' blocks older
// than an output, then the output can be spent again.
//
// A second feature of the wallet is the transaction builder, which is a series
// of functions that can be used to build independent transactions for use with
// untrusted parties. The transactions can be cobbled together piece by piece
// and then signed. When using the transaction builder, the wallet will always
// have exact outputs (by creating another transaction first if needed) and
// thus the transaction does not need to be spent for the transaction builder
// to be able to use any refunds.
type Wallet struct {
	state            modules.ConsensusSet
	tpool            modules.TransactionPool
	unconfirmedDiffs []modules.SiacoinOutputDiff

	// Location of the wallet directory, for saving and loading keys.
	saveDir string

	// A key contains all the information necessary to spend a particular
	// address, as well as all the known outputs that use the address.
	//
	// age is a tool to determine whether or not an output can be spent. When
	// an output is spent by the wallet, the age of the output is marked equal
	// to the age of the wallet. It will not be spent again until the age is
	// `AgeDelay` less than the wallet. The wallet ages by 1 every block. The
	// wallet can also be manually aged, which is a convenient and efficient
	// way of resetting spent outputs. Transactions are not intended to be
	// broadcast for a while can be given an age that is much greater than the
	// wallet.
	//
	// Timelocked keys is a list of addresses found in `keys` that can't be
	// spent until a certain height. The wallet will use `timelockedKeys` to
	// mark keys as unspendable until the timelock has lifted.
	//
	// Visible keys will be displayed to the user.
	consensusHeight  types.BlockHeight
	age              int
	keys             map[types.UnlockHash]*key
	timelockedKeys   map[types.BlockHeight][]types.UnlockHash
	visibleAddresses map[types.UnlockHash]struct{}

	// transactions is a list of transactions that are currently being built by
	// the wallet. Each transaction has a unique id, which is enforced by the
	// transactionCounter.
	transactionCounter int
	transactions       map[string]*openTransaction

	subscribers []chan struct{}

	mu *sync.RWMutex
}

// New creates a new wallet, loading any known addresses from the input file
// name and then using the file to save in the future.
func New(cs modules.ConsensusSet, tpool modules.TransactionPool, saveDir string) (w *Wallet, err error) {
	if cs == nil {
		err = errors.New("wallet cannot use a nil state")
		return
	}
	if tpool == nil {
		err = errors.New("wallet cannot use a nil transaction pool")
		return
	}

	w = &Wallet{
		state: cs,
		tpool: tpool,

		saveDir: saveDir,

		age:              AgeDelay + 100,
		keys:             make(map[types.UnlockHash]*key),
		timelockedKeys:   make(map[types.BlockHeight][]types.UnlockHash),
		visibleAddresses: make(map[types.UnlockHash]struct{}),

		transactions: make(map[string]*openTransaction),

		mu: sync.New(modules.SafeMutexDelay, 1),
	}

	// Create the wallet folder.
	err = os.MkdirAll(saveDir, 0700)
	if err != nil {
		return
	}

	// Try to load a previously saved wallet file. If it doesn't exist, assume
	// that we're creating a new wallet file.
	// TODO: log warning if no file found?
	err = w.load()
	if os.IsNotExist(err) {
		err = nil
		// No wallet file exists... make a visible address for the user.
		_, _, err = w.coinAddress(true)
		if err != nil {
			return nil, err
		}
	}
	if err != nil {
		err = fmt.Errorf("couldn't load wallet file %s: %v", saveDir, err)
		// TODO: try to recover from wallet.backup?
		return
	}

	w.tpool.TransactionPoolSubscribe(w)

	return
}

func (w *Wallet) Close() error {
	id := w.mu.RLock()
	defer w.mu.RUnlock(id)
	return w.save()
}

// SpendCoins creates a transaction sending 'amount' to 'dest'. The transaction
// is submitted to the transaction pool and is also returned.
func (w *Wallet) SpendCoins(amount types.Currency, dest types.UnlockHash) (t types.Transaction, err error) {
	// Create and send the transaction.
	output := types.SiacoinOutput{
		Value:      amount,
		UnlockHash: dest,
	}
	id, err := w.RegisterTransaction(t)
	if err != nil {
		return
	}
	_, err = w.FundTransaction(id, amount)
	if err != nil {
		return
	}
	_, _, err = w.AddOutput(id, output)
	if err != nil {
		return
	}
	t, err = w.SignTransaction(id, true)
	if err != nil {
		return
	}
	err = w.tpool.AcceptTransaction(t)
	if err != nil {
		return
	}

	return
}
<|endoftext|>


<commit_before>/*
Copyright IBM Corp. All Rights Reserved.

SPDX-License-Identifier: Apache-2.0
*/

package grpcmetrics

import (
	"context"
	"strings"
	"time"

	"github.com/hyperledger/fabric/common/metrics"
	"google.golang.org/grpc"
)

type UnaryMetrics struct {
	RequestDuration   metrics.Histogram
	RequestsReceived  metrics.Counter
	RequestsCompleted metrics.Counter
}

func UnaryServerInterceptor(um *UnaryMetrics) grpc.UnaryServerInterceptor {
	return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
		service, method := serviceMethod(info.FullMethod)
		um.RequestsReceived.With("service", service, "method", method).Add(1)

		startTime := time.Now()
		resp, err := handler(ctx, req)
		duration := time.Since(startTime)

		um.RequestDuration.With(
			"service", service, "method", method, "code", grpc.Code(err).String(),
		).Observe(float64(duration) / float64(time.Second))
		um.RequestsCompleted.With("service", service, "method", method, "code", grpc.Code(err).String()).Add(1)

		return resp, err
	}
}

type StreamMetrics struct {
	RequestDuration   metrics.Histogram
	RequestsReceived  metrics.Counter
	RequestsCompleted metrics.Counter
	MessagesSent      metrics.Counter
	MessagesReceived  metrics.Counter
}

func StreamServerInterceptor(sm *StreamMetrics) grpc.StreamServerInterceptor {
	return func(svc interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
		sm := sm
		service, method := serviceMethod(info.FullMethod)
		sm.RequestsReceived.With("service", service, "method", method).Add(1)

		wrappedStream := &serverStream{
			ServerStream:     stream,
			messagesSent:     sm.MessagesSent.With("service", service, "method", method),
			messagesReceived: sm.MessagesReceived.With("service", service, "method", method),
		}

		startTime := time.Now()
		err := handler(svc, wrappedStream)
		duration := time.Since(startTime)

		sm.RequestDuration.With(
			"service", service, "method", method, "code", grpc.Code(err).String(),
		).Observe(float64(duration) / float64(time.Second))
		sm.RequestsCompleted.With("service", service, "method", method, "code", grpc.Code(err).String()).Add(1)

		return err
	}
}

func serviceMethod(fullMethod string) (service, method string) {
	normalizedMethod := strings.Replace(fullMethod, ".", "_", -1)
	parts := strings.SplitN(normalizedMethod[1:], "/", 2)
	return parts[0], parts[1]
}

type serverStream struct {
	grpc.ServerStream
	messagesSent     metrics.Counter
	messagesReceived metrics.Counter
}

func (ss *serverStream) SendMsg(msg interface{}) error {
	ss.messagesSent.Add(1)
	return ss.ServerStream.SendMsg(msg)
}

func (ss *serverStream) RecvMsg(msg interface{}) error {
	err := ss.ServerStream.RecvMsg(msg)
	if err == nil {
		ss.messagesReceived.Add(1)
	}
	return err
}
<commit_msg>[FAB-12725] defensive treatment of info.FullMethod<commit_after>/*
Copyright IBM Corp. All Rights Reserved.

SPDX-License-Identifier: Apache-2.0
*/

package grpcmetrics

import (
	"context"
	"strings"
	"time"

	"github.com/hyperledger/fabric/common/metrics"
	"google.golang.org/grpc"
)

type UnaryMetrics struct {
	RequestDuration   metrics.Histogram
	RequestsReceived  metrics.Counter
	RequestsCompleted metrics.Counter
}

func UnaryServerInterceptor(um *UnaryMetrics) grpc.UnaryServerInterceptor {
	return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
		service, method := serviceMethod(info.FullMethod)
		um.RequestsReceived.With("service", service, "method", method).Add(1)

		startTime := time.Now()
		resp, err := handler(ctx, req)
		duration := time.Since(startTime)

		um.RequestDuration.With(
			"service", service, "method", method, "code", grpc.Code(err).String(),
		).Observe(float64(duration) / float64(time.Second))
		um.RequestsCompleted.With("service", service, "method", method, "code", grpc.Code(err).String()).Add(1)

		return resp, err
	}
}

type StreamMetrics struct {
	RequestDuration   metrics.Histogram
	RequestsReceived  metrics.Counter
	RequestsCompleted metrics.Counter
	MessagesSent      metrics.Counter
	MessagesReceived  metrics.Counter
}

func StreamServerInterceptor(sm *StreamMetrics) grpc.StreamServerInterceptor {
	return func(svc interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
		sm := sm
		service, method := serviceMethod(info.FullMethod)
		sm.RequestsReceived.With("service", service, "method", method).Add(1)

		wrappedStream := &serverStream{
			ServerStream:     stream,
			messagesSent:     sm.MessagesSent.With("service", service, "method", method),
			messagesReceived: sm.MessagesReceived.With("service", service, "method", method),
		}

		startTime := time.Now()
		err := handler(svc, wrappedStream)
		duration := time.Since(startTime)

		sm.RequestDuration.With(
			"service", service, "method", method, "code", grpc.Code(err).String(),
		).Observe(float64(duration) / float64(time.Second))
		sm.RequestsCompleted.With("service", service, "method", method, "code", grpc.Code(err).String()).Add(1)

		return err
	}
}

func serviceMethod(fullMethod string) (service, method string) {
	normalizedMethod := strings.Replace(fullMethod, ".", "_", -1)
	parts := strings.SplitN(normalizedMethod, "/", -1)
	if len(parts) != 3 {
		return "unknown", "unknown"
	}
	return parts[1], parts[2]
}

type serverStream struct {
	grpc.ServerStream
	messagesSent     metrics.Counter
	messagesReceived metrics.Counter
}

func (ss *serverStream) SendMsg(msg interface{}) error {
	ss.messagesSent.Add(1)
	return ss.ServerStream.SendMsg(msg)
}

func (ss *serverStream) RecvMsg(msg interface{}) error {
	err := ss.ServerStream.RecvMsg(msg)
	if err == nil {
		ss.messagesReceived.Add(1)
	}
	return err
}
<|endoftext|>


<commit_before>// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package gologger

import (
	"bytes"
	"fmt"
	"strings"
	"sync"

	"github.com/luci/luci-go/common/logging"
	gol "github.com/op/go-logging"
	"golang.org/x/net/context"
)

const (
	logMessageFieldPadding = 44
)

// goLoggerWrapper is a synchronized wrapper around a go-logging Logger
// instance.
type goLoggerWrapper struct {
	sync.Mutex             // Lock around wrapped logger properties.
	l          *gol.Logger // Wrapped logger.
}

// loggerImpl implements logging.Logger. It optionally binds a goLoggerWrapper
// to a Context.
type loggerImpl struct {
	*goLoggerWrapper // The logger instance to log through.

	c context.Context // Bound context; may be nil if there is no bound context.
}

func (li *loggerImpl) Debugf(format string, args ...interface{}) {
	li.LogCall(logging.Debug, 1, format, args)
}
func (li *loggerImpl) Infof(format string, args ...interface{}) {
	li.LogCall(logging.Info, 1, format, args)
}
func (li *loggerImpl) Warningf(format string, args ...interface{}) {
	li.LogCall(logging.Warning, 1, format, args)
}
func (li *loggerImpl) Errorf(format string, args ...interface{}) {
	li.LogCall(logging.Error, 1, format, args)
}

func (li *loggerImpl) LogCall(l logging.Level, calldepth int, format string, args []interface{}) {
	// Append the fields to the format string.
	if li.c != nil {
		if !logging.IsLogging(li.c, l) {
			return
		}

		if fields := logging.GetFields(li.c); len(fields) > 0 {
			format = formatWithFields(format, fields, args)
			args = nil
		}
	}

	li.Lock()
	defer li.Unlock()

	li.l.ExtraCalldepth = (calldepth + 1)
	switch l {
	case logging.Debug:
		li.l.Debug(format, args...)
	case logging.Info:
		li.l.Info(format, args...)
	case logging.Warning:
		li.l.Warning(format, args...)
	case logging.Error:
		li.l.Error(format, args...)
	}
}

// formatWithFields renders the supplied format string, adding fields.
//
// '%' characters in the fields string are escaped so they can't be interpreted
// as format characters when appended to the initial format string.
func formatWithFields(format string, fields logging.Fields, args []interface{}) string {
	fieldString := strings.Replace(fields.String(), "%", "%%", -1)

	buf := bytes.Buffer{}
	buf.Grow(len(format) + logMessageFieldPadding + len(fieldString))
	fmt.Fprintf(&buf, format, args...)

	padding := 44 - buf.Len()
	if padding < 1 {
		padding = 1
	}
	for i := 0; i < padding; i++ {
		buf.WriteString(" ")
	}
	buf.WriteString(fieldString)
	return buf.String()
}
<commit_msg>gologger: Use properly-formatted calls.<commit_after>// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package gologger

import (
	"bytes"
	"fmt"
	"strings"
	"sync"

	"github.com/luci/luci-go/common/logging"
	gol "github.com/op/go-logging"
	"golang.org/x/net/context"
)

const (
	logMessageFieldPadding = 44
)

// goLoggerWrapper is a synchronized wrapper around a go-logging Logger
// instance.
type goLoggerWrapper struct {
	sync.Mutex             // Lock around wrapped logger properties.
	l          *gol.Logger // Wrapped logger.
}

// loggerImpl implements logging.Logger. It optionally binds a goLoggerWrapper
// to a Context.
type loggerImpl struct {
	*goLoggerWrapper // The logger instance to log through.

	c context.Context // Bound context; may be nil if there is no bound context.
}

func (li *loggerImpl) Debugf(format string, args ...interface{}) {
	li.LogCall(logging.Debug, 1, format, args)
}
func (li *loggerImpl) Infof(format string, args ...interface{}) {
	li.LogCall(logging.Info, 1, format, args)
}
func (li *loggerImpl) Warningf(format string, args ...interface{}) {
	li.LogCall(logging.Warning, 1, format, args)
}
func (li *loggerImpl) Errorf(format string, args ...interface{}) {
	li.LogCall(logging.Error, 1, format, args)
}

func (li *loggerImpl) LogCall(l logging.Level, calldepth int, format string, args []interface{}) {
	// Append the fields to the format string.
	if li.c != nil {
		if !logging.IsLogging(li.c, l) {
			return
		}

		if fields := logging.GetFields(li.c); len(fields) > 0 {
			format = formatWithFields(format, fields, args)
			args = nil
		}
	}

	li.Lock()
	defer li.Unlock()

	li.l.ExtraCalldepth = (calldepth + 1)
	switch l {
	case logging.Debug:
		li.l.Debugf(format, args...)
	case logging.Info:
		li.l.Infof(format, args...)
	case logging.Warning:
		li.l.Warningf(format, args...)
	case logging.Error:
		li.l.Errorf(format, args...)
	}
}

// formatWithFields renders the supplied format string, adding fields.
//
// '%' characters in the fields string are escaped so they can't be interpreted
// as format characters when appended to the initial format string.
func formatWithFields(format string, fields logging.Fields, args []interface{}) string {
	fieldString := strings.Replace(fields.String(), "%", "%%", -1)

	buf := bytes.Buffer{}
	buf.Grow(len(format) + logMessageFieldPadding + len(fieldString))
	fmt.Fprintf(&buf, format, args...)

	padding := 44 - buf.Len()
	if padding < 1 {
		padding = 1
	}
	for i := 0; i < padding; i++ {
		buf.WriteString(" ")
	}
	buf.WriteString(fieldString)
	return buf.String()
}
<|endoftext|>


<commit_before>package dns

import (
	"fmt"
	"net"
	"sync"
	"time"
)

var wg sync.WaitGroup

func TestHosts(hosts []string, debug bool) {
	for _, host := range hosts {
		_, err := net.ResolveIPAddr("ip4", host) // *IPAddr
		if err != nil {
			fmt.Println("Invalid DNS name or IP address:", host)
			return
		}

		wg.Add(1)
		go TestHost(host, debug)
	}

	wg.Wait()
}

func TestHost(host string, debug bool) {
	defer wg.Done()

	if debug {
		fmt.Println("testing host: " + host)
	}

	var startTime = time.Now()
	addrs, err := net.LookupIP(host)
	var endTime = time.Now()
	var elapsedTime = endTime.Sub(startTime)

	if err != nil {
		fmt.Println(err.Error())
	} else {
		fmt.Printf("addresses for %s: %v\ntime: %f seconds\n\n", host, addrs, elapsedTime.Seconds())
	}
}
<commit_msg>Finished up DNS lookups<commit_after>package dns

import (
	"fmt"
	"net"
	"strings"
	"sync"
	"time"
)

var wg sync.WaitGroup

// Source: http://stackoverflow.com/a/15323988
func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

func TestHosts(hosts []string, debug bool) {
	for _, host := range hosts {
		_, err := net.ResolveIPAddr("ip4", host) // *IPAddr
		if err != nil {
			fmt.Println("Invalid DNS name or IP address:", host)
			return
		}

		wg.Add(1)
		go TestHost(host, debug)
	}

	wg.Wait()
}

func lookupNames(host string) []string {
	names, _ := net.LookupAddr(host)

	return names
}

func lookupIPAddresses(host string) (results []string) {
	addrs, _ := net.LookupIP(host)

	for _, val := range addrs {
		results = append(results, val.String())
	}

	return
}

func TestHost(host string, debug bool) {
	defer wg.Done()

	if debug {
		fmt.Println("testing host: " + host)
	}

	// Start the timer
	var startTime = time.Now()

	// Find IP addresses for the given host string
	ips := lookupIPAddresses(host)

	// Find hostnames for the given host string
	names := lookupNames(host)

	// Stop the timer
	var endTime = time.Now()

	// Calculate elapsed time for the lookups
	var elapsedTime = endTime.Sub(startTime)

	// Combine results into a single array
	var results []string

	// If we successfully resolved a hostname, then the host string was an IP address
	//    and there is no need to merge in the ip addresses since it will be equal to
	//    the host string we tried to look up.
	if len(names) == 0 {
		for _, val := range ips {
			if !stringInSlice(val, results) {
				results = append(results, val)
			}
		}
	}

	// Merge in all hostnames found in the lookup.  This will be empty if the host
	//    string was a hostname to begin with.
	for _, val := range names {
		if !stringInSlice(val, results) {
			var tmpname = strings.TrimRight(val, ".")
			results = append(results, tmpname)
		}
	}

	fmt.Printf("addresses for %s: %s\ntime: %f seconds\n\n", host, results, elapsedTime.Seconds())
}
<|endoftext|>


<commit_before>package resource

import (
	"fmt"
	"github.com/dokku/dokku/plugins/common"
)

// CommandLimit implements resource:limit
func CommandLimit(args []string, processType string, r Resource) error {
	appName, err := getAppName(args)
	if err != nil {
		return err
	}

	return setRequestType(appName, processType, r, "limit")
}

// CommandLimitClear implements resource:limit-clear
func CommandLimitClear(args []string, processType string) error {
	appName, err := getAppName(args)
	if err != nil {
		return err
	}

	return clearByRequestType(appName, processType, "limit")
}

// CommandReserve implements resource:reserve
func CommandReserve(args []string, processType string, r Resource) error {
	appName, err := getAppName(args)
	if err != nil {
		return err
	}

	return setRequestType(appName, processType, r, "reserve")
}

// CommandReserveClear implements resource:reserve-clear
func CommandReserveClear(args []string, processType string) error {
	appName, err := getAppName(args)
	if err != nil {
		return err
	}

	return clearByRequestType(appName, processType, "reserve")
}

func clearByRequestType(appName string, processType string, requestType string) error {
	noun := "limits"
	if requestType == "reserve" {
		noun = "reservation"
	}

	message := fmt.Sprintf("clearing %v %v", appName, noun)
	if processType != "_default_" && processType != "" {
		message = fmt.Sprintf("%v (%v)", message, processType)
	}
	common.LogInfo2Quiet(message)

	if processType == "" {
		resources, err := common.PropertyGetAll("resource", appName)
		if err != nil {
			return err
		}
		for key, _ := range resources {
			err := common.PropertyDelete("resource", appName, key)
			if err != nil {
				return err
			}
		}
	} else {
		resources := []string{
			"cpu",
			"memory",
			"memory-swap",
			"network",
			"network-ingress",
			"network-egress",
		}

		for _, key := range resources {
			property := propertyKey(processType, requestType, key)
			err := common.PropertyDelete("resource", appName, property)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func setRequestType(appName string, processType string, r Resource, requestType string) error {
	if len(processType) == 0 {
		processType = "_default_"
	}

	resources := map[string]string{
		"cpu":             r.CPU,
		"memory":          r.Memory,
		"memory-swap":     r.MemorySwap,
		"network":         r.Network,
		"network-ingress": r.NetworkIngress,
		"network-egress":  r.NetworkEgress,
	}

	hasValues := false
	for _, value := range resources {
		if value != "" {
			hasValues = true
		}
	}

	if !hasValues {
		reportRequestType(appName, processType, requestType)
		return nil
	}

	noun := "limits"
	if requestType == "reserve" {
		noun = "reservation"
	}
	message := fmt.Sprintf("Setting resource %v for %v", noun, appName)
	if processType != "_default_" {
		message = fmt.Sprintf("%v (%v)", message, processType)
	}
	common.LogInfo2Quiet(message)

	for key, value := range resources {
		if value != "" {
			common.LogVerbose(fmt.Sprintf("%v: %v", key, value))
		}

		property := propertyKey(processType, requestType, key)
		err := common.PropertyWrite("resource", appName, property, value)
		if err != nil {
			return err
		}
	}

	return nil
}

func reportRequestType(appName string, processType string, requestType string) {
	noun := "limits"
	if requestType == "reserve" {
		noun = "reservation"
	}

	message := fmt.Sprintf("resource %v %v information", noun, appName)
	if processType != "_default_" {
		message = fmt.Sprintf("%v (%v)", message, processType)
	}
	common.LogInfo2Quiet(message)

	resources := []string{
		"cpu",
		"memory",
		"memory-swap",
		"network",
		"network-ingress",
		"network-egress",
	}

	for _, key := range resources {
		property := propertyKey(processType, requestType, key)
		value := common.PropertyGet("resource", appName, property)
		common.LogVerbose(fmt.Sprintf("%v: %v", key, value))
	}
	return
}
<commit_msg>fix: correct range call<commit_after>package resource

import (
	"fmt"
	"github.com/dokku/dokku/plugins/common"
)

// CommandLimit implements resource:limit
func CommandLimit(args []string, processType string, r Resource) error {
	appName, err := getAppName(args)
	if err != nil {
		return err
	}

	return setRequestType(appName, processType, r, "limit")
}

// CommandLimitClear implements resource:limit-clear
func CommandLimitClear(args []string, processType string) error {
	appName, err := getAppName(args)
	if err != nil {
		return err
	}

	return clearByRequestType(appName, processType, "limit")
}

// CommandReserve implements resource:reserve
func CommandReserve(args []string, processType string, r Resource) error {
	appName, err := getAppName(args)
	if err != nil {
		return err
	}

	return setRequestType(appName, processType, r, "reserve")
}

// CommandReserveClear implements resource:reserve-clear
func CommandReserveClear(args []string, processType string) error {
	appName, err := getAppName(args)
	if err != nil {
		return err
	}

	return clearByRequestType(appName, processType, "reserve")
}

func clearByRequestType(appName string, processType string, requestType string) error {
	noun := "limits"
	if requestType == "reserve" {
		noun = "reservation"
	}

	message := fmt.Sprintf("clearing %v %v", appName, noun)
	if processType != "_default_" && processType != "" {
		message = fmt.Sprintf("%v (%v)", message, processType)
	}
	common.LogInfo2Quiet(message)

	if processType == "" {
		resources, err := common.PropertyGetAll("resource", appName)
		if err != nil {
			return err
		}
		for key := range resources {
			err := common.PropertyDelete("resource", appName, key)
			if err != nil {
				return err
			}
		}
	} else {
		resources := []string{
			"cpu",
			"memory",
			"memory-swap",
			"network",
			"network-ingress",
			"network-egress",
		}

		for _, key := range resources {
			property := propertyKey(processType, requestType, key)
			err := common.PropertyDelete("resource", appName, property)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func setRequestType(appName string, processType string, r Resource, requestType string) error {
	if len(processType) == 0 {
		processType = "_default_"
	}

	resources := map[string]string{
		"cpu":             r.CPU,
		"memory":          r.Memory,
		"memory-swap":     r.MemorySwap,
		"network":         r.Network,
		"network-ingress": r.NetworkIngress,
		"network-egress":  r.NetworkEgress,
	}

	hasValues := false
	for _, value := range resources {
		if value != "" {
			hasValues = true
		}
	}

	if !hasValues {
		reportRequestType(appName, processType, requestType)
		return nil
	}

	noun := "limits"
	if requestType == "reserve" {
		noun = "reservation"
	}
	message := fmt.Sprintf("Setting resource %v for %v", noun, appName)
	if processType != "_default_" {
		message = fmt.Sprintf("%v (%v)", message, processType)
	}
	common.LogInfo2Quiet(message)

	for key, value := range resources {
		if value != "" {
			common.LogVerbose(fmt.Sprintf("%v: %v", key, value))
		}

		property := propertyKey(processType, requestType, key)
		err := common.PropertyWrite("resource", appName, property, value)
		if err != nil {
			return err
		}
	}

	return nil
}

func reportRequestType(appName string, processType string, requestType string) {
	noun := "limits"
	if requestType == "reserve" {
		noun = "reservation"
	}

	message := fmt.Sprintf("resource %v %v information", noun, appName)
	if processType != "_default_" {
		message = fmt.Sprintf("%v (%v)", message, processType)
	}
	common.LogInfo2Quiet(message)

	resources := []string{
		"cpu",
		"memory",
		"memory-swap",
		"network",
		"network-ingress",
		"network-egress",
	}

	for _, key := range resources {
		property := propertyKey(processType, requestType, key)
		value := common.PropertyGet("resource", appName, property)
		common.LogVerbose(fmt.Sprintf("%v: %v", key, value))
	}
	return
}
<|endoftext|>


<commit_before>package suggestionbox

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"path"
	"strings"

	"github.com/pkg/errors"
)

// Model represents a single model inside Suggestionbox.
type Model struct {
	// ID is the ID of the model.
	ID string `json:"id,omitempty"`
	// Name is the human readable name of the Model.
	Name string `json:"name"`
	// Options are optional Model settings to adjust the behaviour
	// of this Model within Suggestionbox.
	Options *ModelOptions `json:"options,omitempty"`
	// Choices are the options this Model will select from.
	Choices []Choice `json:"choices,omitempty"`
}

// NewModel makes a new Model.
func NewModel(id, name string, choices ...Choice) Model {
	return Model{
		ID:      id,
		Name:    name,
		Choices: choices,
	}
}

// Feature represents a single feature, to describe an input or a choice
// for example age:28 or location:"London".
type Feature struct {
	// Key is the name of the Feature.
	Key string `json:"key"`
	// Value is the string value of this Feature.
	Value string `json:"value"`
	// Type is the type of the Feature.
	// Can be "number", "text", "keyword", "list", "image_url" or "image_base64"..
	Type string `json:"type"`
}

// Choice is an option with features.
type Choice struct {
	// ID is a unique ID for this choice.
	ID string `json:"id"`
	// Features holds all the Feature objects that describe
	// this choice.
	Features []Feature `json:"features,omitempty"`
}

// NewChoice creates a new Choice.
func NewChoice(id string, features ...Feature) Choice {
	return Choice{
		ID:       id,
		Features: features,
	}
}

// ModelOptions describes the behaviours of a Model.
type ModelOptions struct {
	// ExpirationSeconds is the number of seconds to wait for the reward before it expires.
	ExpirationSeconds int `json:"expiration_seconds,omitempty"`

	// Epsilon enables proportionate exploiting vs exploring ratio.
	Epsilon float64 `json:"epsilon,omitempty"`

	// SoftmaxLambda enables adaptive exploiting vs exploring ratio.
	SoftmaxLambda float64 `json:"softmax_lambda,omitempty"`

	// Ngrams describes the n-grams for text analysis.
	Ngrams int `json:"ngrams,omitempty"`
	// Skipgrams describes the skip-grams for the text analysis.
	Skipgrams int `json:"skipgrams,omitempty"`
}

// ListModels gets a Model by its ID.
func (c *Client) ListModels(ctx context.Context) ([]Model, error) {
	u, err := url.Parse(c.addr + "/suggestionbox/models")
	if err != nil {
		return nil, err
	}
	if !u.IsAbs() {
		return nil, errors.New("box address must be absolute")
	}
	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
	if err != nil {
		return nil, err
	}
	req = req.WithContext(ctx)
	req.Header.Set("Accept", "application/json; charset=utf-8")
	req.Header.Set("Content-Type", "application/json; charset=utf-8")
	resp, err := c.HTTPClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		return nil, errors.New(resp.Status)
	}
	var response struct {
		Success bool
		Error   string
		Models  []Model
	}
	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
		return nil, errors.Wrap(err, "decoding response")
	}
	if !response.Success {
		return nil, ErrSuggestionbox(response.Error)
	}
	return response.Models, nil
}

// GetModel gets a Model by its ID.
func (c *Client) GetModel(ctx context.Context, modelID string) (Model, error) {
	var model Model
	u, err := url.Parse(c.addr + "/" + path.Join("suggestionbox", "models", modelID))
	if err != nil {
		return model, err
	}
	if !u.IsAbs() {
		return model, errors.New("box address must be absolute")
	}
	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
	if err != nil {
		return model, err
	}
	req = req.WithContext(ctx)
	req.Header.Set("Accept", "application/json; charset=utf-8")
	req.Header.Set("Content-Type", "application/json; charset=utf-8")
	resp, err := c.HTTPClient.Do(req)
	if err != nil {
		return model, err
	}
	defer resp.Body.Close()
	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		return model, errors.New(resp.Status)
	}
	var response struct {
		Success bool
		Error   string
		Model
	}
	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
		return model, errors.Wrap(err, "decoding response")
	}
	if !response.Success {
		return model, ErrSuggestionbox(response.Error)
	}
	return response.Model, nil
}

// DeleteModel gets a Model by its ID.
func (c *Client) DeleteModel(ctx context.Context, modelID string) error {
	u, err := url.Parse(c.addr + "/" + path.Join("suggestionbox", "models", modelID))
	if err != nil {
		return err
	}
	if !u.IsAbs() {
		return errors.New("box address must be absolute")
	}
	req, err := http.NewRequest(http.MethodDelete, u.String(), nil)
	if err != nil {
		return err
	}
	req = req.WithContext(ctx)
	req.Header.Set("Accept", "application/json; charset=utf-8")
	req.Header.Set("Content-Type", "application/json; charset=utf-8")
	resp, err := c.HTTPClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		return errors.New(resp.Status)
	}
	var response struct {
		Success bool
		Error   string
		Model
	}
	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
		return errors.Wrap(err, "decoding response")
	}
	if !response.Success {
		return ErrSuggestionbox(response.Error)
	}
	return nil
}

// CreateModel creates the Model in Suggestionbox.
// If no ID is set, one will be assigned in the return Model.
func (c *Client) CreateModel(ctx context.Context, model Model) (Model, error) {
	u, err := url.Parse(c.addr + "/suggestionbox/models")
	if err != nil {
		return model, err
	}
	if !u.IsAbs() {
		return model, errors.New("box address must be absolute")
	}
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(model); err != nil {
		return model, errors.Wrap(err, "encoding request body")
	}
	req, err := http.NewRequest(http.MethodPost, u.String(), &buf)
	if err != nil {
		return model, err
	}
	req = req.WithContext(ctx)
	req.Header.Set("Accept", "application/json; charset=utf-8")
	req.Header.Set("Content-Type", "application/json; charset=utf-8")
	resp, err := c.HTTPClient.Do(req)
	if err != nil {
		return model, err
	}
	defer resp.Body.Close()
	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		return model, errors.New(resp.Status)
	}
	var response struct {
		Success bool
		Error   string
		Model
	}
	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
		return model, errors.Wrap(err, "decoding response")
	}
	if !response.Success {
		return model, ErrSuggestionbox(response.Error)
	}
	return response.Model, nil
}

// FeatureNumber makes a numerical Feature.
func FeatureNumber(key string, value float64) Feature {
	return Feature{
		Type:  "number",
		Key:   key,
		Value: fmt.Sprintf("%v", value),
	}
}

// FeatureText makes a textual Feature that will be tokenized.
// Use FeatureKeyword for values that should not be tokenized.
func FeatureText(key string, text string) Feature {
	return Feature{
		Type:  "text",
		Key:   key,
		Value: text,
	}
}

// FeatureKeyword makes a textual Feature that will not be tokenized.
// Use FeatureList to provide multiple keywords in a single Feature.
// Use Text for bodies of text that should be tokenized.
func FeatureKeyword(key string, keyword string) Feature {
	return Feature{
		Type:  "keyword",
		Key:   key,
		Value: keyword,
	}
}

// FeatureList makes a Feature made up of multiple keywords.
func FeatureList(key string, keywords ...string) Feature {
	return Feature{
		Type:  "list",
		Key:   key,
		Value: strings.Join(keywords, ","),
	}
}

// FeatureImageURL makes a Feature that points to a hosted image.
func FeatureImageURL(key string, url string) Feature {
	return Feature{
		Type:  "image_url",
		Key:   key,
		Value: url,
	}
}

// FeatureImageBase64 makes a Feature that is base 64 encoded.
func FeatureImageBase64(key string, data string) Feature {
	return Feature{
		Type:  "image_base64",
		Key:   key,
		Value: data,
	}
}
<commit_msg>removed Content-Type where there's no body<commit_after>package suggestionbox

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"path"
	"strings"

	"github.com/pkg/errors"
)

// Model represents a single model inside Suggestionbox.
type Model struct {
	// ID is the ID of the model.
	ID string `json:"id,omitempty"`
	// Name is the human readable name of the Model.
	Name string `json:"name"`
	// Options are optional Model settings to adjust the behaviour
	// of this Model within Suggestionbox.
	Options *ModelOptions `json:"options,omitempty"`
	// Choices are the options this Model will select from.
	Choices []Choice `json:"choices,omitempty"`
}

// NewModel makes a new Model.
func NewModel(id, name string, choices ...Choice) Model {
	return Model{
		ID:      id,
		Name:    name,
		Choices: choices,
	}
}

// Feature represents a single feature, to describe an input or a choice
// for example age:28 or location:"London".
type Feature struct {
	// Key is the name of the Feature.
	Key string `json:"key"`
	// Value is the string value of this Feature.
	Value string `json:"value"`
	// Type is the type of the Feature.
	// Can be "number", "text", "keyword", "list", "image_url" or "image_base64"..
	Type string `json:"type"`
}

// Choice is an option with features.
type Choice struct {
	// ID is a unique ID for this choice.
	ID string `json:"id"`
	// Features holds all the Feature objects that describe
	// this choice.
	Features []Feature `json:"features,omitempty"`
}

// NewChoice creates a new Choice.
func NewChoice(id string, features ...Feature) Choice {
	return Choice{
		ID:       id,
		Features: features,
	}
}

// ModelOptions describes the behaviours of a Model.
type ModelOptions struct {
	// ExpirationSeconds is the number of seconds to wait for the reward before it expires.
	ExpirationSeconds int `json:"expiration_seconds,omitempty"`

	// Epsilon enables proportionate exploiting vs exploring ratio.
	Epsilon float64 `json:"epsilon,omitempty"`

	// SoftmaxLambda enables adaptive exploiting vs exploring ratio.
	SoftmaxLambda float64 `json:"softmax_lambda,omitempty"`

	// Ngrams describes the n-grams for text analysis.
	Ngrams int `json:"ngrams,omitempty"`
	// Skipgrams describes the skip-grams for the text analysis.
	Skipgrams int `json:"skipgrams,omitempty"`
}

// ListModels gets a Model by its ID.
func (c *Client) ListModels(ctx context.Context) ([]Model, error) {
	u, err := url.Parse(c.addr + "/suggestionbox/models")
	if err != nil {
		return nil, err
	}
	if !u.IsAbs() {
		return nil, errors.New("box address must be absolute")
	}
	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
	if err != nil {
		return nil, err
	}
	req = req.WithContext(ctx)
	req.Header.Set("Accept", "application/json; charset=utf-8")
	resp, err := c.HTTPClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		return nil, errors.New(resp.Status)
	}
	var response struct {
		Success bool
		Error   string
		Models  []Model
	}
	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
		return nil, errors.Wrap(err, "decoding response")
	}
	if !response.Success {
		return nil, ErrSuggestionbox(response.Error)
	}
	return response.Models, nil
}

// GetModel gets a Model by its ID.
func (c *Client) GetModel(ctx context.Context, modelID string) (Model, error) {
	var model Model
	u, err := url.Parse(c.addr + "/" + path.Join("suggestionbox", "models", modelID))
	if err != nil {
		return model, err
	}
	if !u.IsAbs() {
		return model, errors.New("box address must be absolute")
	}
	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
	if err != nil {
		return model, err
	}
	req = req.WithContext(ctx)
	req.Header.Set("Accept", "application/json; charset=utf-8")
	resp, err := c.HTTPClient.Do(req)
	if err != nil {
		return model, err
	}
	defer resp.Body.Close()
	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		return model, errors.New(resp.Status)
	}
	var response struct {
		Success bool
		Error   string
		Model
	}
	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
		return model, errors.Wrap(err, "decoding response")
	}
	if !response.Success {
		return model, ErrSuggestionbox(response.Error)
	}
	return response.Model, nil
}

// DeleteModel gets a Model by its ID.
func (c *Client) DeleteModel(ctx context.Context, modelID string) error {
	u, err := url.Parse(c.addr + "/" + path.Join("suggestionbox", "models", modelID))
	if err != nil {
		return err
	}
	if !u.IsAbs() {
		return errors.New("box address must be absolute")
	}
	req, err := http.NewRequest(http.MethodDelete, u.String(), nil)
	if err != nil {
		return err
	}
	req = req.WithContext(ctx)
	req.Header.Set("Accept", "application/json; charset=utf-8")
	resp, err := c.HTTPClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		return errors.New(resp.Status)
	}
	var response struct {
		Success bool
		Error   string
		Model
	}
	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
		return errors.Wrap(err, "decoding response")
	}
	if !response.Success {
		return ErrSuggestionbox(response.Error)
	}
	return nil
}

// CreateModel creates the Model in Suggestionbox.
// If no ID is set, one will be assigned in the return Model.
func (c *Client) CreateModel(ctx context.Context, model Model) (Model, error) {
	u, err := url.Parse(c.addr + "/suggestionbox/models")
	if err != nil {
		return model, err
	}
	if !u.IsAbs() {
		return model, errors.New("box address must be absolute")
	}
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(model); err != nil {
		return model, errors.Wrap(err, "encoding request body")
	}
	req, err := http.NewRequest(http.MethodPost, u.String(), &buf)
	if err != nil {
		return model, err
	}
	req = req.WithContext(ctx)
	req.Header.Set("Accept", "application/json; charset=utf-8")
	req.Header.Set("Content-Type", "application/json; charset=utf-8")
	resp, err := c.HTTPClient.Do(req)
	if err != nil {
		return model, err
	}
	defer resp.Body.Close()
	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		return model, errors.New(resp.Status)
	}
	var response struct {
		Success bool
		Error   string
		Model
	}
	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
		return model, errors.Wrap(err, "decoding response")
	}
	if !response.Success {
		return model, ErrSuggestionbox(response.Error)
	}
	return response.Model, nil
}

// FeatureNumber makes a numerical Feature.
func FeatureNumber(key string, value float64) Feature {
	return Feature{
		Type:  "number",
		Key:   key,
		Value: fmt.Sprintf("%v", value),
	}
}

// FeatureText makes a textual Feature that will be tokenized.
// Use FeatureKeyword for values that should not be tokenized.
func FeatureText(key string, text string) Feature {
	return Feature{
		Type:  "text",
		Key:   key,
		Value: text,
	}
}

// FeatureKeyword makes a textual Feature that will not be tokenized.
// Use FeatureList to provide multiple keywords in a single Feature.
// Use Text for bodies of text that should be tokenized.
func FeatureKeyword(key string, keyword string) Feature {
	return Feature{
		Type:  "keyword",
		Key:   key,
		Value: keyword,
	}
}

// FeatureList makes a Feature made up of multiple keywords.
func FeatureList(key string, keywords ...string) Feature {
	return Feature{
		Type:  "list",
		Key:   key,
		Value: strings.Join(keywords, ","),
	}
}

// FeatureImageURL makes a Feature that points to a hosted image.
func FeatureImageURL(key string, url string) Feature {
	return Feature{
		Type:  "image_url",
		Key:   key,
		Value: url,
	}
}

// FeatureImageBase64 makes a Feature that is base 64 encoded.
func FeatureImageBase64(key string, data string) Feature {
	return Feature{
		Type:  "image_base64",
		Key:   key,
		Value: data,
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package kvm

import (
	"fmt"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/juju/errors"
	"github.com/juju/loggo"
	"github.com/juju/names"

	"github.com/juju/juju/agent"
	"github.com/juju/juju/constraints"
	"github.com/juju/juju/container"
	"github.com/juju/juju/environs/cloudinit"
	"github.com/juju/juju/environs/imagemetadata"
	"github.com/juju/juju/instance"
	"github.com/juju/juju/version"
)

var (
	logger = loggo.GetLogger("juju.container.kvm")

	KvmObjectFactory ContainerFactory = &containerFactory{}
	DefaultKvmBridge                  = "virbr0"

	// In order for Juju to be able to create the hardware characteristics of
	// the kvm machines it creates, we need to be explicit in our definition
	// of memory, cpu-cores and root-disk.  The defaults here have been
	// extracted from the uvt-kvm executable.
	DefaultMemory uint64 = 512 // MB
	DefaultCpu    uint64 = 1
	DefaultDisk   uint64 = 8 // GB

	// There are some values where it doesn't make sense to go below.
	MinMemory uint64 = 512 // MB
	MinCpu    uint64 = 1
	MinDisk   uint64 = 2 // GB
)

// Utilized to provide a hard-coded path to kvm-ok
var kvmPath = "/usr/sbin"

// IsKVMSupported calls into the kvm-ok executable from the cpu-checkers package.
// It is a variable to allow us to overrid behaviour in the tests.
var IsKVMSupported = func() (bool, error) {

	// Prefer the user's $PATH first, but check /usr/sbin if we can't
	// find kvm-ok there
	var foundPath string
	const binName = "kvm-ok"
	if path, err := exec.LookPath(binName); err == nil {
		foundPath = path
	} else if path, err := exec.LookPath(filepath.Join(kvmPath, binName)); err == nil {
		foundPath = path
	} else {
		return false, errors.NotFoundf("%s executable", binName)
	}

	command := exec.Command(foundPath)
	output, err := command.CombinedOutput()

	if err != nil {
		return false, errors.Annotate(err, string(output))
	}
	logger.Debugf("%s output:\n%s", binName, output)
	return command.ProcessState.Success(), nil
}

// NewContainerManager returns a manager object that can start and stop kvm
// containers. The containers that are created are namespaced by the name
// parameter.
func NewContainerManager(conf container.ManagerConfig) (container.Manager, error) {
	name := conf.PopValue(container.ConfigName)
	if name == "" {
		return nil, fmt.Errorf("name is required")
	}
	logDir := conf.PopValue(container.ConfigLogDir)
	if logDir == "" {
		logDir = agent.DefaultLogDir
	}
	conf.WarnAboutUnused()
	return &containerManager{name: name, logdir: logDir}, nil
}

// containerManager handles all of the business logic at the juju specific
// level. It makes sure that the necessary directories are in place, that the
// user-data is written out in the right place.
type containerManager struct {
	name   string
	logdir string
}

var _ container.Manager = (*containerManager)(nil)

// Exposed so tests can observe our side-effects
var startParams StartParams

func (manager *containerManager) CreateContainer(
	machineConfig *cloudinit.MachineConfig,
	series string,
	network *container.NetworkConfig,
) (instance.Instance, *instance.HardwareCharacteristics, error) {

	name := names.NewMachineTag(machineConfig.MachineId).String()
	if manager.name != "" {
		name = fmt.Sprintf("%s-%s", manager.name, name)
	}
	// Note here that the kvmObjectFacotry only returns a valid container
	// object, and doesn't actually construct the underlying kvm container on
	// disk.
	kvmContainer := KvmObjectFactory.New(name)

	// Create the cloud-init.
	directory, err := container.NewDirectory(name)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to create container directory: %v", err)
	}
	logger.Tracef("write cloud-init")
	userDataFilename, err := container.WriteUserData(machineConfig, directory)
	if err != nil {
		return nil, nil, errors.LoggedErrorf(logger, "failed to write user data: %v", err)
	}
	// Create the container.
	startParams = ParseConstraintsToStartParams(machineConfig.Constraints)
	startParams.Arch = version.Current.Arch
	startParams.Series = series
	startParams.Network = network
	startParams.UserDataFile = userDataFilename

	// If the Simplestream requested is anything but released, update
	// our StartParams to request it.
	if machineConfig.ImageStream != imagemetadata.ReleasedStream {
		startParams.ImageDownloadUrl = imagemetadata.UbuntuCloudImagesURL + "/" + machineConfig.ImageStream
	}

	var hardware instance.HardwareCharacteristics
	hardware, err = instance.ParseHardware(
		fmt.Sprintf("arch=%s mem=%vM root-disk=%vG cpu-cores=%v",
			startParams.Arch, startParams.Memory, startParams.RootDisk, startParams.CpuCores))
	if err != nil {
		logger.Warningf("failed to parse hardware: %v", err)
	}

	logger.Tracef("create the container, constraints: %v", machineConfig.Constraints)
	if err := kvmContainer.Start(startParams); err != nil {
		return nil, nil, errors.LoggedErrorf(logger, "kvm container creation failed: %v", err)
	}
	logger.Tracef("kvm container created")
	return &kvmInstance{kvmContainer, name}, &hardware, nil
}

func (manager *containerManager) DestroyContainer(id instance.Id) error {
	name := string(id)
	kvmContainer := KvmObjectFactory.New(name)
	if err := kvmContainer.Stop(); err != nil {
		logger.Errorf("failed to stop kvm container: %v", err)
		return err
	}
	return container.RemoveDirectory(name)
}

func (manager *containerManager) ListContainers() (result []instance.Instance, err error) {
	containers, err := KvmObjectFactory.List()
	if err != nil {
		logger.Errorf("failed getting all instances: %v", err)
		return
	}
	managerPrefix := fmt.Sprintf("%s-", manager.name)
	for _, container := range containers {
		// Filter out those not starting with our name.
		name := container.Name()
		if !strings.HasPrefix(name, managerPrefix) {
			continue
		}
		if container.IsRunning() {
			result = append(result, &kvmInstance{container, name})
		}
	}
	return
}

// ParseConstraintsToStartParams takes a constrants object and returns a bare
// StartParams object that has Memory, Cpu, and Disk populated.  If there are
// no defined values in the constraints for those fields, default values are
// used.  Other constrains cause a warning to be emitted.
func ParseConstraintsToStartParams(cons constraints.Value) StartParams {
	params := StartParams{
		Memory:   DefaultMemory,
		CpuCores: DefaultCpu,
		RootDisk: DefaultDisk,
	}

	if cons.Mem != nil {
		mem := *cons.Mem
		if mem < MinMemory {
			params.Memory = MinMemory
		} else {
			params.Memory = mem
		}
	}
	if cons.CpuCores != nil {
		cores := *cons.CpuCores
		if cores < MinCpu {
			params.CpuCores = MinCpu
		} else {
			params.CpuCores = cores
		}
	}
	if cons.RootDisk != nil {
		size := *cons.RootDisk / 1024
		if size < MinDisk {
			params.RootDisk = MinDisk
		} else {
			params.RootDisk = size
		}
	}
	if cons.Arch != nil {
		logger.Infof("arch constraint of %q being ignored as not supported", *cons.Arch)
	}
	if cons.Container != nil {
		logger.Infof("container constraint of %q being ignored as not supported", *cons.Container)
	}
	if cons.CpuPower != nil {
		logger.Infof("cpu-power constraint of %v being ignored as not supported", *cons.CpuPower)
	}
	if cons.Tags != nil {
		logger.Infof("tags constraint of %q being ignored as not supported", strings.Join(*cons.Tags, ","))
	}

	return params
}
<commit_msg>KVM containers can now be created using a user-provided SimpleStreams URL as a base.<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package kvm

import (
	"fmt"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/juju/errors"
	"github.com/juju/loggo"
	"github.com/juju/names"

	"github.com/juju/juju/agent"
	"github.com/juju/juju/constraints"
	"github.com/juju/juju/container"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/environs/cloudinit"
	"github.com/juju/juju/environs/imagemetadata"
	"github.com/juju/juju/instance"
	"github.com/juju/juju/version"
)

var (
	logger = loggo.GetLogger("juju.container.kvm")

	KvmObjectFactory ContainerFactory = &containerFactory{}
	DefaultKvmBridge                  = "virbr0"

	// In order for Juju to be able to create the hardware characteristics of
	// the kvm machines it creates, we need to be explicit in our definition
	// of memory, cpu-cores and root-disk.  The defaults here have been
	// extracted from the uvt-kvm executable.
	DefaultMemory uint64 = 512 // MB
	DefaultCpu    uint64 = 1
	DefaultDisk   uint64 = 8 // GB

	// There are some values where it doesn't make sense to go below.
	MinMemory uint64 = 512 // MB
	MinCpu    uint64 = 1
	MinDisk   uint64 = 2 // GB
)

// Utilized to provide a hard-coded path to kvm-ok
var kvmPath = "/usr/sbin"

// IsKVMSupported calls into the kvm-ok executable from the cpu-checkers package.
// It is a variable to allow us to overrid behaviour in the tests.
var IsKVMSupported = func() (bool, error) {

	// Prefer the user's $PATH first, but check /usr/sbin if we can't
	// find kvm-ok there
	var foundPath string
	const binName = "kvm-ok"
	if path, err := exec.LookPath(binName); err == nil {
		foundPath = path
	} else if path, err := exec.LookPath(filepath.Join(kvmPath, binName)); err == nil {
		foundPath = path
	} else {
		return false, errors.NotFoundf("%s executable", binName)
	}

	command := exec.Command(foundPath)
	output, err := command.CombinedOutput()

	if err != nil {
		return false, errors.Annotate(err, string(output))
	}
	logger.Debugf("%s output:\n%s", binName, output)
	return command.ProcessState.Success(), nil
}

// NewContainerManager returns a manager object that can start and stop kvm
// containers. The containers that are created are namespaced by the name
// parameter.
func NewContainerManager(conf container.ManagerConfig) (container.Manager, error) {
	name := conf.PopValue(container.ConfigName)
	if name == "" {
		return nil, fmt.Errorf("name is required")
	}
	logDir := conf.PopValue(container.ConfigLogDir)
	if logDir == "" {
		logDir = agent.DefaultLogDir
	}
	conf.WarnAboutUnused()
	return &containerManager{name: name, logdir: logDir}, nil
}

// containerManager handles all of the business logic at the juju specific
// level. It makes sure that the necessary directories are in place, that the
// user-data is written out in the right place.
type containerManager struct {
	name   string
	logdir string
}

var _ container.Manager = (*containerManager)(nil)

// Exposed so tests can observe our side-effects
var startParams StartParams

func (manager *containerManager) CreateContainer(
	machineConfig *cloudinit.MachineConfig,
	series string,
	network *container.NetworkConfig,
) (instance.Instance, *instance.HardwareCharacteristics, error) {

	name := names.NewMachineTag(machineConfig.MachineId).String()
	if manager.name != "" {
		name = fmt.Sprintf("%s-%s", manager.name, name)
	}
	// Note here that the kvmObjectFacotry only returns a valid container
	// object, and doesn't actually construct the underlying kvm container on
	// disk.
	kvmContainer := KvmObjectFactory.New(name)

	// Create the cloud-init.
	directory, err := container.NewDirectory(name)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to create container directory: %v", err)
	}
	logger.Tracef("write cloud-init")
	userDataFilename, err := container.WriteUserData(machineConfig, directory)
	if err != nil {
		return nil, nil, errors.LoggedErrorf(logger, "failed to write user data: %v", err)
	}
	// Create the container.
	startParams = ParseConstraintsToStartParams(machineConfig.Constraints)
	startParams.Arch = version.Current.Arch
	startParams.Series = series
	startParams.Network = network
	startParams.UserDataFile = userDataFilename

	// If the Simplestream requested is anything but released, update
	// our StartParams to request it.
	if machineConfig.ImageStream != imagemetadata.ReleasedStream {
		startParams.ImageDownloadUrl = imagemetadata.UbuntuCloudImagesURL + "/" + machineConfig.ImageStream
	}

	var hardware instance.HardwareCharacteristics
	hardware, err = instance.ParseHardware(
		fmt.Sprintf("arch=%s mem=%vM root-disk=%vG cpu-cores=%v",
			startParams.Arch, startParams.Memory, startParams.RootDisk, startParams.CpuCores))
	if err != nil {
		logger.Warningf("failed to parse hardware: %v", err)
	}

	logger.Tracef("create the container, constraints: %v", machineConfig.Constraints)
	if err := kvmContainer.Start(startParams); err != nil {
		return nil, nil, errors.LoggedErrorf(logger, "kvm container creation failed: %v", err)
	}
	logger.Tracef("kvm container created")
	return &kvmInstance{kvmContainer, name}, &hardware, nil
}

func (manager *containerManager) DestroyContainer(id instance.Id) error {
	name := string(id)
	kvmContainer := KvmObjectFactory.New(name)
	if err := kvmContainer.Stop(); err != nil {
		logger.Errorf("failed to stop kvm container: %v", err)
		return err
	}
	return container.RemoveDirectory(name)
}

func (manager *containerManager) ListContainers() (result []instance.Instance, err error) {
	containers, err := KvmObjectFactory.List()
	if err != nil {
		logger.Errorf("failed getting all instances: %v", err)
		return
	}
	managerPrefix := fmt.Sprintf("%s-", manager.name)
	for _, container := range containers {
		// Filter out those not starting with our name.
		name := container.Name()
		if !strings.HasPrefix(name, managerPrefix) {
			continue
		}
		if container.IsRunning() {
			result = append(result, &kvmInstance{container, name})
		}
	}
	return
}

// ParseConstraintsToStartParams takes a constrants object and returns a bare
// StartParams object that has Memory, Cpu, and Disk populated.  If there are
// no defined values in the constraints for those fields, default values are
// used.  Other constrains cause a warning to be emitted.
func ParseConstraintsToStartParams(cons constraints.Value) StartParams {
	params := StartParams{
		Memory:   DefaultMemory,
		CpuCores: DefaultCpu,
		RootDisk: DefaultDisk,
	}

	if cons.Mem != nil {
		mem := *cons.Mem
		if mem < MinMemory {
			params.Memory = MinMemory
		} else {
			params.Memory = mem
		}
	}
	if cons.CpuCores != nil {
		cores := *cons.CpuCores
		if cores < MinCpu {
			params.CpuCores = MinCpu
		} else {
			params.CpuCores = cores
		}
	}
	if cons.RootDisk != nil {
		size := *cons.RootDisk / 1024
		if size < MinDisk {
			params.RootDisk = MinDisk
		} else {
			params.RootDisk = size
		}
	}
	if cons.Arch != nil {
		logger.Infof("arch constraint of %q being ignored as not supported", *cons.Arch)
	}
	if cons.Container != nil {
		logger.Infof("container constraint of %q being ignored as not supported", *cons.Container)
	}
	if cons.CpuPower != nil {
		logger.Infof("cpu-power constraint of %v being ignored as not supported", *cons.CpuPower)
	}
	if cons.Tags != nil {
		logger.Infof("tags constraint of %q being ignored as not supported", strings.Join(*cons.Tags, ","))
	}

	return params
}
<|endoftext|>