<commit_before>// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package passert

import (
	"fmt"
	"reflect"
	"sort"
	"strings"

	"github.com/apache/beam/sdks/go/pkg/beam"
	"github.com/apache/beam/sdks/go/pkg/beam/core/util/reflectx"
	"github.com/apache/beam/sdks/go/pkg/beam/internal/errors"
)

// Equals float calls into TryEqualsFloat, checkong that two PCollections of non-complex
// numeric types are equal, with each element being within a provided threshold of an
// expected value. Panics if TryEqualsFloat returns an error.
func EqualsFloat(s beam.Scope, observed, expected beam.PCollection, threshold float64) {
	if err := TryEqualsFloat(s, observed, expected, threshold); err != nil {
		panic(fmt.Sprintf("TryEqualsFloat failed: %v", err))
	}
}

// TryEqualsFloat checks that two PCollections of floats are equal, with each element
// being within a specified threshold of its corresponding element. Both PCollections
// are loaded into memory, sorted, and compared element by element. Returns an error if
// the PCollection types are complex or non-numeric.
func TryEqualsFloat(s beam.Scope, observed, expected beam.PCollection, threshold float64) error {
	errorStrings := []string{}
	observedT := beam.ValidateNonCompositeType(observed)
	if obsErr := validateNonComplexNumber(observedT.Type()); obsErr != nil {
		errorStrings = append(errorStrings, fmt.Sprintf("observed PCollection has incompatible type: %v", obsErr))
	}
	expectedT := beam.ValidateNonCompositeType(expected)
	validateNonComplexNumber(expectedT.Type())
	if expErr := validateNonComplexNumber(expectedT.Type()); expErr != nil {
		errorStrings = append(errorStrings, fmt.Sprintf("expected PCollection has incompatible type: %v", expErr))
	}
	if len(errorStrings) != 0 {
		return errors.New(strings.Join(errorStrings, "\n"))
	}
	s = s.Scope(fmt.Sprintf("passert.EqualsFloat[%v]", threshold))
	beam.ParDo0(s, &thresholdFn{Threshold: threshold}, beam.Impulse(s), beam.SideInput{Input: observed}, beam.SideInput{Input: expected})
	return nil
}

type thresholdFn struct {
	Threshold float64
}

func (f *thresholdFn) ProcessElement(_ []byte, observed, expected func(*beam.T) bool) error {
	var observedValues, expectedValues []float64
	var observedInput, expectedInput beam.T
	for observed(&observedInput) {
		val := toFloat(observedInput)
		observedValues = append(observedValues, val)
	}
	for expected(&expectedInput) {
		val := toFloat(expectedInput)
		expectedValues = append(expectedValues, val)
	}
	if len(observedValues) != len(expectedValues) {
		return errors.Errorf("PCollections of different lengths, got %v expected %v", len(observedValues), len(expectedValues))
	}
	sort.Float64s(observedValues)
	sort.Float64s(expectedValues)
	var tooLow, tooHigh []string
	for i := 0; i < len(observedValues); i++ {
		delta := observedValues[i] - expectedValues[i]
		if delta > f.Threshold {
			tooHigh = append(tooHigh, fmt.Sprintf("%v > %v,", observedValues[i], expectedValues[i]))
		} else if delta < f.Threshold*-1 {
			tooLow = append(tooLow, fmt.Sprintf("%v < %v,", observedValues[i], expectedValues[i]))
		}
	}
	if len(tooLow)+len(tooHigh) == 0 {
		return nil
	}
	errorStrings := []string{}
	if len(tooLow) != 0 {
		errorStrings = append(errorStrings, fmt.Sprintf("values below expected: %v", tooLow))
	}
	if len(tooHigh) != 0 {
		errorStrings = append(errorStrings, fmt.Sprintf("values above expected: %v", tooHigh))
	}
	return errors.New(strings.Join(errorStrings, "\n"))
}

// AllWithinBounds checks that a PCollection of numeric types is within the bounds
// [lo, high]. Checks for case where bounds are flipped and swaps them so the bounds
// passed to the doFn are always lo <= hi.
func AllWithinBounds(s beam.Scope, col beam.PCollection, lo, hi float64) {
	t := beam.ValidateNonCompositeType(col)
	validateNonComplexNumber(t.Type())
	if lo > hi {
		lo, hi = hi, lo
	}
	s = s.Scope(fmt.Sprintf("passert.AllWithinBounds([%v, %v])", lo, hi))
	beam.ParDo0(s, &boundsFn{lo: lo, hi: hi}, beam.Impulse(s), beam.SideInput{Input: col})
}

type boundsFn struct {
	lo, hi float64
}

func (f *boundsFn) ProcessElement(_ []byte, col func(*beam.T) bool) error {
	var tooLow, tooHigh []float64
	var input beam.T
	for col(&input) {
		val := toFloat(input)
		if val < f.lo {
			tooLow = append(tooLow, val)
		} else if val > f.hi {
			tooHigh = append(tooHigh, val)
		}
	}
	if len(tooLow)+len(tooHigh) == 0 {
		return nil
	}
	errorStrings := []string{}
	if len(tooLow) != 0 {
		sort.Float64s(tooLow)
		errorStrings = append(errorStrings, fmt.Sprintf("values below minimum value %v: %v", f.lo, tooLow))
	}
	if len(tooHigh) != 0 {
		sort.Float64s(tooHigh)
		errorStrings = append(errorStrings, fmt.Sprintf("values above maximum value %v: %v", f.hi, tooHigh))
	}
	return errors.New(strings.Join(errorStrings, "\n"))
}

func toFloat(input beam.T) float64 {
	return reflect.ValueOf(input.(interface{})).Convert(reflectx.Float64).Interface().(float64)
}

func validateNonComplexNumber(t reflect.Type) error {
	if !reflectx.IsNumber(t) || reflectx.IsComplex(t) {
		return errors.Errorf("type must be a non-complex number: %v", t)
	}
	return nil
}
<commit_msg>[BEAM-12548] Fix malformed comment for EqualsFloat (#15200)<commit_after>// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package passert

import (
	"fmt"
	"reflect"
	"sort"
	"strings"

	"github.com/apache/beam/sdks/go/pkg/beam"
	"github.com/apache/beam/sdks/go/pkg/beam/core/util/reflectx"
	"github.com/apache/beam/sdks/go/pkg/beam/internal/errors"
)

// EqualsFloat calls into TryEqualsFloat, checkong that two PCollections of non-complex
// numeric types are equal, with each element being within a provided threshold of an
// expected value. Panics if TryEqualsFloat returns an error.
func EqualsFloat(s beam.Scope, observed, expected beam.PCollection, threshold float64) {
	if err := TryEqualsFloat(s, observed, expected, threshold); err != nil {
		panic(fmt.Sprintf("TryEqualsFloat failed: %v", err))
	}
}

// TryEqualsFloat checks that two PCollections of floats are equal, with each element
// being within a specified threshold of its corresponding element. Both PCollections
// are loaded into memory, sorted, and compared element by element. Returns an error if
// the PCollection types are complex or non-numeric.
func TryEqualsFloat(s beam.Scope, observed, expected beam.PCollection, threshold float64) error {
	errorStrings := []string{}
	observedT := beam.ValidateNonCompositeType(observed)
	if obsErr := validateNonComplexNumber(observedT.Type()); obsErr != nil {
		errorStrings = append(errorStrings, fmt.Sprintf("observed PCollection has incompatible type: %v", obsErr))
	}
	expectedT := beam.ValidateNonCompositeType(expected)
	validateNonComplexNumber(expectedT.Type())
	if expErr := validateNonComplexNumber(expectedT.Type()); expErr != nil {
		errorStrings = append(errorStrings, fmt.Sprintf("expected PCollection has incompatible type: %v", expErr))
	}
	if len(errorStrings) != 0 {
		return errors.New(strings.Join(errorStrings, "\n"))
	}
	s = s.Scope(fmt.Sprintf("passert.EqualsFloat[%v]", threshold))
	beam.ParDo0(s, &thresholdFn{Threshold: threshold}, beam.Impulse(s), beam.SideInput{Input: observed}, beam.SideInput{Input: expected})
	return nil
}

type thresholdFn struct {
	Threshold float64
}

func (f *thresholdFn) ProcessElement(_ []byte, observed, expected func(*beam.T) bool) error {
	var observedValues, expectedValues []float64
	var observedInput, expectedInput beam.T
	for observed(&observedInput) {
		val := toFloat(observedInput)
		observedValues = append(observedValues, val)
	}
	for expected(&expectedInput) {
		val := toFloat(expectedInput)
		expectedValues = append(expectedValues, val)
	}
	if len(observedValues) != len(expectedValues) {
		return errors.Errorf("PCollections of different lengths, got %v expected %v", len(observedValues), len(expectedValues))
	}
	sort.Float64s(observedValues)
	sort.Float64s(expectedValues)
	var tooLow, tooHigh []string
	for i := 0; i < len(observedValues); i++ {
		delta := observedValues[i] - expectedValues[i]
		if delta > f.Threshold {
			tooHigh = append(tooHigh, fmt.Sprintf("%v > %v,", observedValues[i], expectedValues[i]))
		} else if delta < f.Threshold*-1 {
			tooLow = append(tooLow, fmt.Sprintf("%v < %v,", observedValues[i], expectedValues[i]))
		}
	}
	if len(tooLow)+len(tooHigh) == 0 {
		return nil
	}
	errorStrings := []string{}
	if len(tooLow) != 0 {
		errorStrings = append(errorStrings, fmt.Sprintf("values below expected: %v", tooLow))
	}
	if len(tooHigh) != 0 {
		errorStrings = append(errorStrings, fmt.Sprintf("values above expected: %v", tooHigh))
	}
	return errors.New(strings.Join(errorStrings, "\n"))
}

// AllWithinBounds checks that a PCollection of numeric types is within the bounds
// [lo, high]. Checks for case where bounds are flipped and swaps them so the bounds
// passed to the doFn are always lo <= hi.
func AllWithinBounds(s beam.Scope, col beam.PCollection, lo, hi float64) {
	t := beam.ValidateNonCompositeType(col)
	validateNonComplexNumber(t.Type())
	if lo > hi {
		lo, hi = hi, lo
	}
	s = s.Scope(fmt.Sprintf("passert.AllWithinBounds([%v, %v])", lo, hi))
	beam.ParDo0(s, &boundsFn{lo: lo, hi: hi}, beam.Impulse(s), beam.SideInput{Input: col})
}

type boundsFn struct {
	lo, hi float64
}

func (f *boundsFn) ProcessElement(_ []byte, col func(*beam.T) bool) error {
	var tooLow, tooHigh []float64
	var input beam.T
	for col(&input) {
		val := toFloat(input)
		if val < f.lo {
			tooLow = append(tooLow, val)
		} else if val > f.hi {
			tooHigh = append(tooHigh, val)
		}
	}
	if len(tooLow)+len(tooHigh) == 0 {
		return nil
	}
	errorStrings := []string{}
	if len(tooLow) != 0 {
		sort.Float64s(tooLow)
		errorStrings = append(errorStrings, fmt.Sprintf("values below minimum value %v: %v", f.lo, tooLow))
	}
	if len(tooHigh) != 0 {
		sort.Float64s(tooHigh)
		errorStrings = append(errorStrings, fmt.Sprintf("values above maximum value %v: %v", f.hi, tooHigh))
	}
	return errors.New(strings.Join(errorStrings, "\n"))
}

func toFloat(input beam.T) float64 {
	return reflect.ValueOf(input.(interface{})).Convert(reflectx.Float64).Interface().(float64)
}

func validateNonComplexNumber(t reflect.Type) error {
	if !reflectx.IsNumber(t) || reflectx.IsComplex(t) {
		return errors.Errorf("type must be a non-complex number: %v", t)
	}
	return nil
}
<|endoftext|>


<commit_before>// Package writeaheadlog defines and implements a general purpose, high
// performance write-ahead-log for performing ACID transactions to disk without
// sacrificing speed or latency more than fundamentally required.
package writeaheadlog

import (
	"bytes"
	"encoding/binary"
	"os"
	"sort"
	"sync"
	"sync/atomic"

	"github.com/NebulousLabs/errors"
)

// WAL is a general purpose, high performance write-ahead-log for performing
// ACID transactions to disk without sacrificing speed or latency more than
// fundamentally required.
type WAL struct {
	// atomicNextTxnNum is used to give every transaction a unique transaction
	// number. The transaction will then wait until atomicTransactionCounter allows
	// the transaction to be committed. This ensures that transactions are committed
	// in the correct order.
	atomicNextTxnNum uint64

	// atomicUnfinishedTxns counts how many transactions were created but not
	// released yet. This counter needs to be 0 for the wal to exit cleanly.
	atomicUnfinishedTxns int64

	// availablePages lists the offset of file pages which currently have completed or
	// voided updates in them. The pages are in no particular order.
	availablePages []uint64

	// filePageCount indicates the number of pages total in the file. If the
	// number of availablePages ever drops below the number of pages required
	// for a new transaction, then the file is extended, new pages are added,
	// and the availablePages array is updated to include the extended pages.
	filePageCount int

	// logFile contains all of the persistent data associated with the log.
	logFile file

	// path is the path to the underlying logFile
	path string

	// mu is used to lock the availablePages field of the wal
	mu sync.Mutex

	// syncCond is used to schedule the calls to fsync
	syncCond *sync.Cond

	// syncCount is a counter that indicates how many transactions are
	// currently waiting for a fsync
	syncCount uint64

	// stopChan is a channel that is used to signal a shutdown
	stopChan chan struct{}

	// syncing indicates if the syncing thread is currently being executed
	syncing bool

	// syncErr is the error returned by the most recent fsync call
	syncErr error

	// recoveryComplete indicates if the caller signalled that the recovery is complete
	recoveryComplete bool

	// dependencies are used to inject special behaviour into the wal by providing
	// custom dependencies when the wal is created and calling deps.disrupt(setting).
	// The following settings are currently available
	deps dependencies
}

// allocatePages creates new pages and adds them to the available pages of the wal
func (w *WAL) allocatePages(numPages int) {
	// Starting at index 1 because the first page is reserved for metadata
	start := w.filePageCount + 1
	for i := start; i < start+numPages; i++ {
		w.availablePages = append(w.availablePages, uint64(i)*pageSize)
	}
	w.filePageCount += numPages
}

// newWal initializes and returns a wal.
func newWal(path string, deps dependencies) (u []Update, w *WAL, err error) {
	// Create a new WAL
	newWal := &WAL{
		deps:     deps,
		stopChan: make(chan struct{}),
		syncCond: sync.NewCond(new(sync.Mutex)),
		path:     path,
	}

	// Create a condition for the wal
	// Try opening the WAL file.
	data, err := deps.readFile(path)
	if err == nil {
		// Reuse the existing wal
		newWal.logFile, err = deps.openFile(path, os.O_RDWR, 0600)
		if err != nil {
			return nil, nil, err
		}

		// Recover WAL and return updates
		updates, err := newWal.recoverWAL(data)
		if err != nil {
			err = errors.Compose(err, newWal.logFile.Close())
			return nil, nil, err
		}
		if len(updates) == 0 {
			// if there are no updates to apply, set the recovery to complete
			newWal.recoveryComplete = true
		}
		return updates, newWal, nil

	} else if !os.IsNotExist(err) {
		// the file exists but couldn't be opened
		return nil, nil, errors.Extend(err, errors.New("walFile was not opened successfully"))
	}

	// Create new empty WAL
	newWal.logFile, err = deps.create(path)
	if err != nil {
		return nil, nil, errors.Extend(err, errors.New("walFile could not be created"))
	}
	// Write the metadata to the WAL
	if err = writeWALMetadata(newWal.logFile); err != nil {
		return nil, nil, errors.Extend(err, errors.New("Failed to write metadata to file"))
	}
	// No recovery needs to be performed.
	newWal.recoveryComplete = true
	return nil, newWal, nil
}

// readWALMetadata reads WAL metadata from the input file, returning an error
// if the result is unexpected.
func readWALMetadata(data []byte) (uint16, error) {
	// The metadata should at least long enough to contain all the fields.
	if len(data) < len(metadataHeader)+len(metadataVersion)+metadataStatusSize {
		return 0, errors.New("unable to read wal metadata")
	}

	// Check that the header and version match.
	if !bytes.Equal(data[:len(metadataHeader)], metadataHeader[:]) {
		return 0, errors.New("file header is incorrect")
	}
	if !bytes.Equal(data[len(metadataHeader):len(metadataHeader)+len(metadataVersion)], metadataVersion[:]) {
		return 0, errors.New("file version is unrecognized - maybe you need to upgrade")
	}
	// Determine and return the current status of the file.
	fileState := uint16(data[len(metadataHeader)+len(metadataVersion)])
	if fileState <= 0 || fileState > 3 {
		fileState = recoveryStateUnclean
	}
	return fileState, nil
}

// recoverWAL recovers a WAL and returns comitted but not finished updates
func (w *WAL) recoverWAL(data []byte) ([]Update, error) {
	// Validate metadata
	recoveryState, err := readWALMetadata(data[0:])
	if err != nil {
		return nil, err
	}

	if recoveryState == recoveryStateClean {
		if err := w.writeRecoveryState(recoveryStateUnclean); err != nil {
			return nil, errors.Extend(err, errors.New("unable to write WAL recovery state"))
		}
		w.recoveryComplete = true
		return nil, nil
	}

	// If recoveryState is set to wipe we don't need to recover but we have to
	// wipe the wal and change the state
	if recoveryState == recoveryStateWipe {
		if err := w.wipeWAL(); err != nil {
			return nil, err
		}
		if err := w.writeRecoveryState(recoveryStateUnclean); err != nil {
			return nil, errors.Extend(err, errors.New("unable to write WAL recovery state"))
		}
		w.recoveryComplete = true
		return nil, w.logFile.Sync()
	}

	// load all normal pages
	type diskPage struct {
		page
		nextPageOffset uint64
	}
	pageSet := make(map[uint64]*diskPage) // keyed by offset
	for i := uint64(pageSize); i+pageMetaSize < uint64(len(data)); i += pageSize {
		nextOffset := binary.LittleEndian.Uint64(data[i:])
		if nextOffset < pageSize {
			// nextOffset is actually a transaction status
			continue
		}
		payloadSize := binary.LittleEndian.Uint64(data[i+8:])
		if payloadSize > MaxPayloadSize {
			continue
		}
		payload := data[i+16 : i+16+payloadSize]

		pageSet[i] = &diskPage{
			page: page{
				offset:  i,
				payload: payload,
			},
			nextPageOffset: nextOffset,
		}
	}

	// fill in each nextPage pointer
	for _, p := range pageSet {
		if nextDiskPage, ok := pageSet[p.nextPageOffset]; ok {
			p.nextPage = &nextDiskPage.page
		}
	}

	// reconstruct transactions
	var txns []Transaction
	for i := pageSize; i+firstPageMetaSize < len(data); i += pageSize {
		status := binary.LittleEndian.Uint64(data[i:])
		if status != txnStatusComitted {
			continue
		}
		// decode metadata and first page
		seq := binary.LittleEndian.Uint64(data[i+8:])
		var diskChecksum checksum
		n := copy(diskChecksum[:], data[i+16:])
		nextPageOffset := binary.LittleEndian.Uint64(data[i+16+n:])
		payloadSize := binary.LittleEndian.Uint64(data[i+16+n+8:])
		if payloadSize > maxFirstPayloadSize {
			continue
		}
		firstPage := &page{
			payload: data[i+firstPageMetaSize : i+firstPageMetaSize+int(payloadSize)],
		}
		if nextDiskPage, ok := pageSet[nextPageOffset]; ok {
			firstPage.nextPage = &nextDiskPage.page
		}

		txn := Transaction{
			status:         status,
			sequenceNumber: seq,
			firstPage:      firstPage,
		}

		// validate checksum
		if txn.checksum() != diskChecksum {
			continue
		}

		// decode updates
		var updateBytes []byte
		for page := txn.firstPage; page != nil; page = page.nextPage {
			updateBytes = append(updateBytes, page.payload...)
		}
		updates, err := unmarshalUpdates(updateBytes)
		if err != nil {
			continue
		}
		txn.Updates = updates

		txns = append(txns, txn)
	}

	// sort txns by sequence number
	sort.Slice(txns, func(i, j int) bool {
		return txns[i].sequenceNumber < txns[j].sequenceNumber
	})

	// concatenate the updates of each transaction
	var updates []Update
	for _, txn := range txns {
		updates = append(updates, txn.Updates...)
	}
	return updates, nil
}

// writeRecoveryState is a helper function that changes the recoveryState on disk
func (w *WAL) writeRecoveryState(state uint16) error {
	_, err := w.logFile.WriteAt([]byte{byte(state)}, int64(len(metadataHeader)+len(metadataVersion)))
	if err != nil {
		return err
	}
	return w.logFile.Sync()
}

// RecoveryComplete is called after a wal is recovered to signal that it is
// safe to reset the wal
func (w *WAL) RecoveryComplete() error {
	// Set the metadata to wipe
	if err := w.writeRecoveryState(recoveryStateWipe); err != nil {
		return err
	}

	// Sync before we start wiping
	if err := w.logFile.Sync(); err != nil {
		return err
	}

	// Simulate crash after recovery state was written
	if w.deps.disrupt("RecoveryFail") {
		return nil
	}

	// Wipe the wal
	if err := w.wipeWAL(); err != nil {
		return err
	}

	// Set the metadata to unclean again
	if err := w.writeRecoveryState(recoveryStateUnclean); err != nil {
		return err
	}

	w.recoveryComplete = true
	return nil
}

// managedReservePages reserves pages for a given payload and links them
// together, allocating new pages if necessary. It returns the first page in
// the chain.
func (w *WAL) managedReservePages(data []byte) *page {
	// Find out how many pages are needed for the payload
	numPages := len(data) / MaxPayloadSize
	if len(data)%MaxPayloadSize != 0 {
		numPages++
	}

	w.mu.Lock()
	defer w.mu.Unlock()
	// allocate more pages if necessary
	if pagesNeeded := numPages - len(w.availablePages); pagesNeeded > 0 {
		w.allocatePages(pagesNeeded)

		// sanity check: the number of available pages should now equal the number of required ones
		if len(w.availablePages) != numPages {
			panic(errors.New("sanity check failed: num of available pages != num of required pages"))
		}
	}

	// Reserve some pages and remove them from the available ones
	reservedPages := w.availablePages[len(w.availablePages)-numPages:]
	w.availablePages = w.availablePages[:len(w.availablePages)-numPages]

	// Set the fields of each page
	buf := bytes.NewBuffer(data)
	pages := make([]page, numPages)
	for i := range pages {
		// Set nextPage if the current page isn't the last one
		if i+1 < numPages {
			pages[i].nextPage = &pages[i+1]
		}

		// Set offset according to the index in reservedPages
		pages[i].offset = reservedPages[i]

		// Copy part of the update into the payload
		pages[i].payload = buf.Next(MaxPayloadSize)
	}

	return &pages[0]
}

// wipeWAL sets all the pages of the WAL to applied so that they can be reused.
func (w *WAL) wipeWAL() error {
	// Marshal the txnStatusApplied
	txnAppliedBytes := make([]byte, 8)
	binary.LittleEndian.PutUint64(txnAppliedBytes, txnStatusApplied)

	// Get the length of the file.
	stat, err := w.logFile.Stat()
	if err != nil {
		return err
	}
	length := stat.Size()

	// Set all pages to applied.
	for offset := int64(pageSize); offset < length; offset += pageSize {
		if _, err := w.logFile.WriteAt(txnAppliedBytes, offset); err != nil {
			return err
		}
	}

	// Sync the wipe to disk
	return w.logFile.Sync()
}

// writeWALMetadata writes WAL metadata to the input file.
func writeWALMetadata(f file) error {
	// Create the metadata.
	data := make([]byte, 0, len(metadataHeader)+len(metadataVersion)+metadataStatusSize)
	data = append(data, metadataHeader[:]...)
	data = append(data, metadataVersion[:]...)
	// Penultimate byte is the recovery state, and final byte is a newline.
	data = append(data, byte(recoveryStateUnclean))
	data = append(data, byte('\n'))
	_, err := f.WriteAt(data, 0)
	return err
}

// Close closes the wal, frees used resources and checks for active
// transactions.
func (w *WAL) Close() error {
	// Check if there are unfinished transactions
	var err1 error
	if atomic.LoadInt64(&w.atomicUnfinishedTxns) != 0 {
		err1 = errors.New("There are still non-released transactions left")
	}

	// Write the recovery state to indicate clean shutdown if no error occured
	if err1 == nil && !w.deps.disrupt("UncleanShutdown") {
		err1 = w.writeRecoveryState(recoveryStateClean)
	}

	// Close the logFile and stopChan
	err2 := w.logFile.Close()
	close(w.stopChan)

	return errors.Compose(err1, err2)
}

// New will open a WAL. If the previous run did not shut down cleanly, a set of
// updates will be returned which got committed successfully to the WAL, but
// were never signaled as fully completed.
//
// If no WAL exists, a new one will be created.
//
// If in debugging mode, the WAL may return a series of updates multiple times,
// simulating multiple consecutive unclean shutdowns. If the updates are
// properly idempotent, there should be no functional difference between the
// multiple appearances and them just being loaded a single time correctly.
func New(path string) (u []Update, w *WAL, err error) {
	// Create a wal with production dependencies
	return newWal(path, &prodDependencies{})
}
<commit_msg>use pageMetaSize instead of 16<commit_after>// Package writeaheadlog defines and implements a general purpose, high
// performance write-ahead-log for performing ACID transactions to disk without
// sacrificing speed or latency more than fundamentally required.
package writeaheadlog

import (
	"bytes"
	"encoding/binary"
	"os"
	"sort"
	"sync"
	"sync/atomic"

	"github.com/NebulousLabs/errors"
)

// WAL is a general purpose, high performance write-ahead-log for performing
// ACID transactions to disk without sacrificing speed or latency more than
// fundamentally required.
type WAL struct {
	// atomicNextTxnNum is used to give every transaction a unique transaction
	// number. The transaction will then wait until atomicTransactionCounter allows
	// the transaction to be committed. This ensures that transactions are committed
	// in the correct order.
	atomicNextTxnNum uint64

	// atomicUnfinishedTxns counts how many transactions were created but not
	// released yet. This counter needs to be 0 for the wal to exit cleanly.
	atomicUnfinishedTxns int64

	// availablePages lists the offset of file pages which currently have completed or
	// voided updates in them. The pages are in no particular order.
	availablePages []uint64

	// filePageCount indicates the number of pages total in the file. If the
	// number of availablePages ever drops below the number of pages required
	// for a new transaction, then the file is extended, new pages are added,
	// and the availablePages array is updated to include the extended pages.
	filePageCount int

	// logFile contains all of the persistent data associated with the log.
	logFile file

	// path is the path to the underlying logFile
	path string

	// mu is used to lock the availablePages field of the wal
	mu sync.Mutex

	// syncCond is used to schedule the calls to fsync
	syncCond *sync.Cond

	// syncCount is a counter that indicates how many transactions are
	// currently waiting for a fsync
	syncCount uint64

	// stopChan is a channel that is used to signal a shutdown
	stopChan chan struct{}

	// syncing indicates if the syncing thread is currently being executed
	syncing bool

	// syncErr is the error returned by the most recent fsync call
	syncErr error

	// recoveryComplete indicates if the caller signalled that the recovery is complete
	recoveryComplete bool

	// dependencies are used to inject special behaviour into the wal by providing
	// custom dependencies when the wal is created and calling deps.disrupt(setting).
	// The following settings are currently available
	deps dependencies
}

// allocatePages creates new pages and adds them to the available pages of the wal
func (w *WAL) allocatePages(numPages int) {
	// Starting at index 1 because the first page is reserved for metadata
	start := w.filePageCount + 1
	for i := start; i < start+numPages; i++ {
		w.availablePages = append(w.availablePages, uint64(i)*pageSize)
	}
	w.filePageCount += numPages
}

// newWal initializes and returns a wal.
func newWal(path string, deps dependencies) (u []Update, w *WAL, err error) {
	// Create a new WAL
	newWal := &WAL{
		deps:     deps,
		stopChan: make(chan struct{}),
		syncCond: sync.NewCond(new(sync.Mutex)),
		path:     path,
	}

	// Create a condition for the wal
	// Try opening the WAL file.
	data, err := deps.readFile(path)
	if err == nil {
		// Reuse the existing wal
		newWal.logFile, err = deps.openFile(path, os.O_RDWR, 0600)
		if err != nil {
			return nil, nil, err
		}

		// Recover WAL and return updates
		updates, err := newWal.recoverWAL(data)
		if err != nil {
			err = errors.Compose(err, newWal.logFile.Close())
			return nil, nil, err
		}
		if len(updates) == 0 {
			// if there are no updates to apply, set the recovery to complete
			newWal.recoveryComplete = true
		}
		return updates, newWal, nil

	} else if !os.IsNotExist(err) {
		// the file exists but couldn't be opened
		return nil, nil, errors.Extend(err, errors.New("walFile was not opened successfully"))
	}

	// Create new empty WAL
	newWal.logFile, err = deps.create(path)
	if err != nil {
		return nil, nil, errors.Extend(err, errors.New("walFile could not be created"))
	}
	// Write the metadata to the WAL
	if err = writeWALMetadata(newWal.logFile); err != nil {
		return nil, nil, errors.Extend(err, errors.New("Failed to write metadata to file"))
	}
	// No recovery needs to be performed.
	newWal.recoveryComplete = true
	return nil, newWal, nil
}

// readWALMetadata reads WAL metadata from the input file, returning an error
// if the result is unexpected.
func readWALMetadata(data []byte) (uint16, error) {
	// The metadata should at least long enough to contain all the fields.
	if len(data) < len(metadataHeader)+len(metadataVersion)+metadataStatusSize {
		return 0, errors.New("unable to read wal metadata")
	}

	// Check that the header and version match.
	if !bytes.Equal(data[:len(metadataHeader)], metadataHeader[:]) {
		return 0, errors.New("file header is incorrect")
	}
	if !bytes.Equal(data[len(metadataHeader):len(metadataHeader)+len(metadataVersion)], metadataVersion[:]) {
		return 0, errors.New("file version is unrecognized - maybe you need to upgrade")
	}
	// Determine and return the current status of the file.
	fileState := uint16(data[len(metadataHeader)+len(metadataVersion)])
	if fileState <= 0 || fileState > 3 {
		fileState = recoveryStateUnclean
	}
	return fileState, nil
}

// recoverWAL recovers a WAL and returns comitted but not finished updates
func (w *WAL) recoverWAL(data []byte) ([]Update, error) {
	// Validate metadata
	recoveryState, err := readWALMetadata(data[0:])
	if err != nil {
		return nil, err
	}

	if recoveryState == recoveryStateClean {
		if err := w.writeRecoveryState(recoveryStateUnclean); err != nil {
			return nil, errors.Extend(err, errors.New("unable to write WAL recovery state"))
		}
		w.recoveryComplete = true
		return nil, nil
	}

	// If recoveryState is set to wipe we don't need to recover but we have to
	// wipe the wal and change the state
	if recoveryState == recoveryStateWipe {
		if err := w.wipeWAL(); err != nil {
			return nil, err
		}
		if err := w.writeRecoveryState(recoveryStateUnclean); err != nil {
			return nil, errors.Extend(err, errors.New("unable to write WAL recovery state"))
		}
		w.recoveryComplete = true
		return nil, w.logFile.Sync()
	}

	// load all normal pages
	type diskPage struct {
		page
		nextPageOffset uint64
	}
	pageSet := make(map[uint64]*diskPage) // keyed by offset
	for i := uint64(pageSize); i+pageMetaSize < uint64(len(data)); i += pageSize {
		nextOffset := binary.LittleEndian.Uint64(data[i:])
		if nextOffset < pageSize {
			// nextOffset is actually a transaction status
			continue
		}
		payloadSize := binary.LittleEndian.Uint64(data[i+8:])
		if payloadSize > MaxPayloadSize {
			continue
		}
		payload := data[i+pageMetaSize : i+pageMetaSize+payloadSize]

		pageSet[i] = &diskPage{
			page: page{
				offset:  i,
				payload: payload,
			},
			nextPageOffset: nextOffset,
		}
	}

	// fill in each nextPage pointer
	for _, p := range pageSet {
		if nextDiskPage, ok := pageSet[p.nextPageOffset]; ok {
			p.nextPage = &nextDiskPage.page
		}
	}

	// reconstruct transactions
	var txns []Transaction
	for i := pageSize; i+firstPageMetaSize < len(data); i += pageSize {
		status := binary.LittleEndian.Uint64(data[i:])
		if status != txnStatusComitted {
			continue
		}
		// decode metadata and first page
		seq := binary.LittleEndian.Uint64(data[i+8:])
		var diskChecksum checksum
		n := copy(diskChecksum[:], data[i+16:])
		nextPageOffset := binary.LittleEndian.Uint64(data[i+16+n:])
		payloadSize := binary.LittleEndian.Uint64(data[i+16+n+8:])
		if payloadSize > maxFirstPayloadSize {
			continue
		}
		firstPage := &page{
			payload: data[i+firstPageMetaSize : i+firstPageMetaSize+int(payloadSize)],
		}
		if nextDiskPage, ok := pageSet[nextPageOffset]; ok {
			firstPage.nextPage = &nextDiskPage.page
		}

		txn := Transaction{
			status:         status,
			sequenceNumber: seq,
			firstPage:      firstPage,
		}

		// validate checksum
		if txn.checksum() != diskChecksum {
			continue
		}

		// decode updates
		var updateBytes []byte
		for page := txn.firstPage; page != nil; page = page.nextPage {
			updateBytes = append(updateBytes, page.payload...)
		}
		updates, err := unmarshalUpdates(updateBytes)
		if err != nil {
			continue
		}
		txn.Updates = updates

		txns = append(txns, txn)
	}

	// sort txns by sequence number
	sort.Slice(txns, func(i, j int) bool {
		return txns[i].sequenceNumber < txns[j].sequenceNumber
	})

	// concatenate the updates of each transaction
	var updates []Update
	for _, txn := range txns {
		updates = append(updates, txn.Updates...)
	}
	return updates, nil
}

// writeRecoveryState is a helper function that changes the recoveryState on disk
func (w *WAL) writeRecoveryState(state uint16) error {
	_, err := w.logFile.WriteAt([]byte{byte(state)}, int64(len(metadataHeader)+len(metadataVersion)))
	if err != nil {
		return err
	}
	return w.logFile.Sync()
}

// RecoveryComplete is called after a wal is recovered to signal that it is
// safe to reset the wal
func (w *WAL) RecoveryComplete() error {
	// Set the metadata to wipe
	if err := w.writeRecoveryState(recoveryStateWipe); err != nil {
		return err
	}

	// Sync before we start wiping
	if err := w.logFile.Sync(); err != nil {
		return err
	}

	// Simulate crash after recovery state was written
	if w.deps.disrupt("RecoveryFail") {
		return nil
	}

	// Wipe the wal
	if err := w.wipeWAL(); err != nil {
		return err
	}

	// Set the metadata to unclean again
	if err := w.writeRecoveryState(recoveryStateUnclean); err != nil {
		return err
	}

	w.recoveryComplete = true
	return nil
}

// managedReservePages reserves pages for a given payload and links them
// together, allocating new pages if necessary. It returns the first page in
// the chain.
func (w *WAL) managedReservePages(data []byte) *page {
	// Find out how many pages are needed for the payload
	numPages := len(data) / MaxPayloadSize
	if len(data)%MaxPayloadSize != 0 {
		numPages++
	}

	w.mu.Lock()
	defer w.mu.Unlock()
	// allocate more pages if necessary
	if pagesNeeded := numPages - len(w.availablePages); pagesNeeded > 0 {
		w.allocatePages(pagesNeeded)

		// sanity check: the number of available pages should now equal the number of required ones
		if len(w.availablePages) != numPages {
			panic(errors.New("sanity check failed: num of available pages != num of required pages"))
		}
	}

	// Reserve some pages and remove them from the available ones
	reservedPages := w.availablePages[len(w.availablePages)-numPages:]
	w.availablePages = w.availablePages[:len(w.availablePages)-numPages]

	// Set the fields of each page
	buf := bytes.NewBuffer(data)
	pages := make([]page, numPages)
	for i := range pages {
		// Set nextPage if the current page isn't the last one
		if i+1 < numPages {
			pages[i].nextPage = &pages[i+1]
		}

		// Set offset according to the index in reservedPages
		pages[i].offset = reservedPages[i]

		// Copy part of the update into the payload
		pages[i].payload = buf.Next(MaxPayloadSize)
	}

	return &pages[0]
}

// wipeWAL sets all the pages of the WAL to applied so that they can be reused.
func (w *WAL) wipeWAL() error {
	// Marshal the txnStatusApplied
	txnAppliedBytes := make([]byte, 8)
	binary.LittleEndian.PutUint64(txnAppliedBytes, txnStatusApplied)

	// Get the length of the file.
	stat, err := w.logFile.Stat()
	if err != nil {
		return err
	}
	length := stat.Size()

	// Set all pages to applied.
	for offset := int64(pageSize); offset < length; offset += pageSize {
		if _, err := w.logFile.WriteAt(txnAppliedBytes, offset); err != nil {
			return err
		}
	}

	// Sync the wipe to disk
	return w.logFile.Sync()
}

// writeWALMetadata writes WAL metadata to the input file.
func writeWALMetadata(f file) error {
	// Create the metadata.
	data := make([]byte, 0, len(metadataHeader)+len(metadataVersion)+metadataStatusSize)
	data = append(data, metadataHeader[:]...)
	data = append(data, metadataVersion[:]...)
	// Penultimate byte is the recovery state, and final byte is a newline.
	data = append(data, byte(recoveryStateUnclean))
	data = append(data, byte('\n'))
	_, err := f.WriteAt(data, 0)
	return err
}

// Close closes the wal, frees used resources and checks for active
// transactions.
func (w *WAL) Close() error {
	// Check if there are unfinished transactions
	var err1 error
	if atomic.LoadInt64(&w.atomicUnfinishedTxns) != 0 {
		err1 = errors.New("There are still non-released transactions left")
	}

	// Write the recovery state to indicate clean shutdown if no error occured
	if err1 == nil && !w.deps.disrupt("UncleanShutdown") {
		err1 = w.writeRecoveryState(recoveryStateClean)
	}

	// Close the logFile and stopChan
	err2 := w.logFile.Close()
	close(w.stopChan)

	return errors.Compose(err1, err2)
}

// New will open a WAL. If the previous run did not shut down cleanly, a set of
// updates will be returned which got committed successfully to the WAL, but
// were never signaled as fully completed.
//
// If no WAL exists, a new one will be created.
//
// If in debugging mode, the WAL may return a series of updates multiple times,
// simulating multiple consecutive unclean shutdowns. If the updates are
// properly idempotent, there should be no functional difference between the
// multiple appearances and them just being loaded a single time correctly.
func New(path string) (u []Update, w *WAL, err error) {
	// Create a wal with production dependencies
	return newWal(path, &prodDependencies{})
}
<|endoftext|>


<commit_before>package stan

////////////////////////////////////////////////////////////////////////////////
// Benchmarks
////////////////////////////////////////////////////////////////////////////////

import (
	"fmt"
	"sync/atomic"
	"testing"
	"time"
)

func BenchmarkPublish(b *testing.B) {
	b.StopTimer()

	// Run a STAN server
	s := RunServer(clusterName)
	defer s.Shutdown()
	sc, err := Connect(clusterName, clientName)
	defer sc.Close()
	if err != nil {
		b.Fatalf("Expected to connect correctly, got err %v\n", err)
	}
	hw := []byte("Hello World")

	b.StartTimer()
	b.ReportAllocs()

	for i := 0; i < b.N; i++ {
		if err := sc.Publish("foo", hw); err != nil {
			b.Fatalf("Got error on publish: %v\n", err)
		}
	}
}

func BenchmarkPublishAsync(b *testing.B) {
	b.StopTimer()

	// Run a STAN server
	s := RunServer(clusterName)
	defer s.Shutdown()
	sc, err := Connect(clusterName, clientName)
	defer sc.Close()
	if err != nil {
		b.Fatalf("Expected to connect correctly, got err %v\n", err)
	}
	hw := []byte("Hello World")

	ch := make(chan bool)
	received := int32(0)

	ah := func(guid string, err error) {
		if err != nil {
			b.Fatalf("Received an error in ack callback: %v\n", err)
		}
		if nr := atomic.AddInt32(&received, 1); nr >= int32(b.N) {
			ch <- true
		}
	}
	b.StartTimer()
	b.ReportAllocs()

	for i := 0; i < b.N; i++ {
		if _, err := sc.PublishAsync("foo", hw, ah); err != nil {
			fmt.Printf("Client status %v, Server status %v\n", s.nc.Status(), (sc.(*conn)).nc.Status())
			fmt.Printf("len(ackmap) = %d\n", len(sc.(*conn).pubAckMap))

			b.Fatalf("Error from PublishAsync: %v\n", err)
		}
	}

	err = WaitTime(ch, 10*time.Second)
	if err != nil {
		fmt.Printf("sc error is %v\n", sc.(*conn).nc.LastError())
		b.Fatal("Timed out waiting for ack messages")
	} else if atomic.LoadInt32(&received) != int32(b.N) {
		b.Fatalf("Received: %d", received)
	}

	//	msgs, bytes, _ := sc.(*conn).ackSubscription.MaxPending()
	//	fmt.Printf("max pending msgs:%d bytes:%d\n", msgs, bytes)
}

func BenchmarkSubscribe(b *testing.B) {
	b.StopTimer()

	// Run a STAN server
	s := RunServer(clusterName)
	defer s.Shutdown()
	sc, err := Connect(clusterName, clientName)
	defer sc.Close()
	if err != nil {
		b.Fatalf("Expected to connect correctly, got err %v\n", err)
	}

	hw := []byte("Hello World")
	pch := make(chan bool)

	// Queue up all the messages. Keep this outside of the timing.
	for i := 0; i < b.N; i++ {
		if i == b.N-1 {
			// last one
			sc.PublishAsync("foo", hw, func(lguid string, err error) {
				if err != nil {
					b.Fatalf("Got an error from ack handler, %v", err)
				}
				pch <- true
			})
		} else {
			sc.PublishAsync("foo", hw, nil)
		}
	}

	// Wait for published to finish
	if err := WaitTime(pch, 10*time.Second); err != nil {
		b.Fatalf("Error waiting for publish to finish\n")
	}

	ch := make(chan bool)
	received := int32(0)

	b.StartTimer()
	b.ReportAllocs()

	sc.Subscribe("foo", func(m *Msg) {
		if nr := atomic.AddInt32(&received, 1); nr >= int32(b.N) {
			ch <- true
		}
	}, DeliverAllAvailable())

	err = WaitTime(ch, 10*time.Second)
	nr := atomic.LoadInt32(&received)
	if err != nil {
		b.Fatalf("Timed out waiting for messages, received only %d of %d\n", nr, b.N)
	} else if nr != int32(b.N) {
		b.Fatalf("Only Received: %d of %d", received, b.N)
	}
}

func BenchmarkQueueSubscribe(b *testing.B) {
	b.StopTimer()

	// Run a STAN server
	s := RunServer(clusterName)
	defer s.Shutdown()
	sc, err := Connect(clusterName, clientName)
	defer sc.Close()
	if err != nil {
		b.Fatalf("Expected to connect correctly, got err %v\n", err)
	}

	hw := []byte("Hello World")
	pch := make(chan bool)

	// Queue up all the messages. Keep this outside of the timing.
	for i := 0; i < b.N; i++ {
		if i == b.N-1 {
			// last one
			sc.PublishAsync("foo", hw, func(lguid string, err error) {
				if err != nil {
					b.Fatalf("Got an error from ack handler, %v", err)
				}
				pch <- true
			})
		} else {
			sc.PublishAsync("foo", hw, nil)
		}
	}

	// Wait for published to finish
	if err := WaitTime(pch, 10*time.Second); err != nil {
		b.Fatalf("Error waiting for publish to finish\n")
	}

	ch := make(chan bool)
	received := int32(0)

	b.StartTimer()
	b.ReportAllocs()

	mcb := func(m *Msg) {
		if nr := atomic.AddInt32(&received, 1); nr >= int32(b.N) {
			ch <- true
		}
	}

	sc.QueueSubscribe("foo", "bar", mcb, DeliverAllAvailable())
	sc.QueueSubscribe("foo", "bar", mcb, DeliverAllAvailable())
	sc.QueueSubscribe("foo", "bar", mcb, DeliverAllAvailable())
	sc.QueueSubscribe("foo", "bar", mcb, DeliverAllAvailable())

	err = WaitTime(ch, 20*time.Second)
	nr := atomic.LoadInt32(&received)
	if err != nil {
		b.Fatalf("Timed out waiting for messages, received only %d of %d\n", nr, b.N)
	} else if nr != int32(b.N) {
		b.Fatalf("Only Received: %d of %d", received, b.N)
	}
}

func BenchmarkPublishSubscribe(b *testing.B) {
	b.StopTimer()

	// Run a STAN server
	s := RunServer(clusterName)
	defer s.Shutdown()
	sc, err := Connect(clusterName, clientName)
	defer sc.Close()
	if err != nil {
		b.Fatalf("Expected to connect correctly, got err %v\n", err)
	}
	hw := []byte("Hello World")

	ch := make(chan bool)
	received := int32(0)

	// Subscribe callback, counts msgs received.
	_, err = sc.Subscribe("foo", func(m *Msg) {
		if nr := atomic.AddInt32(&received, 1); nr >= int32(b.N) {
			ch <- true
		}
	}, DeliverAllAvailable())

	if err != nil {
		b.Fatalf("Error subscribing, %v", err)
	}

	b.StartTimer()
	b.ReportAllocs()

	for i := 0; i < b.N; i++ {
		_, err := sc.PublishAsync("foo", hw, func(guid string, err error) {
			if err != nil {
				b.Fatalf("Received an error in publish ack callback: %v\n", err)
			}
		})
		if err != nil {
			b.Fatalf("Error publishing %v\n", err)
		}
	}

	err = WaitTime(ch, 30*time.Second)
	nr := atomic.LoadInt32(&received)
	if err != nil {
		b.Fatalf("Timed out waiting for messages, received only %d of %d\n", nr, b.N)
	} else if nr != int32(b.N) {
		b.Fatalf("Only Received: %d of %d", received, b.N)
	}
}

func BenchmarkTimeNow(b *testing.B) {
	for i := 0; i < b.N; i++ {
		now := time.Now()
		now.Add(10 * time.Nanosecond)
	}
}
<commit_msg>Fix defer in benchmark tests<commit_after>package stan

////////////////////////////////////////////////////////////////////////////////
// Benchmarks
////////////////////////////////////////////////////////////////////////////////

import (
	"fmt"
	"sync/atomic"
	"testing"
	"time"
)

func BenchmarkPublish(b *testing.B) {
	b.StopTimer()

	// Run a STAN server
	s := RunServer(clusterName)
	defer s.Shutdown()
	sc := NewDefaultConnection(b)
	defer sc.Close()

	hw := []byte("Hello World")

	b.StartTimer()
	b.ReportAllocs()

	for i := 0; i < b.N; i++ {
		if err := sc.Publish("foo", hw); err != nil {
			b.Fatalf("Got error on publish: %v\n", err)
		}
	}
}

func BenchmarkPublishAsync(b *testing.B) {
	b.StopTimer()

	// Run a STAN server
	s := RunServer(clusterName)
	defer s.Shutdown()
	sc := NewDefaultConnection(b)
	defer sc.Close()

	hw := []byte("Hello World")

	ch := make(chan bool)
	received := int32(0)

	ah := func(guid string, err error) {
		if err != nil {
			b.Fatalf("Received an error in ack callback: %v\n", err)
		}
		if nr := atomic.AddInt32(&received, 1); nr >= int32(b.N) {
			ch <- true
		}
	}
	b.StartTimer()
	b.ReportAllocs()

	for i := 0; i < b.N; i++ {
		if _, err := sc.PublishAsync("foo", hw, ah); err != nil {
			fmt.Printf("Client status %v, Server status %v\n", s.nc.Status(), (sc.(*conn)).nc.Status())
			fmt.Printf("len(ackmap) = %d\n", len(sc.(*conn).pubAckMap))

			b.Fatalf("Error from PublishAsync: %v\n", err)
		}
	}

	err := WaitTime(ch, 10*time.Second)
	if err != nil {
		fmt.Printf("sc error is %v\n", sc.(*conn).nc.LastError())
		b.Fatal("Timed out waiting for ack messages")
	} else if atomic.LoadInt32(&received) != int32(b.N) {
		b.Fatalf("Received: %d", received)
	}

	//	msgs, bytes, _ := sc.(*conn).ackSubscription.MaxPending()
	//	fmt.Printf("max pending msgs:%d bytes:%d\n", msgs, bytes)
}

func BenchmarkSubscribe(b *testing.B) {
	b.StopTimer()

	// Run a STAN server
	s := RunServer(clusterName)
	defer s.Shutdown()
	sc := NewDefaultConnection(b)
	defer sc.Close()

	hw := []byte("Hello World")
	pch := make(chan bool)

	// Queue up all the messages. Keep this outside of the timing.
	for i := 0; i < b.N; i++ {
		if i == b.N-1 {
			// last one
			sc.PublishAsync("foo", hw, func(lguid string, err error) {
				if err != nil {
					b.Fatalf("Got an error from ack handler, %v", err)
				}
				pch <- true
			})
		} else {
			sc.PublishAsync("foo", hw, nil)
		}
	}

	// Wait for published to finish
	if err := WaitTime(pch, 10*time.Second); err != nil {
		b.Fatalf("Error waiting for publish to finish\n")
	}

	ch := make(chan bool)
	received := int32(0)

	b.StartTimer()
	b.ReportAllocs()

	sc.Subscribe("foo", func(m *Msg) {
		if nr := atomic.AddInt32(&received, 1); nr >= int32(b.N) {
			ch <- true
		}
	}, DeliverAllAvailable())

	err := WaitTime(ch, 10*time.Second)
	nr := atomic.LoadInt32(&received)
	if err != nil {
		b.Fatalf("Timed out waiting for messages, received only %d of %d\n", nr, b.N)
	} else if nr != int32(b.N) {
		b.Fatalf("Only Received: %d of %d", received, b.N)
	}
}

func BenchmarkQueueSubscribe(b *testing.B) {
	b.StopTimer()

	// Run a STAN server
	s := RunServer(clusterName)
	defer s.Shutdown()
	sc := NewDefaultConnection(b)
	defer sc.Close()

	hw := []byte("Hello World")
	pch := make(chan bool)

	// Queue up all the messages. Keep this outside of the timing.
	for i := 0; i < b.N; i++ {
		if i == b.N-1 {
			// last one
			sc.PublishAsync("foo", hw, func(lguid string, err error) {
				if err != nil {
					b.Fatalf("Got an error from ack handler, %v", err)
				}
				pch <- true
			})
		} else {
			sc.PublishAsync("foo", hw, nil)
		}
	}

	// Wait for published to finish
	if err := WaitTime(pch, 10*time.Second); err != nil {
		b.Fatalf("Error waiting for publish to finish\n")
	}

	ch := make(chan bool)
	received := int32(0)

	b.StartTimer()
	b.ReportAllocs()

	mcb := func(m *Msg) {
		if nr := atomic.AddInt32(&received, 1); nr >= int32(b.N) {
			ch <- true
		}
	}

	sc.QueueSubscribe("foo", "bar", mcb, DeliverAllAvailable())
	sc.QueueSubscribe("foo", "bar", mcb, DeliverAllAvailable())
	sc.QueueSubscribe("foo", "bar", mcb, DeliverAllAvailable())
	sc.QueueSubscribe("foo", "bar", mcb, DeliverAllAvailable())

	err := WaitTime(ch, 20*time.Second)
	nr := atomic.LoadInt32(&received)
	if err != nil {
		b.Fatalf("Timed out waiting for messages, received only %d of %d\n", nr, b.N)
	} else if nr != int32(b.N) {
		b.Fatalf("Only Received: %d of %d", received, b.N)
	}
}

func BenchmarkPublishSubscribe(b *testing.B) {
	b.StopTimer()

	// Run a STAN server
	s := RunServer(clusterName)
	defer s.Shutdown()
	sc := NewDefaultConnection(b)
	defer sc.Close()

	hw := []byte("Hello World")

	ch := make(chan bool)
	received := int32(0)

	// Subscribe callback, counts msgs received.
	_, err := sc.Subscribe("foo", func(m *Msg) {
		if nr := atomic.AddInt32(&received, 1); nr >= int32(b.N) {
			ch <- true
		}
	}, DeliverAllAvailable())

	if err != nil {
		b.Fatalf("Error subscribing, %v", err)
	}

	b.StartTimer()
	b.ReportAllocs()

	for i := 0; i < b.N; i++ {
		_, err := sc.PublishAsync("foo", hw, func(guid string, err error) {
			if err != nil {
				b.Fatalf("Received an error in publish ack callback: %v\n", err)
			}
		})
		if err != nil {
			b.Fatalf("Error publishing %v\n", err)
		}
	}

	err = WaitTime(ch, 30*time.Second)
	nr := atomic.LoadInt32(&received)
	if err != nil {
		b.Fatalf("Timed out waiting for messages, received only %d of %d\n", nr, b.N)
	} else if nr != int32(b.N) {
		b.Fatalf("Only Received: %d of %d", received, b.N)
	}
}

func BenchmarkTimeNow(b *testing.B) {
	for i := 0; i < b.N; i++ {
		now := time.Now()
		now.Add(10 * time.Nanosecond)
	}
}
<|endoftext|>


<commit_before>package main

import (
  "log"
  "net"
  "time"
  "github.com/leesper/tao"
  "github.com/leesper/tao/examples/echo"
)

func init() {
  log.SetFlags(log.Lshortfile | log.LstdFlags)
}

func main() {
  tao.MessageMap.Register(echo.EchoMessage{}.MessageNumber(), tao.UnmarshalFunctionType(echo.UnmarshalEchoMessage))
  // tao.HandlerMap.Register(echo.EchoMessage{}.MessageNumber(), tao.NewHandlerFunctionType(echo.NewEchoMessageHandler))

  serverAddr, err := net.ResolveTCPAddr("tcp", "127.0.0.1:18341")
  if err != nil {
    log.Fatalln(err)
  }

  tcpConn, err := net.DialTCP("tcp", nil, serverAddr)
  if err != nil {
    log.Fatalln(err)
  }

  tcpConnection := tao.NewTcpConnection(nil, tcpConn)

  tcpConnection.SetOnConnectCallback(func() bool {
    log.Printf("On connect\n")
    return true
  })

  tcpConnection.SetOnErrorCallback(func() {
    log.Printf("On error\n")
  })

  tcpConnection.SetOnCloseCallback(func(client *tao.TcpConnection) {
    log.Printf("On close\n")
  })

  tcpConnection.SetOnMessageCallback(func(msg tao.Message, client *tao.TcpConnection) {
    echoMessage := msg.(echo.EchoMessage)
    log.Printf("%s\n", echoMessage.Message)
  })

  echoMessage := echo.EchoMessage{
    Message: "hello, world",
  }

  tcpConnection.Do()

  for i := 0; i < 3; i++ {
    err = tcpConnection.Write(echoMessage)
    if err != nil {
      log.Println(err)
    }
  }
  time.Sleep(5 * time.Second)
}
<commit_msg>clean up comment<commit_after>package main

import (
  "log"
  "net"
  "time"
  "github.com/leesper/tao"
  "github.com/leesper/tao/examples/echo"
)

func init() {
  log.SetFlags(log.Lshortfile | log.LstdFlags)
}

func main() {
  tao.MessageMap.Register(echo.EchoMessage{}.MessageNumber(), tao.UnmarshalFunctionType(echo.UnmarshalEchoMessage))

  serverAddr, err := net.ResolveTCPAddr("tcp", "127.0.0.1:18341")
  if err != nil {
    log.Fatalln(err)
  }

  tcpConn, err := net.DialTCP("tcp", nil, serverAddr)
  if err != nil {
    log.Fatalln(err)
  }

  tcpConnection := tao.NewTcpConnection(nil, tcpConn)

  tcpConnection.SetOnConnectCallback(func() bool {
    log.Printf("On connect\n")
    return true
  })

  tcpConnection.SetOnErrorCallback(func() {
    log.Printf("On error\n")
  })

  tcpConnection.SetOnCloseCallback(func(client *tao.TcpConnection) {
    log.Printf("On close\n")
  })

  tcpConnection.SetOnMessageCallback(func(msg tao.Message, client *tao.TcpConnection) {
    echoMessage := msg.(echo.EchoMessage)
    log.Printf("%s\n", echoMessage.Message)
  })

  echoMessage := echo.EchoMessage{
    Message: "hello, world",
  }

  tcpConnection.Do()

  for i := 0; i < 3; i++ {
    err = tcpConnection.Write(echoMessage)
    if err != nil {
      log.Println(err)
    }
  }
  time.Sleep(5 * time.Second)
}
<|endoftext|>


<commit_before>package statsd

import (
	"bytes"
	"sync"
)

type bufferPool struct {
	sync.Pool
}

func newBufferPool() *bufferPool {
	bp := &bufferPool{}

	bp.New = func() interface{} {
		return &bytes.Buffer{}
	}

	return bp
}

func (bp *bufferPool) Get() *bytes.Buffer {
	b := (bp.Pool.Get()).(*bytes.Buffer)
	b.Truncate(0)
	return b
}

func (bp *bufferPool) Put(b *bytes.Buffer) {
	bp.Pool.Put(b)
}
<commit_msg>Move truncate to bufferPool.Put<commit_after>package statsd

import (
	"bytes"
	"sync"
)

type bufferPool struct {
	sync.Pool
}

func newBufferPool() *bufferPool {
	bp := &bufferPool{}

	bp.New = func() interface{} {
		return &bytes.Buffer{}
	}

	return bp
}

func (bp *bufferPool) Get() *bytes.Buffer {
	return (bp.Pool.Get()).(*bytes.Buffer)
}

func (bp *bufferPool) Put(b *bytes.Buffer) {
	b.Truncate(0)
	bp.Pool.Put(b)
}
<|endoftext|>


<commit_before>package writer

import (
	"bufio"
	"context"
	"fmt"
	"io"
	"os"
	"path"
	"sync"
	"sync/atomic"
	"time"

	"github.com/lomik/carbon-clickhouse/helper/RowBinary"
	"github.com/lomik/carbon-clickhouse/helper/config"
	"github.com/lomik/carbon-clickhouse/helper/stop"
	"github.com/lomik/zapwriter"
	"github.com/pierrec/lz4"
	"go.uber.org/zap"
)

type compWriter interface {
	Write([]byte) (int, error)
	Flush() error
	Close() error
}

// Writer dumps all received data in prepared for clickhouse format
type Writer struct {
	stop.Struct
	sync.RWMutex
	stat struct {
		writtenBytes  uint32
		unhandled     uint32
		chunkInterval uint32
	}
	inputChan    chan *RowBinary.WriteBuffer
	path         string
	autoInterval *config.ChunkAutoInterval
	compAlgo     config.CompAlgo
	compLevel    int
	lz4Header    lz4.Header
	inProgress   map[string]bool // current writing files
	logger       *zap.Logger
	uploaders    []string
	onFinish     func(string) error
}

func New(in chan *RowBinary.WriteBuffer, path string, autoInterval *config.ChunkAutoInterval, compAlgo config.CompAlgo, compLevel int, uploaders []string, onFinish func(string) error) *Writer {
	finishCallback := func(fn string) error {
		if err := Link(fn, uploaders); err != nil {
			return err
		}

		if onFinish != nil {
			return onFinish(fn)
		}

		return nil
	}

	wr := &Writer{
		inputChan:    in,
		path:         path,
		autoInterval: autoInterval,
		compAlgo:     compAlgo,
		compLevel:    compLevel,
		inProgress:   make(map[string]bool),
		logger:       zapwriter.Logger("writer"),
		uploaders:    uploaders,
		onFinish:     finishCallback,
	}

	switch compAlgo {
	case config.CompAlgoLZ4:
		wr.lz4Header = lz4.Header{
			Size:             1024,
			CompressionLevel: compLevel,
		}
	}

	return wr
}

func (w *Writer) Start() error {
	return w.StartFunc(func() error {
		// link pre-existing files
		if err := w.LinkAll(); err != nil {
			return err
		}
		if err := w.Cleanup(); err != nil {
			return err
		}
		w.Go(w.worker)
		w.Go(w.cleaner)
		return nil
	})
}

func (w *Writer) Stat(send func(metric string, value float64)) {
	writtenBytes := atomic.LoadUint32(&w.stat.writtenBytes)
	atomic.AddUint32(&w.stat.writtenBytes, -writtenBytes)
	send("writtenBytes", float64(writtenBytes))

	send("unhandled", float64(atomic.LoadUint32(&w.stat.unhandled)))
	send("chunkInterval_s", float64(atomic.LoadUint32(&w.stat.chunkInterval)))
}

func (w *Writer) IsInProgress(filename string) bool {
	w.RLock()
	v := w.inProgress[filename]
	w.RUnlock()
	return v
}

func (w *Writer) worker(ctx context.Context) {
	var out *os.File
	var cwr compWriter
	var outBuf *bufio.Writer
	var fn string // current filename

	defer func() {
		if out != nil {
			out.Close()
		}
	}()

	// close old file, open new
	rotate := func() {
		if out != nil {
			outBuf.Flush()

			if cwr != nil {
				cwr.Flush()
				cwr.Close()
			}

			out.Close()

			out = nil
			cwr = nil
			outBuf = nil
		}

		var err error

	OpenLoop:
		for {
			go func(filename string) {
				if filename == "" || w.onFinish == nil {
					return
				}

				err = w.onFinish(filename)
				if err != nil {
					w.logger.Error("onFinish callback failed", zap.String("filename", filename), zap.Error(err))
				}
			}(fn)

			// replace fn in inProgress
			w.Lock()
			delete(w.inProgress, fn)
			fn = path.Join(w.path, fmt.Sprintf("default.%d", time.Now().UnixNano()))
			w.inProgress[fn] = true
			w.Unlock()

			out, err = os.OpenFile(fn, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)

			if err != nil {
				w.logger.Error("create failed", zap.String("filename", fn), zap.Error(err))

				// check exit channel
				select {
				case <-ctx.Done():
					break OpenLoop
				default:
				}

				// try and spam to error log every second
				time.Sleep(time.Second)

				continue OpenLoop
			}

			var wr io.Writer
			switch w.compAlgo {
			case config.CompAlgoNone:
				wr = out
			case config.CompAlgoLZ4:
				lz4w := lz4.NewWriter(out)
				lz4w.Header = w.lz4Header
				cwr = lz4w
				wr = lz4w
			}

			outBuf = bufio.NewWriterSize(wr, 1024*1024)
			break OpenLoop
		}
	}

	// open first file
	rotate()

	tickerC := make(chan struct{}, 1)

	go func() {
		prevInterval := w.autoInterval.GetDefault()
		for {
			u := int(atomic.LoadUint32(&w.stat.unhandled))
			interval := w.autoInterval.GetInterval(u)
			if interval != prevInterval {
				w.logger.Info("chunk interval changed", zap.String("interval", interval.String()))
				prevInterval = interval
			}
			atomic.StoreUint32(&w.stat.chunkInterval, uint32(interval.Seconds()))

			select {
			case <-ctx.Done():
				return
			case <-time.After(interval):
				select {
				case tickerC <- struct{}{}:
					// pass
				case <-ctx.Done():
					return
				}
			}
		}
	}()

	write := func(b *RowBinary.WriteBuffer) {
		_, err := outBuf.Write(b.Body[:b.Used])
		if b.ConfirmRequired() {
			if err != nil {
				b.Fail(err)
			} else {
				err := outBuf.Flush()
				if err != nil {
					b.Fail(err)
				} else {
					b.Confirm()
				}
			}
		}
		// @TODO: log error?
		atomic.AddUint32(&w.stat.writtenBytes, uint32(b.Used))
		b.Release()
	}

	for {
		select {
		case b := <-w.inputChan:
			write(b)
		case <-tickerC:
			rotate()
		case <-ctx.Done():
			return
		default: // outBuf flush if nothing received
			outBuf.Flush()

			select {
			case b := <-w.inputChan:
				write(b)
			case <-tickerC:
				rotate()
			case <-ctx.Done():
				return
			}
		}
	}
}

func (w *Writer) cleaner(ctx context.Context) {
	ticker := time.NewTicker(w.autoInterval.GetDefault())
	defer ticker.Stop()

	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			w.Cleanup()
		}
	}
}
<commit_msg>lz4 tuning<commit_after>package writer

import (
	"bufio"
	"context"
	"fmt"
	"io"
	"os"
	"path"
	"sync"
	"sync/atomic"
	"time"

	"github.com/lomik/carbon-clickhouse/helper/RowBinary"
	"github.com/lomik/carbon-clickhouse/helper/config"
	"github.com/lomik/carbon-clickhouse/helper/stop"
	"github.com/lomik/zapwriter"
	"github.com/pierrec/lz4"
	"go.uber.org/zap"
)

type compWriter interface {
	Write([]byte) (int, error)
	Flush() error
	Close() error
}

// Writer dumps all received data in prepared for clickhouse format
type Writer struct {
	stop.Struct
	sync.RWMutex
	stat struct {
		writtenBytes  uint32
		unhandled     uint32
		chunkInterval uint32
	}
	inputChan    chan *RowBinary.WriteBuffer
	path         string
	autoInterval *config.ChunkAutoInterval
	compAlgo     config.CompAlgo
	compLevel    int
	lz4Header    lz4.Header
	inProgress   map[string]bool // current writing files
	logger       *zap.Logger
	uploaders    []string
	onFinish     func(string) error
}

func New(in chan *RowBinary.WriteBuffer, path string, autoInterval *config.ChunkAutoInterval, compAlgo config.CompAlgo, compLevel int, uploaders []string, onFinish func(string) error) *Writer {
	finishCallback := func(fn string) error {
		if err := Link(fn, uploaders); err != nil {
			return err
		}

		if onFinish != nil {
			return onFinish(fn)
		}

		return nil
	}

	wr := &Writer{
		inputChan:    in,
		path:         path,
		autoInterval: autoInterval,
		compAlgo:     compAlgo,
		compLevel:    compLevel,
		inProgress:   make(map[string]bool),
		logger:       zapwriter.Logger("writer"),
		uploaders:    uploaders,
		onFinish:     finishCallback,
	}

	switch compAlgo {
	case config.CompAlgoLZ4:
		wr.lz4Header = lz4.Header{
			Size:             128,
			CompressionLevel: compLevel,
		}
	}

	return wr
}

func (w *Writer) Start() error {
	return w.StartFunc(func() error {
		// link pre-existing files
		if err := w.LinkAll(); err != nil {
			return err
		}
		if err := w.Cleanup(); err != nil {
			return err
		}
		w.Go(w.worker)
		w.Go(w.cleaner)
		return nil
	})
}

func (w *Writer) Stat(send func(metric string, value float64)) {
	writtenBytes := atomic.LoadUint32(&w.stat.writtenBytes)
	atomic.AddUint32(&w.stat.writtenBytes, -writtenBytes)
	send("writtenBytes", float64(writtenBytes))

	send("unhandled", float64(atomic.LoadUint32(&w.stat.unhandled)))
	send("chunkInterval_s", float64(atomic.LoadUint32(&w.stat.chunkInterval)))
}

func (w *Writer) IsInProgress(filename string) bool {
	w.RLock()
	v := w.inProgress[filename]
	w.RUnlock()
	return v
}

func (w *Writer) worker(ctx context.Context) {
	var out *os.File
	var cwr compWriter
	var outBuf *bufio.Writer
	var fn string // current filename

	cwrClose := func() {
		if cwr != nil {
			if err := cwr.Close(); err != nil {
				w.logger.Error("CompWriter close failed", zap.Error(err))
			}
		}
	}

	defer func() {
		if out != nil {
			outBuf.Flush()
			cwrClose()
			out.Close()
		}
	}()

	// close old file, open new
	rotate := func() {
		if out != nil {
			outBuf.Flush()
			cwrClose()
			out.Close()

			out = nil
			cwr = nil
			outBuf = nil
		}

		var err error

	OpenLoop:
		for {
			go func(filename string) {
				if filename == "" || w.onFinish == nil {
					return
				}

				err = w.onFinish(filename)
				if err != nil {
					w.logger.Error("onFinish callback failed", zap.String("filename", filename), zap.Error(err))
				}
			}(fn)

			// replace fn in inProgress
			w.Lock()
			delete(w.inProgress, fn)
			fn = path.Join(w.path, fmt.Sprintf("default.%d", time.Now().UnixNano()))
			w.inProgress[fn] = true
			w.Unlock()

			out, err = os.OpenFile(fn, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)

			if err != nil {
				w.logger.Error("create failed", zap.String("filename", fn), zap.Error(err))

				// check exit channel
				select {
				case <-ctx.Done():
					break OpenLoop
				default:
				}

				// try and spam to error log every second
				time.Sleep(time.Second)

				continue OpenLoop
			}

			var wr io.Writer
			switch w.compAlgo {
			case config.CompAlgoNone:
				wr = out
			case config.CompAlgoLZ4:
				lz4w := lz4.NewWriter(out)
				lz4w.Header = w.lz4Header
				cwr = lz4w
				wr = lz4w
			}

			outBuf = bufio.NewWriterSize(wr, 1024*1024)
			break OpenLoop
		}
	}

	// open first file
	rotate()

	tickerC := make(chan struct{}, 1)

	go func() {
		prevInterval := w.autoInterval.GetDefault()
		for {
			u := int(atomic.LoadUint32(&w.stat.unhandled))
			interval := w.autoInterval.GetInterval(u)
			if interval != prevInterval {
				w.logger.Info("chunk interval changed", zap.String("interval", interval.String()))
				prevInterval = interval
			}
			atomic.StoreUint32(&w.stat.chunkInterval, uint32(interval.Seconds()))

			select {
			case <-ctx.Done():
				return
			case <-time.After(interval):
				select {
				case tickerC <- struct{}{}:
					// pass
				case <-ctx.Done():
					return
				}
			}
		}
	}()

	write := func(b *RowBinary.WriteBuffer) {
		_, err := outBuf.Write(b.Body[:b.Used])
		if b.ConfirmRequired() {
			if err != nil {
				b.Fail(err)
			} else {
				err := outBuf.Flush()
				if err != nil {
					b.Fail(err)
				} else {
					b.Confirm()
				}
			}
		}
		// @TODO: log error?
		atomic.AddUint32(&w.stat.writtenBytes, uint32(b.Used))
		b.Release()
	}

	for {
		select {
		case b := <-w.inputChan:
			write(b)
		case <-tickerC:
			rotate()
		case <-ctx.Done():
			return
		default: // outBuf flush if nothing received
			outBuf.Flush()

			if cwr != nil {
				if err := cwr.Flush(); err != nil {
					w.logger.Error("CompWriter Flush() failed", zap.Error(err))
				}
			}

			select {
			case b := <-w.inputChan:
				write(b)
			case <-tickerC:
				rotate()
			case <-ctx.Done():
				return
			}
		}
	}
}

func (w *Writer) cleaner(ctx context.Context) {
	ticker := time.NewTicker(w.autoInterval.GetDefault())
	defer ticker.Stop()

	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			w.Cleanup()
		}
	}
}
<|endoftext|>


<commit_before>package storage

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"

	"github.com/containers/storage/pkg/ioutils"
	"github.com/containers/storage/pkg/stringid"
)

var (
	// ErrContainerUnknown indicates that there was no container with the specified name or ID
	ErrContainerUnknown = errors.New("container not known")
)

// A Container is a reference to a read-write layer with metadata.
type Container struct {
	// ID is either one which was specified at create-time, or a random
	// value which was generated by the library.
	ID string `json:"id"`

	// Names is an optional set of user-defined convenience values.  The
	// container can be referred to by its ID or any of its names.  Names
	// are unique among containers.
	Names []string `json:"names,omitempty"`

	// ImageID is the ID of the image which was used to create the container.
	ImageID string `json:"image"`

	// LayerID is the ID of the read-write layer for the container itself.
	// It is assumed that the image's top layer is the parent of the container's
	// read-write layer.
	LayerID string `json:"layer"`

	// Metadata is data we keep for the convenience of the caller.  It is not
	// expected to be large, since it is kept in memory.
	Metadata string `json:"metadata,omitempty"`

	// BigDataNames is a list of names of data items that we keep for the
	// convenience of the caller.  They can be large, and are only in
	// memory when being read from or written to disk.
	BigDataNames []string `json:"big-data-names,omitempty"`

	Flags map[string]interface{} `json:"flags,omitempty"`
}

// ContainerStore provides bookkeeping for information about Containers.
type ContainerStore interface {
	FileBasedStore
	MetadataStore
	BigDataStore
	FlaggableStore

	// Create creates a container that has a specified ID (or generates a
	// random one if an empty value is supplied) and optional names,
	// based on the specified image, using the specified layer as its
	// read-write layer.
	Create(id string, names []string, image, layer, metadata string) (*Container, error)

	// SetNames updates the list of names associated with the container
	// with the specified ID.
	SetNames(id string, names []string) error

	// Get retrieves information about a container given an ID or name.
	Get(id string) (*Container, error)

	// Exists checks if there is a container with the given ID or name.
	Exists(id string) bool

	// Delete removes the record of the container.
	Delete(id string) error

	// Wipe removes records of all containers.
	Wipe() error

	// Lookup attempts to translate a name to an ID.  Most methods do this
	// implicitly.
	Lookup(name string) (string, error)

	// Containers returns a slice enumerating the known containers.
	Containers() ([]Container, error)
}

type containerStore struct {
	lockfile   Locker
	dir        string
	containers []Container
	byid       map[string]*Container
	bylayer    map[string]*Container
	byname     map[string]*Container
}

func (r *containerStore) Containers() ([]Container, error) {
	return r.containers, nil
}

func (r *containerStore) containerspath() string {
	return filepath.Join(r.dir, "containers.json")
}

func (r *containerStore) datadir(id string) string {
	return filepath.Join(r.dir, id)
}

func (r *containerStore) datapath(id, key string) string {
	return filepath.Join(r.datadir(id), makeBigDataBaseName(key))
}

func (r *containerStore) Load() error {
	needSave := false
	rpath := r.containerspath()
	data, err := ioutil.ReadFile(rpath)
	if err != nil && !os.IsNotExist(err) {
		return err
	}
	containers := []Container{}
	layers := make(map[string]*Container)
	ids := make(map[string]*Container)
	names := make(map[string]*Container)
	if err = json.Unmarshal(data, &containers); len(data) == 0 || err == nil {
		for n, container := range containers {
			ids[container.ID] = &containers[n]
			layers[container.LayerID] = &containers[n]
			for _, name := range container.Names {
				if conflict, ok := names[name]; ok {
					r.removeName(conflict, name)
					needSave = true
				}
				names[name] = &containers[n]
			}
		}
	}
	r.containers = containers
	r.byid = ids
	r.bylayer = layers
	r.byname = names
	if needSave {
		r.Touch()
		return r.Save()
	}
	return nil
}

func (r *containerStore) Save() error {
	rpath := r.containerspath()
	jdata, err := json.Marshal(&r.containers)
	if err != nil {
		return err
	}
	return ioutils.AtomicWriteFile(rpath, jdata, 0600)
}

func newContainerStore(dir string) (ContainerStore, error) {
	if err := os.MkdirAll(dir, 0700); err != nil {
		return nil, err
	}
	lockfile, err := GetLockfile(filepath.Join(dir, "containers.lock"))
	if err != nil {
		return nil, err
	}
	lockfile.Lock()
	defer lockfile.Unlock()
	cstore := containerStore{
		lockfile:   lockfile,
		dir:        dir,
		containers: []Container{},
		byid:       make(map[string]*Container),
		bylayer:    make(map[string]*Container),
		byname:     make(map[string]*Container),
	}
	if err := cstore.Load(); err != nil {
		return nil, err
	}
	return &cstore, nil
}

func (r *containerStore) ClearFlag(id string, flag string) error {
	if container, ok := r.byname[id]; ok {
		id = container.ID
	} else if container, ok := r.bylayer[id]; ok {
		id = container.ID
	}
	if _, ok := r.byid[id]; !ok {
		return ErrImageUnknown
	}
	container := r.byid[id]
	delete(container.Flags, flag)
	return r.Save()
}

func (r *containerStore) SetFlag(id string, flag string, value interface{}) error {
	if container, ok := r.byname[id]; ok {
		id = container.ID
	} else if container, ok := r.bylayer[id]; ok {
		id = container.ID
	}
	if _, ok := r.byid[id]; !ok {
		return ErrImageUnknown
	}
	container := r.byid[id]
	container.Flags[flag] = value
	return r.Save()
}

func (r *containerStore) Create(id string, names []string, image, layer, metadata string) (container *Container, err error) {
	if id == "" {
		id = stringid.GenerateRandomID()
		_, idInUse := r.byid[id]
		for idInUse {
			id = stringid.GenerateRandomID()
			_, idInUse = r.byid[id]
		}
	}
	if _, idInUse := r.byid[id]; idInUse {
		return nil, ErrDuplicateID
	}
	for _, name := range names {
		if _, nameInUse := r.byname[name]; nameInUse {
			return nil, ErrDuplicateName
		}
	}
	if err == nil {
		newContainer := Container{
			ID:           id,
			Names:        names,
			ImageID:      image,
			LayerID:      layer,
			Metadata:     metadata,
			BigDataNames: []string{},
			Flags:        make(map[string]interface{}),
		}
		r.containers = append(r.containers, newContainer)
		container = &r.containers[len(r.containers)-1]
		r.byid[id] = container
		r.bylayer[layer] = container
		for _, name := range names {
			r.byname[name] = container
		}
		err = r.Save()
	}
	return container, err
}

func (r *containerStore) GetMetadata(id string) (string, error) {
	if container, ok := r.byname[id]; ok {
		id = container.ID
	} else if container, ok := r.bylayer[id]; ok {
		id = container.ID
	}
	if container, ok := r.byid[id]; ok {
		return container.Metadata, nil
	}
	return "", ErrContainerUnknown
}

func (r *containerStore) SetMetadata(id, metadata string) error {
	if container, ok := r.byname[id]; ok {
		id = container.ID
	} else if container, ok := r.bylayer[id]; ok {
		id = container.ID
	}
	if container, ok := r.byid[id]; ok {
		container.Metadata = metadata
		return r.Save()
	}
	return ErrContainerUnknown
}

func (r *containerStore) removeName(container *Container, name string) {
	newNames := []string{}
	for _, oldName := range container.Names {
		if oldName != name {
			newNames = append(newNames, oldName)
		}
	}
	container.Names = newNames
}

func (r *containerStore) SetNames(id string, names []string) error {
	if container, ok := r.byname[id]; ok {
		id = container.ID
	} else if container, ok := r.bylayer[id]; ok {
		id = container.ID
	}
	if container, ok := r.byid[id]; ok {
		for _, name := range container.Names {
			delete(r.byname, name)
		}
		for _, name := range names {
			if otherContainer, ok := r.byname[name]; ok {
				r.removeName(otherContainer, name)
			}
			r.byname[name] = container
		}
		container.Names = names
		return r.Save()
	}
	return ErrContainerUnknown
}

func (r *containerStore) Delete(id string) error {
	if container, ok := r.byname[id]; ok {
		id = container.ID
	} else if container, ok := r.bylayer[id]; ok {
		id = container.ID
	}
	if _, ok := r.byid[id]; !ok {
		return ErrContainerUnknown
	}
	if container, ok := r.byid[id]; ok {
		newContainers := []Container{}
		for _, candidate := range r.containers {
			if candidate.ID != id {
				newContainers = append(newContainers, candidate)
			}
		}
		delete(r.byid, container.ID)
		delete(r.bylayer, container.LayerID)
		for _, name := range container.Names {
			delete(r.byname, name)
		}
		r.containers = newContainers
		if err := r.Save(); err != nil {
			return err
		}
		if err := os.RemoveAll(r.datadir(id)); err != nil {
			return err
		}
	}
	return nil
}

func (r *containerStore) Get(id string) (*Container, error) {
	if c, ok := r.byname[id]; ok {
		return c, nil
	} else if c, ok := r.bylayer[id]; ok {
		return c, nil
	}
	if c, ok := r.byid[id]; ok {
		return c, nil
	}
	return nil, ErrContainerUnknown
}

func (r *containerStore) Lookup(name string) (id string, err error) {
	container, ok := r.byname[name]
	if !ok {
		container, ok = r.byid[name]
		if !ok {
			return "", ErrContainerUnknown
		}
	}
	return container.ID, nil
}

func (r *containerStore) Exists(id string) bool {
	if _, ok := r.byname[id]; ok {
		return true
	}
	if _, ok := r.bylayer[id]; ok {
		return true
	}
	if _, ok := r.byid[id]; ok {
		return true
	}
	return false
}

func (r *containerStore) GetBigData(id, key string) ([]byte, error) {
	if img, ok := r.byname[id]; ok {
		id = img.ID
	}
	if _, ok := r.byid[id]; !ok {
		return nil, ErrImageUnknown
	}
	return ioutil.ReadFile(r.datapath(id, key))
}

func (r *containerStore) GetBigDataNames(id string) ([]string, error) {
	if img, ok := r.byname[id]; ok {
		id = img.ID
	}
	if _, ok := r.byid[id]; !ok {
		return nil, ErrImageUnknown
	}
	return r.byid[id].BigDataNames, nil
}

func (r *containerStore) SetBigData(id, key string, data []byte) error {
	if img, ok := r.byname[id]; ok {
		id = img.ID
	}
	if _, ok := r.byid[id]; !ok {
		return ErrImageUnknown
	}
	if err := os.MkdirAll(r.datadir(id), 0700); err != nil {
		return err
	}
	err := ioutils.AtomicWriteFile(r.datapath(id, key), data, 0600)
	if err == nil {
		add := true
		for _, name := range r.byid[id].BigDataNames {
			if name == key {
				add = false
				break
			}
		}
		if add {
			r.byid[id].BigDataNames = append(r.byid[id].BigDataNames, key)
			err = r.Save()
		}
	}
	return err
}

func (r *containerStore) Wipe() error {
	ids := []string{}
	for id := range r.byid {
		ids = append(ids, id)
	}
	for _, id := range ids {
		if err := r.Delete(id); err != nil {
			return err
		}
	}
	return nil
}

func (r *containerStore) Lock() {
	r.lockfile.Lock()
}

func (r *containerStore) Unlock() {
	r.lockfile.Unlock()
}

func (r *containerStore) Touch() error {
	return r.lockfile.Touch()
}

func (r *containerStore) Modified() (bool, error) {
	return r.lockfile.Modified()
}

func (r *containerStore) TouchedSince(when time.Time) bool {
	return r.lockfile.TouchedSince(when)
}
<commit_msg>Don't return image errors for container calls<commit_after>package storage

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"

	"github.com/containers/storage/pkg/ioutils"
	"github.com/containers/storage/pkg/stringid"
)

var (
	// ErrContainerUnknown indicates that there was no container with the specified name or ID
	ErrContainerUnknown = errors.New("container not known")
)

// A Container is a reference to a read-write layer with metadata.
type Container struct {
	// ID is either one which was specified at create-time, or a random
	// value which was generated by the library.
	ID string `json:"id"`

	// Names is an optional set of user-defined convenience values.  The
	// container can be referred to by its ID or any of its names.  Names
	// are unique among containers.
	Names []string `json:"names,omitempty"`

	// ImageID is the ID of the image which was used to create the container.
	ImageID string `json:"image"`

	// LayerID is the ID of the read-write layer for the container itself.
	// It is assumed that the image's top layer is the parent of the container's
	// read-write layer.
	LayerID string `json:"layer"`

	// Metadata is data we keep for the convenience of the caller.  It is not
	// expected to be large, since it is kept in memory.
	Metadata string `json:"metadata,omitempty"`

	// BigDataNames is a list of names of data items that we keep for the
	// convenience of the caller.  They can be large, and are only in
	// memory when being read from or written to disk.
	BigDataNames []string `json:"big-data-names,omitempty"`

	Flags map[string]interface{} `json:"flags,omitempty"`
}

// ContainerStore provides bookkeeping for information about Containers.
type ContainerStore interface {
	FileBasedStore
	MetadataStore
	BigDataStore
	FlaggableStore

	// Create creates a container that has a specified ID (or generates a
	// random one if an empty value is supplied) and optional names,
	// based on the specified image, using the specified layer as its
	// read-write layer.
	Create(id string, names []string, image, layer, metadata string) (*Container, error)

	// SetNames updates the list of names associated with the container
	// with the specified ID.
	SetNames(id string, names []string) error

	// Get retrieves information about a container given an ID or name.
	Get(id string) (*Container, error)

	// Exists checks if there is a container with the given ID or name.
	Exists(id string) bool

	// Delete removes the record of the container.
	Delete(id string) error

	// Wipe removes records of all containers.
	Wipe() error

	// Lookup attempts to translate a name to an ID.  Most methods do this
	// implicitly.
	Lookup(name string) (string, error)

	// Containers returns a slice enumerating the known containers.
	Containers() ([]Container, error)
}

type containerStore struct {
	lockfile   Locker
	dir        string
	containers []Container
	byid       map[string]*Container
	bylayer    map[string]*Container
	byname     map[string]*Container
}

func (r *containerStore) Containers() ([]Container, error) {
	return r.containers, nil
}

func (r *containerStore) containerspath() string {
	return filepath.Join(r.dir, "containers.json")
}

func (r *containerStore) datadir(id string) string {
	return filepath.Join(r.dir, id)
}

func (r *containerStore) datapath(id, key string) string {
	return filepath.Join(r.datadir(id), makeBigDataBaseName(key))
}

func (r *containerStore) Load() error {
	needSave := false
	rpath := r.containerspath()
	data, err := ioutil.ReadFile(rpath)
	if err != nil && !os.IsNotExist(err) {
		return err
	}
	containers := []Container{}
	layers := make(map[string]*Container)
	ids := make(map[string]*Container)
	names := make(map[string]*Container)
	if err = json.Unmarshal(data, &containers); len(data) == 0 || err == nil {
		for n, container := range containers {
			ids[container.ID] = &containers[n]
			layers[container.LayerID] = &containers[n]
			for _, name := range container.Names {
				if conflict, ok := names[name]; ok {
					r.removeName(conflict, name)
					needSave = true
				}
				names[name] = &containers[n]
			}
		}
	}
	r.containers = containers
	r.byid = ids
	r.bylayer = layers
	r.byname = names
	if needSave {
		r.Touch()
		return r.Save()
	}
	return nil
}

func (r *containerStore) Save() error {
	rpath := r.containerspath()
	jdata, err := json.Marshal(&r.containers)
	if err != nil {
		return err
	}
	return ioutils.AtomicWriteFile(rpath, jdata, 0600)
}

func newContainerStore(dir string) (ContainerStore, error) {
	if err := os.MkdirAll(dir, 0700); err != nil {
		return nil, err
	}
	lockfile, err := GetLockfile(filepath.Join(dir, "containers.lock"))
	if err != nil {
		return nil, err
	}
	lockfile.Lock()
	defer lockfile.Unlock()
	cstore := containerStore{
		lockfile:   lockfile,
		dir:        dir,
		containers: []Container{},
		byid:       make(map[string]*Container),
		bylayer:    make(map[string]*Container),
		byname:     make(map[string]*Container),
	}
	if err := cstore.Load(); err != nil {
		return nil, err
	}
	return &cstore, nil
}

func (r *containerStore) ClearFlag(id string, flag string) error {
	if container, ok := r.byname[id]; ok {
		id = container.ID
	} else if container, ok := r.bylayer[id]; ok {
		id = container.ID
	}
	if _, ok := r.byid[id]; !ok {
		return ErrContainerUnknown
	}
	container := r.byid[id]
	delete(container.Flags, flag)
	return r.Save()
}

func (r *containerStore) SetFlag(id string, flag string, value interface{}) error {
	if container, ok := r.byname[id]; ok {
		id = container.ID
	} else if container, ok := r.bylayer[id]; ok {
		id = container.ID
	}
	if _, ok := r.byid[id]; !ok {
		return ErrContainerUnknown
	}
	container := r.byid[id]
	container.Flags[flag] = value
	return r.Save()
}

func (r *containerStore) Create(id string, names []string, image, layer, metadata string) (container *Container, err error) {
	if id == "" {
		id = stringid.GenerateRandomID()
		_, idInUse := r.byid[id]
		for idInUse {
			id = stringid.GenerateRandomID()
			_, idInUse = r.byid[id]
		}
	}
	if _, idInUse := r.byid[id]; idInUse {
		return nil, ErrDuplicateID
	}
	for _, name := range names {
		if _, nameInUse := r.byname[name]; nameInUse {
			return nil, ErrDuplicateName
		}
	}
	if err == nil {
		newContainer := Container{
			ID:           id,
			Names:        names,
			ImageID:      image,
			LayerID:      layer,
			Metadata:     metadata,
			BigDataNames: []string{},
			Flags:        make(map[string]interface{}),
		}
		r.containers = append(r.containers, newContainer)
		container = &r.containers[len(r.containers)-1]
		r.byid[id] = container
		r.bylayer[layer] = container
		for _, name := range names {
			r.byname[name] = container
		}
		err = r.Save()
	}
	return container, err
}

func (r *containerStore) GetMetadata(id string) (string, error) {
	if container, ok := r.byname[id]; ok {
		id = container.ID
	} else if container, ok := r.bylayer[id]; ok {
		id = container.ID
	}
	if container, ok := r.byid[id]; ok {
		return container.Metadata, nil
	}
	return "", ErrContainerUnknown
}

func (r *containerStore) SetMetadata(id, metadata string) error {
	if container, ok := r.byname[id]; ok {
		id = container.ID
	} else if container, ok := r.bylayer[id]; ok {
		id = container.ID
	}
	if container, ok := r.byid[id]; ok {
		container.Metadata = metadata
		return r.Save()
	}
	return ErrContainerUnknown
}

func (r *containerStore) removeName(container *Container, name string) {
	newNames := []string{}
	for _, oldName := range container.Names {
		if oldName != name {
			newNames = append(newNames, oldName)
		}
	}
	container.Names = newNames
}

func (r *containerStore) SetNames(id string, names []string) error {
	if container, ok := r.byname[id]; ok {
		id = container.ID
	} else if container, ok := r.bylayer[id]; ok {
		id = container.ID
	}
	if container, ok := r.byid[id]; ok {
		for _, name := range container.Names {
			delete(r.byname, name)
		}
		for _, name := range names {
			if otherContainer, ok := r.byname[name]; ok {
				r.removeName(otherContainer, name)
			}
			r.byname[name] = container
		}
		container.Names = names
		return r.Save()
	}
	return ErrContainerUnknown
}

func (r *containerStore) Delete(id string) error {
	if container, ok := r.byname[id]; ok {
		id = container.ID
	} else if container, ok := r.bylayer[id]; ok {
		id = container.ID
	}
	if _, ok := r.byid[id]; !ok {
		return ErrContainerUnknown
	}
	if container, ok := r.byid[id]; ok {
		newContainers := []Container{}
		for _, candidate := range r.containers {
			if candidate.ID != id {
				newContainers = append(newContainers, candidate)
			}
		}
		delete(r.byid, container.ID)
		delete(r.bylayer, container.LayerID)
		for _, name := range container.Names {
			delete(r.byname, name)
		}
		r.containers = newContainers
		if err := r.Save(); err != nil {
			return err
		}
		if err := os.RemoveAll(r.datadir(id)); err != nil {
			return err
		}
	}
	return nil
}

func (r *containerStore) Get(id string) (*Container, error) {
	if c, ok := r.byname[id]; ok {
		return c, nil
	} else if c, ok := r.bylayer[id]; ok {
		return c, nil
	}
	if c, ok := r.byid[id]; ok {
		return c, nil
	}
	return nil, ErrContainerUnknown
}

func (r *containerStore) Lookup(name string) (id string, err error) {
	container, ok := r.byname[name]
	if !ok {
		container, ok = r.byid[name]
		if !ok {
			return "", ErrContainerUnknown
		}
	}
	return container.ID, nil
}

func (r *containerStore) Exists(id string) bool {
	if _, ok := r.byname[id]; ok {
		return true
	}
	if _, ok := r.bylayer[id]; ok {
		return true
	}
	if _, ok := r.byid[id]; ok {
		return true
	}
	return false
}

func (r *containerStore) GetBigData(id, key string) ([]byte, error) {
	if img, ok := r.byname[id]; ok {
		id = img.ID
	}
	if _, ok := r.byid[id]; !ok {
		return nil, ErrContainerUnknown
	}
	return ioutil.ReadFile(r.datapath(id, key))
}

func (r *containerStore) GetBigDataNames(id string) ([]string, error) {
	if img, ok := r.byname[id]; ok {
		id = img.ID
	}
	if _, ok := r.byid[id]; !ok {
		return nil, ErrContainerUnknown
	}
	return r.byid[id].BigDataNames, nil
}

func (r *containerStore) SetBigData(id, key string, data []byte) error {
	if img, ok := r.byname[id]; ok {
		id = img.ID
	}
	if _, ok := r.byid[id]; !ok {
		return ErrContainerUnknown
	}
	if err := os.MkdirAll(r.datadir(id), 0700); err != nil {
		return err
	}
	err := ioutils.AtomicWriteFile(r.datapath(id, key), data, 0600)
	if err == nil {
		add := true
		for _, name := range r.byid[id].BigDataNames {
			if name == key {
				add = false
				break
			}
		}
		if add {
			r.byid[id].BigDataNames = append(r.byid[id].BigDataNames, key)
			err = r.Save()
		}
	}
	return err
}

func (r *containerStore) Wipe() error {
	ids := []string{}
	for id := range r.byid {
		ids = append(ids, id)
	}
	for _, id := range ids {
		if err := r.Delete(id); err != nil {
			return err
		}
	}
	return nil
}

func (r *containerStore) Lock() {
	r.lockfile.Lock()
}

func (r *containerStore) Unlock() {
	r.lockfile.Unlock()
}

func (r *containerStore) Touch() error {
	return r.lockfile.Touch()
}

func (r *containerStore) Modified() (bool, error) {
	return r.lockfile.Modified()
}

func (r *containerStore) TouchedSince(when time.Time) bool {
	return r.lockfile.TouchedSince(when)
}
<|endoftext|>


<commit_before>package xml

import (
	"bytes"
	_ "github.com/crabmusket/gosunspec/models/model101"
	"testing"
)

// Examples from the SunSpec Data Exchange Specification version 1.2.
// http://sunspec.org/wp-content/uploads/2015/06/SunSpec-Model-Data-Exchange-12021.pdf

// Page 10
const example = `
<SunSpecData v="1">
	<d ns="mac" lid="11:22:33:44:55:66" man="gsc" mod="r300" sn="123456" t="2011-05-12T09:21:49Z" cid="2">
		<m id="1">

		</m>
		<m id="101" x="1">

3043

2216

-1

6701.3

60.01

126973

14.28

469

6805

32.94

4

		</m>
	</d>
</SunSpecData>
`

func TestXmlParse(t *testing.T) {
	buffer := bytes.NewBuffer([]byte(example))
	data, err := parseXML(buffer)
	if err != nil {
		t.Fatal("could not parse example", err.Error())
	}
	data0 := data
	if data0.Version != "1" {
		t.Error("wrong version found")
	}
	if len(data0.Devices) != 1 {
		t.Fatal("wrong number of devices found")
	}
	if len(data0.Devices[0].Models) != 2 {
		t.Fatal("wrong number of models found")
	}
	if len(data0.Devices[0].Models[1].Points) != 11 {
		t.Fatal("wrong number of points found")
	}
	if id := data0.Devices[0].Models[1].Points[0].Id; id != "A" {
		t.Error("wrong id in first point:", id)
	}
	if value := data0.Devices[0].Models[1].Points[0].Value; value != "3043" {
		t.Error("wrong value in first point:", value)
	}
	if scale := data0.Devices[0].Models[1].Points[0].ScaleFactor; scale != -2 {
		t.Error("wrong scale factor in first point:", scale)
	}
	if units := data0.Devices[0].Models[1].Points[3].Unit; units != "Watts" {
		t.Error("wrong units in third point:", units)
	}
}
<commit_msg>fix test: 6701.3 can't be represented in an int16 with a scale factor of -1.<commit_after>package xml

import (
	"bytes"
	_ "github.com/crabmusket/gosunspec/models/model101"
	"testing"
)

// Examples from the SunSpec Data Exchange Specification version 1.2.
// http://sunspec.org/wp-content/uploads/2015/06/SunSpec-Model-Data-Exchange-12021.pdf

// Page 10
const example = `
<SunSpecData v="1">
	<d ns="mac" lid="11:22:33:44:55:66" man="gsc" mod="r300" sn="123456" t="2011-05-12T09:21:49Z" cid="2">
		<m id="1">

		</m>
		<m id="101" x="1">

3043

2216

-1

6501.3

60.01

126973

14.28

469

6805

32.94

4

		</m>
	</d>
</SunSpecData>
`

func TestXmlParse(t *testing.T) {
	buffer := bytes.NewBuffer([]byte(example))
	data, err := parseXML(buffer)
	if err != nil {
		t.Fatal("could not parse example", err.Error())
	}
	data0 := data
	if data0.Version != "1" {
		t.Error("wrong version found")
	}
	if len(data0.Devices) != 1 {
		t.Fatal("wrong number of devices found")
	}
	if len(data0.Devices[0].Models) != 2 {
		t.Fatal("wrong number of models found")
	}
	if len(data0.Devices[0].Models[1].Points) != 11 {
		t.Fatal("wrong number of points found")
	}
	if id := data0.Devices[0].Models[1].Points[0].Id; id != "A" {
		t.Error("wrong id in first point:", id)
	}
	if value := data0.Devices[0].Models[1].Points[0].Value; value != "3043" {
		t.Error("wrong value in first point:", value)
	}
	if scale := data0.Devices[0].Models[1].Points[0].ScaleFactor; scale != -2 {
		t.Error("wrong scale factor in first point:", scale)
	}
	if units := data0.Devices[0].Models[1].Points[3].Unit; units != "Watts" {
		t.Error("wrong units in third point:", units)
	}
}
<|endoftext|>