<commit_before>package py

/*
#include "Python.h"
#include "datetime.h"

int IsPyTypeTrue(PyObject *o) {
  return o == Py_True;
}

int IsPyTypeFalse(PyObject *o) {
  return o == Py_False;
}

int IsPyTypeInt(PyObject *o) {
  return PyInt_CheckExact(o);
}

int IsPyTypeFloat(PyObject *o) {
  return PyFloat_CheckExact(o);
}

int IsPyTypeByteArray(PyObject *o) {
  return PyByteArray_CheckExact(o);
}

int IsPyTypeString(PyObject *o) {
  return PyString_CheckExact(o);
}

int IsPyTypeList(PyObject *o) {
  return PyList_CheckExact(o);
}

int IsPyTypeDict(PyObject *o) {
  return PyDict_CheckExact(o);
}

int IsPyTypeTuple(PyObject *o) {
  return PyTuple_CheckExact(o);
}

int IsPyTypeNone(PyObject *o) {
  return o == Py_None;
}

int IsPyTypeUnicode(PyObject *o) {
  return PyUnicode_CheckExact(o);
}

PyTypeObject* GetTypeObject(PyObject *o) {
  return (PyTypeObject*)PyObject_Type(o);
}

const char* GetTypeName(PyTypeObject *t) {
  return t->tp_name;
}

void DecRefTypeObject(PyTypeObject *t) {
  Py_DECREF(t);
}
*/
import "C"
import (
	"fmt"
	"gopkg.in/sensorbee/sensorbee.v0/data"
	"time"
	"unsafe"
)

func fromPyTypeObject(o *C.PyObject) (data.Value, error) {
	switch {
	case C.IsPyTypeTrue(o) > 0:
		return data.Bool(true), nil

	case C.IsPyTypeFalse(o) > 0:
		return data.Bool(false), nil

	case C.IsPyTypeInt(o) > 0:
		return data.Int(C.PyInt_AsLong(o)), nil

	case C.IsPyTypeFloat(o) > 0:
		return data.Float(C.PyFloat_AsDouble(o)), nil

	case C.IsPyTypeByteArray(o) > 0:
		bytePtr := C.PyByteArray_FromObject(o)
		charPtr := C.PyByteArray_AsString(bytePtr)
		l := C.PyByteArray_Size(o)
		return data.Blob(C.GoBytes(unsafe.Pointer(charPtr), C.int(l))), nil

	case C.IsPyTypeString(o) > 0:
		size := C.int(C.PyString_Size(o))
		charPtr := C.PyString_AsString(o)
		return data.String(string(C.GoBytes(unsafe.Pointer(charPtr), size))), nil

	case C.IsPyTypeUnicode(o) > 0:
		// Use unicode string as UTF-8 in py because
		// Go's source code is defined to be UTF-8 text and string literal is too.
		utf8 := C.CString("UTF-8")
		defer C.free(unsafe.Pointer(utf8))

		strObj := C.PyUnicode_AsEncodedString(o, utf8, nil)
		if strObj == nil {
			return data.Null{}, getPyErr()
		}
		str := Object{p: strObj}
		defer str.decRef()

		return fromPyTypeObject(str.p)

	case isPyTypeDateTime(o):
		return fromTimestamp(o), nil

	case C.IsPyTypeList(o) > 0:
		return fromPyArray(o)

	case C.IsPyTypeDict(o) > 0:
		return fromPyMap(o)

	case C.IsPyTypeTuple(o) > 0:
		return fromPyTuple(o)

	case C.IsPyTypeNone(o) > 0:
		return data.Null{}, nil

	}

	t := C.GetTypeObject(o)
	tn := C.GoString(C.GetTypeName(t))
	defer C.DecRefTypeObject(t)
	return data.Null{}, fmt.Errorf("unsupported type in sensorbee/py: %v", tn)
}

func fromPyArray(ls *C.PyObject) (data.Array, error) {
	size := int(C.PyList_Size(ls))
	array := make(data.Array, size)
	for i := 0; i < size; i++ {
		o := C.PyList_GetItem(ls, C.Py_ssize_t(i))
		v, err := fromPyTypeObject(o)
		if err != nil {
			return nil, err
		}
		array[i] = v
	}
	return array, nil
}

func fromPyMap(o *C.PyObject) (data.Map, error) {
	m := data.Map{}

	var key, value *C.PyObject
	pos := C.Py_ssize_t(C.int(0))

	for C.int(C.PyDict_Next(o, &pos, &key, &value)) > 0 {
		// data.Map's key is only allowed string or unicode
		if C.IsPyTypeString(key) == 0 && C.IsPyTypeUnicode(key) == 0 {
			continue
		}
		k, _ := fromPyTypeObject(key)
		key, _ := data.ToString(k)
		v, err := fromPyTypeObject(value)
		if err != nil {
			return nil, err
		}
		m[key] = v
	}

	return m, nil
}

func fromPyTuple(o *C.PyObject) (data.Array, error) {
	size := int(C.PyTuple_Size(o))
	array := make(data.Array, size)
	for i := 0; i < size; i++ {
		o := C.PyTuple_GetItem(o, C.Py_ssize_t(i))
		v, err := fromPyTypeObject(o)
		if err != nil {
			return nil, err
		}
		array[i] = v
	}
	return array, nil
}

func fromTimestamp(o *C.PyObject) data.Timestamp {
	// FIXME: this internal code should not use
	d := (*C.PyDateTime_DateTime)(unsafe.Pointer(o))
	t := time.Date(int(d.data[0])<<8|int(d.data[1]), time.Month(int(d.data[2])+1),
		int(d.data[3]), int(d.data[4]), int(d.data[5]), int(d.data[6]),
		(int(d.data[7])<<16|int(d.data[8])<<8|int(d.data[9]))*1000,
		time.UTC)

	if d.hastzinfo <= 0 {
		return data.Timestamp(t)
	}

	return fromTimestampWithTimezone(o, t)
}

// fromTimestampWithTimezone converts into data.Timestamp with UTC time zone
// from datetime with tzinfo.  All of datetime passed to Go from Python API
// must be unified into UTC time zone by this function.
//
// This function calls `utcoffset` method to acquire offset from UTC for
// adjusting time zone.
func fromTimestampWithTimezone(o *C.PyObject, t time.Time) data.Timestamp {
	pyFunc, err := getPyFunc(o, "utcoffset")
	if err != nil {
		// Cannot get `utcoffset` function
		return data.Timestamp(t)
	}
	defer pyFunc.decRef()

	ret, err := pyFunc.callObject(Object{})
	if ret.p == nil && err != nil {
		// Failed to execute `utcoffset` function
		return data.Timestamp(t)
	}

	if !isPyTypeTimeDelta(ret.p) {
		// Cannot get `datetime.timedelta` instance
		return data.Timestamp(t)
	}

	// Adjust for time zone
	delta := (*C.PyDateTime_Delta)(unsafe.Pointer(ret.p))
	t = t.AddDate(0, 0, -int(delta.days))
	t = t.Add(time.Duration(-delta.seconds)*time.Second +
		time.Duration(-delta.microseconds)*time.Microsecond)
	return data.Timestamp(t)
}
<commit_msg>add nil check<commit_after>package py

/*
#include "Python.h"
#include "datetime.h"

int IsPyTypeTrue(PyObject *o) {
  return o == Py_True;
}

int IsPyTypeFalse(PyObject *o) {
  return o == Py_False;
}

int IsPyTypeInt(PyObject *o) {
  return PyInt_CheckExact(o);
}

int IsPyTypeFloat(PyObject *o) {
  return PyFloat_CheckExact(o);
}

int IsPyTypeByteArray(PyObject *o) {
  return PyByteArray_CheckExact(o);
}

int IsPyTypeString(PyObject *o) {
  return PyString_CheckExact(o);
}

int IsPyTypeList(PyObject *o) {
  return PyList_CheckExact(o);
}

int IsPyTypeDict(PyObject *o) {
  return PyDict_CheckExact(o);
}

int IsPyTypeTuple(PyObject *o) {
  return PyTuple_CheckExact(o);
}

int IsPyTypeNone(PyObject *o) {
  return o == Py_None;
}

int IsPyTypeUnicode(PyObject *o) {
  return PyUnicode_CheckExact(o);
}

PyTypeObject* GetTypeObject(PyObject *o) {
  return (PyTypeObject*)PyObject_Type(o);
}

const char* GetTypeName(PyTypeObject *t) {
  return t->tp_name;
}

void DecRefTypeObject(PyTypeObject *t) {
  Py_DECREF(t);
}
*/
import "C"
import (
	"fmt"
	"gopkg.in/sensorbee/sensorbee.v0/data"
	"time"
	"unsafe"
)

func fromPyTypeObject(o *C.PyObject) (data.Value, error) {
	switch {
	case C.IsPyTypeTrue(o) > 0:
		return data.Bool(true), nil

	case C.IsPyTypeFalse(o) > 0:
		return data.Bool(false), nil

	case C.IsPyTypeInt(o) > 0:
		return data.Int(C.PyInt_AsLong(o)), nil

	case C.IsPyTypeFloat(o) > 0:
		return data.Float(C.PyFloat_AsDouble(o)), nil

	case C.IsPyTypeByteArray(o) > 0:
		bytePtr := C.PyByteArray_FromObject(o)
		charPtr := C.PyByteArray_AsString(bytePtr)
		l := C.PyByteArray_Size(o)
		return data.Blob(C.GoBytes(unsafe.Pointer(charPtr), C.int(l))), nil

	case C.IsPyTypeString(o) > 0:
		size := C.int(C.PyString_Size(o))
		charPtr := C.PyString_AsString(o)
		return data.String(string(C.GoBytes(unsafe.Pointer(charPtr), size))), nil

	case C.IsPyTypeUnicode(o) > 0:
		// Use unicode string as UTF-8 in py because
		// Go's source code is defined to be UTF-8 text and string literal is too.
		utf8 := C.CString("UTF-8")
		defer C.free(unsafe.Pointer(utf8))

		strObj := C.PyUnicode_AsEncodedString(o, utf8, nil)
		if strObj == nil {
			return data.Null{}, getPyErr()
		}
		str := Object{p: strObj}
		defer str.decRef()

		return fromPyTypeObject(str.p)

	case isPyTypeDateTime(o):
		return fromTimestamp(o), nil

	case C.IsPyTypeList(o) > 0:
		return fromPyArray(o)

	case C.IsPyTypeDict(o) > 0:
		return fromPyMap(o)

	case C.IsPyTypeTuple(o) > 0:
		return fromPyTuple(o)

	case C.IsPyTypeNone(o) > 0:
		return data.Null{}, nil

	}

	t := C.GetTypeObject(o)
	if t == nil {
		return data.Null{}, fmt.Errorf("unsupported type in sensorbee/py (cannot detect python object type)")
	}
	tn := C.GoString(C.GetTypeName(t))
	defer C.DecRefTypeObject(t)
	return data.Null{}, fmt.Errorf("unsupported type in sensorbee/py: %v", tn)
}

func fromPyArray(ls *C.PyObject) (data.Array, error) {
	size := int(C.PyList_Size(ls))
	array := make(data.Array, size)
	for i := 0; i < size; i++ {
		o := C.PyList_GetItem(ls, C.Py_ssize_t(i))
		v, err := fromPyTypeObject(o)
		if err != nil {
			return nil, err
		}
		array[i] = v
	}
	return array, nil
}

func fromPyMap(o *C.PyObject) (data.Map, error) {
	m := data.Map{}

	var key, value *C.PyObject
	pos := C.Py_ssize_t(C.int(0))

	for C.int(C.PyDict_Next(o, &pos, &key, &value)) > 0 {
		// data.Map's key is only allowed string or unicode
		if C.IsPyTypeString(key) == 0 && C.IsPyTypeUnicode(key) == 0 {
			continue
		}
		k, _ := fromPyTypeObject(key)
		key, _ := data.ToString(k)
		v, err := fromPyTypeObject(value)
		if err != nil {
			return nil, err
		}
		m[key] = v
	}

	return m, nil
}

func fromPyTuple(o *C.PyObject) (data.Array, error) {
	size := int(C.PyTuple_Size(o))
	array := make(data.Array, size)
	for i := 0; i < size; i++ {
		o := C.PyTuple_GetItem(o, C.Py_ssize_t(i))
		v, err := fromPyTypeObject(o)
		if err != nil {
			return nil, err
		}
		array[i] = v
	}
	return array, nil
}

func fromTimestamp(o *C.PyObject) data.Timestamp {
	// FIXME: this internal code should not use
	d := (*C.PyDateTime_DateTime)(unsafe.Pointer(o))
	t := time.Date(int(d.data[0])<<8|int(d.data[1]), time.Month(int(d.data[2])+1),
		int(d.data[3]), int(d.data[4]), int(d.data[5]), int(d.data[6]),
		(int(d.data[7])<<16|int(d.data[8])<<8|int(d.data[9]))*1000,
		time.UTC)

	if d.hastzinfo <= 0 {
		return data.Timestamp(t)
	}

	return fromTimestampWithTimezone(o, t)
}

// fromTimestampWithTimezone converts into data.Timestamp with UTC time zone
// from datetime with tzinfo.  All of datetime passed to Go from Python API
// must be unified into UTC time zone by this function.
//
// This function calls `utcoffset` method to acquire offset from UTC for
// adjusting time zone.
func fromTimestampWithTimezone(o *C.PyObject, t time.Time) data.Timestamp {
	pyFunc, err := getPyFunc(o, "utcoffset")
	if err != nil {
		// Cannot get `utcoffset` function
		return data.Timestamp(t)
	}
	defer pyFunc.decRef()

	ret, err := pyFunc.callObject(Object{})
	if ret.p == nil && err != nil {
		// Failed to execute `utcoffset` function
		return data.Timestamp(t)
	}

	if !isPyTypeTimeDelta(ret.p) {
		// Cannot get `datetime.timedelta` instance
		return data.Timestamp(t)
	}

	// Adjust for time zone
	delta := (*C.PyDateTime_Delta)(unsafe.Pointer(ret.p))
	t = t.AddDate(0, 0, -int(delta.days))
	t = t.Add(time.Duration(-delta.seconds)*time.Second +
		time.Duration(-delta.microseconds)*time.Microsecond)
	return data.Timestamp(t)
}
<|endoftext|>


<commit_before>package migration

import (
	"fmt"
	"io"

	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/ioprogress"
	"github.com/lxc/lxd/shared/units"
)

// Type represents the migration transport type. It indicates the method by which the migration can
// take place and what optional features are available.
type Type struct {
	FSType   MigrationFSType // Transport mode selected.
	Features []string        // Feature hints for selected FSType transport mode.
}

// VolumeSourceArgs represents the arguments needed to setup a volume migration source.
type VolumeSourceArgs struct {
	Name          string
	Snapshots     []string
	MigrationType Type
	TrackProgress bool
	MultiSync     bool
	FinalSync     bool
	Data          interface{} // Optional store to persist storage driver state between MultiSync phases.
	ContentType   string
}

// VolumeTargetArgs represents the arguments needed to setup a volume migration sink.
type VolumeTargetArgs struct {
	Name          string
	Description   string
	Config        map[string]string
	Snapshots     []string
	MigrationType Type
	TrackProgress bool
	Refresh       bool
	Live          bool
	VolumeSize    int64
	ContentType   string
}

// TypesToHeader converts one or more Types to a MigrationHeader. It uses the first type argument
// supplied to indicate the preferred migration method and sets the MigrationHeader's Fs type
// to that. If the preferred type is ZFS then it will also set the header's optional ZfsFeatures.
// If the fallback Rsync type is present in any of the types even if it is not preferred, then its
// optional features are added to the header's RsyncFeatures, allowing for fallback negotiation to
// take place on the farside.
func TypesToHeader(types ...Type) MigrationHeader {
	missingFeature := false
	hasFeature := true
	var preferredType Type

	if len(types) > 0 {
		preferredType = types[0]
	}

	header := MigrationHeader{Fs: &preferredType.FSType}

	// Add ZFS features if preferred type is ZFS.
	if preferredType.FSType == MigrationFSType_ZFS {
		features := ZfsFeatures{
			Compress: &missingFeature,
		}
		for _, feature := range preferredType.Features {
			if feature == "compress" {
				features.Compress = &hasFeature
			}
		}

		header.ZfsFeatures = &features
	}

	// Add BTRFS features if preferred type is BTRFS.
	if preferredType.FSType == MigrationFSType_BTRFS {
		features := BtrfsFeatures{
			MigrationHeader:  &missingFeature,
			HeaderSubvolumes: &missingFeature,
		}
		for _, feature := range preferredType.Features {
			if feature == BTRFSFeatureMigrationHeader {
				features.MigrationHeader = &hasFeature
			} else if feature == BTRFSFeatureSubvolumes {
				features.HeaderSubvolumes = &hasFeature
			}
		}

		header.BtrfsFeatures = &features
	}

	// Check all the types for an Rsync method, if found add its features to the header's RsyncFeatures list.
	for _, t := range types {
		if t.FSType != MigrationFSType_RSYNC && t.FSType != MigrationFSType_BLOCK_AND_RSYNC {
			continue
		}

		features := RsyncFeatures{
			Xattrs:        &missingFeature,
			Delete:        &missingFeature,
			Compress:      &missingFeature,
			Bidirectional: &missingFeature,
		}

		for _, feature := range t.Features {
			if feature == "xattrs" {
				features.Xattrs = &hasFeature
			} else if feature == "delete" {
				features.Delete = &hasFeature
			} else if feature == "compress" {
				features.Compress = &hasFeature
			} else if feature == "bidirectional" {
				features.Bidirectional = &hasFeature
			}
		}

		header.RsyncFeatures = &features
		break // Only use the first rsync transport type found to generate rsync features list.
	}

	return header
}

// MatchTypes attempts to find matching migration transport types between an offered type sent from a remote
// source and the types supported by a local storage pool. If matches are found then one or more Types are
// returned containing the method and the matching optional features present in both. The function also takes a
// fallback type which is used as an additional offer type preference in case the preferred remote type is not
// compatible with the local type available. It is expected that both sides of the migration will support the
// fallback type for the volume's content type that is being migrated.
func MatchTypes(offer MigrationHeader, fallbackType MigrationFSType, ourTypes []Type) ([]Type, error) {
	// Generate an offer types slice from the preferred type supplied from remote and the
	// fallback type supplied based on the content type of the transfer.
	offeredFSTypes := []MigrationFSType{offer.GetFs(), fallbackType}

	matchedTypes := []Type{}

	// Find first matching type.
	for _, ourType := range ourTypes {
		for _, offerFSType := range offeredFSTypes {
			if offerFSType != ourType.FSType {
				continue // Not a match, try the next one.
			}

			// We got a match, now extract the relevant offered features.
			var offeredFeatures []string
			if offerFSType == MigrationFSType_ZFS {
				offeredFeatures = offer.GetZfsFeaturesSlice()
			} else if offerFSType == MigrationFSType_BTRFS {
				offeredFeatures = offer.GetBtrfsFeaturesSlice()
			} else if offerFSType == MigrationFSType_RSYNC {
				offeredFeatures = offer.GetRsyncFeaturesSlice()
				if !shared.StringInSlice("bidirectional", offeredFeatures) {
					// If no bi-directional support, this means we are getting a response from
					// an old LXD server that doesn't support bidirectional negotiation, so
					// assume LXD 3.7 level. NOTE: Do NOT extend this list of arguments.
					offeredFeatures = []string{"xattrs", "delete", "compress"}
				}
			}

			// Find common features in both our type and offered type.
			commonFeatures := []string{}
			for _, ourFeature := range ourType.Features {
				if shared.StringInSlice(ourFeature, offeredFeatures) {
					commonFeatures = append(commonFeatures, ourFeature)
				}
			}

			// Append type with combined features.
			matchedTypes = append(matchedTypes, Type{
				FSType:   ourType.FSType,
				Features: commonFeatures,
			})
		}
	}

	if len(matchedTypes) < 1 {
		// No matching transport type found, generate an error with offered types and our types.
		offeredTypeStrings := make([]string, 0, len(offeredFSTypes))
		for _, offerFSType := range offeredFSTypes {
			offeredTypeStrings = append(offeredTypeStrings, offerFSType.String())
		}

		ourTypeStrings := make([]string, 0, len(ourTypes))
		for _, ourType := range ourTypes {
			ourTypeStrings = append(ourTypeStrings, ourType.FSType.String())
		}

		return matchedTypes, fmt.Errorf("No matching migration types found. Offered types: %v, our types: %v", offeredTypeStrings, ourTypeStrings)
	}

	return matchedTypes, nil
}

func progressWrapperRender(op *operations.Operation, key string, description string, progressInt int64, speedInt int64) {
	meta := op.Metadata()
	if meta == nil {
		meta = make(map[string]interface{})
	}

	progress := fmt.Sprintf("%s (%s/s)", units.GetByteSizeString(progressInt, 2), units.GetByteSizeString(speedInt, 2))
	if description != "" {
		progress = fmt.Sprintf("%s: %s (%s/s)", description, units.GetByteSizeString(progressInt, 2), units.GetByteSizeString(speedInt, 2))
	}

	if meta[key] != progress {
		meta[key] = progress
		op.UpdateMetadata(meta)
	}
}

// ProgressReader reports the read progress.
func ProgressReader(op *operations.Operation, key string, description string) func(io.ReadCloser) io.ReadCloser {
	return func(reader io.ReadCloser) io.ReadCloser {
		if op == nil {
			return reader
		}

		progress := func(progressInt int64, speedInt int64) {
			progressWrapperRender(op, key, description, progressInt, speedInt)
		}

		readPipe := &ioprogress.ProgressReader{
			ReadCloser: reader,
			Tracker: &ioprogress.ProgressTracker{
				Handler: progress,
			},
		}

		return readPipe
	}
}

// ProgressWriter reports the write progress.
func ProgressWriter(op *operations.Operation, key string, description string) func(io.WriteCloser) io.WriteCloser {
	return func(writer io.WriteCloser) io.WriteCloser {
		if op == nil {
			return writer
		}

		progress := func(progressInt int64, speedInt int64) {
			progressWrapperRender(op, key, description, progressInt, speedInt)
		}

		writePipe := &ioprogress.ProgressWriter{
			WriteCloser: writer,
			Tracker: &ioprogress.ProgressTracker{
				Handler: progress,
			},
		}

		return writePipe
	}
}

// ProgressTracker returns a migration I/O tracker
func ProgressTracker(op *operations.Operation, key string, description string) *ioprogress.ProgressTracker {
	progress := func(progressInt int64, speedInt int64) {
		progressWrapperRender(op, key, description, progressInt, speedInt)
	}

	tracker := &ioprogress.ProgressTracker{
		Handler: progress,
	}

	return tracker
}
<commit_msg>lxd/migration/migration/volumes: Updates TypesToHeader and MatchTypes to use a pointer to MigrationHeader<commit_after>package migration

import (
	"fmt"
	"io"

	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/ioprogress"
	"github.com/lxc/lxd/shared/units"
)

// Type represents the migration transport type. It indicates the method by which the migration can
// take place and what optional features are available.
type Type struct {
	FSType   MigrationFSType // Transport mode selected.
	Features []string        // Feature hints for selected FSType transport mode.
}

// VolumeSourceArgs represents the arguments needed to setup a volume migration source.
type VolumeSourceArgs struct {
	Name          string
	Snapshots     []string
	MigrationType Type
	TrackProgress bool
	MultiSync     bool
	FinalSync     bool
	Data          interface{} // Optional store to persist storage driver state between MultiSync phases.
	ContentType   string
}

// VolumeTargetArgs represents the arguments needed to setup a volume migration sink.
type VolumeTargetArgs struct {
	Name          string
	Description   string
	Config        map[string]string
	Snapshots     []string
	MigrationType Type
	TrackProgress bool
	Refresh       bool
	Live          bool
	VolumeSize    int64
	ContentType   string
}

// TypesToHeader converts one or more Types to a MigrationHeader. It uses the first type argument
// supplied to indicate the preferred migration method and sets the MigrationHeader's Fs type
// to that. If the preferred type is ZFS then it will also set the header's optional ZfsFeatures.
// If the fallback Rsync type is present in any of the types even if it is not preferred, then its
// optional features are added to the header's RsyncFeatures, allowing for fallback negotiation to
// take place on the farside.
func TypesToHeader(types ...Type) *MigrationHeader {
	missingFeature := false
	hasFeature := true
	var preferredType Type

	if len(types) > 0 {
		preferredType = types[0]
	}

	header := MigrationHeader{Fs: &preferredType.FSType}

	// Add ZFS features if preferred type is ZFS.
	if preferredType.FSType == MigrationFSType_ZFS {
		features := ZfsFeatures{
			Compress: &missingFeature,
		}
		for _, feature := range preferredType.Features {
			if feature == "compress" {
				features.Compress = &hasFeature
			}
		}

		header.ZfsFeatures = &features
	}

	// Add BTRFS features if preferred type is BTRFS.
	if preferredType.FSType == MigrationFSType_BTRFS {
		features := BtrfsFeatures{
			MigrationHeader:  &missingFeature,
			HeaderSubvolumes: &missingFeature,
		}
		for _, feature := range preferredType.Features {
			if feature == BTRFSFeatureMigrationHeader {
				features.MigrationHeader = &hasFeature
			} else if feature == BTRFSFeatureSubvolumes {
				features.HeaderSubvolumes = &hasFeature
			}
		}

		header.BtrfsFeatures = &features
	}

	// Check all the types for an Rsync method, if found add its features to the header's RsyncFeatures list.
	for _, t := range types {
		if t.FSType != MigrationFSType_RSYNC && t.FSType != MigrationFSType_BLOCK_AND_RSYNC {
			continue
		}

		features := RsyncFeatures{
			Xattrs:        &missingFeature,
			Delete:        &missingFeature,
			Compress:      &missingFeature,
			Bidirectional: &missingFeature,
		}

		for _, feature := range t.Features {
			if feature == "xattrs" {
				features.Xattrs = &hasFeature
			} else if feature == "delete" {
				features.Delete = &hasFeature
			} else if feature == "compress" {
				features.Compress = &hasFeature
			} else if feature == "bidirectional" {
				features.Bidirectional = &hasFeature
			}
		}

		header.RsyncFeatures = &features
		break // Only use the first rsync transport type found to generate rsync features list.
	}

	return &header
}

// MatchTypes attempts to find matching migration transport types between an offered type sent from a remote
// source and the types supported by a local storage pool. If matches are found then one or more Types are
// returned containing the method and the matching optional features present in both. The function also takes a
// fallback type which is used as an additional offer type preference in case the preferred remote type is not
// compatible with the local type available. It is expected that both sides of the migration will support the
// fallback type for the volume's content type that is being migrated.
func MatchTypes(offer *MigrationHeader, fallbackType MigrationFSType, ourTypes []Type) ([]Type, error) {
	// Generate an offer types slice from the preferred type supplied from remote and the
	// fallback type supplied based on the content type of the transfer.
	offeredFSTypes := []MigrationFSType{offer.GetFs(), fallbackType}

	matchedTypes := []Type{}

	// Find first matching type.
	for _, ourType := range ourTypes {
		for _, offerFSType := range offeredFSTypes {
			if offerFSType != ourType.FSType {
				continue // Not a match, try the next one.
			}

			// We got a match, now extract the relevant offered features.
			var offeredFeatures []string
			if offerFSType == MigrationFSType_ZFS {
				offeredFeatures = offer.GetZfsFeaturesSlice()
			} else if offerFSType == MigrationFSType_BTRFS {
				offeredFeatures = offer.GetBtrfsFeaturesSlice()
			} else if offerFSType == MigrationFSType_RSYNC {
				offeredFeatures = offer.GetRsyncFeaturesSlice()
				if !shared.StringInSlice("bidirectional", offeredFeatures) {
					// If no bi-directional support, this means we are getting a response from
					// an old LXD server that doesn't support bidirectional negotiation, so
					// assume LXD 3.7 level. NOTE: Do NOT extend this list of arguments.
					offeredFeatures = []string{"xattrs", "delete", "compress"}
				}
			}

			// Find common features in both our type and offered type.
			commonFeatures := []string{}
			for _, ourFeature := range ourType.Features {
				if shared.StringInSlice(ourFeature, offeredFeatures) {
					commonFeatures = append(commonFeatures, ourFeature)
				}
			}

			// Append type with combined features.
			matchedTypes = append(matchedTypes, Type{
				FSType:   ourType.FSType,
				Features: commonFeatures,
			})
		}
	}

	if len(matchedTypes) < 1 {
		// No matching transport type found, generate an error with offered types and our types.
		offeredTypeStrings := make([]string, 0, len(offeredFSTypes))
		for _, offerFSType := range offeredFSTypes {
			offeredTypeStrings = append(offeredTypeStrings, offerFSType.String())
		}

		ourTypeStrings := make([]string, 0, len(ourTypes))
		for _, ourType := range ourTypes {
			ourTypeStrings = append(ourTypeStrings, ourType.FSType.String())
		}

		return matchedTypes, fmt.Errorf("No matching migration types found. Offered types: %v, our types: %v", offeredTypeStrings, ourTypeStrings)
	}

	return matchedTypes, nil
}

func progressWrapperRender(op *operations.Operation, key string, description string, progressInt int64, speedInt int64) {
	meta := op.Metadata()
	if meta == nil {
		meta = make(map[string]interface{})
	}

	progress := fmt.Sprintf("%s (%s/s)", units.GetByteSizeString(progressInt, 2), units.GetByteSizeString(speedInt, 2))
	if description != "" {
		progress = fmt.Sprintf("%s: %s (%s/s)", description, units.GetByteSizeString(progressInt, 2), units.GetByteSizeString(speedInt, 2))
	}

	if meta[key] != progress {
		meta[key] = progress
		op.UpdateMetadata(meta)
	}
}

// ProgressReader reports the read progress.
func ProgressReader(op *operations.Operation, key string, description string) func(io.ReadCloser) io.ReadCloser {
	return func(reader io.ReadCloser) io.ReadCloser {
		if op == nil {
			return reader
		}

		progress := func(progressInt int64, speedInt int64) {
			progressWrapperRender(op, key, description, progressInt, speedInt)
		}

		readPipe := &ioprogress.ProgressReader{
			ReadCloser: reader,
			Tracker: &ioprogress.ProgressTracker{
				Handler: progress,
			},
		}

		return readPipe
	}
}

// ProgressWriter reports the write progress.
func ProgressWriter(op *operations.Operation, key string, description string) func(io.WriteCloser) io.WriteCloser {
	return func(writer io.WriteCloser) io.WriteCloser {
		if op == nil {
			return writer
		}

		progress := func(progressInt int64, speedInt int64) {
			progressWrapperRender(op, key, description, progressInt, speedInt)
		}

		writePipe := &ioprogress.ProgressWriter{
			WriteCloser: writer,
			Tracker: &ioprogress.ProgressTracker{
				Handler: progress,
			},
		}

		return writePipe
	}
}

// ProgressTracker returns a migration I/O tracker
func ProgressTracker(op *operations.Operation, key string, description string) *ioprogress.ProgressTracker {
	progress := func(progressInt int64, speedInt int64) {
		progressWrapperRender(op, key, description, progressInt, speedInt)
	}

	tracker := &ioprogress.ProgressTracker{
		Handler: progress,
	}

	return tracker
}
<|endoftext|>


<commit_before>package consensus

import (
	"errors"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/persist"
	"github.com/NebulousLabs/Sia/types"
	"github.com/boltdb/bolt"
)

var meta = persist.Metadata{
	Version: "0.4.0",
	Header:  "Consensus Set Database",
}

var (
	errBadSetInsert = errors.New("attempting to add an already existing item to the consensus set")
	errNilBucket    = errors.New("using a bucket that does not exist")
	errNilItem      = errors.New("requested item does not exist")
	errNotGuarded   = errors.New("database modification not protected by guard")
)

// setDB is a wrapper around the persist bolt db which backs the
// consensus set
type setDB struct {
	*persist.BoltDatabase
	// The open flag is used to prevent reading from the database
	// after closing sia when the loading loop is still running
	open bool // DEPRECATED
}

// openDB loads the set database and populates it with the necessary buckets
func openDB(filename string) (*setDB, error) {
	db, err := persist.OpenDatabase(meta, filename)
	if err != nil {
		return nil, err
	}

	var buckets []string = []string{
		"Path",
		"BlockMap",
		"Metadata",
	}

	// Create buckets
	err = db.Update(func(tx *bolt.Tx) error {
		for _, bucketName := range buckets {
			_, err := tx.CreateBucketIfNotExists([]byte(bucketName))
			if err != nil {
				return err
			}
		}
		// Initilize the consistency guards
		b := tx.Bucket([]byte("Metadata"))
		err := b.Put([]byte("GuardA"), encoding.Marshal(0))
		if err != nil {
			return err
		}
		return b.Put([]byte("GuardB"), encoding.Marshal(0))
	})
	return &setDB{db, true}, err
}

// startConsistencyGuard increments the first guard. If this is not
// equal to the second, a transaction is taking place in the database
func (db *setDB) startConsistencyGuard() {
	err := db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("Metadata"))
		var i int
		err := encoding.Unmarshal(b.Get([]byte("GuardA")), &i)
		if err != nil {
			return err
		}
		return b.Put([]byte("GuardA"), encoding.Marshal(i+1))
	})
	if err != nil {
		panic(err)
	}
}

// startConsistencyGuard increments the first guard. If this is not
// equal to the second, a transaction is taking place in the database
func (db *setDB) stopConsistencyGuard() {
	err := db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("Metadata"))
		var i int
		err := encoding.Unmarshal(b.Get([]byte("GuardB")), &i)
		if err != nil {
			return err
		}
		return b.Put([]byte("GuardB"), encoding.Marshal(i+1))
	})
	if err != nil {
		panic(err)
	}
}

// checkConsistencyGuard checks the two guards and returns true if
// they differ. This signifies that thaer there is a transaction
// taking place.
func (db *setDB) checkConsistencyGuard() bool {
	var guarded bool
	err := db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("Metadata"))
		var x, y int
		err := encoding.Unmarshal(b.Get([]byte("GuardA")), &x)
		if err != nil {
			return err
		}
		err = encoding.Unmarshal(b.Get([]byte("GuardB")), &y)
		if err != nil {
			return err
		}
		guarded = x != y
		return nil
	})
	if err != nil {
		panic(err)
	}
	return guarded
}

// addItem should only be called from this file, and adds a new item
// to the database
//
// addItem and getItem are part of consensus due to stricter error
// conditions than a generic bolt implementation
func (db *setDB) addItem(bucket string, key, value interface{}) error {
	// Check that this transaction is guarded by consensusGuard.
	// However, allow direct database modifications when testing
	if build.DEBUG && !db.checkConsistencyGuard() && build.Release != "testing" {
		panic(errNotGuarded)
	}
	v := encoding.Marshal(value)
	k := encoding.Marshal(key)
	return db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(bucket))
		// Sanity check: make sure the buckets exists and that
		// you are not inserting something that already exists
		if build.DEBUG {
			if b == nil {
				panic(errNilBucket)
			}
			i := b.Get(k)
			if i != nil {
				panic(errBadSetInsert)
			}
		}
		return b.Put(k, v)
	})
}

// getItem is a generic function to insert an item into the set database
func (db *setDB) getItem(bucket string, key interface{}) (item []byte, err error) {
	k := encoding.Marshal(key)
	err = db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(bucket))
		// Sanity check to make sure the bucket exists.
		if build.DEBUG {
			if b == nil {
				panic(errNilBucket)
			}
		}
		item = b.Get(k)
		// Sanity check to make sure the item requested exists
		if build.DEBUG {
			if item == nil {
				panic(errNilItem)
			}
		}
		return nil
	})
	return item, err
}

// rmItem removes an item from a bucket
func (db *setDB) rmItem(bucket string, key interface{}) error {
	if build.DEBUG && !db.checkConsistencyGuard() && build.Release != "testing" {
		panic(errNotGuarded)
	}
	k := encoding.Marshal(key)
	return db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(bucket))
		if build.DEBUG {
			// Sanity check to make sure the bucket exists.
			if b == nil {
				panic(errNilBucket)
			}
			// Sanity check to make sure you are deleting an item that exists
			item := b.Get(k)
			if item == nil {
				panic(errNilItem)
			}
		}
		return b.Delete(k)
	})
}

// inBucket checks if an item with the given key is in the bucket
func (db *setDB) inBucket(bucket string, key interface{}) bool {
	exists, err := db.Exists(bucket, encoding.Marshal(key))
	if build.DEBUG && err != nil {
		panic(err)
	}
	return exists
}

// lenBucket is a simple wrapper for bucketSize that panics on error
func (db *setDB) lenBucket(bucket String) uint64 {
	s, err := db.bucketSize(bucket)
	if build.DEBUG && err != nil {
		panic(err)
	}
	return s
}

// pushPath inserts a block into the database at the "end" of the chain, i.e.
// the current height + 1.
func (db *setDB) pushPath(bid types.BlockID) error {
	if build.DEBUG && !db.checkConsistencyGuard() && build.Release != "testing" {
		panic(errNotGuarded)
	}
	value := encoding.Marshal(bid)
	return db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("Path"))
		key := encoding.EncUint64(uint64(b.Stats().KeyN))
		return b.Put(key, value)
	})
}

// popPath removes a block from the "end" of the chain, i.e. the block
// with the largest height.
func (db *setDB) popPath() error {
	if build.DEBUG && !db.checkConsistencyGuard() && build.Release != "testing" {
		panic(errNotGuarded)
	}
	return db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("Path"))
		key := encoding.EncUint64(uint64(b.Stats().KeyN - 1))
		return b.Delete(key)
	})
}

// getPath retreives the block id of a block at a given hegiht from the path
func (db *setDB) getPath(h types.BlockHeight) (id types.BlockID) {
	idBytes, err := db.getItem("Path", h)
	if err != nil {
		panic(err)
	}
	err = encoding.Unmarshal(idBytes, &id)
	if err != nil {
		panic(err)
	}
	return
}

// pathHeight returns the size of the current path
func (db *setDB) pathHeight() types.BlockHeight {
	return db.lenBucket("Path")
}

// addBlockMap adds a processedBlock to the block map
// This will eventually take a processed block as an argument
func (db *setDB) addBlockMap(pb *processedBlock) error {
	return db.addItem("BlockMap", pb.Block.ID(), *pb)
}

// getBlockMap queries the set database to return a processedBlock
// with the given ID
func (db *setDB) getBlockMap(id types.BlockID) *processedBlock {
	bnBytes, err := db.getItem("BlockMap", id)
	if build.DEBUG && err != nil {
		panic(err)
	}
	var pb processedBlock
	err = encoding.Unmarshal(bnBytes, &pb)
	if build.DEBUG && err != nil {
		panic(err)
	}
	return &pb
}

// inBlockMap checks for the existance of a block with a given ID in
// the consensus set
func (db *setDB) inBlockMap(id types.BlockID) bool {
	return db.inBucket("BlockMap", id)
}

// rmBlockMap removes a processedBlock from the blockMap bucket
func (db *setDB) rmBlockMap(id types.BlockID) error {
	return db.rmItem("BlockMap", id)
}

// updateBlockMap is a wrapper function for modification of
func (db *setDB) updateBlockMap(pb *processedBlock) {
	// These errors will only be caused by an error by bolt
	// e.g. database being closed.
	err := db.rmBlockMap(pb.Block.ID())
	if build.DEBUG && err != nil {
		panic(err)
	}
	err = db.addBlockMap(pb)
	if build.DEBUG && err != nil {
		panic(err)
	}
}

// addSiafundOutputs is a wrapper around addItem for adding a siafundOutput.
func (db *setDB) addSiafundOutputs(id types.SiafundOutputID, output types.SiafundOutput) error {
	return db.addItem("SiafundOutputs", id, output)
}

// getSiafundOutputs is a wrapper around getItem which decodes the
// result into a siafundOutput
func (db *setDB) getSiafundOutputs(id types.SiafundOutputID) types.SiafundOutput {
	sfoBytes := db.getItem("SiafundOutputs", id)
	if build.DEBUG && err != nil {
		panic(err)
	}
	var sfo types.SiafundOutput
	err := encoding.Unmarshal(sfoBytes, &sfo)
	if build.DEBUG && err != nil {
		panic(err)
	}
	return sfo
}

// inSiafundOutputs is a wrapper around inBucket which returns a true
// if an output with the given id is in the database
func (db *setDB) inSiafundOutputs(id types.SiafundOutputID) bool {
	return db.inBucket("SiafundOutputs", id)
}

// nrmSiafundOutputs removes a siafund output from the database
func (db *setDB) rmSiafundOutputs(id types.SiafundOutputID) error {
	return db.rmItem("SiafundOutputs", id)
}

// lenSiafundOutputs returns the size of the siafundOutputs map
func (db *setDB) lenSiafundOutputs() uint64 {
	return lenBucket("SiafundOutputs")
}
<commit_msg>Create for each function for a bucket and sfo's<commit_after>package consensus

import (
	"errors"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/persist"
	"github.com/NebulousLabs/Sia/types"
	"github.com/boltdb/bolt"
)

var meta = persist.Metadata{
	Version: "0.4.0",
	Header:  "Consensus Set Database",
}

var (
	errBadSetInsert = errors.New("attempting to add an already existing item to the consensus set")
	errNilBucket    = errors.New("using a bucket that does not exist")
	errNilItem      = errors.New("requested item does not exist")
	errNotGuarded   = errors.New("database modification not protected by guard")
)

// setDB is a wrapper around the persist bolt db which backs the
// consensus set
type setDB struct {
	*persist.BoltDatabase
	// The open flag is used to prevent reading from the database
	// after closing sia when the loading loop is still running
	open bool // DEPRECATED
}

// openDB loads the set database and populates it with the necessary buckets
func openDB(filename string) (*setDB, error) {
	db, err := persist.OpenDatabase(meta, filename)
	if err != nil {
		return nil, err
	}

	var buckets []string = []string{
		"Path",
		"BlockMap",
		"Metadata",
	}

	// Create buckets
	err = db.Update(func(tx *bolt.Tx) error {
		for _, bucketName := range buckets {
			_, err := tx.CreateBucketIfNotExists([]byte(bucketName))
			if err != nil {
				return err
			}
		}
		// Initilize the consistency guards
		b := tx.Bucket([]byte("Metadata"))
		err := b.Put([]byte("GuardA"), encoding.Marshal(0))
		if err != nil {
			return err
		}
		return b.Put([]byte("GuardB"), encoding.Marshal(0))
	})
	return &setDB{db, true}, err
}

// startConsistencyGuard increments the first guard. If this is not
// equal to the second, a transaction is taking place in the database
func (db *setDB) startConsistencyGuard() {
	err := db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("Metadata"))
		var i int
		err := encoding.Unmarshal(b.Get([]byte("GuardA")), &i)
		if err != nil {
			return err
		}
		return b.Put([]byte("GuardA"), encoding.Marshal(i+1))
	})
	if err != nil {
		panic(err)
	}
}

// startConsistencyGuard increments the first guard. If this is not
// equal to the second, a transaction is taking place in the database
func (db *setDB) stopConsistencyGuard() {
	err := db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("Metadata"))
		var i int
		err := encoding.Unmarshal(b.Get([]byte("GuardB")), &i)
		if err != nil {
			return err
		}
		return b.Put([]byte("GuardB"), encoding.Marshal(i+1))
	})
	if err != nil {
		panic(err)
	}
}

// checkConsistencyGuard checks the two guards and returns true if
// they differ. This signifies that thaer there is a transaction
// taking place.
func (db *setDB) checkConsistencyGuard() bool {
	var guarded bool
	err := db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("Metadata"))
		var x, y int
		err := encoding.Unmarshal(b.Get([]byte("GuardA")), &x)
		if err != nil {
			return err
		}
		err = encoding.Unmarshal(b.Get([]byte("GuardB")), &y)
		if err != nil {
			return err
		}
		guarded = x != y
		return nil
	})
	if err != nil {
		panic(err)
	}
	return guarded
}

// addItem should only be called from this file, and adds a new item
// to the database
//
// addItem and getItem are part of consensus due to stricter error
// conditions than a generic bolt implementation
func (db *setDB) addItem(bucket string, key, value interface{}) error {
	// Check that this transaction is guarded by consensusGuard.
	// However, allow direct database modifications when testing
	if build.DEBUG && !db.checkConsistencyGuard() && build.Release != "testing" {
		panic(errNotGuarded)
	}
	v := encoding.Marshal(value)
	k := encoding.Marshal(key)
	return db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(bucket))
		// Sanity check: make sure the buckets exists and that
		// you are not inserting something that already exists
		if build.DEBUG {
			if b == nil {
				panic(errNilBucket)
			}
			i := b.Get(k)
			if i != nil {
				panic(errBadSetInsert)
			}
		}
		return b.Put(k, v)
	})
}

// getItem is a generic function to insert an item into the set database
func (db *setDB) getItem(bucket string, key interface{}) (item []byte, err error) {
	k := encoding.Marshal(key)
	err = db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(bucket))
		// Sanity check to make sure the bucket exists.
		if build.DEBUG {
			if b == nil {
				panic(errNilBucket)
			}
		}
		item = b.Get(k)
		// Sanity check to make sure the item requested exists
		if build.DEBUG {
			if item == nil {
				panic(errNilItem)
			}
		}
		return nil
	})
	return item, err
}

// rmItem removes an item from a bucket
func (db *setDB) rmItem(bucket string, key interface{}) error {
	if build.DEBUG && !db.checkConsistencyGuard() && build.Release != "testing" {
		panic(errNotGuarded)
	}
	k := encoding.Marshal(key)
	return db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(bucket))
		if build.DEBUG {
			// Sanity check to make sure the bucket exists.
			if b == nil {
				panic(errNilBucket)
			}
			// Sanity check to make sure you are deleting an item that exists
			item := b.Get(k)
			if item == nil {
				panic(errNilItem)
			}
		}
		return b.Delete(k)
	})
}

// inBucket checks if an item with the given key is in the bucket
func (db *setDB) inBucket(bucket string, key interface{}) bool {
	exists, err := db.Exists(bucket, encoding.Marshal(key))
	if build.DEBUG && err != nil {
		panic(err)
	}
	return exists
}

// lenBucket is a simple wrapper for bucketSize that panics on error
func (db *setDB) lenBucket(bucket string) uint64 {
	s, err := db.BucketSize(bucket)
	if build.DEBUG && err != nil {
		panic(err)
	}
	return s
}

// forEachInBucket runs a given function on every element in a given
// bucket name, and will panic on any error
func (db *setDB) forEachItem(bucket string, fn func(k, v []byte) error) {
	err := db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(bucket))
		if build.DEBUG && b == nil {
			panic(errNilBucket)
		}
		return b.ForEach(fn)
	})
	if err != nil {
		panic(err)
	}
}

// pushPath inserts a block into the database at the "end" of the chain, i.e.
// the current height + 1.
func (db *setDB) pushPath(bid types.BlockID) error {
	if build.DEBUG && !db.checkConsistencyGuard() && build.Release != "testing" {
		panic(errNotGuarded)
	}
	value := encoding.Marshal(bid)
	return db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("Path"))
		key := encoding.EncUint64(uint64(b.Stats().KeyN))
		return b.Put(key, value)
	})
}

// popPath removes a block from the "end" of the chain, i.e. the block
// with the largest height.
func (db *setDB) popPath() error {
	if build.DEBUG && !db.checkConsistencyGuard() && build.Release != "testing" {
		panic(errNotGuarded)
	}
	return db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("Path"))
		key := encoding.EncUint64(uint64(b.Stats().KeyN - 1))
		return b.Delete(key)
	})
}

// getPath retreives the block id of a block at a given hegiht from the path
func (db *setDB) getPath(h types.BlockHeight) (id types.BlockID) {
	idBytes, err := db.getItem("Path", h)
	if err != nil {
		panic(err)
	}
	err = encoding.Unmarshal(idBytes, &id)
	if err != nil {
		panic(err)
	}
	return
}

// pathHeight returns the size of the current path
func (db *setDB) pathHeight() types.BlockHeight {
	return types.BlockHeight(db.lenBucket("Path"))
}

// addBlockMap adds a processedBlock to the block map
// This will eventually take a processed block as an argument
func (db *setDB) addBlockMap(pb *processedBlock) error {
	return db.addItem("BlockMap", pb.Block.ID(), *pb)
}

// getBlockMap queries the set database to return a processedBlock
// with the given ID
func (db *setDB) getBlockMap(id types.BlockID) *processedBlock {
	bnBytes, err := db.getItem("BlockMap", id)
	if build.DEBUG && err != nil {
		panic(err)
	}
	var pb processedBlock
	err = encoding.Unmarshal(bnBytes, &pb)
	if build.DEBUG && err != nil {
		panic(err)
	}
	return &pb
}

// inBlockMap checks for the existance of a block with a given ID in
// the consensus set
func (db *setDB) inBlockMap(id types.BlockID) bool {
	return db.inBucket("BlockMap", id)
}

// rmBlockMap removes a processedBlock from the blockMap bucket
func (db *setDB) rmBlockMap(id types.BlockID) error {
	return db.rmItem("BlockMap", id)
}

// updateBlockMap is a wrapper function for modification of
func (db *setDB) updateBlockMap(pb *processedBlock) {
	// These errors will only be caused by an error by bolt
	// e.g. database being closed.
	err := db.rmBlockMap(pb.Block.ID())
	if build.DEBUG && err != nil {
		panic(err)
	}
	err = db.addBlockMap(pb)
	if build.DEBUG && err != nil {
		panic(err)
	}
}

// addSiafundOutputs is a wrapper around addItem for adding a siafundOutput.
func (db *setDB) addSiafundOutputs(id types.SiafundOutputID, output types.SiafundOutput) error {
	return db.addItem("SiafundOutputs", id, output)
}

// getSiafundOutputs is a wrapper around getItem which decodes the
// result into a siafundOutput
func (db *setDB) getSiafundOutputs(id types.SiafundOutputID) types.SiafundOutput {
	sfoBytes, err := db.getItem("SiafundOutputs", id)
	if build.DEBUG && err != nil {
		panic(err)
	}
	var sfo types.SiafundOutput
	err = encoding.Unmarshal(sfoBytes, &sfo)
	if build.DEBUG && err != nil {
		panic(err)
	}
	return sfo
}

// inSiafundOutputs is a wrapper around inBucket which returns a true
// if an output with the given id is in the database
func (db *setDB) inSiafundOutputs(id types.SiafundOutputID) bool {
	return db.inBucket("SiafundOutputs", id)
}

// nrmSiafundOutputs removes a siafund output from the database
func (db *setDB) rmSiafundOutputs(id types.SiafundOutputID) error {
	return db.rmItem("SiafundOutputs", id)
}

// lenSiafundOutputs returns the size of the siafundOutputs map
func (db *setDB) lenSiafundOutputs() uint64 {
	return db.lenBucket("SiafundOutputs")
}

func (db *setDB) forEachSiafundOutputs(fn func(k types.SiafundOutputID, v types.SiafundOutput) error) {
	db.forEachItem("SiafundOutputs", func(kb, vb []byte) error {
		var key types.SiafundOutputID
		var value types.SiafundOutput
		err := encoding.Unmarshal(kb, &key)
		if err != nil {
			return err
		}
		err = encoding.Unmarshal(vb, &value)
		if err != nil {
			return err
		}
		return fn(key, value)
	})
}
<|endoftext|>


<commit_before>package explorer

import (
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

// ProcessConsensusChange follows the most recent changes to the consensus set,
// including parsing new blocks and updating the utxo sets.
func (e *Explorer) ProcessConsensusChange(cc modules.ConsensusChange) {
	e.mu.Lock()
	defer e.mu.Unlock()

	// Update cumulative stats for reverted blocks.
	for _, block := range cc.RevertedBlocks {
		// Delete the block from the list of active blocks.
		e.blockchainHeight -= 1
		delete(e.blockHashes, block.ID())
		delete(e.transactionHashes, types.TransactionID(block.ID())) // Miner payouts are a transaction.

		// Catalog the removed miner payouts.
		for j, payout := range block.MinerPayouts {
			delete(e.siacoinOutputIDs[block.MinerPayoutID(uint64(j))], types.TransactionID(block.ID()))
			delete(e.unlockHashes[payout.UnlockHash], types.TransactionID(block.ID()))
		}

		// Update cumulative stats for reverted transcations.
		for _, txn := range block.Transactions {
			// Add the transction to the list of active transactions.
			e.transactionCount--
			delete(e.transactionHashes, txn.ID())

			for _, sci := range txn.SiacoinInputs {
				delete(e.siacoinOutputIDs[sci.ParentID], txn.ID())
				delete(e.unlockHashes[sci.UnlockConditions.UnlockHash()], txn.ID())
				e.siacoinInputCount--
			}
			for k, sco := range txn.SiacoinOutputs {
				delete(e.siacoinOutputIDs[txn.SiacoinOutputID(uint64(k))], txn.ID())
				delete(e.unlockHashes[sco.UnlockHash], txn.ID())
				e.siacoinOutputCount--
			}
			for k, fc := range txn.FileContracts {
				fcid := txn.FileContractID(uint64(k))
				delete(e.fileContractIDs[fcid], txn.ID())
				delete(e.unlockHashes[fc.UnlockHash], txn.ID())
				for l, sco := range fc.ValidProofOutputs {
					delete(e.siacoinOutputIDs[fcid.StorageProofOutputID(types.ProofValid, uint64(l))], txn.ID())
					delete(e.unlockHashes[sco.UnlockHash], txn.ID())
				}
				for l, sco := range fc.MissedProofOutputs {
					delete(e.siacoinOutputIDs[fcid.StorageProofOutputID(types.ProofMissed, uint64(l))], txn.ID())
					delete(e.unlockHashes[sco.UnlockHash], txn.ID())
				}
				e.fileContractCount--
				e.totalContractCost = e.totalContractCost.Sub(fc.Payout)
				e.totalContractSize = e.totalContractSize.Sub(types.NewCurrency64(fc.FileSize))
			}
			for _, fcr := range txn.FileContractRevisions {
				delete(e.fileContractIDs[fcr.ParentID], txn.ID())
				delete(e.unlockHashes[fcr.UnlockConditions.UnlockHash()], txn.ID())
				delete(e.unlockHashes[fcr.NewUnlockHash], txn.ID())
				for l, sco := range fcr.NewValidProofOutputs {
					delete(e.siacoinOutputIDs[fcr.ParentID.StorageProofOutputID(types.ProofValid, uint64(l))], txn.ID())
					delete(e.unlockHashes[sco.UnlockHash], txn.ID())
				}
				for l, sco := range fcr.NewMissedProofOutputs {
					delete(e.siacoinOutputIDs[fcr.ParentID.StorageProofOutputID(types.ProofMissed, uint64(l))], txn.ID())
					delete(e.unlockHashes[sco.UnlockHash], txn.ID())
				}
				e.fileContractRevisionCount--
				e.totalContractSize = e.totalContractSize.Sub(types.NewCurrency64(fcr.NewFileSize))
				e.totalRevisionVolume = e.totalRevisionVolume.Sub(types.NewCurrency64(fcr.NewFileSize))
			}
			for _, sp := range txn.StorageProofs {
				delete(e.fileContractIDs[sp.ParentID], txn.ID())
				e.storageProofCount--
			}
			for _, sfi := range txn.SiafundInputs {
				delete(e.siafundOutputIDs[sfi.ParentID], txn.ID())
				delete(e.unlockHashes[sfi.UnlockConditions.UnlockHash()], txn.ID())
				delete(e.unlockHashes[sfi.ClaimUnlockHash], txn.ID())
				e.siafundInputCount--
			}
			for k, sfo := range txn.SiafundOutputs {
				delete(e.siafundOutputIDs[txn.SiafundOutputID(uint64(k))], txn.ID())
				delete(e.unlockHashes[sfo.UnlockHash], txn.ID())
				e.siafundOutputCount--
			}
			for _ = range txn.MinerFees {
				e.minerFeeCount--
			}
			for _ = range txn.ArbitraryData {
				e.arbitraryDataCount--
			}
			for _ = range txn.TransactionSignatures {
				e.transactionSignatureCount--
			}
		}
	}

	// Update cumulative stats for applied blocks.
	for _, block := range cc.AppliedBlocks {
		// Add the block to the list of active blocks.
		e.blockchainHeight++
		e.blockHashes[block.ID()] = e.blockchainHeight
		e.transactionHashes[types.TransactionID(block.ID())] = e.blockchainHeight // Miner payouts are a transaciton.

		// Catalog the new miner payouts.
		for j, payout := range block.MinerPayouts {
			_, exists := e.siacoinOutputIDs[block.MinerPayoutID(uint64(j))]
			if !exists {
				e.siacoinOutputIDs[block.MinerPayoutID(uint64(j))] = make(map[types.TransactionID]struct{})
			}
			e.siacoinOutputIDs[block.MinerPayoutID(uint64(j))][types.TransactionID(block.ID())] = struct{}{}
			_, exists = e.unlockHashes[payout.UnlockHash]
			if !exists {
				e.unlockHashes[payout.UnlockHash] = make(map[types.TransactionID]struct{})
			}
			e.unlockHashes[payout.UnlockHash][types.TransactionID(block.ID())] = struct{}{}
		}

		// Update cumulative stats for applied transactions.
		for _, txn := range block.Transactions {
			// Add the transaction to the list of active transactions.
			e.transactionCount++
			e.transactionHashes[txn.ID()] = e.blockchainHeight

			for _, sci := range txn.SiacoinInputs {
				_, exists := e.siacoinOutputIDs[sci.ParentID]
				if !exists {
					e.siacoinOutputIDs[sci.ParentID] = make(map[types.TransactionID]struct{})
				}
				e.siacoinOutputIDs[sci.ParentID][txn.ID()] = struct{}{}
				_, exists = e.unlockHashes[sci.UnlockConditions.UnlockHash()]
				if !exists {
					e.unlockHashes[sci.UnlockConditions.UnlockHash()] = make(map[types.TransactionID]struct{})
				}
				e.unlockHashes[sci.UnlockConditions.UnlockHash()][txn.ID()] = struct{}{}
				e.siacoinInputCount++
			}
			for j, sco := range txn.SiacoinOutputs {
				_, exists := e.siacoinOutputIDs[txn.SiacoinOutputID(uint64(j))]
				if !exists {
					e.siacoinOutputIDs[txn.SiacoinOutputID(uint64(j))] = make(map[types.TransactionID]struct{})
				}
				e.siacoinOutputIDs[txn.SiacoinOutputID(uint64(j))][txn.ID()] = struct{}{}
				_, exists = e.unlockHashes[sco.UnlockHash]
				if !exists {
					e.unlockHashes[sco.UnlockHash] = make(map[types.TransactionID]struct{})
				}
				e.unlockHashes[sco.UnlockHash][txn.ID()] = struct{}{}
				e.siacoinOutputCount++
			}
			for _, fc := range txn.FileContracts {
				e.fileContractCount++
				e.totalContractCost = e.totalContractCost.Add(fc.Payout)
				e.totalContractSize = e.totalContractSize.Add(types.NewCurrency64(fc.FileSize))
			}
			for _, fcr := range txn.FileContractRevisions {
				e.fileContractRevisionCount++
				e.totalContractSize = e.totalContractSize.Add(types.NewCurrency64(fcr.NewFileSize))
				e.totalRevisionVolume = e.totalRevisionVolume.Add(types.NewCurrency64(fcr.NewFileSize))
			}
			for _ = range txn.StorageProofs {
				e.storageProofCount++
			}
			for _ = range txn.SiafundInputs {
				e.siafundInputCount++
			}
			for _ = range txn.SiafundOutputs {
				e.siafundOutputCount++
			}
			for _ = range txn.MinerFees {
				e.minerFeeCount++
			}
			for _ = range txn.ArbitraryData {
				e.arbitraryDataCount++
			}
			for _ = range txn.TransactionSignatures {
				e.transactionSignatureCount++
			}
		}
	}

	// Compute the changes in the active set.
	for _, diff := range cc.FileContractDiffs {
		if diff.Direction == modules.DiffApply {
			e.activeContractCount += 1
			e.activeContractCost = e.activeContractCost.Add(diff.FileContract.Payout)
			e.activeContractSize = e.activeContractSize.Add(types.NewCurrency64(diff.FileContract.FileSize))
		} else {
			e.activeContractCount -= 1
			e.activeContractCost = e.activeContractCost.Sub(diff.FileContract.Payout)
			e.activeContractSize = e.activeContractSize.Sub(types.NewCurrency64(diff.FileContract.FileSize))
		}
	}

	// Set the id of the current block.
	e.currentBlock = cc.AppliedBlocks[len(cc.AppliedBlocks)-1].ID()
}
<commit_msg>finish hash lookups<commit_after>package explorer

import (
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

// ProcessConsensusChange follows the most recent changes to the consensus set,
// including parsing new blocks and updating the utxo sets.
func (e *Explorer) ProcessConsensusChange(cc modules.ConsensusChange) {
	e.mu.Lock()
	defer e.mu.Unlock()

	// Update cumulative stats for reverted blocks.
	for _, block := range cc.RevertedBlocks {
		// Delete the block from the list of active blocks.
		bid := block.ID()
		tbid := types.TransactionID(bid)
		e.blockchainHeight -= 1
		delete(e.blockHashes, bid)
		delete(e.transactionHashes, tbid)// Miner payouts are a transaction.

		// Catalog the removed miner payouts.
		for j, payout := range block.MinerPayouts {
			scoid := block.MinerPayoutID(uint64(j))
			delete(e.siacoinOutputIDs[scoid], tbid)
			delete(e.unlockHashes[payout.UnlockHash], tbid)
		}

		// Update cumulative stats for reverted transcations.
		for _, txn := range block.Transactions {
			// Add the transction to the list of active transactions.
			txid := txn.ID()
			e.transactionCount--
			delete(e.transactionHashes, txid)

			for _, sci := range txn.SiacoinInputs {
				delete(e.siacoinOutputIDs[sci.ParentID], txid)
				delete(e.unlockHashes[sci.UnlockConditions.UnlockHash()], txid)
				e.siacoinInputCount--
			}
			for k, sco := range txn.SiacoinOutputs {
				delete(e.siacoinOutputIDs[txn.SiacoinOutputID(uint64(k))], txid)
				delete(e.unlockHashes[sco.UnlockHash], txid)
				e.siacoinOutputCount--
			}
			for k, fc := range txn.FileContracts {
				fcid := txn.FileContractID(uint64(k))
				delete(e.fileContractIDs[fcid], txid)
				delete(e.unlockHashes[fc.UnlockHash], txid)
				for l, sco := range fc.ValidProofOutputs {
					scoid := fcid.StorageProofOutputID(types.ProofValid, uint64(l))
					delete(e.siacoinOutputIDs[scoid], txid)
					delete(e.unlockHashes[sco.UnlockHash], txid)
				}
				for l, sco := range fc.MissedProofOutputs {
					scoid := fcid.StorageProofOutputID(types.ProofMissed, uint64(l))
					delete(e.siacoinOutputIDs[scoid], txid)
					delete(e.unlockHashes[sco.UnlockHash], txid)
				}
				e.fileContractCount--
				e.totalContractCost = e.totalContractCost.Sub(fc.Payout)
				e.totalContractSize = e.totalContractSize.Sub(types.NewCurrency64(fc.FileSize))
			}
			for _, fcr := range txn.FileContractRevisions {
				delete(e.fileContractIDs[fcr.ParentID], txid)
				delete(e.unlockHashes[fcr.UnlockConditions.UnlockHash()], txid)
				delete(e.unlockHashes[fcr.NewUnlockHash], txid)
				for l, sco := range fcr.NewValidProofOutputs {
					scoid := fcr.ParentID.StorageProofOutputID(types.ProofValid, uint64(l))
					delete(e.siacoinOutputIDs[scoid], txid)
					delete(e.unlockHashes[sco.UnlockHash], txid)
				}
				for l, sco := range fcr.NewMissedProofOutputs {
					scoid := fcr.ParentID.StorageProofOutputID(types.ProofMissed, uint64(l))
					delete(e.siacoinOutputIDs[scoid], txid)
					delete(e.unlockHashes[sco.UnlockHash], txid)
				}
				e.fileContractRevisionCount--
				e.totalContractSize = e.totalContractSize.Sub(types.NewCurrency64(fcr.NewFileSize))
				e.totalRevisionVolume = e.totalRevisionVolume.Sub(types.NewCurrency64(fcr.NewFileSize))
			}
			for _, sp := range txn.StorageProofs {
				delete(e.fileContractIDs[sp.ParentID], txid)
				e.storageProofCount--
			}
			for _, sfi := range txn.SiafundInputs {
				delete(e.siafundOutputIDs[sfi.ParentID], txid)
				delete(e.unlockHashes[sfi.UnlockConditions.UnlockHash()], txid)
				delete(e.unlockHashes[sfi.ClaimUnlockHash], txid)
				e.siafundInputCount--
			}
			for k, sfo := range txn.SiafundOutputs {
				sfoid := txn.SiafundOutputID(uint64(k))
				delete(e.siafundOutputIDs[sfoid], txid)
				delete(e.unlockHashes[sfo.UnlockHash], txid)
				e.siafundOutputCount--
			}
			for _ = range txn.MinerFees {
				e.minerFeeCount--
			}
			for _ = range txn.ArbitraryData {
				e.arbitraryDataCount--
			}
			for _ = range txn.TransactionSignatures {
				e.transactionSignatureCount--
			}
		}
	}

	// Update cumulative stats for applied blocks.
	for _, block := range cc.AppliedBlocks {
		// Add the block to the list of active blocks.
		bid := block.ID()
		tbid := types.TransactionID(bid)
		e.blockchainHeight++
		e.blockHashes[bid] = e.blockchainHeight
		e.transactionHashes[tbid] = e.blockchainHeight // Miner payouts are a transaciton.

		// Catalog the new miner payouts.
		for j, payout := range block.MinerPayouts {
			scoid := block.MinerPayoutID(uint64(j))
			_, exists := e.siacoinOutputIDs[scoid]
			if !exists {
				e.siacoinOutputIDs[scoid] = make(map[types.TransactionID]struct{})
			}
			e.siacoinOutputIDs[scoid][tbid] = struct{}{}
			_, exists = e.unlockHashes[payout.UnlockHash]
			if !exists {
				e.unlockHashes[payout.UnlockHash] = make(map[types.TransactionID]struct{})
			}
			e.unlockHashes[payout.UnlockHash][tbid] = struct{}{}
		}

		// Update cumulative stats for applied transactions.
		for _, txn := range block.Transactions {
			// Add the transaction to the list of active transactions.
			txid := txn.ID()
			e.transactionCount++
			e.transactionHashes[txid] = e.blockchainHeight

			for _, sci := range txn.SiacoinInputs {
				_, exists := e.siacoinOutputIDs[sci.ParentID]
				if !exists {
					panic("siacoin input without siacoin output")
				}
				e.siacoinOutputIDs[sci.ParentID][txid] = struct{}{}
				_, exists = e.unlockHashes[sci.UnlockConditions.UnlockHash()]
				if !exists {
					panic("unlock conditions without a parent unlock hash")
				}
				e.unlockHashes[sci.UnlockConditions.UnlockHash()][txid] = struct{}{}
				e.siacoinInputCount++
			}
			for j, sco := range txn.SiacoinOutputs {
				scoid := txn.SiacoinOutputID(uint64(j))
				_, exists := e.siacoinOutputIDs[scoid]
				if !exists {
					e.siacoinOutputIDs[scoid] = make(map[types.TransactionID]struct{})
				}
				e.siacoinOutputIDs[scoid][txid] = struct{}{}
				_, exists = e.unlockHashes[sco.UnlockHash]
				if !exists {
					e.unlockHashes[sco.UnlockHash] = make(map[types.TransactionID]struct{})
				}
				e.unlockHashes[sco.UnlockHash][txn.ID()] = struct{}{}
				e.siacoinOutputCount++
			}
			for k, fc := range txn.FileContracts {
				fcid := txn.FileContractID(uint64(k))
				_, exists := e.fileContractIDs[fcid]
				if !exists {
					e.fileContractIDs[fcid] = make(map[types.TransactionID]struct{})
				}
				e.fileContractIDs[fcid][txid] = struct{}{}
				_, exists = e.unlockHashes[fc.UnlockHash]
				if !exists {
					e.unlockHashes[fc.UnlockHash] = make(map[types.TransactionID]struct{})
				}
				e.unlockHashes[fc.UnlockHash][txid] = struct{}{}
				for l, sco := range fc.ValidProofOutputs {
					scoid := fcid.StorageProofOutputID(types.ProofValid, uint64(l))
					_, exists = e.siacoinOutputIDs[scoid]
					if !exists {
						e.siacoinOutputIDs[scoid] = make(map[types.TransactionID]struct{})
					}
					e.siacoinOutputIDs[scoid][txid] = struct{}{}
					_, exists = e.unlockHashes[sco.UnlockHash]
					if !exists {
						e.unlockHashes[sco.UnlockHash] = make(map[types.TransactionID]struct{})
					}
					e.unlockHashes[sco.UnlockHash][txid] = struct{}{}
				}
				for l, sco := range fc.MissedProofOutputs {
					scoid := fcid.StorageProofOutputID(types.ProofMissed, uint64(l))
					_, exists = e.siacoinOutputIDs[scoid]
					if !exists {
						e.siacoinOutputIDs[scoid] = make(map[types.TransactionID]struct{})
					}
					e.siacoinOutputIDs[scoid][txid] = struct{}{}
					_, exists = e.unlockHashes[sco.UnlockHash]
					if !exists {
						e.unlockHashes[sco.UnlockHash] = make(map[types.TransactionID]struct{})
					}
					e.unlockHashes[sco.UnlockHash][txid] = struct{}{}
				}
				e.fileContractCount++
				e.totalContractCost = e.totalContractCost.Add(fc.Payout)
				e.totalContractSize = e.totalContractSize.Add(types.NewCurrency64(fc.FileSize))
			}
			for _, fcr := range txn.FileContractRevisions {
				_, exists := e.fileContractIDs[fcr.ParentID]
				if !exists {
					panic("revision without entry in file contract list")
				}
				e.fileContractIDs[fcr.ParentID][txid] = struct{}{}
				_, exists = e.unlockHashes[fcr.UnlockConditions.UnlockHash()]
				if !exists {
					panic("unlock conditions without unlock hash")
				}
				e.unlockHashes[fcr.UnlockConditions.UnlockHash()][txid] = struct{}{}
				_, exists = e.unlockHashes[fcr.NewUnlockHash]
				if !exists {
					e.unlockHashes[fcr.NewUnlockHash] = make(map[types.TransactionID]struct{})
				}
				e.unlockHashes[fcr.NewUnlockHash][txid] = struct{}{}
				for l, sco := range fcr.NewValidProofOutputs {
					scoid := fcr.ParentID.StorageProofOutputID(types.ProofValid, uint64(l))
					_, exists = e.siacoinOutputIDs[scoid]
					if !exists {
						e.siacoinOutputIDs[scoid] = make(map[types.TransactionID]struct{})
					}
					e.siacoinOutputIDs[scoid][txid] = struct{}{}
					_, exists = e.unlockHashes[sco.UnlockHash]
					if !exists {
						e.unlockHashes[sco.UnlockHash] = make(map[types.TransactionID]struct{})
					}
					e.unlockHashes[sco.UnlockHash][txid] = struct{}{}
				}
				for l, sco := range fcr.NewMissedProofOutputs {
					scoid := fcr.ParentID.StorageProofOutputID(types.ProofMissed, uint64(l))
					_, exists = e.siacoinOutputIDs[scoid]
					if !exists {
						e.siacoinOutputIDs[scoid] = make(map[types.TransactionID]struct{})
					}
					e.siacoinOutputIDs[scoid][txid] = struct{}{}
					_, exists = e.unlockHashes[sco.UnlockHash]
					if !exists {
						e.unlockHashes[sco.UnlockHash] = make(map[types.TransactionID]struct{})
					}
					e.unlockHashes[sco.UnlockHash][txid] = struct{}{}
				}
				e.fileContractRevisionCount++
				e.totalContractSize = e.totalContractSize.Add(types.NewCurrency64(fcr.NewFileSize))
				e.totalRevisionVolume = e.totalRevisionVolume.Add(types.NewCurrency64(fcr.NewFileSize))
			}
			for _, sp := range txn.StorageProofs {
				_, exists := e.fileContractIDs[sp.ParentID]
				if !exists {
					panic("storage proof without file contract parent")
				}
				e.fileContractIDs[sp.ParentID][txid] = struct{}{}
				e.storageProofCount++
			}
			for _, sfi := range txn.SiafundInputs {
				_, exists := e.siafundOutputIDs[sfi.ParentID]
				if !exists {
					panic("siafund input without corresponding output")
				}
				e.siafundOutputIDs[sfi.ParentID][txid] = struct{}{}
				_, exists = e.unlockHashes[sfi.UnlockConditions.UnlockHash()]
				if !exists {
					panic("unlock conditions without unlock hash")
				}
				e.unlockHashes[sfi.UnlockConditions.UnlockHash()][txid] = struct{}{}
				_, exists = e.unlockHashes[sfi.ClaimUnlockHash]
				if !exists {
					e.unlockHashes[sfi.ClaimUnlockHash] = make(map[types.TransactionID]struct{})
				}
				e.unlockHashes[sfi.ClaimUnlockHash][txid] = struct{}{}
				e.siafundInputCount++
			}
			for k, sfo := range txn.SiafundOutputs {
				sfoid := txn.SiafundOutputID(uint64(k))
				_, exists := e.siafundOutputIDs[sfoid]
				if !exists {
					e.siafundOutputIDs[sfoid] = make(map[types.TransactionID]struct{})
				}
				e.siafundOutputIDs[sfoid][txid] = struct{}{}
				_, exists = e.unlockHashes[sfo.UnlockHash]
				if !exists {
					e.unlockHashes[sfo.UnlockHash] = make(map[types.TransactionID]struct{})
				}
				e.unlockHashes[sfo.UnlockHash][txid] = struct{}{}
				e.siafundOutputCount++
			}
			for _ = range txn.MinerFees {
				e.minerFeeCount++
			}
			for _ = range txn.ArbitraryData {
				e.arbitraryDataCount++
			}
			for _ = range txn.TransactionSignatures {
				e.transactionSignatureCount++
			}
		}
	}

	// Compute the changes in the active set.
	for _, diff := range cc.FileContractDiffs {
		if diff.Direction == modules.DiffApply {
			e.activeContractCount += 1
			e.activeContractCost = e.activeContractCost.Add(diff.FileContract.Payout)
			e.activeContractSize = e.activeContractSize.Add(types.NewCurrency64(diff.FileContract.FileSize))
		} else {
			e.activeContractCount -= 1
			e.activeContractCost = e.activeContractCost.Sub(diff.FileContract.Payout)
			e.activeContractSize = e.activeContractSize.Sub(types.NewCurrency64(diff.FileContract.FileSize))
		}
	}

	// Set the id of the current block.
	e.currentBlock = cc.AppliedBlocks[len(cc.AppliedBlocks)-1].ID()
}
<|endoftext|>


<commit_before>/*
Copyright The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Code generated by client-gen. DO NOT EDIT.

package versioned

import (
	"fmt"

	discovery "k8s.io/client-go/discovery"
	rest "k8s.io/client-go/rest"
	flowcontrol "k8s.io/client-go/util/flowcontrol"
	samplecontrollerv1alpha1 "k8s.io/sample-controller/pkg/generated/clientset/versioned/typed/samplecontroller/v1alpha1"
)

type Interface interface {
	Discovery() discovery.DiscoveryInterface
	SamplecontrollerV1alpha1() samplecontrollerv1alpha1.SamplecontrollerV1alpha1Interface
}

// Clientset contains the clients for groups. Each group has exactly one
// version included in a Clientset.
type Clientset struct {
	*discovery.DiscoveryClient
	samplecontrollerV1alpha1 *samplecontrollerv1alpha1.SamplecontrollerV1alpha1Client
}

// SamplecontrollerV1alpha1 retrieves the SamplecontrollerV1alpha1Client
func (c *Clientset) SamplecontrollerV1alpha1() samplecontrollerv1alpha1.SamplecontrollerV1alpha1Interface {
	return c.samplecontrollerV1alpha1
}

// Discovery retrieves the DiscoveryClient
func (c *Clientset) Discovery() discovery.DiscoveryInterface {
	if c == nil {
		return nil
	}
	return c.DiscoveryClient
}

// NewForConfig creates a new Clientset for the given config.
// If config's RateLimiter is not set and QPS and Burst are acceptable,
// NewForConfig will generate a rate-limiter in configShallowCopy.
func NewForConfig(c *rest.Config) (*Clientset, error) {
	configShallowCopy := *c
	if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 {
		if configShallowCopy.Burst <= 0 {
			return nil, fmt.Errorf("Burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0")
		}
		configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst)
	}
	var cs Clientset
	var err error
	cs.samplecontrollerV1alpha1, err = samplecontrollerv1alpha1.NewForConfig(&configShallowCopy)
	if err != nil {
		return nil, err
	}

	cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfig(&configShallowCopy)
	if err != nil {
		return nil, err
	}
	return &cs, nil
}

// NewForConfigOrDie creates a new Clientset for the given config and
// panics if there is an error in the config.
func NewForConfigOrDie(c *rest.Config) *Clientset {
	var cs Clientset
	cs.samplecontrollerV1alpha1 = samplecontrollerv1alpha1.NewForConfigOrDie(c)

	cs.DiscoveryClient = discovery.NewDiscoveryClientForConfigOrDie(c)
	return &cs
}

// New creates a new Clientset for the given RESTClient.
func New(c rest.Interface) *Clientset {
	var cs Clientset
	cs.samplecontrollerV1alpha1 = samplecontrollerv1alpha1.New(c)

	cs.DiscoveryClient = discovery.NewDiscoveryClient(c)
	return &cs
}
<commit_msg>Fix error-string-capitalization in clientset generator.<commit_after>/*
Copyright The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Code generated by client-gen. DO NOT EDIT.

package versioned

import (
	"fmt"

	discovery "k8s.io/client-go/discovery"
	rest "k8s.io/client-go/rest"
	flowcontrol "k8s.io/client-go/util/flowcontrol"
	samplecontrollerv1alpha1 "k8s.io/sample-controller/pkg/generated/clientset/versioned/typed/samplecontroller/v1alpha1"
)

type Interface interface {
	Discovery() discovery.DiscoveryInterface
	SamplecontrollerV1alpha1() samplecontrollerv1alpha1.SamplecontrollerV1alpha1Interface
}

// Clientset contains the clients for groups. Each group has exactly one
// version included in a Clientset.
type Clientset struct {
	*discovery.DiscoveryClient
	samplecontrollerV1alpha1 *samplecontrollerv1alpha1.SamplecontrollerV1alpha1Client
}

// SamplecontrollerV1alpha1 retrieves the SamplecontrollerV1alpha1Client
func (c *Clientset) SamplecontrollerV1alpha1() samplecontrollerv1alpha1.SamplecontrollerV1alpha1Interface {
	return c.samplecontrollerV1alpha1
}

// Discovery retrieves the DiscoveryClient
func (c *Clientset) Discovery() discovery.DiscoveryInterface {
	if c == nil {
		return nil
	}
	return c.DiscoveryClient
}

// NewForConfig creates a new Clientset for the given config.
// If config's RateLimiter is not set and QPS and Burst are acceptable,
// NewForConfig will generate a rate-limiter in configShallowCopy.
func NewForConfig(c *rest.Config) (*Clientset, error) {
	configShallowCopy := *c
	if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 {
		if configShallowCopy.Burst <= 0 {
			return nil, fmt.Errorf("burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0")
		}
		configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst)
	}
	var cs Clientset
	var err error
	cs.samplecontrollerV1alpha1, err = samplecontrollerv1alpha1.NewForConfig(&configShallowCopy)
	if err != nil {
		return nil, err
	}

	cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfig(&configShallowCopy)
	if err != nil {
		return nil, err
	}
	return &cs, nil
}

// NewForConfigOrDie creates a new Clientset for the given config and
// panics if there is an error in the config.
func NewForConfigOrDie(c *rest.Config) *Clientset {
	var cs Clientset
	cs.samplecontrollerV1alpha1 = samplecontrollerv1alpha1.NewForConfigOrDie(c)

	cs.DiscoveryClient = discovery.NewDiscoveryClientForConfigOrDie(c)
	return &cs
}

// New creates a new Clientset for the given RESTClient.
func New(c rest.Interface) *Clientset {
	var cs Clientset
	cs.samplecontrollerV1alpha1 = samplecontrollerv1alpha1.New(c)

	cs.DiscoveryClient = discovery.NewDiscoveryClient(c)
	return &cs
}
<|endoftext|>


<commit_before>package api

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Defaults", func() {

	It("should set q35 machine type", func() {
		domain := &Domain{}
		SetDefaults_OSType(&domain.Spec.OS.Type)
		Expect(domain.Spec.OS.Type.Machine).To(Equal("q35"))
	})
})
<commit_msg>virt-handler: add domain type check to machine type check<commit_after>package api

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Defaults", func() {

	It("should set q35 machine type and hvm domain type", func() {
		domain := &Domain{}
		SetDefaults_OSType(&domain.Spec.OS.Type)
		Expect(domain.Spec.OS.Type.Machine).To(Equal("q35"))
		Expect(domain.Spec.OS.Type.OS).To(Equal("hvm"))
	})
})
<|endoftext|>


<commit_before>package middleware

import (
	"bufio"
	"errors"
	"fmt"
	"github.com/DeedleFake/Go-PhysicsFS/physfs"
	"github.com/carbonsrv/carbon/modules/glue"
	"github.com/carbonsrv/carbon/modules/helpers"
	"github.com/carbonsrv/carbon/modules/scheduler"
	"github.com/carbonsrv/carbon/modules/static"
	"github.com/fzzy/radix/redis"
	"github.com/gin-gonic/gin"
	"github.com/pmylund/go-cache"
	"github.com/shurcooL/github_flavored_markdown"
	"github.com/vifino/contrib/gzip"
	"github.com/vifino/golua/lua"
	"github.com/vifino/luar"
	"io/ioutil"
	"net"
	"os"
	"path/filepath"
	"time"
)

func Bind(L *lua.State) {
	BindCarbon(L)
	BindMiddleware(L)
	BindRedis(L)
	BindKVStore(L)
	BindPhysFS(L)
	BindIOEnhancements(L)
	BindOSEnhancements(L)
	BindThread(L)
	BindNet(L)
	BindConversions(L)
	BindComs(L)
	BindMarkdown(L)
	BindOther(L)
}

func BindCarbon(L *lua.State) {
	luar.Register(L, "carbon", luar.Map{ // Carbon specific API
		"glue": glue.GetGlue,
	})
}

func BindEngine(L *lua.State) {
	luar.Register(L, "carbon", luar.Map{
		"_gin_new": gin.New,
	})
}

func BindMiddleware(L *lua.State) {
	luar.Register(L, "mw", luar.Map{
		"Lua": Lua,
		"ExtRoute": (func(plan map[string]interface{}) func(*gin.Context) {
			newplan := make(Plan, len(plan))
			for k, v := range plan {
				newplan[k] = v.(func(*gin.Context))
			}
			return ExtRoute(newplan)
		}),
		"VHOST": (func(plan map[string]interface{}) func(*gin.Context) {
			newplan := make(Plan, len(plan))
			for k, v := range plan {
				newplan[k] = v.(func(*gin.Context))
			}
			return VHOST(newplan)
		}),
		"VHOST_Middleware": (func(plan map[string]interface{}) gin.HandlerFunc {
			newplan := make(Plan, len(plan))
			for k, v := range plan {
				newplan[k] = v.(gin.HandlerFunc)
			}
			return VHOST_Middleware(newplan)
		}),
		"Logger":   gin.Logger,
		"Recovery": gin.Recovery,
		"GZip": func() func(*gin.Context) {
			return gzip.Gzip(gzip.DefaultCompression)
		},
		"DLR_NS":    DLR_NS,
		"DLR_RUS":   DLR_RUS,
		"DLRWS_RUS": DLRWS_RUS,
		"Echo":      EchoHTML,
		"EchoText":  Echo,
	})
	luar.Register(L, "carbon", luar.Map{
		"_mw_CGI": CGI,
		"_mw_combine": (func(middlewares map[string]interface{}) func(*gin.Context) {
			newmiddlewares := make([]func(*gin.Context), len(middlewares))
			for k, v := range plan {
				newmiddlewares[k] = v.(func(*gin.Context))
			}
			return Combine(newmiddlewares)
		}),
	})
	L.DoString(glue.RouteGlue())
}

func BindPhysFS(L *lua.State) {
	luar.Register(L, "fs", luar.Map{ // PhysFS
		"mount":       physfs.Mount,
		"exits":       physfs.Exists,
		"getFS":       physfs.FileSystem,
		"mkdir":       physfs.Mkdir,
		"umount":      physfs.RemoveFromSearchPath,
		"delete":      physfs.Delete,
		"setWriteDir": physfs.SetWriteDir,
		"getWriteDir": physfs.GetWriteDir,
	})
}

func BindIOEnhancements(L *lua.State) {
	luar.Register(L, "carbon", luar.Map{ // Small enhancements to the io stuff.
		"_io_list": (func(path string) ([]string, error) {
			files, err := ioutil.ReadDir(path)
			if err != nil {
				return make([]string, 1), err
			} else {
				list := make([]string, len(files))
				for i := range files {
					list[i] = files[i].Name()
				}
				return list, nil
			}
		}),
		"_io_glob": filepath.Glob,
		"_io_modtime": (func(path string) (int, error) {
			info, err := os.Stat(path)
			if err != nil {
				return -1, err
			} else {
				return int(info.ModTime().UTC().Unix()), nil
			}
		}),
	})
}

func BindOSEnhancements(L *lua.State) {
	luar.Register(L, "carbon", luar.Map{ // Small enhancements to the io stuff.
		"_os_exists": (func(path string) bool {
			if _, err := os.Stat(path); err == nil {
				return true
			} else {
				return false
			}
		}),
		"_os_chdir":   os.Chdir,
		"_os_abspath": filepath.Abs,
	})
}

func BindRedis(L *lua.State) {
	luar.Register(L, "redis", luar.Map{
		"connectTimeout": (func(host string, timeout int) (*redis.Client, error) {
			return redis.DialTimeout("tcp", host, time.Duration(timeout)*time.Second)
		}),
		"connect": (func(host string) (*redis.Client, error) {
			return redis.Dial("tcp", host)
		}),
	})
}

func BindKVStore(L *lua.State) { // Thread safe Key Value Store that doesn't persist.
	luar.Register(L, "kvstore", luar.Map{
		"_set": (func(k string, v interface{}) {
			kvstore.Set(k, v, -1)
		}),
		"_del": (func(k string) {
			kvstore.Delete(k)
		}),
		"_get": (func(k string) interface{} {
			res, found := kvstore.Get(k)
			if found {
				return res
			} else {
				return nil
			}
		}),
		"_inc": (func(k string, n int64) error {
			return kvstore.Increment(k, n)
		}),
		"_dec": (func(k string, n int64) error {
			return kvstore.Decrement(k, n)
		}),
	})
}

func BindThread(L *lua.State) {
	luar.Register(L, "thread", luar.Map{
		"_spawn": (func(bcode string, dobind bool, vals map[string]interface{}) error {
			L := luar.Init()
			Bind(L)
			err := L.DoString(glue.MainGlue())
			if err != nil {
				panic(err)
			}

			if dobind {
				luar.Register(L, "", vals)
			}

			if L.LoadBuffer(bcode, len(bcode), "thread") != 0 {
				return errors.New(L.ToString(-1))
			}

			scheduler.Add(func() {
				if L.Pcall(0, 0, 0) != 0 { // != 0 means error in execution
					// Silently error because reasons. ._.
				}
			})
			return nil
		}),
	})
}

func BindComs(L *lua.State) {
	luar.Register(L, "com", luar.Map{
		"create": (func() chan interface{} {
			return make(chan interface{})
		}),
		"createBuffered": (func(buffer int) chan interface{} {
			return make(chan interface{}, buffer)
		}),
		"receive": (func(c chan interface{}) interface{} {
			return <-c
		}),
		"send": (func(c chan interface{}, val interface{}) {
			c <- val
		}),
	})
}

func BindNet(L *lua.State) {
	luar.Register(L, "net", luar.Map{
		"dial": net.Dial,
		"write": (func(con net.Conn, str string) {
			fmt.Fprintf(con, str)
		}),
		"readline": (func(con net.Conn) (string, error) {
			return bufio.NewReader(con).ReadString('\n')
		}),
	})
}

func BindConversions(L *lua.State) {
	luar.Register(L, "convert", luar.Map{
		"stringtocharslice": (func(x string) []byte {
			return []byte(x)
		}),
		"charslicetostring": (func(x []byte) string {
			return string(x)
		}),
	})
}

func BindContext(L *lua.State, context *gin.Context) {
	luar.Register(L, "", luar.Map{
		"context":    context,
		"req":        context.Request,
		"_paramfunc": context.Param,
		"_formfunc":  context.PostForm,
		"_queryfunc": context.Query,
	})
}
func BindStatic(L *lua.State, cfe *cache.Cache) {
	luar.Register(L, "carbon", luar.Map{
		"_staticserve": (func(path, prefix string) func(*gin.Context) {
			return staticServe.ServeCached(prefix, staticServe.PhysFS(path, prefix, true, true), cfe)
		}),
	})
}

func BindMarkdown(L *lua.State) {
	luar.Register(L, "markdown", luar.Map{
		"github": (func(source string) string {
			return string(github_flavored_markdown.Markdown([]byte(source)))
		}),
	})
}

func BindOther(L *lua.State) {
	luar.Register(L, "", luar.Map{
		"unixtime": (func() int {
			return int(time.Now().UTC().Unix())
		}),
	})
	luar.Register(L, "carbon", luar.Map{
		"_syntaxhl": helpers.SyntaxHL,
	})
}
<commit_msg>Note to self: Check the code you've written before you commit.<commit_after>package middleware

import (
	"bufio"
	"errors"
	"fmt"
	"github.com/DeedleFake/Go-PhysicsFS/physfs"
	"github.com/carbonsrv/carbon/modules/glue"
	"github.com/carbonsrv/carbon/modules/helpers"
	"github.com/carbonsrv/carbon/modules/scheduler"
	"github.com/carbonsrv/carbon/modules/static"
	"github.com/fzzy/radix/redis"
	"github.com/gin-gonic/gin"
	"github.com/pmylund/go-cache"
	"github.com/shurcooL/github_flavored_markdown"
	"github.com/vifino/contrib/gzip"
	"github.com/vifino/golua/lua"
	"github.com/vifino/luar"
	"io/ioutil"
	"net"
	"os"
	"path/filepath"
	"time"
)

func Bind(L *lua.State) {
	BindCarbon(L)
	BindMiddleware(L)
	BindRedis(L)
	BindKVStore(L)
	BindPhysFS(L)
	BindIOEnhancements(L)
	BindOSEnhancements(L)
	BindThread(L)
	BindNet(L)
	BindConversions(L)
	BindComs(L)
	BindMarkdown(L)
	BindOther(L)
}

func BindCarbon(L *lua.State) {
	luar.Register(L, "carbon", luar.Map{ // Carbon specific API
		"glue": glue.GetGlue,
	})
}

func BindEngine(L *lua.State) {
	luar.Register(L, "carbon", luar.Map{
		"_gin_new": gin.New,
	})
}

func BindMiddleware(L *lua.State) {
	luar.Register(L, "mw", luar.Map{
		"Lua": Lua,
		"ExtRoute": (func(plan map[string]interface{}) func(*gin.Context) {
			newplan := make(Plan, len(plan))
			for k, v := range plan {
				newplan[k] = v.(func(*gin.Context))
			}
			return ExtRoute(newplan)
		}),
		"VHOST": (func(plan map[string]interface{}) func(*gin.Context) {
			newplan := make(Plan, len(plan))
			for k, v := range plan {
				newplan[k] = v.(func(*gin.Context))
			}
			return VHOST(newplan)
		}),
		"VHOST_Middleware": (func(plan map[string]interface{}) gin.HandlerFunc {
			newplan := make(Plan, len(plan))
			for k, v := range plan {
				newplan[k] = v.(gin.HandlerFunc)
			}
			return VHOST_Middleware(newplan)
		}),
		"Logger":   gin.Logger,
		"Recovery": gin.Recovery,
		"GZip": func() func(*gin.Context) {
			return gzip.Gzip(gzip.DefaultCompression)
		},
		"DLR_NS":    DLR_NS,
		"DLR_RUS":   DLR_RUS,
		"DLRWS_RUS": DLRWS_RUS,
		"Echo":      EchoHTML,
		"EchoText":  Echo,
	})
	luar.Register(L, "carbon", luar.Map{
		"_mw_CGI": CGI,
		"_mw_combine": (func(middlewares map[string]interface{}) func(*gin.Context) {
			newmiddlewares := make([]func(*gin.Context), len(middlewares))
			for k, v := range middlewares {
				newmiddlewares[k] = v.(func(*gin.Context))
			}
			return Combine(newmiddlewares)
		}),
	})
	L.DoString(glue.RouteGlue())
}

func BindPhysFS(L *lua.State) {
	luar.Register(L, "fs", luar.Map{ // PhysFS
		"mount":       physfs.Mount,
		"exits":       physfs.Exists,
		"getFS":       physfs.FileSystem,
		"mkdir":       physfs.Mkdir,
		"umount":      physfs.RemoveFromSearchPath,
		"delete":      physfs.Delete,
		"setWriteDir": physfs.SetWriteDir,
		"getWriteDir": physfs.GetWriteDir,
	})
}

func BindIOEnhancements(L *lua.State) {
	luar.Register(L, "carbon", luar.Map{ // Small enhancements to the io stuff.
		"_io_list": (func(path string) ([]string, error) {
			files, err := ioutil.ReadDir(path)
			if err != nil {
				return make([]string, 1), err
			} else {
				list := make([]string, len(files))
				for i := range files {
					list[i] = files[i].Name()
				}
				return list, nil
			}
		}),
		"_io_glob": filepath.Glob,
		"_io_modtime": (func(path string) (int, error) {
			info, err := os.Stat(path)
			if err != nil {
				return -1, err
			} else {
				return int(info.ModTime().UTC().Unix()), nil
			}
		}),
	})
}

func BindOSEnhancements(L *lua.State) {
	luar.Register(L, "carbon", luar.Map{ // Small enhancements to the io stuff.
		"_os_exists": (func(path string) bool {
			if _, err := os.Stat(path); err == nil {
				return true
			} else {
				return false
			}
		}),
		"_os_chdir":   os.Chdir,
		"_os_abspath": filepath.Abs,
	})
}

func BindRedis(L *lua.State) {
	luar.Register(L, "redis", luar.Map{
		"connectTimeout": (func(host string, timeout int) (*redis.Client, error) {
			return redis.DialTimeout("tcp", host, time.Duration(timeout)*time.Second)
		}),
		"connect": (func(host string) (*redis.Client, error) {
			return redis.Dial("tcp", host)
		}),
	})
}

func BindKVStore(L *lua.State) { // Thread safe Key Value Store that doesn't persist.
	luar.Register(L, "kvstore", luar.Map{
		"_set": (func(k string, v interface{}) {
			kvstore.Set(k, v, -1)
		}),
		"_del": (func(k string) {
			kvstore.Delete(k)
		}),
		"_get": (func(k string) interface{} {
			res, found := kvstore.Get(k)
			if found {
				return res
			} else {
				return nil
			}
		}),
		"_inc": (func(k string, n int64) error {
			return kvstore.Increment(k, n)
		}),
		"_dec": (func(k string, n int64) error {
			return kvstore.Decrement(k, n)
		}),
	})
}

func BindThread(L *lua.State) {
	luar.Register(L, "thread", luar.Map{
		"_spawn": (func(bcode string, dobind bool, vals map[string]interface{}) error {
			L := luar.Init()
			Bind(L)
			err := L.DoString(glue.MainGlue())
			if err != nil {
				panic(err)
			}

			if dobind {
				luar.Register(L, "", vals)
			}

			if L.LoadBuffer(bcode, len(bcode), "thread") != 0 {
				return errors.New(L.ToString(-1))
			}

			scheduler.Add(func() {
				if L.Pcall(0, 0, 0) != 0 { // != 0 means error in execution
					// Silently error because reasons. ._.
				}
			})
			return nil
		}),
	})
}

func BindComs(L *lua.State) {
	luar.Register(L, "com", luar.Map{
		"create": (func() chan interface{} {
			return make(chan interface{})
		}),
		"createBuffered": (func(buffer int) chan interface{} {
			return make(chan interface{}, buffer)
		}),
		"receive": (func(c chan interface{}) interface{} {
			return <-c
		}),
		"send": (func(c chan interface{}, val interface{}) {
			c <- val
		}),
	})
}

func BindNet(L *lua.State) {
	luar.Register(L, "net", luar.Map{
		"dial": net.Dial,
		"write": (func(con net.Conn, str string) {
			fmt.Fprintf(con, str)
		}),
		"readline": (func(con net.Conn) (string, error) {
			return bufio.NewReader(con).ReadString('\n')
		}),
	})
}

func BindConversions(L *lua.State) {
	luar.Register(L, "convert", luar.Map{
		"stringtocharslice": (func(x string) []byte {
			return []byte(x)
		}),
		"charslicetostring": (func(x []byte) string {
			return string(x)
		}),
	})
}

func BindContext(L *lua.State, context *gin.Context) {
	luar.Register(L, "", luar.Map{
		"context":    context,
		"req":        context.Request,
		"_paramfunc": context.Param,
		"_formfunc":  context.PostForm,
		"_queryfunc": context.Query,
	})
}
func BindStatic(L *lua.State, cfe *cache.Cache) {
	luar.Register(L, "carbon", luar.Map{
		"_staticserve": (func(path, prefix string) func(*gin.Context) {
			return staticServe.ServeCached(prefix, staticServe.PhysFS(path, prefix, true, true), cfe)
		}),
	})
}

func BindMarkdown(L *lua.State) {
	luar.Register(L, "markdown", luar.Map{
		"github": (func(source string) string {
			return string(github_flavored_markdown.Markdown([]byte(source)))
		}),
	})
}

func BindOther(L *lua.State) {
	luar.Register(L, "", luar.Map{
		"unixtime": (func() int {
			return int(time.Now().UTC().Unix())
		}),
	})
	luar.Register(L, "carbon", luar.Map{
		"_syntaxhl": helpers.SyntaxHL,
	})
}
<|endoftext|>


<commit_before>// Copyright 2018 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package app

import (
	"net/http"

	"github.com/issue9/web/dependency"
)

// Module 表示模块信息
type Module struct {
	Name        string
	Deps        []string
	Description string
	Routes      []*Route

	inits []dependency.InitFunc
}

// Route 表示模块信息中的路由信息
type Route struct {
	Path    string
	Handler http.Handler
	Methods []string
}

// Modules 获取当前的所有模块信息
func (app *App) Modules() []*Module {
	return app.modules
}

// AddModule 注册一个新的模块。
func (app *App) AddModule(m *Module) *App {
	app.modules = append(app.modules, m)
	return app
}

func (app *App) initDependency() error {
	dep := dependency.New()

	for _, module := range app.modules {
		dep.Add(module.Name, app.getInit(module), module.Deps...)
	}

	return dep.Init()
}

func (app *App) getInit(m *Module) dependency.InitFunc {
	return func() error {
		for _, init := range m.inits {
			if err := init(); err != nil {
				return err
			}
		}

		for _, r := range m.Routes {
			if err := app.router.Handle(r.Path, r.Handler, r.Methods...); err != nil {
				return err
			}
		}

		return nil
	}
}

// NewModule 声明一个新的模块
func NewModule(name, desc string, deps ...string) *Module {
	return &Module{
		Name:        name,
		Deps:        deps,
		Description: desc,
		Routes:      make([]*Route, 0, 10),
		inits:       make([]dependency.InitFunc, 0, 5),
	}
}

// AddInit 添加一个初始化函数
func (m *Module) AddInit(f dependency.InitFunc) *Module {
	m.inits = append(m.inits, f)
	return m
}

// AddRoute 添加一个路由项
func (m *Module) AddRoute(path string, h http.Handler, methods ...string) *Module {
	m.Routes = append(m.Routes, &Route{
		Methods: methods,
		Path:    path,
		Handler: h,
	})

	return m
}

// Get 指定一个 GET 请求
func (m *Module) Get(path string, h http.Handler) *Module {
	return m.AddRoute(path, h, http.MethodGet)
}

// Post 指定个 POST 请求处理
func (m *Module) Post(path string, h http.Handler) *Module {
	return m.AddRoute(path, h, http.MethodPost)
}

// Delete 指定个 Delete 请求处理
func (m *Module) Delete(path string, h http.Handler) *Module {
	return m.AddRoute(path, h, http.MethodDelete)
}

// Put 指定个 Put 请求处理
func (m *Module) Put(path string, h http.Handler) *Module {
	return m.AddRoute(path, h, http.MethodPut)
}

// Patch 指定个 Patch 请求处理
func (m *Module) Patch(path string, h http.Handler) *Module {
	return m.AddRoute(path, h, http.MethodPatch)
}

// GetFunc 指定一个 GET 请求
func (m *Module) GetFunc(path string, h func(w http.ResponseWriter, r *http.Request)) *Module {
	return m.AddRoute(path, http.HandlerFunc(h), http.MethodGet)
}

// PostFunc 指定一个 GET 请求
func (m *Module) PostFunc(path string, h func(w http.ResponseWriter, r *http.Request)) *Module {
	return m.AddRoute(path, http.HandlerFunc(h), http.MethodPost)
}

// DeleteFunc 指定一个 GET 请求
func (m *Module) DeleteFunc(path string, h func(w http.ResponseWriter, r *http.Request)) *Module {
	return m.AddRoute(path, http.HandlerFunc(h), http.MethodDelete)
}

// PutFunc 指定一个 GET 请求
func (m *Module) PutFunc(path string, h func(w http.ResponseWriter, r *http.Request)) *Module {
	return m.AddRoute(path, http.HandlerFunc(h), http.MethodPut)
}

// PatchFunc 指定一个 GET 请求
func (m *Module) PatchFunc(path string, h func(w http.ResponseWriter, r *http.Request)) *Module {
	return m.AddRoute(path, http.HandlerFunc(h), http.MethodPatch)
}
<commit_msg>添加插件模式<commit_after>// Copyright 2018 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package app

import (
	"errors"
	"net/http"
	"plugin"

	"github.com/issue9/web/dependency"
)

// 表示模块的类型。
const (
	ModuleTypeAll     = iota
	ModuleTypeDefault // 默认的方式，即和代码一起编译
	ModuleTypePlugin  // 加载以 buildmode=plugin 方式加载的模块
)

// Module 表示模块信息
type Module struct {
	Name        string
	Deps        []string
	Description string
	Routes      []*Route
	Type        int

	inits []dependency.InitFunc
}

// Route 表示模块信息中的路由信息
type Route struct {
	Path    string
	Handler http.Handler
	Methods []string
}

// Modules 获取当前的所有模块信息
func (app *App) Modules() []*Module {
	return app.modules
}

// AddModule 注册一个新的模块。
func (app *App) AddModule(m *Module) *App {
	app.modules = append(app.modules, m)
	return app
}

// LoadPlugin 加载插件。
//
// 必须符合以下要求：
// 一个插件必须为一个模块；
// 公开一个类型为 *Module 的变量 Module。
func (app *App) LoadPlugin(path string) error {
	p, err := plugin.Open(path)
	if err != nil {
		return err
	}

	symbol, err := p.Lookup("M")
	if err != nil {
		return err
	}

	module, ok := symbol.(*Module)
	if !ok {
		return errors.New("无法转换成 Module")
	}

	module.Type = ModuleTypePlugin
	app.AddModule(module)

	return nil
}

func (app *App) initDependency() error {
	dep := dependency.New()

	for _, module := range app.modules {
		dep.Add(module.Name, app.getInit(module), module.Deps...)
	}

	return dep.Init()
}

func (app *App) getInit(m *Module) dependency.InitFunc {
	return func() error {
		for _, init := range m.inits {
			if err := init(); err != nil {
				return err
			}
		}

		for _, r := range m.Routes {
			if err := app.router.Handle(r.Path, r.Handler, r.Methods...); err != nil {
				return err
			}
		}

		return nil
	}
}

// NewModule 声明一个新的模块
func NewModule(name, desc string, deps ...string) *Module {
	return &Module{
		Name:        name,
		Deps:        deps,
		Description: desc,
		Routes:      make([]*Route, 0, 10),
		inits:       make([]dependency.InitFunc, 0, 5),
	}
}

// AddInit 添加一个初始化函数
func (m *Module) AddInit(f dependency.InitFunc) *Module {
	m.inits = append(m.inits, f)
	return m
}

// AddRoute 添加一个路由项
func (m *Module) AddRoute(path string, h http.Handler, methods ...string) *Module {
	m.Routes = append(m.Routes, &Route{
		Methods: methods,
		Path:    path,
		Handler: h,
	})

	return m
}

// Get 指定一个 GET 请求
func (m *Module) Get(path string, h http.Handler) *Module {
	return m.AddRoute(path, h, http.MethodGet)
}

// Post 指定个 POST 请求处理
func (m *Module) Post(path string, h http.Handler) *Module {
	return m.AddRoute(path, h, http.MethodPost)
}

// Delete 指定个 Delete 请求处理
func (m *Module) Delete(path string, h http.Handler) *Module {
	return m.AddRoute(path, h, http.MethodDelete)
}

// Put 指定个 Put 请求处理
func (m *Module) Put(path string, h http.Handler) *Module {
	return m.AddRoute(path, h, http.MethodPut)
}

// Patch 指定个 Patch 请求处理
func (m *Module) Patch(path string, h http.Handler) *Module {
	return m.AddRoute(path, h, http.MethodPatch)
}

// GetFunc 指定一个 GET 请求
func (m *Module) GetFunc(path string, h func(w http.ResponseWriter, r *http.Request)) *Module {
	return m.AddRoute(path, http.HandlerFunc(h), http.MethodGet)
}

// PostFunc 指定一个 GET 请求
func (m *Module) PostFunc(path string, h func(w http.ResponseWriter, r *http.Request)) *Module {
	return m.AddRoute(path, http.HandlerFunc(h), http.MethodPost)
}

// DeleteFunc 指定一个 GET 请求
func (m *Module) DeleteFunc(path string, h func(w http.ResponseWriter, r *http.Request)) *Module {
	return m.AddRoute(path, http.HandlerFunc(h), http.MethodDelete)
}

// PutFunc 指定一个 GET 请求
func (m *Module) PutFunc(path string, h func(w http.ResponseWriter, r *http.Request)) *Module {
	return m.AddRoute(path, http.HandlerFunc(h), http.MethodPut)
}

// PatchFunc 指定一个 GET 请求
func (m *Module) PatchFunc(path string, h func(w http.ResponseWriter, r *http.Request)) *Module {
	return m.AddRoute(path, http.HandlerFunc(h), http.MethodPatch)
}
<|endoftext|>