<commit_before>// Copyright ©2015 The Gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gonum

import (
	"gonum.org/v1/gonum/blas"
	"gonum.org/v1/gonum/blas/blas64"
)

// Dgetri computes the inverse of the matrix A using the LU factorization computed
// by Dgetrf. On entry, a contains the PLU decomposition of A as computed by
// Dgetrf and on exit contains the reciprocal of the original matrix.
//
// Dgetri will not perform the inversion if the matrix is singular, and returns
// a boolean indicating whether the inversion was successful.
//
// work is temporary storage, and lwork specifies the usable memory length.
// At minimum, lwork >= n and this function will panic otherwise.
// Dgetri is a blocked inversion, but the block size is limited
// by the temporary space available. If lwork == -1, instead of performing Dgetri,
// the optimal work length will be stored into work[0].
func (impl Implementation) Dgetri(n int, a []float64, lda int, ipiv []int, work []float64, lwork int) (ok bool) {
	checkMatrix(n, n, a, lda)
	if len(ipiv) < n {
		panic(badIpiv)
	}
	nb := impl.Ilaenv(1, "DGETRI", " ", n, -1, -1, -1)
	if lwork == -1 {
		work[0] = float64(n * nb)
		return true
	}
	if lwork < n {
		panic(badWork)
	}
	if len(work) < lwork {
		panic(badWork)
	}
	if n == 0 {
		return true
	}
	ok = impl.Dtrtri(blas.Upper, blas.NonUnit, n, a, lda)
	if !ok {
		return false
	}
	nbmin := 2
	ldwork := nb
	if nb > 1 && nb < n {
		iws := max(ldwork*n, 1)
		if lwork < iws {
			nb = lwork / n
			nbmin = max(2, impl.Ilaenv(2, "DGETRI", " ", n, -1, -1, -1))
		}
	}
	bi := blas64.Implementation()
	// TODO(btracey): Replace this with a more row-major oriented algorithm.
	if nb < nbmin || nb >= n {
		// Unblocked code.
		for j := n - 1; j >= 0; j-- {
			for i := j + 1; i < n; i++ {
				work[i*ldwork] = a[i*lda+j]
				a[i*lda+j] = 0
			}
			if j < n {
				bi.Dgemv(blas.NoTrans, n, n-j-1, -1, a[(j+1):], lda, work[(j+1)*ldwork:], ldwork, 1, a[j:], lda)
			}
		}
	} else {
		nn := ((n - 1) / nb) * nb
		for j := nn; j >= 0; j -= nb {
			jb := min(nb, n-j)
			for jj := j; jj < j+jb-1; jj++ {
				for i := jj + 1; i < n; i++ {
					work[i*ldwork+(jj-j)] = a[i*lda+jj]
					a[i*lda+jj] = 0
				}
			}
			if j+jb < n {
				bi.Dgemm(blas.NoTrans, blas.NoTrans, n, jb, n-j-jb, -1, a[(j+jb):], lda, work[(j+jb)*ldwork:], ldwork, 1, a[j:], lda)
				bi.Dtrsm(blas.Right, blas.Lower, blas.NoTrans, blas.Unit, n, jb, 1, work[j*ldwork:], ldwork, a[j:], lda)
			}
		}
	}
	for j := n - 2; j >= 0; j-- {
		jp := ipiv[j]
		if jp != j {
			bi.Dswap(n, a[j:], lda, a[jp:], lda)
		}
	}
	return true
}
<commit_msg>lapack/gonum: fix where Dtrsm is called in Dgetri<commit_after>// Copyright ©2015 The Gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gonum

import (
	"gonum.org/v1/gonum/blas"
	"gonum.org/v1/gonum/blas/blas64"
)

// Dgetri computes the inverse of the matrix A using the LU factorization computed
// by Dgetrf. On entry, a contains the PLU decomposition of A as computed by
// Dgetrf and on exit contains the reciprocal of the original matrix.
//
// Dgetri will not perform the inversion if the matrix is singular, and returns
// a boolean indicating whether the inversion was successful.
//
// work is temporary storage, and lwork specifies the usable memory length.
// At minimum, lwork >= n and this function will panic otherwise.
// Dgetri is a blocked inversion, but the block size is limited
// by the temporary space available. If lwork == -1, instead of performing Dgetri,
// the optimal work length will be stored into work[0].
func (impl Implementation) Dgetri(n int, a []float64, lda int, ipiv []int, work []float64, lwork int) (ok bool) {
	checkMatrix(n, n, a, lda)
	if len(ipiv) < n {
		panic(badIpiv)
	}
	nb := impl.Ilaenv(1, "DGETRI", " ", n, -1, -1, -1)
	if lwork == -1 {
		work[0] = float64(n * nb)
		return true
	}
	if lwork < n {
		panic(badWork)
	}
	if len(work) < lwork {
		panic(badWork)
	}
	if n == 0 {
		return true
	}
	ok = impl.Dtrtri(blas.Upper, blas.NonUnit, n, a, lda)
	if !ok {
		return false
	}
	nbmin := 2
	ldwork := nb
	if nb > 1 && nb < n {
		iws := max(ldwork*n, 1)
		if lwork < iws {
			nb = lwork / n
			nbmin = max(2, impl.Ilaenv(2, "DGETRI", " ", n, -1, -1, -1))
		}
	}
	bi := blas64.Implementation()
	// TODO(btracey): Replace this with a more row-major oriented algorithm.
	if nb < nbmin || nb >= n {
		// Unblocked code.
		for j := n - 1; j >= 0; j-- {
			for i := j + 1; i < n; i++ {
				work[i*ldwork] = a[i*lda+j]
				a[i*lda+j] = 0
			}
			if j < n {
				bi.Dgemv(blas.NoTrans, n, n-j-1, -1, a[(j+1):], lda, work[(j+1)*ldwork:], ldwork, 1, a[j:], lda)
			}
		}
	} else {
		nn := ((n - 1) / nb) * nb
		for j := nn; j >= 0; j -= nb {
			jb := min(nb, n-j)
			for jj := j; jj < j+jb-1; jj++ {
				for i := jj + 1; i < n; i++ {
					work[i*ldwork+(jj-j)] = a[i*lda+jj]
					a[i*lda+jj] = 0
				}
			}
			if j+jb < n {
				bi.Dgemm(blas.NoTrans, blas.NoTrans, n, jb, n-j-jb, -1, a[(j+jb):], lda, work[(j+jb)*ldwork:], ldwork, 1, a[j:], lda)
			}
			bi.Dtrsm(blas.Right, blas.Lower, blas.NoTrans, blas.Unit, n, jb, 1, work[j*ldwork:], ldwork, a[j:], lda)
		}
	}
	for j := n - 2; j >= 0; j-- {
		jp := ipiv[j]
		if jp != j {
			bi.Dswap(n, a[j:], lda, a[jp:], lda)
		}
	}
	return true
}
<|endoftext|>


<commit_before>package middlewares

import (
	"net/http"
	"strconv"
	"strings"
	"sync/atomic"
	"time"
	"unicode/utf8"

	"github.com/containous/traefik/log"
	"github.com/containous/traefik/metrics"
	gokitmetrics "github.com/go-kit/kit/metrics"
	"github.com/urfave/negroni"
)

const (
	protoHTTP      = "http"
	protoSSE       = "sse"
	protoWebsocket = "websocket"
)

// NewEntryPointMetricsMiddleware creates a new metrics middleware for an Entrypoint.
func NewEntryPointMetricsMiddleware(registry metrics.Registry, entryPointName string) negroni.Handler {
	return &metricsMiddleware{
		reqsCounter:          registry.EntrypointReqsCounter(),
		reqDurationHistogram: registry.EntrypointReqDurationHistogram(),
		openConnsGauge:       registry.EntrypointOpenConnsGauge(),
		baseLabels:           []string{"entrypoint", entryPointName},
	}
}

// NewBackendMetricsMiddleware creates a new metrics middleware for a Backend.
func NewBackendMetricsMiddleware(registry metrics.Registry, backendName string) negroni.Handler {
	return &metricsMiddleware{
		reqsCounter:          registry.BackendReqsCounter(),
		reqDurationHistogram: registry.BackendReqDurationHistogram(),
		openConnsGauge:       registry.BackendOpenConnsGauge(),
		baseLabels:           []string{"backend", backendName},
	}
}

type metricsMiddleware struct {
	reqsCounter          gokitmetrics.Counter
	reqDurationHistogram gokitmetrics.Histogram
	openConnsGauge       gokitmetrics.Gauge
	baseLabels           []string
	openConns            int64
}

func (m *metricsMiddleware) ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	labels := []string{"method", getMethod(r), "protocol", getRequestProtocol(r)}
	labels = append(labels, m.baseLabels...)

	openConns := atomic.AddInt64(&m.openConns, 1)
	m.openConnsGauge.With(labels...).Set(float64(openConns))
	defer func(labelValues []string) {
		openConns := atomic.AddInt64(&m.openConns, -1)
		m.openConnsGauge.With(labelValues...).Set(float64(openConns))
	}(labels)

	start := time.Now()
	recorder := &responseRecorder{rw, http.StatusOK}
	next(recorder, r)

	labels = append(labels, "code", strconv.Itoa(recorder.statusCode))
	m.reqsCounter.With(labels...).Add(1)
	m.reqDurationHistogram.With(labels...).Observe(float64(time.Since(start).Seconds()))
}

func getRequestProtocol(req *http.Request) string {
	switch {
	case isWebsocketRequest(req):
		return protoWebsocket
	case isSSERequest(req):
		return protoSSE
	default:
		return protoHTTP
	}
}

// isWebsocketRequest determines if the specified HTTP request is a websocket handshake request.
func isWebsocketRequest(req *http.Request) bool {
	return containsHeader(req, "Connection", "upgrade") && containsHeader(req, "Upgrade", "websocket")
}

// isSSERequest determines if the specified HTTP request is a request for an event subscription.
func isSSERequest(req *http.Request) bool {
	return containsHeader(req, "Accept", "text/event-stream")
}

func containsHeader(req *http.Request, name, value string) bool {
	items := strings.Split(req.Header.Get(name), ",")
	for _, item := range items {
		if value == strings.ToLower(strings.TrimSpace(item)) {
			return true
		}
	}
	return false
}

func getMethod(r *http.Request) string {
	if !utf8.ValidString(r.Method) {
		log.Warnf("Invalid HTTP method encoding: %s", r.Method)
		return "NON_UTF8_HTTP_METHOD"
	}
	return r.Method
}

type retryMetrics interface {
	BackendRetriesCounter() gokitmetrics.Counter
}

// NewMetricsRetryListener instantiates a MetricsRetryListener with the given retryMetrics.
func NewMetricsRetryListener(retryMetrics retryMetrics, backendName string) RetryListener {
	return &MetricsRetryListener{retryMetrics: retryMetrics, backendName: backendName}
}

// MetricsRetryListener is an implementation of the RetryListener interface to
// record RequestMetrics about retry attempts.
type MetricsRetryListener struct {
	retryMetrics retryMetrics
	backendName  string
}

// Retried tracks the retry in the RequestMetrics implementation.
func (m *MetricsRetryListener) Retried(req *http.Request, attempt int) {
	m.retryMetrics.BackendRetriesCounter().With("backend", m.backendName).Add(1)
}
<commit_msg>Remove unnecessary conversion<commit_after>package middlewares

import (
	"net/http"
	"strconv"
	"strings"
	"sync/atomic"
	"time"
	"unicode/utf8"

	"github.com/containous/traefik/log"
	"github.com/containous/traefik/metrics"
	gokitmetrics "github.com/go-kit/kit/metrics"
	"github.com/urfave/negroni"
)

const (
	protoHTTP      = "http"
	protoSSE       = "sse"
	protoWebsocket = "websocket"
)

// NewEntryPointMetricsMiddleware creates a new metrics middleware for an Entrypoint.
func NewEntryPointMetricsMiddleware(registry metrics.Registry, entryPointName string) negroni.Handler {
	return &metricsMiddleware{
		reqsCounter:          registry.EntrypointReqsCounter(),
		reqDurationHistogram: registry.EntrypointReqDurationHistogram(),
		openConnsGauge:       registry.EntrypointOpenConnsGauge(),
		baseLabels:           []string{"entrypoint", entryPointName},
	}
}

// NewBackendMetricsMiddleware creates a new metrics middleware for a Backend.
func NewBackendMetricsMiddleware(registry metrics.Registry, backendName string) negroni.Handler {
	return &metricsMiddleware{
		reqsCounter:          registry.BackendReqsCounter(),
		reqDurationHistogram: registry.BackendReqDurationHistogram(),
		openConnsGauge:       registry.BackendOpenConnsGauge(),
		baseLabels:           []string{"backend", backendName},
	}
}

type metricsMiddleware struct {
	reqsCounter          gokitmetrics.Counter
	reqDurationHistogram gokitmetrics.Histogram
	openConnsGauge       gokitmetrics.Gauge
	baseLabels           []string
	openConns            int64
}

func (m *metricsMiddleware) ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	labels := []string{"method", getMethod(r), "protocol", getRequestProtocol(r)}
	labels = append(labels, m.baseLabels...)

	openConns := atomic.AddInt64(&m.openConns, 1)
	m.openConnsGauge.With(labels...).Set(float64(openConns))
	defer func(labelValues []string) {
		openConns := atomic.AddInt64(&m.openConns, -1)
		m.openConnsGauge.With(labelValues...).Set(float64(openConns))
	}(labels)

	start := time.Now()
	recorder := &responseRecorder{rw, http.StatusOK}
	next(recorder, r)

	labels = append(labels, "code", strconv.Itoa(recorder.statusCode))
	m.reqsCounter.With(labels...).Add(1)
	m.reqDurationHistogram.With(labels...).Observe(time.Since(start).Seconds())
}

func getRequestProtocol(req *http.Request) string {
	switch {
	case isWebsocketRequest(req):
		return protoWebsocket
	case isSSERequest(req):
		return protoSSE
	default:
		return protoHTTP
	}
}

// isWebsocketRequest determines if the specified HTTP request is a websocket handshake request.
func isWebsocketRequest(req *http.Request) bool {
	return containsHeader(req, "Connection", "upgrade") && containsHeader(req, "Upgrade", "websocket")
}

// isSSERequest determines if the specified HTTP request is a request for an event subscription.
func isSSERequest(req *http.Request) bool {
	return containsHeader(req, "Accept", "text/event-stream")
}

func containsHeader(req *http.Request, name, value string) bool {
	items := strings.Split(req.Header.Get(name), ",")
	for _, item := range items {
		if value == strings.ToLower(strings.TrimSpace(item)) {
			return true
		}
	}
	return false
}

func getMethod(r *http.Request) string {
	if !utf8.ValidString(r.Method) {
		log.Warnf("Invalid HTTP method encoding: %s", r.Method)
		return "NON_UTF8_HTTP_METHOD"
	}
	return r.Method
}

type retryMetrics interface {
	BackendRetriesCounter() gokitmetrics.Counter
}

// NewMetricsRetryListener instantiates a MetricsRetryListener with the given retryMetrics.
func NewMetricsRetryListener(retryMetrics retryMetrics, backendName string) RetryListener {
	return &MetricsRetryListener{retryMetrics: retryMetrics, backendName: backendName}
}

// MetricsRetryListener is an implementation of the RetryListener interface to
// record RequestMetrics about retry attempts.
type MetricsRetryListener struct {
	retryMetrics retryMetrics
	backendName  string
}

// Retried tracks the retry in the RequestMetrics implementation.
func (m *MetricsRetryListener) Retried(req *http.Request, attempt int) {
	m.retryMetrics.BackendRetriesCounter().With("backend", m.backendName).Add(1)
}
<|endoftext|>


<commit_before>package table

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/freneticmonkey/migrate/migrate/metadata"
	"github.com/freneticmonkey/migrate/migrate/util"
)

// Tables Helper type for a slice of Table structs
type Tables []Table

// Column Stores the properties for a Column field
type Column struct {
	ID       string `yaml:"id"`
	Name     string
	Type     string
	Size     int
	Nullable bool
	AutoInc  bool

	// Binary      bool
	// Unique      bool
	// Unsigned    bool
	// ZeroFilled  bool

	Metadata metadata.Metadata `yaml:"-"`
}

// ToSQL Formats the column into its SQL representation
func (c Column) ToSQL() string {

	var params util.Params

	if !c.Nullable {
		params.Add("NOT NULL")
	}

	if c.AutoInc {
		params.Add("AUTO_INCREMENT")
	}
	return fmt.Sprintf("%s %s(%d) %s", c.Name, c.Type, c.Size, params.String())
}

// Index Stores the properties for a Index field
type Index struct {
	ID        string `yaml:"id"`
	Name      string
	Columns   []string
	IsPrimary bool
	IsUnique  bool
	Metadata  metadata.Metadata `yaml:"-"`
}

// ToSQL Formats the index into its SQL representation
func (i Index) ToSQL() string {

	name := ""
	columns := ""

	if i.IsPrimary {
		name = "PRIMARY KEY"
	} else {
		isUnique := ""
		if i.IsUnique {
			isUnique = "UNIQUE"
		}
		name = fmt.Sprintf("%s KEY `%s` ", isUnique, i.Name)
	}

	columns = strings.Join(i.Columns, ", ")

	return fmt.Sprintf("%s (%s)", name, columns)
}

// Table Stores the fields and properties representing a Table parsed from YAML
// or from a MySQL CREATE TABLE statement
type Table struct {
	ID               string `yaml:"id"`
	Name             string
	Engine           string
	AutoInc          int64
	CharSet          string
	Columns          []Column
	PrimaryIndex     Index
	SecondaryIndexes []Index

	namespace []string
	Filename  string            `yaml:"-"`
	Metadata  metadata.Metadata `yaml:"-"`
}

// SetNamespace Use the path and filename parameters to rename the table
// into an underscore delimited namespace
func (t *Table) SetNamespace(path string, filename string) (err error) {
	wd, err := os.Getwd()

	t.Filename = filepath.Join(path, filename)

	relativePath, err := filepath.Rel(filepath.Join(wd, path), filename)

	dir, _ := filepath.Split(relativePath)

	var ns []string

	if len(dir) > 0 {
		// TODO: Cross platform support
		ns = strings.Split(dir, "/")
		t.namespace = ns[:len(ns)-1]

		// rewrite tablenames
		t.Name = fmt.Sprintf("%s_%s", strings.Join(t.namespace, "_"), t.Name)
	}

	return err
}

// RemoveNamespace Remove the underscore delimited namespace from the table
// to return it to a regular table name
func (t *Table) RemoveNamespace() {
	ns := strings.Split(t.Name, "_")
	t.Name = ns[len(ns)-1]
}

// syncMetadata Check if the Metadata exists in the DB and insert it if it doesn't
func syncMetadata(md *metadata.Metadata) {
	// var dbmd metadata.Metadata
	dbmd, err := metadata.GetByName(md.Name, md.ParentID)
	if err != nil {
		err = md.Insert()
		util.ErrorCheckf(err, "Problem inserting %s Metdata for %s for with PropertyID: [%s]", md.Type, md.Name, md.PropertyID)
	} else {
		md.MDID = dbmd.MDID
		md.DB = dbmd.DB
	}
}

// SyncDBMetadata Helper function to insert new Metadata and retrieves existing Metadata from the DB
func (t *Table) SyncDBMetadata() (err error) {

	syncMetadata(&t.Metadata)

	syncMetadata(&t.PrimaryIndex.Metadata)

	for i := 0; i < len(t.Columns); i++ {
		syncMetadata(&t.Columns[i].Metadata)
	}

	for i := 0; i < len(t.SecondaryIndexes); i++ {
		syncMetadata(&t.SecondaryIndexes[i].Metadata)
	}

	return err
}
<commit_msg>Fixed Table Filename value<commit_after>package table

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/freneticmonkey/migrate/migrate/metadata"
	"github.com/freneticmonkey/migrate/migrate/util"
)

// Tables Helper type for a slice of Table structs
type Tables []Table

// Column Stores the properties for a Column field
type Column struct {
	ID       string `yaml:"id"`
	Name     string
	Type     string
	Size     int
	Nullable bool
	AutoInc  bool

	// Binary      bool
	// Unique      bool
	// Unsigned    bool
	// ZeroFilled  bool

	Metadata metadata.Metadata `yaml:"-"`
}

// ToSQL Formats the column into its SQL representation
func (c Column) ToSQL() string {

	var params util.Params

	if !c.Nullable {
		params.Add("NOT NULL")
	}

	if c.AutoInc {
		params.Add("AUTO_INCREMENT")
	}
	return fmt.Sprintf("%s %s(%d) %s", c.Name, c.Type, c.Size, params.String())
}

// Index Stores the properties for a Index field
type Index struct {
	ID        string `yaml:"id"`
	Name      string
	Columns   []string
	IsPrimary bool
	IsUnique  bool
	Metadata  metadata.Metadata `yaml:"-"`
}

// ToSQL Formats the index into its SQL representation
func (i Index) ToSQL() string {

	name := ""
	columns := ""

	if i.IsPrimary {
		name = "PRIMARY KEY"
	} else {
		isUnique := ""
		if i.IsUnique {
			isUnique = "UNIQUE"
		}
		name = fmt.Sprintf("%s KEY `%s` ", isUnique, i.Name)
	}

	columns = strings.Join(i.Columns, ", ")

	return fmt.Sprintf("%s (%s)", name, columns)
}

// Table Stores the fields and properties representing a Table parsed from YAML
// or from a MySQL CREATE TABLE statement
type Table struct {
	ID               string `yaml:"id"`
	Name             string
	Engine           string
	AutoInc          int64
	CharSet          string
	Columns          []Column
	PrimaryIndex     Index
	SecondaryIndexes []Index

	namespace []string
	Filename  string            `yaml:"-"`
	Metadata  metadata.Metadata `yaml:"-"`
}

// SetNamespace Use the path and filename parameters to rename the table
// into an underscore delimited namespace
func (t *Table) SetNamespace(path string, filename string) (err error) {
	wd, err := os.Getwd()

	t.Filename = filename

	relativePath, err := filepath.Rel(filepath.Join(wd, path), filename)

	dir, _ := filepath.Split(relativePath)

	var ns []string

	if len(dir) > 0 {
		// TODO: Cross platform support
		ns = strings.Split(dir, "/")
		t.namespace = ns[:len(ns)-1]

		// rewrite tablenames
		t.Name = fmt.Sprintf("%s_%s", strings.Join(t.namespace, "_"), t.Name)
	}

	return err
}

// RemoveNamespace Remove the underscore delimited namespace from the table
// to return it to a regular table name
func (t *Table) RemoveNamespace() {
	ns := strings.Split(t.Name, "_")
	t.Name = ns[len(ns)-1]
}

// syncMetadata Check if the Metadata exists in the DB and insert it if it doesn't
func syncMetadata(md *metadata.Metadata) {
	// var dbmd metadata.Metadata
	dbmd, err := metadata.GetByName(md.Name, md.ParentID)
	if err != nil {
		err = md.Insert()
		util.ErrorCheckf(err, "Problem inserting %s Metdata for %s for with PropertyID: [%s]", md.Type, md.Name, md.PropertyID)
	} else {
		md.MDID = dbmd.MDID
		md.DB = dbmd.DB
	}
}

// SyncDBMetadata Helper function to insert new Metadata and retrieves existing Metadata from the DB
func (t *Table) SyncDBMetadata() (err error) {

	syncMetadata(&t.Metadata)

	syncMetadata(&t.PrimaryIndex.Metadata)

	for i := 0; i < len(t.Columns); i++ {
		syncMetadata(&t.Columns[i].Metadata)
	}

	for i := 0; i < len(t.SecondaryIndexes); i++ {
		syncMetadata(&t.SecondaryIndexes[i].Metadata)
	}

	return err
}
<|endoftext|>


<commit_before>package pulse

/*
#cgo pkg-config: libpulse-simple

#include <stdlib.h>
#include <pulse/sample.h>
*/
import "C"
import "unsafe"

const (
	CHANNELS_MAX = C.PA_CHANNELS_MAX // Maximum number of allowed channels
	RATE_MAX     = C.PA_RATE_MAX     // Maximum allowed sample rate
)

type SampleFormat C.pa_sample_format_t

const (
	SAMPLE_U8        SampleFormat = C.PA_SAMPLE_U8
	SAMPLE_ALAW      SampleFormat = C.PA_SAMPLE_ALAW
	SAMPLE_ULAW      SampleFormat = C.PA_SAMPLE_ULAW
	SAMPLE_S16LE     SampleFormat = C.PA_SAMPLE_S16LE
	SAMPLE_S16BE     SampleFormat = C.PA_SAMPLE_S16BE
	SAMPLE_FLOAT32LE SampleFormat = C.PA_SAMPLE_FLOAT32LE
	SAMPLE_FLOAT32BE SampleFormat = C.PA_SAMPLE_FLOAT32BE
	SAMPLE_S32LE     SampleFormat = C.PA_SAMPLE_S32LE
	SAMPLE_S32BE     SampleFormat = C.PA_SAMPLE_S32BE
	SAMPLE_S24LE     SampleFormat = C.PA_SAMPLE_S24LE
	SAMPLE_S24BE     SampleFormat = C.PA_SAMPLE_S24BE
	SAMPLE_S24_32LE  SampleFormat = C.PA_SAMPLE_S24_32LE
	SAMPLE_S24_32BE  SampleFormat = C.PA_SAMPLE_S24_32BE
	SAMPLE_MAX       SampleFormat = C.PA_SAMPLE_MAX
	SAMPLE_INVALID   SampleFormat = C.PA_SAMPLE_INVALID
)

const (
	SAMPLE_S16NE     SampleFormat = C.PA_SAMPLE_S16NE
	SAMPLE_FLOAT32NE SampleFormat = C.PA_SAMPLE_FLOAT32NE
	SAMPLE_S32NE     SampleFormat = C.PA_SAMPLE_S32NE
	SAMPLE_S24NE     SampleFormat = C.PA_SAMPLE_S24NE
	SAMPLE_S24_32NE  SampleFormat = C.PA_SAMPLE_S24_32NE
	SAMPLE_S16RE     SampleFormat = C.PA_SAMPLE_S16RE
	SAMPLE_FLOAT32RE SampleFormat = C.PA_SAMPLE_FLOAT32RE
	SAMPLE_S32RE     SampleFormat = C.PA_SAMPLE_S32RE
	SAMPLE_S24RE     SampleFormat = C.PA_SAMPLE_S24RE
	SAMPLE_S24_32RE  SampleFormat = C.PA_SAMPLE_S24_32RE
)

const SAMPLE_FLOAT32 SampleFormat = C.PA_SAMPLE_FLOAT32

type SampleSpec struct {
	Format   SampleFormat
	Rate     uint32
	Channels uint8
}

func (spec *SampleSpec) toC() *C.pa_sample_spec {
	if m == nil {
		return nil
	}
	return &C.pa_sample_spec{
		format:   C.pa_sample_format_t(spec.Format),
		rate:     C.uint32_t(spec.Rate),
		channels: C.uint8_t(spec.Channels),
	}
}

// SampleSpec.BytesPerSecond returns the number of bytes per second of audio.
func (spec *SampleSpec) BytesPerSecond() uint {
	return uint(C.pa_bytes_per_second(spec.toC()))
}

// SampleSpec.FrameSize returns the size of a single audio frame in bytes.
func (spec *SampleSpec) FrameSize() uint {
	return uint(C.pa_frame_size(spec.toC()))
}

// SampleSpec.SampleSize returns the size of a single sample in bytes.
func (spec *SampleSpec) SampleSize() uint {
	return uint(C.pa_sample_size(spec.toC()))
}

// SampleFormat.SampleSize returns the size of a single sample in bytes.
func (f SampleFormat) SampleSize() uint {
	return uint(C.pa_sample_size_of_format(C.pa_sample_format_t(f)))
}

// SampleSpec.BytesToUsec returns the number of microseconds taken
// to play the given number of bytes as audio.
//
// The return value will always be rounded down for non-integral values.
func (spec *SampleSpec) BytesToUsec(bytes uint) uint64 {
	return uint64(C.pa_bytes_to_usec(C.uint64_t(bytes), spec.toC()))
}

// SampleSpec.UsecToBytes returns the number of bytes required
// for the given number of microseconds of audio.
//
// The return value will always be rounded down for non-integral values.
func (spec *SampleSpec) UsecToBytes(usec uint64) uint {
	return uint(C.pa_usec_to_bytes(C.pa_usec_t(usec), spec.toC()))
}

/* TODO?
// Initialize the specified sample spec and return a pointer to
// it. The sample spec will have a defined state but
// pa_sample_spec_valid() will fail for it. \since 0.9.13
pa_sample_spec* pa_sample_spec_init(pa_sample_spec *spec);
*/

// SampleSpec.Valid returns whether or not the given sample spec is valid.
func (spec *SampleSpec) Valid() bool {
	if C.pa_sample_spec_valid(spec.toC()) == 0 {
		return false
	}
	return true
}

// SampleSpec.Equal returns whether or not the given sample specs match.
func (spec *SampleSpec) Equal(other *SampleSpec) bool {
	if C.pa_sample_spec_equal(spec.toC(), other.toC()) == 0 {
		return false
	}
	return true
}

// SampleFormat.String returns a string describing the format.
func (f SampleFormat) String() string {
	cstr := C.pa_sample_format_to_string(C.pa_sample_format_t(f))
	return C.GoString(cstr)
}

// ParseSampleFormat returns the SampleFormat described by the given string.
//
// The string should be as returned by SampleFormat.String().
func ParseSampleFormat(s string) SampleFormat {
	cstr := C.CString(s)
	defer C.free(unsafe.Pointer(cstr))
	return SampleFormat(C.pa_parse_sample_format(cstr))
}

/* TODO
// Maximum required string length for
// pa_sample_spec_snprint(). Please note that this value can change
// with any release without warning and without being considered API
// or ABI breakage. You should not use this definition anywhere where
// it might become part of an ABI.
#define PA_SAMPLE_SPEC_SNPRINT_MAX 32

// Pretty print a sample type specification to a string
char* pa_sample_spec_snprint(char *s, size_t l, const pa_sample_spec *spec);

// Maximum required string length for pa_bytes_snprint(). Please note
// that this value can change with any release without warning and
// without being considered API or ABI breakage. You should not use
// this definition anywhere where it might become part of an
// ABI. \since 0.9.16
#define PA_BYTES_SNPRINT_MAX 11

// Pretty print a byte size value (i.e.\ "2.5 MiB")
char* pa_bytes_snprint(char *s, size_t l, unsigned v);
*/

// SampleFormat.IsLe returns 1 when the format is little endian.
//
// Returns -1 when endianness does not apply to this format.
func (f SampleFormat) IsLe() int {
	return int(C.pa_sample_format_is_le(C.pa_sample_format_t(f)))
}

// SampleFormat.IsBe returns 1 when the format is big endian.
//
// Returns -1 when endianness does not apply to this format.
func (f SampleFormat) IsBe() int {
	return int(C.pa_sample_format_is_be(C.pa_sample_format_t(f)))
}

// SampleFormat.IsNe returns 1 when the format is native endian.
//
// Returns -1 when endianness does not apply to this format.
func (f SampleFormat) IsNe() int {
	// note: C.pa_sample_format_is_ne() doesn't seem to work
	if SAMPLE_S16NE == SAMPLE_S16LE {
		return int(C.pa_sample_format_is_le(C.pa_sample_format_t(f)))
	}
	return int(C.pa_sample_format_is_be(C.pa_sample_format_t(f)))
}

// SampleFormat.IsRe returns 1 when the format is reverse endian.
//
// Returns -1 when endianness does not apply to this format.
func (f SampleFormat) IsRe() int {
	// note: C.pa_sample_format_is_re() doesn't seem to work
	if SAMPLE_S16NE == SAMPLE_S16LE {
		return int(C.pa_sample_format_is_be(C.pa_sample_format_t(f)))
	}
	return int(C.pa_sample_format_is_le(C.pa_sample_format_t(f)))
}
<commit_msg>Fix silly error.<commit_after>package pulse

/*
#cgo pkg-config: libpulse-simple

#include <stdlib.h>
#include <pulse/sample.h>
*/
import "C"
import "unsafe"

const (
	CHANNELS_MAX = C.PA_CHANNELS_MAX // Maximum number of allowed channels
	RATE_MAX     = C.PA_RATE_MAX     // Maximum allowed sample rate
)

type SampleFormat C.pa_sample_format_t

const (
	SAMPLE_U8        SampleFormat = C.PA_SAMPLE_U8
	SAMPLE_ALAW      SampleFormat = C.PA_SAMPLE_ALAW
	SAMPLE_ULAW      SampleFormat = C.PA_SAMPLE_ULAW
	SAMPLE_S16LE     SampleFormat = C.PA_SAMPLE_S16LE
	SAMPLE_S16BE     SampleFormat = C.PA_SAMPLE_S16BE
	SAMPLE_FLOAT32LE SampleFormat = C.PA_SAMPLE_FLOAT32LE
	SAMPLE_FLOAT32BE SampleFormat = C.PA_SAMPLE_FLOAT32BE
	SAMPLE_S32LE     SampleFormat = C.PA_SAMPLE_S32LE
	SAMPLE_S32BE     SampleFormat = C.PA_SAMPLE_S32BE
	SAMPLE_S24LE     SampleFormat = C.PA_SAMPLE_S24LE
	SAMPLE_S24BE     SampleFormat = C.PA_SAMPLE_S24BE
	SAMPLE_S24_32LE  SampleFormat = C.PA_SAMPLE_S24_32LE
	SAMPLE_S24_32BE  SampleFormat = C.PA_SAMPLE_S24_32BE
	SAMPLE_MAX       SampleFormat = C.PA_SAMPLE_MAX
	SAMPLE_INVALID   SampleFormat = C.PA_SAMPLE_INVALID
)

const (
	SAMPLE_S16NE     SampleFormat = C.PA_SAMPLE_S16NE
	SAMPLE_FLOAT32NE SampleFormat = C.PA_SAMPLE_FLOAT32NE
	SAMPLE_S32NE     SampleFormat = C.PA_SAMPLE_S32NE
	SAMPLE_S24NE     SampleFormat = C.PA_SAMPLE_S24NE
	SAMPLE_S24_32NE  SampleFormat = C.PA_SAMPLE_S24_32NE
	SAMPLE_S16RE     SampleFormat = C.PA_SAMPLE_S16RE
	SAMPLE_FLOAT32RE SampleFormat = C.PA_SAMPLE_FLOAT32RE
	SAMPLE_S32RE     SampleFormat = C.PA_SAMPLE_S32RE
	SAMPLE_S24RE     SampleFormat = C.PA_SAMPLE_S24RE
	SAMPLE_S24_32RE  SampleFormat = C.PA_SAMPLE_S24_32RE
)

const SAMPLE_FLOAT32 SampleFormat = C.PA_SAMPLE_FLOAT32

type SampleSpec struct {
	Format   SampleFormat
	Rate     uint32
	Channels uint8
}

func (spec *SampleSpec) toC() *C.pa_sample_spec {
	if spec == nil {
		return nil
	}
	return &C.pa_sample_spec{
		format:   C.pa_sample_format_t(spec.Format),
		rate:     C.uint32_t(spec.Rate),
		channels: C.uint8_t(spec.Channels),
	}
}

// SampleSpec.BytesPerSecond returns the number of bytes per second of audio.
func (spec *SampleSpec) BytesPerSecond() uint {
	return uint(C.pa_bytes_per_second(spec.toC()))
}

// SampleSpec.FrameSize returns the size of a single audio frame in bytes.
func (spec *SampleSpec) FrameSize() uint {
	return uint(C.pa_frame_size(spec.toC()))
}

// SampleSpec.SampleSize returns the size of a single sample in bytes.
func (spec *SampleSpec) SampleSize() uint {
	return uint(C.pa_sample_size(spec.toC()))
}

// SampleFormat.SampleSize returns the size of a single sample in bytes.
func (f SampleFormat) SampleSize() uint {
	return uint(C.pa_sample_size_of_format(C.pa_sample_format_t(f)))
}

// SampleSpec.BytesToUsec returns the number of microseconds taken
// to play the given number of bytes as audio.
//
// The return value will always be rounded down for non-integral values.
func (spec *SampleSpec) BytesToUsec(bytes uint) uint64 {
	return uint64(C.pa_bytes_to_usec(C.uint64_t(bytes), spec.toC()))
}

// SampleSpec.UsecToBytes returns the number of bytes required
// for the given number of microseconds of audio.
//
// The return value will always be rounded down for non-integral values.
func (spec *SampleSpec) UsecToBytes(usec uint64) uint {
	return uint(C.pa_usec_to_bytes(C.pa_usec_t(usec), spec.toC()))
}

/* TODO?
// Initialize the specified sample spec and return a pointer to
// it. The sample spec will have a defined state but
// pa_sample_spec_valid() will fail for it. \since 0.9.13
pa_sample_spec* pa_sample_spec_init(pa_sample_spec *spec);
*/

// SampleSpec.Valid returns whether or not the given sample spec is valid.
func (spec *SampleSpec) Valid() bool {
	if C.pa_sample_spec_valid(spec.toC()) == 0 {
		return false
	}
	return true
}

// SampleSpec.Equal returns whether or not the given sample specs match.
func (spec *SampleSpec) Equal(other *SampleSpec) bool {
	if C.pa_sample_spec_equal(spec.toC(), other.toC()) == 0 {
		return false
	}
	return true
}

// SampleFormat.String returns a string describing the format.
func (f SampleFormat) String() string {
	cstr := C.pa_sample_format_to_string(C.pa_sample_format_t(f))
	return C.GoString(cstr)
}

// ParseSampleFormat returns the SampleFormat described by the given string.
//
// The string should be as returned by SampleFormat.String().
func ParseSampleFormat(s string) SampleFormat {
	cstr := C.CString(s)
	defer C.free(unsafe.Pointer(cstr))
	return SampleFormat(C.pa_parse_sample_format(cstr))
}

/* TODO
// Maximum required string length for
// pa_sample_spec_snprint(). Please note that this value can change
// with any release without warning and without being considered API
// or ABI breakage. You should not use this definition anywhere where
// it might become part of an ABI.
#define PA_SAMPLE_SPEC_SNPRINT_MAX 32

// Pretty print a sample type specification to a string
char* pa_sample_spec_snprint(char *s, size_t l, const pa_sample_spec *spec);

// Maximum required string length for pa_bytes_snprint(). Please note
// that this value can change with any release without warning and
// without being considered API or ABI breakage. You should not use
// this definition anywhere where it might become part of an
// ABI. \since 0.9.16
#define PA_BYTES_SNPRINT_MAX 11

// Pretty print a byte size value (i.e.\ "2.5 MiB")
char* pa_bytes_snprint(char *s, size_t l, unsigned v);
*/

// SampleFormat.IsLe returns 1 when the format is little endian.
//
// Returns -1 when endianness does not apply to this format.
func (f SampleFormat) IsLe() int {
	return int(C.pa_sample_format_is_le(C.pa_sample_format_t(f)))
}

// SampleFormat.IsBe returns 1 when the format is big endian.
//
// Returns -1 when endianness does not apply to this format.
func (f SampleFormat) IsBe() int {
	return int(C.pa_sample_format_is_be(C.pa_sample_format_t(f)))
}

// SampleFormat.IsNe returns 1 when the format is native endian.
//
// Returns -1 when endianness does not apply to this format.
func (f SampleFormat) IsNe() int {
	// note: C.pa_sample_format_is_ne() doesn't seem to work
	if SAMPLE_S16NE == SAMPLE_S16LE {
		return int(C.pa_sample_format_is_le(C.pa_sample_format_t(f)))
	}
	return int(C.pa_sample_format_is_be(C.pa_sample_format_t(f)))
}

// SampleFormat.IsRe returns 1 when the format is reverse endian.
//
// Returns -1 when endianness does not apply to this format.
func (f SampleFormat) IsRe() int {
	// note: C.pa_sample_format_is_re() doesn't seem to work
	if SAMPLE_S16NE == SAMPLE_S16LE {
		return int(C.pa_sample_format_is_be(C.pa_sample_format_t(f)))
	}
	return int(C.pa_sample_format_is_le(C.pa_sample_format_t(f)))
}
<|endoftext|>


<commit_before>package mq

import (
	"bytes"
	"encoding/json"
	"github.com/Cepave/open-falcon-backend/modules/fe/g"
	"github.com/streadway/amqp"
	"io/ioutil"
	"log"
	"net/http"
	"time"
)

var RetriedLimit = 10
var SleepTimePeriod = time.Duration(60)
var ExitStringPrefix = "Exit mq goroutine safely: "
var LogStringFormat = "%s: %s"

type Message struct {
	Hostname string `json:"hostname"`
	Mute     bool   `json:"mute"`
}

func setup(url string) (*amqp.Connection, *amqp.Channel, error) {
	conn, err := amqp.Dial(url)
	if err != nil {
		log.Printf(LogStringFormat, "Failed to connect to RabbitMQ", err)
		return nil, nil, err
	}

	ch, err := conn.Channel()
	if err != nil {
		log.Printf(LogStringFormat, "Failed to open a channel", err)
		defer conn.Close()
		return nil, nil, err
	}
	return conn, ch, nil
}

func Start() {
	mq := g.Config().Mq
	nodes := map[string]interface{}{}

	// Retry RetriedLimit times if there is some problem during connecting
	var ch *amqp.Channel
	var conn *amqp.Connection
	var err error
	for i := 0; i < RetriedLimit; i++ {
		if conn, ch, err = setup(mq.Queue); err != nil {
			time.Sleep(time.Second * SleepTimePeriod)
		}
	}
	if err != nil {
		log.Println(ExitStringPrefix + "retried too many times.")
		return
	}
	defer conn.Close()
	defer ch.Close()

	q, err := ch.QueueDeclare(
		"mute", // name
		false,  // durable
		false,  // delete when usused
		false,  // exclusive
		false,  // no-wait
		nil,    // arguments
	)
	if err != nil {
		log.Printf(LogStringFormat, ExitStringPrefix+"failed to declare a queue", err)
		return
	}

	msgs, err := ch.Consume(
		q.Name, // queue
		"",     // consumer
		true,   // auto-ack
		false,  // exclusive
		false,  // no-local
		false,  // no-wait
		nil,    // args
	)
	if err != nil {
		log.Printf(LogStringFormat, ExitStringPrefix+"failed to register a consumer", err)
		return
	}

	forever := make(chan bool)

	var pkt Message
	go func() {
		for d := range msgs {
			json.Unmarshal(d.Body, &pkt)
			log.Printf("Received a message: %v", pkt)
			params := map[string]string{
				"host": pkt.Hostname,
				"mute": "",
			}
			if pkt.Mute {
				params["mute"] = "1"
			} else {
				params["mute"] = "0"
			}
			payload := map[string]interface{}{
				"method": "host.update",
				"params": params,
			}
			log.Println("payload =", payload)

			s, err := json.Marshal(payload)
			if err != nil {
				log.Println("json.Marshal Error:", err.Error())
			}

			url := mq.Consumer
			log.Println("url =", url)
			reqPost, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(s)))
			if err != nil {
				log.Println("reqPost Error:", err.Error())
			}
			reqPost.Header.Set("Content-Type", "application/json")

			client := &http.Client{}
			resp, err := client.Do(reqPost)
			if err != nil {
				log.Println("resp Error:", err.Error())
			} else {
				defer resp.Body.Close()
				body, _ := ioutil.ReadAll(resp.Body)
				err = json.Unmarshal(body, &nodes)
				if err != nil {
					log.Println("Unmarshal Error:", err.Error())
				}
				log.Println("nodes =", nodes)
			}
		}
	}()
	log.Println("mq.Start ok. Waiting for messages.")
	<-forever
}
<commit_msg>[OWL-819][fe] mq has abnormal delay.<commit_after>package mq

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"
	"time"

	"github.com/Cepave/open-falcon-backend/modules/fe/g"
	"github.com/streadway/amqp"
)

var RetriedLimit = 10
var SleepTimePeriod = time.Duration(60)
var ExitStringPrefix = "Exit mq goroutine safely: "
var LogStringFormat = "%s: %s"

type Message struct {
	Hostname string `json:"hostname"`
	Mute     bool   `json:"mute"`
}

func setup(url string) (*amqp.Connection, *amqp.Channel, error) {
	conn, err := amqp.Dial(url)
	if err != nil {
		log.Printf(LogStringFormat, "Failed to connect to RabbitMQ", err)
		defer conn.Close()
		return nil, nil, err
	}

	ch, err := conn.Channel()
	if err != nil {
		log.Printf(LogStringFormat, "Failed to open a channel", err)
		defer conn.Close()
		return nil, nil, err
	}
	log.Println("mq: consumer succcessfully opens a channel and setup return err as nil.")
	return conn, ch, nil
}

func Start() {
	log.Println("mq: start mq consumer.")
	mq := g.Config().Mq
	nodes := map[string]interface{}{}

	// Retry RetriedLimit times if there is some problem during connecting
	var ch *amqp.Channel
	var conn *amqp.Connection
	var err error
	for i := 0; i < RetriedLimit; i++ {
		if conn, ch, err = setup(mq.Queue); err != nil {
			time.Sleep(time.Second * SleepTimePeriod)
		} else {
			break
		}
	}
	if err != nil {
		log.Println(ExitStringPrefix + "retried too many times.")
		return
	}
	defer conn.Close()
	defer ch.Close()

	q, err := ch.QueueDeclare(
		"mute", // name
		false,  // durable
		false,  // delete when usused
		false,  // exclusive
		false,  // no-wait
		nil,    // arguments
	)
	if err != nil {
		log.Printf(LogStringFormat, ExitStringPrefix+"failed to declare a queue", err)
		return
	}

	msgs, err := ch.Consume(
		q.Name, // queue
		"",     // consumer
		true,   // auto-ack
		false,  // exclusive
		false,  // no-local
		false,  // no-wait
		nil,    // args
	)
	if err != nil {
		log.Printf(LogStringFormat, ExitStringPrefix+"failed to register a consumer", err)
		return
	}

	forever := make(chan bool)

	var pkt Message
	go func() {
		for d := range msgs {
			json.Unmarshal(d.Body, &pkt)
			log.Printf("Received a message: %v", pkt)
			params := map[string]string{
				"host": pkt.Hostname,
				"mute": "",
			}
			if pkt.Mute {
				params["mute"] = "1"
			} else {
				params["mute"] = "0"
			}
			payload := map[string]interface{}{
				"method": "host.update",
				"params": params,
			}
			log.Println("payload =", payload)

			s, err := json.Marshal(payload)
			if err != nil {
				log.Println("json.Marshal Error:", err.Error())
			}

			url := mq.Consumer
			log.Println("url =", url)
			reqPost, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(s)))
			if err != nil {
				log.Println("reqPost Error:", err.Error())
			}
			reqPost.Header.Set("Content-Type", "application/json")

			client := &http.Client{}
			resp, err := client.Do(reqPost)
			if err != nil {
				log.Println("resp Error:", err.Error())
			} else {
				defer resp.Body.Close()
				body, _ := ioutil.ReadAll(resp.Body)
				err = json.Unmarshal(body, &nodes)
				if err != nil {
					log.Println("Unmarshal Error:", err.Error())
				}
				log.Println("nodes =", nodes)
			}
		}
	}()
	log.Println("mq.Start ok. Waiting for messages.")
	<-forever
}
<|endoftext|>


<commit_before>// +build acceptance

package acceptance

import (
	"fmt"
	"strings"
	"testing"

	tfstatehelper "github.com/hashicorp/terraform-provider-kubernetes/manifest/test/helper/state"
)

func TestKubernetesManifest_CustomResourceDefinition(t *testing.T) {
	kind := strings.Title(randString(8))
	plural := strings.ToLower(kind) + "s"
	group := "terraform.io"
	version := "v1"
	name := fmt.Sprintf("%s.%s", plural, group)

	tf := tfhelper.RequireNewWorkingDir(t)
	tf.SetReattachInfo(reattachInfo)
	defer func() {
		tf.RequireDestroy(t)
		tf.Close()
		k8shelper.AssertResourceDoesNotExist(t, "apiextensions.k8s.io/v1", "customresourcedefinitions", name)
	}()

	tfvars := TFVARS{
		"kind":       kind,
		"plural":     plural,
		"group":      group,
		"cr_version": version,
	}
	tfconfig := loadTerraformConfig(t, "CustomResourceDefinition/customresourcedefinition.tf", tfvars)
	tf.RequireSetConfig(t, tfconfig)
	tf.RequireInit(t)
	tf.RequireApply(t)

	k8shelper.AssertResourceExists(t, "apiextensions.k8s.io/v1", "customresourcedefinitions", name)

	tfstate := tfstatehelper.NewHelper(tf.RequireState(t))
	tfstate.AssertAttributeValues(t, tfstatehelper.AttributeValues{
		"kubernetes_manifest.test.object.metadata.name":           name,
		"kubernetes_manifest.test.object.spec.group":              group,
		"kubernetes_manifest.test.object.spec.names.kind":         kind,
		"kubernetes_manifest.test.object.spec.names.plural":       plural,
		"kubernetes_manifest.test.object.spec.scope":              "Namespaced",
		"kubernetes_manifest.test.object.spec.versions.0.name":    version,
		"kubernetes_manifest.test.object.spec.versions.0.served":  true,
		"kubernetes_manifest.test.object.spec.versions.0.storage": true,
		"kubernetes_manifest.test.object.spec.versions.0.schema": map[string]interface{}{
			"openAPIV3Schema": map[string]interface{}{
				"type": "object",
				"properties": map[string]interface{}{
					"data": map[string]interface{}{
						"type": "string",
					},
					"refs": map[string]interface{}{
						"type": "number",
					},
				},
			},
		},
		"kubernetes_manifest.test.object.spec.versions.1.name":    version + "beta1",
		"kubernetes_manifest.test.object.spec.versions.1.served":  true,
		"kubernetes_manifest.test.object.spec.versions.1.storage": false,
		"kubernetes_manifest.test.object.spec.versions.1.schema": map[string]interface{}{
			"openAPIV3Schema": map[string]interface{}{
				"type": "object",
				"properties": map[string]interface{}{
					"data": map[string]interface{}{
						"type": "string",
					},
					"otherData": map[string]interface{}{
						"type": "string",
					},
					"refs": map[string]interface{}{
						"type": "number",
					},
				},
			},
		},
	})
}
<commit_msg>Update CRD structure assertions<commit_after>//go:build acceptance
// +build acceptance

package acceptance

import (
	"fmt"
	"strings"
	"testing"

	tfstatehelper "github.com/hashicorp/terraform-provider-kubernetes/manifest/test/helper/state"
)

func TestKubernetesManifest_CustomResourceDefinition(t *testing.T) {
	kind := strings.Title(randString(8))
	plural := strings.ToLower(kind) + "s"
	group := "terraform.io"
	version := "v1"
	name := fmt.Sprintf("%s.%s", plural, group)

	tf := tfhelper.RequireNewWorkingDir(t)
	tf.SetReattachInfo(reattachInfo)
	defer func() {
		tf.RequireDestroy(t)
		tf.Close()
		k8shelper.AssertResourceDoesNotExist(t, "apiextensions.k8s.io/v1", "customresourcedefinitions", name)
	}()

	tfvars := TFVARS{
		"kind":       kind,
		"plural":     plural,
		"group":      group,
		"cr_version": version,
	}
	tfconfig := loadTerraformConfig(t, "CustomResourceDefinition/customresourcedefinition.tf", tfvars)
	tf.RequireSetConfig(t, tfconfig)
	tf.RequireInit(t)
	tf.RequireApply(t)

	k8shelper.AssertResourceExists(t, "apiextensions.k8s.io/v1", "customresourcedefinitions", name)

	tfstate := tfstatehelper.NewHelper(tf.RequireState(t))
	tfstate.AssertAttributeValues(t, tfstatehelper.AttributeValues{
		"kubernetes_manifest.test.object.metadata.name":           name,
		"kubernetes_manifest.test.object.spec.group":              group,
		"kubernetes_manifest.test.object.spec.names.kind":         kind,
		"kubernetes_manifest.test.object.spec.names.plural":       plural,
		"kubernetes_manifest.test.object.spec.scope":              "Namespaced",
		"kubernetes_manifest.test.object.spec.versions.0.name":    version,
		"kubernetes_manifest.test.object.spec.versions.0.served":  true,
		"kubernetes_manifest.test.object.spec.versions.0.storage": true,
		"kubernetes_manifest.test.object.spec.versions.0.schema": map[string]interface{}{
			"openAPIV3Schema": map[string]interface{}{
				"type": "object",
				"properties": map[string]interface{}{
					"data": map[string]interface{}{
						"type": "string",
					},
					"refs": map[string]interface{}{
						"type": "number",
					},
					"stuff": map[string]interface{}{
						"type": "array",
						"items": map[string]interface{}{
							"type": "object",
							"properties": map[string]interface{}{
								"foo": map[string]interface{}{
									"type": "string",
								},
							},
						},
					},
				},
			},
		},
		"kubernetes_manifest.test.object.spec.versions.1.name":    version + "beta1",
		"kubernetes_manifest.test.object.spec.versions.1.served":  true,
		"kubernetes_manifest.test.object.spec.versions.1.storage": false,
		"kubernetes_manifest.test.object.spec.versions.1.schema": map[string]interface{}{
			"openAPIV3Schema": map[string]interface{}{
				"type": "object",
				"properties": map[string]interface{}{
					"data": map[string]interface{}{
						"type": "string",
					},
					"otherData": map[string]interface{}{
						"type": "string",
					},
					"refs": map[string]interface{}{
						"type": "number",
					},
				},
			},
		},
	})
}
<|endoftext|>


<commit_before>// Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
// Licensed under the Mozilla Public License v2.0

package oci

import (
	"context"

	"github.com/hashicorp/terraform/helper/schema"
	oci_object_storage "github.com/oracle/oci-go-sdk/objectstorage"
)

func init() {
	RegisterDatasource("oci_objectstorage_object_lifecycle_policy", ObjectStorageObjectLifecyclePolicyDataSource())
}

func ObjectStorageObjectLifecyclePolicyDataSource() *schema.Resource {
	fieldMap := make(map[string]*schema.Schema)
	fieldMap["bucket"] = &schema.Schema{
		Type:     schema.TypeString,
		Required: true,
	}
	fieldMap["namespace"] = &schema.Schema{
		Type:     schema.TypeString,
		Required: true,
	}
	return GetSingularDataSourceItemSchema(ObjectStorageObjectLifecyclePolicyResource(), fieldMap, readSingularObjectStorageObjectLifecyclePolicy)
}

func readSingularObjectStorageObjectLifecyclePolicy(d *schema.ResourceData, m interface{}) error {
	sync := &ObjectStorageObjectLifecyclePolicyDataSourceCrud{}
	sync.D = d
	sync.Client = m.(*OracleClients).objectStorageClient()

	return ReadResource(sync)
}

type ObjectStorageObjectLifecyclePolicyDataSourceCrud struct {
	D      *schema.ResourceData
	Client *oci_object_storage.ObjectStorageClient
	Res    *oci_object_storage.GetObjectLifecyclePolicyResponse
}

func (s *ObjectStorageObjectLifecyclePolicyDataSourceCrud) VoidState() {
	s.D.SetId("")
}

func (s *ObjectStorageObjectLifecyclePolicyDataSourceCrud) Get() error {
	request := oci_object_storage.GetObjectLifecyclePolicyRequest{}

	if bucket, ok := s.D.GetOkExists("bucket"); ok {
		tmp := bucket.(string)
		request.BucketName = &tmp
	}

	if namespace, ok := s.D.GetOkExists("namespace"); ok {
		tmp := namespace.(string)
		request.NamespaceName = &tmp
	}

	request.RequestMetadata.RetryPolicy = getRetryPolicy(false, "object_storage")

	response, err := s.Client.GetObjectLifecyclePolicy(context.Background(), request)
	if err != nil {
		return err
	}

	s.Res = &response
	return nil
}

func (s *ObjectStorageObjectLifecyclePolicyDataSourceCrud) SetData() error {
	if s.Res == nil {
		return nil
	}

	s.D.SetId(GenerateDataSourceID())

	rules := []interface{}{}
	for _, item := range s.Res.Items {
		var objectLifecycleRuleMap = ObjectLifecycleRuleToMap(item)
		fixupObjectNameFilterInclusionPrefixesAsList(objectLifecycleRuleMap)
		rules = append(rules, objectLifecycleRuleMap)
	}
	s.D.Set("rules", rules)

	if s.Res.TimeCreated != nil {
		s.D.Set("time_created", s.Res.TimeCreated.String())
	}

	return nil
}

func fixupObjectNameFilterInclusionPrefixesAsList(objectLifecycleRuleMap map[string]interface{}) {
	objectNameFilterList := objectLifecycleRuleMap["object_name_filter"].([]interface{})
	if objectNameFilterList != nil {
		firstElement := objectNameFilterList[0]
		objectNameFilterMap := firstElement.(map[string]interface{})
		inclusionPrefixesSet := objectNameFilterMap["inclusion_prefixes"].(*schema.Set)
		objectNameFilterMap["inclusion_prefixes"] = inclusionPrefixesSet.List()

		inclusionPatternsSet := objectNameFilterMap["inclusion_patterns"].(*schema.Set)
		objectNameFilterMap["inclusion_patterns"] = inclusionPatternsSet.List()

		exclusionPatternsSet := objectNameFilterMap["exclusion_patterns"].(*schema.Set)
		objectNameFilterMap["exclusion_patterns"] = exclusionPatternsSet.List()
	}
}
<commit_msg>Add error handling for get on map and type conversion in object lifecycle policy data source<commit_after>// Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
// Licensed under the Mozilla Public License v2.0

package oci

import (
	"context"

	"github.com/hashicorp/terraform/helper/schema"
	oci_object_storage "github.com/oracle/oci-go-sdk/objectstorage"
)

func init() {
	RegisterDatasource("oci_objectstorage_object_lifecycle_policy", ObjectStorageObjectLifecyclePolicyDataSource())
}

func ObjectStorageObjectLifecyclePolicyDataSource() *schema.Resource {
	fieldMap := make(map[string]*schema.Schema)
	fieldMap["bucket"] = &schema.Schema{
		Type:     schema.TypeString,
		Required: true,
	}
	fieldMap["namespace"] = &schema.Schema{
		Type:     schema.TypeString,
		Required: true,
	}
	return GetSingularDataSourceItemSchema(ObjectStorageObjectLifecyclePolicyResource(), fieldMap, readSingularObjectStorageObjectLifecyclePolicy)
}

func readSingularObjectStorageObjectLifecyclePolicy(d *schema.ResourceData, m interface{}) error {
	sync := &ObjectStorageObjectLifecyclePolicyDataSourceCrud{}
	sync.D = d
	sync.Client = m.(*OracleClients).objectStorageClient()

	return ReadResource(sync)
}

type ObjectStorageObjectLifecyclePolicyDataSourceCrud struct {
	D      *schema.ResourceData
	Client *oci_object_storage.ObjectStorageClient
	Res    *oci_object_storage.GetObjectLifecyclePolicyResponse
}

func (s *ObjectStorageObjectLifecyclePolicyDataSourceCrud) VoidState() {
	s.D.SetId("")
}

func (s *ObjectStorageObjectLifecyclePolicyDataSourceCrud) Get() error {
	request := oci_object_storage.GetObjectLifecyclePolicyRequest{}

	if bucket, ok := s.D.GetOkExists("bucket"); ok {
		tmp := bucket.(string)
		request.BucketName = &tmp
	}

	if namespace, ok := s.D.GetOkExists("namespace"); ok {
		tmp := namespace.(string)
		request.NamespaceName = &tmp
	}

	request.RequestMetadata.RetryPolicy = getRetryPolicy(false, "object_storage")

	response, err := s.Client.GetObjectLifecyclePolicy(context.Background(), request)
	if err != nil {
		return err
	}

	s.Res = &response
	return nil
}

func (s *ObjectStorageObjectLifecyclePolicyDataSourceCrud) SetData() error {
	if s.Res == nil {
		return nil
	}

	s.D.SetId(GenerateDataSourceID())

	rules := []interface{}{}
	for _, item := range s.Res.Items {
		var objectLifecycleRuleMap = ObjectLifecycleRuleToMap(item)
		fixupObjectNameFilterInclusionPrefixesAsList(objectLifecycleRuleMap)
		rules = append(rules, objectLifecycleRuleMap)
	}
	s.D.Set("rules", rules)

	if s.Res.TimeCreated != nil {
		s.D.Set("time_created", s.Res.TimeCreated.String())
	}

	return nil
}

func fixupObjectNameFilterInclusionPrefixesAsList(objectLifecycleRuleMap map[string]interface{}) {
	if objectNameFilterList, exists := objectLifecycleRuleMap["object_name_filter"]; exists {
		if objectNameFilterList, ok := objectNameFilterList.([]interface{}); ok && len(objectNameFilterList) > 0 {
			firstElement := objectNameFilterList[0]

			if objectNameFilterMap, ok := firstElement.(map[string]interface{}); ok {
				if inclusionPrefixesSet, ok := objectNameFilterMap["inclusion_prefixes"].(*schema.Set); ok {
					objectNameFilterMap["inclusion_prefixes"] = inclusionPrefixesSet.List()
				}

				if inclusionPatternsSet, ok := objectNameFilterMap["inclusion_patterns"].(*schema.Set); ok {
					objectNameFilterMap["inclusion_patterns"] = inclusionPatternsSet.List()
				}

				if exclusionPatternsSet, ok := objectNameFilterMap["exclusion_patterns"].(*schema.Set); ok {
					objectNameFilterMap["exclusion_patterns"] = exclusionPatternsSet.List()
				}
			}

		}

	}
}
<|endoftext|>


<commit_before>package transloadit

import (
	"os"
	"strings"
	"testing"
)

var assemblyId string
var assemblyUrl string

func TestAssembly(t *testing.T) {

	client := setup(t)

	assembly := client.CreateAssembly()

	file, err := os.Open("./fixtures/lol_cat.jpg")
	if err != nil {
		t.Fatal(err)
	}

	file2, err := os.Open("./fixtures/mona_lisa.jpg")
	if err != nil {
		t.Fatal(err)
	}

	assembly.AddReader("image", "lol_cat.jpg", file)
	assembly.AddReader("image2", "mona_lisa.jpg", file2)

	assembly.AddStep("resize", map[string]interface{}{
		"robot":           "/image/resize",
		"width":           75,
		"height":          75,
		"resize_strategy": "pad",
		"background":      "#000000",
	})

	assembly.NotifyUrl = "http://requestb.in/1kwp6lx1"

	info, err := assembly.Upload()
	if err != nil {
		t.Fatal(err)
	}

	if info.AssemblyId == "" {
		t.Fatal("response doesn't contain assembly_id")
	}

	if info.NotifyUrl != "http://requestb.in/1kwp6lx1" {
		t.Fatal("wrong notify url")
	}

	if len(info.Uploads) != 2 {
		t.Fatal("wrong number of uploads")
	}

	if info.Uploads[0].Name == "lol_cat.jpg" {
		if info.Uploads[0].Field != "image" {
			t.Fatal("wrong field name")
		}
	} else if info.Uploads[1].Name == "lol_cat.jpg" {
		if info.Uploads[1].Field != "image" {
			t.Fatal("wrong field name")
		}
	} else {
		t.Fatal("lol_cat.jpg not found in uploads")
	}

	if info.ClientAgent != "Transloadit Go SDK v1" {
		t.Fatal("wrong user agent")
	}

	assemblyId = info.AssemblyId
	assemblyUrl = info.AssemblyUrl
}

func TestAssemblyFail(t *testing.T) {

	config := DefaultConfig
	config.AuthKey = "does not exist"
	config.AuthSecret = "does not matter"

	client, err := NewClient(&config)
	if err != nil {
		t.Fatal(err)
	}

	assembly := client.CreateAssembly()

	file, err := os.Open("./fixtures/lol_cat.jpg")
	if err != nil {
		t.Fatal(err)
	}

	assembly.AddReader("image", "lol_cat.jpg", file)

	assembly.AddStep("resize", map[string]interface{}{
		"robot":           "/image/resize",
		"width":           75,
		"height":          75,
		"resize_strategy": "pad",
		"background":      "#000000",
	})

	info, err := assembly.Upload()
	if err != nil {
		t.Fatal(err)
	}

	if info.Error != "GET_ACCOUNT_UNKNOWN_AUTH_KEY" {
		t.Fatal("reponse doesn't contain error message")
	}

}

func TestAssemblyBlocking(t *testing.T) {

	client := setup(t)

	assembly := client.CreateAssembly()

	file, err := os.Open("./fixtures/lol_cat.jpg")
	if err != nil {
		t.Fatal(err)
	}

	assembly.AddReader("image", "lol_cat.jpg", file)

	assembly.AddStep("resize", map[string]interface{}{
		"robot":           "/image/resize",
		"width":           75,
		"height":          75,
		"resize_strategy": "pad",
		"background":      "#000000",
	})

	assembly.Blocking = true

	info, err := assembly.Upload()
	if err != nil {
		t.Fatal(err)
	}

	if info.Ok != "ASSEMBLY_COMPLETED" {
		t.Fatal("wrong assembly status")
	}

	if info.AssemblyId == "" {
		t.Fatal("response doesn't contain assembly_id")
	}

	if len(info.Uploads) != 1 {
		t.Fatal("wrong number of uploads")
	}
}

func TestGetAssembly(t *testing.T) {

	client := setup(t)

	assembly, err := client.GetAssembly(assemblyUrl)
	if err != nil {
		t.Fatal(err)
	}

	if assembly.AssemblyId == "" {
		t.Fatal("assembly id not contained")
	}

	if assembly.AssemblyUrl != assemblyUrl {
		t.Fatal("assembly urls don't match")
	}

}

func TestReplayAssembly(t *testing.T) {

	client := setup(t)

	assembly := client.ReplayAssembly(assemblyId)

	assembly.NotifyUrl = "http://requestb.in/1kwp6lx1"
	assembly.ReparseTemplate()

	info, err := assembly.Start()
	if err != nil {
		t.Fatal(err)
	}

	if info.Ok != "ASSEMBLY_REPLAYING" {
		t.Fatal("wrong status code returned")
	}

	if info.NotifyUrl != "http://requestb.in/1kwp6lx1" {
		t.Fatal("wrong notify url")
	}

}

func TestReplayAssemblyBlocking(t *testing.T) {

	client := setup(t)

	assembly := client.ReplayAssembly(assemblyId)

	assembly.Blocking = true

	info, err := assembly.Start()
	if err != nil {
		t.Fatal(err)
	}

	if info.Ok != "ASSEMBLY_COMPLETED" {
		t.Fatal("wrong status code returned")
	}

}

func TestAssemblyUsingTemplate(t *testing.T) {

	client := setup(t)

	assembly := client.CreateAssembly()

	assembly.TemplateId = "64c11b20308811e4b5548d4f316c150f"

	info, err := assembly.Upload()
	if err != nil {
		t.Fatal(err)
	}

	if info.AssemblyId == "" {
		t.Fatal("response doesn't contain assembly_id")
	}

	if !strings.Contains(info.Params, "64c11b20308811e4b5548d4f316c150f") {
		t.Fatal("template id not as parameter submitted")
	}
}

func TestCancelAssembly(t *testing.T) {

	client := setup(t)

	assembly := client.CreateAssembly()

	assembly.AddStep("import", map[string]interface{}{
		"robot": "/http/import",
		"url":   "http://mirror.nl.leaseweb.net/speedtest/10000mb.bin",
	})

	info, err := assembly.Upload()
	if err != nil {
		t.Fatal(err)
	}

	if info.AssemblyUrl == "" {
		t.Fatal("response doesn't contain assembly_url")
	}

	_, err = client.CancelAssembly(info.AssemblyUrl)
	if err != nil {
		t.Fatal(err)
	}

}

func TestListAssemblies(t *testing.T) {

	client := setup(t)

	assemblies, err := client.ListAssemblies(&ListOptions{
		PageSize: 3,
	})
	if err != nil {
		t.Fatal(err)
	}

	if len(assemblies.Assemblies) < 3 {
		t.Fatal("wrong number of assemblies")
	}

	if assemblies.Count == 0 {
		t.Fatal("wrong count")
	}

	if assemblies.Assemblies[0].AssemblyId == "" {
		t.Fatal("wrong template name")
	}

}
<commit_msg>Easier debugging if we show the error #13<commit_after>package transloadit

import (
	"fmt"
	"os"
	"strings"
	"testing"
)

var assemblyId string
var assemblyUrl string

func TestAssembly(t *testing.T) {

	client := setup(t)

	assembly := client.CreateAssembly()

	file, err := os.Open("./fixtures/lol_cat.jpg")
	if err != nil {
		t.Fatal(err)
	}

	file2, err := os.Open("./fixtures/mona_lisa.jpg")
	if err != nil {
		t.Fatal(err)
	}

	assembly.AddReader("image", "lol_cat.jpg", file)
	assembly.AddReader("image2", "mona_lisa.jpg", file2)

	assembly.AddStep("resize", map[string]interface{}{
		"robot":           "/image/resize",
		"width":           75,
		"height":          75,
		"resize_strategy": "pad",
		"background":      "#000000",
	})

	assembly.NotifyUrl = "http://requestb.in/1kwp6lx1"

	info, err := assembly.Upload()
	if err != nil {
		t.Fatal(err)
	}

	if info.AssemblyId == "" {
		t.Fatal("response doesn't contain assembly_id")
	}

	if info.NotifyUrl != "http://requestb.in/1kwp6lx1" {
		t.Fatal("wrong notify url")
	}

	if len(info.Uploads) != 2 {
		t.Fatal("wrong number of uploads")
	}

	if info.Uploads[0].Name == "lol_cat.jpg" {
		if info.Uploads[0].Field != "image" {
			t.Fatal("wrong field name")
		}
	} else if info.Uploads[1].Name == "lol_cat.jpg" {
		if info.Uploads[1].Field != "image" {
			t.Fatal("wrong field name")
		}
	} else {
		t.Fatal("lol_cat.jpg not found in uploads")
	}

	if info.ClientAgent != "Transloadit Go SDK v1" {
		t.Fatal("wrong user agent")
	}

	assemblyId = info.AssemblyId
	assemblyUrl = info.AssemblyUrl
}

func TestAssemblyFail(t *testing.T) {

	config := DefaultConfig
	config.AuthKey = "does not exist"
	config.AuthSecret = "does not matter"

	client, err := NewClient(&config)
	if err != nil {
		t.Fatal(err)
	}

	assembly := client.CreateAssembly()

	file, err := os.Open("./fixtures/lol_cat.jpg")
	if err != nil {
		t.Fatal(err)
	}

	assembly.AddReader("image", "lol_cat.jpg", file)

	assembly.AddStep("resize", map[string]interface{}{
		"robot":           "/image/resize",
		"width":           75,
		"height":          75,
		"resize_strategy": "pad",
		"background":      "#000000",
	})

	info, err := assembly.Upload()
	if err != nil {
		t.Fatal(err)
	}

	if info.Error != "GET_ACCOUNT_UNKNOWN_AUTH_KEY" {
		t.Fatal("reponse doesn't contain error message")
	}

}

func TestAssemblyBlocking(t *testing.T) {

	client := setup(t)

	assembly := client.CreateAssembly()

	file, err := os.Open("./fixtures/lol_cat.jpg")
	if err != nil {
		t.Fatal(err)
	}

	assembly.AddReader("image", "lol_cat.jpg", file)

	assembly.AddStep("resize", map[string]interface{}{
		"robot":           "/image/resize",
		"width":           75,
		"height":          75,
		"resize_strategy": "pad",
		"background":      "#000000",
	})

	assembly.Blocking = true

	info, err := assembly.Upload()
	if err != nil {
		t.Fatal(err)
	}

	if info.Ok != "ASSEMBLY_COMPLETED" {
		t.Fatal("wrong assembly status")
	}

	if info.AssemblyId == "" {
		t.Fatal("response doesn't contain assembly_id")
	}

	if len(info.Uploads) != 1 {
		t.Fatal("wrong number of uploads")
	}
}

func TestGetAssembly(t *testing.T) {

	client := setup(t)

	assembly, err := client.GetAssembly(assemblyUrl)
	if err != nil {
		t.Fatal(err)
	}

	if assembly.AssemblyId == "" {
		t.Fatal("assembly id not contained")
	}

	if assembly.AssemblyUrl != assemblyUrl {
		t.Fatal("assembly urls don't match")
	}

}

func TestReplayAssembly(t *testing.T) {

	client := setup(t)

	assembly := client.ReplayAssembly(assemblyId)

	assembly.NotifyUrl = "http://requestb.in/1kwp6lx1"
	assembly.ReparseTemplate()

	info, err := assembly.Start()
	if err != nil {
		t.Fatal(err)
	}

	if info.Ok != "ASSEMBLY_REPLAYING" {
		t.Fatal("wrong status code returned")
	}

	if info.NotifyUrl != "http://requestb.in/1kwp6lx1" {
		t.Fatal("wrong notify url")
	}

}

func TestReplayAssemblyBlocking(t *testing.T) {

	client := setup(t)

	assembly := client.ReplayAssembly(assemblyId)

	assembly.Blocking = true

	info, err := assembly.Start()
	if err != nil {
		t.Fatal(err)
	}

	if info.Ok != "ASSEMBLY_COMPLETED" {
		t.Fatal("wrong status code returned")
	}

}

func TestAssemblyUsingTemplate(t *testing.T) {

	client := setup(t)

	assembly := client.CreateAssembly()

	assembly.TemplateId = "64c11b20308811e4b5548d4f316c150f"

	info, err := assembly.Upload()
	if err != nil {
		t.Fatal(err)
	}

	if info.AssemblyId == "" {
		t.Fatal(fmt.Sprintf("response doesn't contain assembly_id. %s", info.Error))
	}

	if !strings.Contains(info.Params, "64c11b20308811e4b5548d4f316c150f") {
		t.Fatal("template id not as parameter submitted")
	}
}

func TestCancelAssembly(t *testing.T) {

	client := setup(t)

	assembly := client.CreateAssembly()

	assembly.AddStep("import", map[string]interface{}{
		"robot": "/http/import",
		"url":   "http://mirror.nl.leaseweb.net/speedtest/10000mb.bin",
	})

	info, err := assembly.Upload()
	if err != nil {
		t.Fatal(err)
	}

	if info.AssemblyUrl == "" {
		t.Fatal("response doesn't contain assembly_url")
	}

	_, err = client.CancelAssembly(info.AssemblyUrl)
	if err != nil {
		t.Fatal(err)
	}

}

func TestListAssemblies(t *testing.T) {

	client := setup(t)

	assemblies, err := client.ListAssemblies(&ListOptions{
		PageSize: 3,
	})
	if err != nil {
		t.Fatal(err)
	}

	if len(assemblies.Assemblies) < 3 {
		t.Fatal("wrong number of assemblies")
	}

	if assemblies.Count == 0 {
		t.Fatal("wrong count")
	}

	if assemblies.Assemblies[0].AssemblyId == "" {
		t.Fatal("wrong template name")
	}

}
<|endoftext|>


<commit_before>package host

// TODO: Need to check that 'RevisionConfirmed' is sensitive to whether or not
// it was the *most recent* revision that got confirmed.

import (
	"encoding/binary"
	"encoding/json"

	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"

	"github.com/NebulousLabs/bolt"
)

// initRescan is a helper function of initConsensusSubscribe, and is called when
// the host and the consensus set have become desynchronized. Desynchronization
// typically happens if the user is replacing or altering the persistent files
// in the consensus set or the host.
func (h *Host) initRescan() error {
	// Reset all of the variables that have relevance to the consensus set.
	var allObligations []*storageObligation
	err := func() error {
		h.mu.Lock()
		defer h.mu.Unlock()

		// Reset all of the consensus-relevant variables in the host.
		h.blockHeight = 0

		// Reset all of the storage obligations.
		err := h.db.Update(func(tx *bolt.Tx) error {
			bsu := tx.Bucket(bucketStorageObligations)
			c := bsu.Cursor()
			for k, soBytes := c.First(); soBytes != nil; k, soBytes = c.Next() {
				var so storageObligation
				err := json.Unmarshal(soBytes, &so)
				if err != nil {
					return err
				}
				so.OriginConfirmed = false
				so.RevisionConfirmed = false
				so.ProofConfirmed = false
				allObligations = append(allObligations, &so)
				soBytes, err = json.Marshal(so)
				if err != nil {
					return err
				}
				err = bsu.Put(k, soBytes)
				if err != nil {
					return err
				}
			}
			return nil
		})
		if err != nil {
			return err
		}

		return h.save()
	}()
	if err != nil {
		return err
	}

	// Subscribe to the consensus set. This is a blocking call that will not
	// return until the host has fully caught up to the current block.
	err = h.cs.ConsensusSetSubscribe(h, modules.ConsensusChangeBeginning)
	if err != nil {
		return err
	}

	// Re-queue all of the action items for the storage obligations.
	for _, so := range allObligations {
		soid := so.id()
		err0 := h.tpool.AcceptTransactionSet(so.OriginTransactionSet)
		err1 := h.queueActionItem(h.blockHeight+resubmissionTimeout, soid)
		err2 := h.queueActionItem(so.expiration()-revisionSubmissionBuffer, soid)
		err3 := h.queueActionItem(so.expiration()+resubmissionTimeout, soid)
		err = composeErrors(err0, err1, err2, err3)
		if err != nil {
			return composeErrors(err, h.removeStorageObligation(so, obligationRejected))
		}
	}
	return nil
}

// initConsensusSubscription subscribes the host to the consensus set.
func (h *Host) initConsensusSubscription() error {
	err := h.cs.ConsensusSetSubscribe(h, h.recentChange)
	if err == modules.ErrInvalidConsensusChangeID {
		// Perform a rescan of the consensus set if the change id that the host
		// has is unrecognized by the consensus set. This will typically only
		// happen if the user has been replacing files inside the Sia folder
		// structure.
		return h.initRescan()
	}
	return err
}

// ProcessConsensusChange will be called by the consensus set every time there
// is a change to the blockchain.
func (h *Host) ProcessConsensusChange(cc modules.ConsensusChange) {
	h.mu.Lock()
	defer h.mu.Unlock()
	err := h.tg.Add()
	if err != nil {
		return
	}
	defer h.tg.Done()

	// Wrap the whole parsing into a single large database tx to keep things
	// efficient.
	var actionItems []*storageObligation
	err = h.db.Update(func(tx *bolt.Tx) error {
		for _, block := range cc.RevertedBlocks {
			// Look for transactions relevant to open storage obligations.
			for _, txn := range block.Transactions {
				// Check for file contracts.
				if len(txn.FileContracts) > 0 {
					for j := range txn.FileContracts {
						fcid := txn.FileContractID(uint64(j))
						so, err := getStorageObligation(tx, fcid)
						if err != nil {
							// The storage folder may not exist, or the disk
							// may be having trouble. Either way, we ignore the
							// problem. If the disk is having trouble, the user
							// will have to perform a rescan.
							continue
						}
						so.OriginConfirmed = false
						err = putStorageObligation(tx, so)
						if err != nil {
							continue
						}
					}
				}

				// Check for file contract revisions.
				if len(txn.FileContractRevisions) > 0 {
					for _, fcr := range txn.FileContractRevisions {
						so, err := getStorageObligation(tx, fcr.ParentID)
						if err != nil {
							// The storage folder may not exist, or the disk
							// may be having trouble. Either way, we ignore the
							// problem. If the disk is having trouble, the user
							// will have to perform a rescan.
							continue
						}
						so.RevisionConfirmed = false
						err = putStorageObligation(tx, so)
						if err != nil {
							continue
						}
					}
				}

				// Check for storage proofs.
				if len(txn.StorageProofs) > 0 {
					for _, sp := range txn.StorageProofs {
						// Check database for relevant storage proofs.
						so, err := getStorageObligation(tx, sp.ParentID)
						if err != nil {
							// The storage folder may not exist, or the disk
							// may be having trouble. Either way, we ignore the
							// problem. If the disk is having trouble, the user
							// will have to perform a rescan.
							continue
						}
						so.ProofConfirmed = false
						err = putStorageObligation(tx, so)
						if err != nil {
							continue
						}
					}
				}
			}

			// Height is not adjusted when dealing with the genesis block because
			// the default height is 0 and the genesis block height is 0. If
			// removing the genesis block, height will already be at height 0 and
			// should not update, lest an underflow occur.
			if block.ID() != types.GenesisID {
				h.blockHeight--
			}
		}
		for _, block := range cc.AppliedBlocks {
			// Look for transactions relevant to open storage obligations.
			for _, txn := range block.Transactions {
				// Check for file contracts.
				if len(txn.FileContracts) > 0 {
					for i := range txn.FileContracts {
						fcid := txn.FileContractID(uint64(i))
						so, err := getStorageObligation(tx, fcid)
						if err != nil {
							// The storage folder may not exist, or the disk
							// may be having trouble. Either way, we ignore the
							// problem. If the disk is having trouble, the user
							// will have to perform a rescan.
							continue
						}
						so.OriginConfirmed = true
						err = putStorageObligation(tx, so)
						if err != nil {
							continue
						}
					}
				}

				// Check for file contract revisions.
				if len(txn.FileContractRevisions) > 0 {
					for _, fcr := range txn.FileContractRevisions {
						so, err := getStorageObligation(tx, fcr.ParentID)
						if err != nil {
							// The storage folder may not exist, or the disk
							// may be having trouble. Either way, we ignore the
							// problem. If the disk is having trouble, the user
							// will have to perform a rescan.
							continue
						}
						so.RevisionConfirmed = true
						err = putStorageObligation(tx, so)
						if err != nil {
							continue
						}
					}
				}

				// Check for storage proofs.
				if len(txn.StorageProofs) > 0 {
					for _, sp := range txn.StorageProofs {
						so, err := getStorageObligation(tx, sp.ParentID)
						if err != nil {
							// The storage folder may not exist, or the disk
							// may be having trouble. Either way, we ignore the
							// problem. If the disk is having trouble, the user
							// will have to perform a rescan.
							continue
						}
						so.ProofConfirmed = true
						err = putStorageObligation(tx, so)
						if err != nil {
							continue
						}
					}
				}
			}

			// Height is not adjusted when dealing with the genesis block because
			// the default height is 0 and the genesis block height is 0. If adding
			// the genesis block, height will already be at height 0 and should not
			// update.
			if block.ID() != types.GenesisID {
				h.blockHeight++
			}

			// Handle any action items relevant to the current height.
			bai := tx.Bucket(bucketActionItems)
			heightBytes := make([]byte, 8)
			binary.BigEndian.PutUint64(heightBytes, uint64(h.blockHeight)) // BigEndian used so bolt will keep things sorted automatically.
			existingItems := bai.Get(heightBytes)

			// From the existing items, pull out a storage obligation.
			knownActionItems := make(map[types.FileContractID]struct{})
			obligationIDs := make([]types.FileContractID, len(existingItems)/crypto.HashSize)
			for i := 0; i < len(existingItems); i += crypto.HashSize {
				copy(obligationIDs[i/crypto.HashSize][:], existingItems[i:i+crypto.HashSize])
			}
			bso := tx.Bucket(bucketStorageObligations)
			for _, soid := range obligationIDs {
				soBytes := bso.Get(soid[:])
				var so storageObligation
				err := json.Unmarshal(soBytes, &so)
				if err != nil {
					return err
				}
				_, exists := knownActionItems[soid]
				if !exists {
					actionItems = append(actionItems, &so)
					knownActionItems[soid] = struct{}{}
				}
			}
		}
		return nil
	})
	if err != nil {
		h.log.Println(err)
	}

	// Handle the list of action items.
	for _, ai := range actionItems {
		h.handleActionItem(ai)
	}

	// Update the host's recent change pointer to point to the most recent
	// change.
	h.recentChange = cc.ID

	// Save the host.
	err = h.save()
	if err != nil {
		h.log.Println("ERROR: could not save during ProcessConsensusChange:", err)
	}
}
<commit_msg>more gracefully handle missing storage obligation<commit_after>package host

// TODO: Need to check that 'RevisionConfirmed' is sensitive to whether or not
// it was the *most recent* revision that got confirmed.

import (
	"encoding/binary"
	"encoding/json"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"

	"github.com/NebulousLabs/bolt"
)

// initRescan is a helper function of initConsensusSubscribe, and is called when
// the host and the consensus set have become desynchronized. Desynchronization
// typically happens if the user is replacing or altering the persistent files
// in the consensus set or the host.
func (h *Host) initRescan() error {
	// Reset all of the variables that have relevance to the consensus set.
	var allObligations []*storageObligation
	err := func() error {
		h.mu.Lock()
		defer h.mu.Unlock()

		// Reset all of the consensus-relevant variables in the host.
		h.blockHeight = 0

		// Reset all of the storage obligations.
		err := h.db.Update(func(tx *bolt.Tx) error {
			bsu := tx.Bucket(bucketStorageObligations)
			c := bsu.Cursor()
			for k, soBytes := c.First(); soBytes != nil; k, soBytes = c.Next() {
				var so storageObligation
				err := json.Unmarshal(soBytes, &so)
				if err != nil {
					return err
				}
				so.OriginConfirmed = false
				so.RevisionConfirmed = false
				so.ProofConfirmed = false
				allObligations = append(allObligations, &so)
				soBytes, err = json.Marshal(so)
				if err != nil {
					return err
				}
				err = bsu.Put(k, soBytes)
				if err != nil {
					return err
				}
			}
			return nil
		})
		if err != nil {
			return err
		}

		return h.save()
	}()
	if err != nil {
		return err
	}

	// Subscribe to the consensus set. This is a blocking call that will not
	// return until the host has fully caught up to the current block.
	err = h.cs.ConsensusSetSubscribe(h, modules.ConsensusChangeBeginning)
	if err != nil {
		return err
	}

	// Re-queue all of the action items for the storage obligations.
	for _, so := range allObligations {
		soid := so.id()
		err0 := h.tpool.AcceptTransactionSet(so.OriginTransactionSet)
		err1 := h.queueActionItem(h.blockHeight+resubmissionTimeout, soid)
		err2 := h.queueActionItem(so.expiration()-revisionSubmissionBuffer, soid)
		err3 := h.queueActionItem(so.expiration()+resubmissionTimeout, soid)
		err = composeErrors(err0, err1, err2, err3)
		if err != nil {
			return composeErrors(err, h.removeStorageObligation(so, obligationRejected))
		}
	}
	return nil
}

// initConsensusSubscription subscribes the host to the consensus set.
func (h *Host) initConsensusSubscription() error {
	err := h.cs.ConsensusSetSubscribe(h, h.recentChange)
	if err == modules.ErrInvalidConsensusChangeID {
		// Perform a rescan of the consensus set if the change id that the host
		// has is unrecognized by the consensus set. This will typically only
		// happen if the user has been replacing files inside the Sia folder
		// structure.
		return h.initRescan()
	}
	return err
}

// ProcessConsensusChange will be called by the consensus set every time there
// is a change to the blockchain.
func (h *Host) ProcessConsensusChange(cc modules.ConsensusChange) {
	h.mu.Lock()
	defer h.mu.Unlock()
	err := h.tg.Add()
	if err != nil {
		return
	}
	defer h.tg.Done()

	// Wrap the whole parsing into a single large database tx to keep things
	// efficient.
	var actionItems []*storageObligation
	err = h.db.Update(func(tx *bolt.Tx) error {
		for _, block := range cc.RevertedBlocks {
			// Look for transactions relevant to open storage obligations.
			for _, txn := range block.Transactions {
				// Check for file contracts.
				if len(txn.FileContracts) > 0 {
					for j := range txn.FileContracts {
						fcid := txn.FileContractID(uint64(j))
						so, err := getStorageObligation(tx, fcid)
						if err != nil {
							// The storage folder may not exist, or the disk
							// may be having trouble. Either way, we ignore the
							// problem. If the disk is having trouble, the user
							// will have to perform a rescan.
							continue
						}
						so.OriginConfirmed = false
						err = putStorageObligation(tx, so)
						if err != nil {
							continue
						}
					}
				}

				// Check for file contract revisions.
				if len(txn.FileContractRevisions) > 0 {
					for _, fcr := range txn.FileContractRevisions {
						so, err := getStorageObligation(tx, fcr.ParentID)
						if err != nil {
							// The storage folder may not exist, or the disk
							// may be having trouble. Either way, we ignore the
							// problem. If the disk is having trouble, the user
							// will have to perform a rescan.
							continue
						}
						so.RevisionConfirmed = false
						err = putStorageObligation(tx, so)
						if err != nil {
							continue
						}
					}
				}

				// Check for storage proofs.
				if len(txn.StorageProofs) > 0 {
					for _, sp := range txn.StorageProofs {
						// Check database for relevant storage proofs.
						so, err := getStorageObligation(tx, sp.ParentID)
						if err != nil {
							// The storage folder may not exist, or the disk
							// may be having trouble. Either way, we ignore the
							// problem. If the disk is having trouble, the user
							// will have to perform a rescan.
							continue
						}
						so.ProofConfirmed = false
						err = putStorageObligation(tx, so)
						if err != nil {
							continue
						}
					}
				}
			}

			// Height is not adjusted when dealing with the genesis block because
			// the default height is 0 and the genesis block height is 0. If
			// removing the genesis block, height will already be at height 0 and
			// should not update, lest an underflow occur.
			if block.ID() != types.GenesisID {
				h.blockHeight--
			}
		}
		for _, block := range cc.AppliedBlocks {
			// Look for transactions relevant to open storage obligations.
			for _, txn := range block.Transactions {
				// Check for file contracts.
				if len(txn.FileContracts) > 0 {
					for i := range txn.FileContracts {
						fcid := txn.FileContractID(uint64(i))
						so, err := getStorageObligation(tx, fcid)
						if err != nil {
							// The storage folder may not exist, or the disk
							// may be having trouble. Either way, we ignore the
							// problem. If the disk is having trouble, the user
							// will have to perform a rescan.
							continue
						}
						so.OriginConfirmed = true
						err = putStorageObligation(tx, so)
						if err != nil {
							continue
						}
					}
				}

				// Check for file contract revisions.
				if len(txn.FileContractRevisions) > 0 {
					for _, fcr := range txn.FileContractRevisions {
						so, err := getStorageObligation(tx, fcr.ParentID)
						if err != nil {
							// The storage folder may not exist, or the disk
							// may be having trouble. Either way, we ignore the
							// problem. If the disk is having trouble, the user
							// will have to perform a rescan.
							continue
						}
						so.RevisionConfirmed = true
						err = putStorageObligation(tx, so)
						if err != nil {
							continue
						}
					}
				}

				// Check for storage proofs.
				if len(txn.StorageProofs) > 0 {
					for _, sp := range txn.StorageProofs {
						so, err := getStorageObligation(tx, sp.ParentID)
						if err != nil {
							// The storage folder may not exist, or the disk
							// may be having trouble. Either way, we ignore the
							// problem. If the disk is having trouble, the user
							// will have to perform a rescan.
							continue
						}
						so.ProofConfirmed = true
						err = putStorageObligation(tx, so)
						if err != nil {
							continue
						}
					}
				}
			}

			// Height is not adjusted when dealing with the genesis block because
			// the default height is 0 and the genesis block height is 0. If adding
			// the genesis block, height will already be at height 0 and should not
			// update.
			if block.ID() != types.GenesisID {
				h.blockHeight++
			}

			// Handle any action items relevant to the current height.
			bai := tx.Bucket(bucketActionItems)
			heightBytes := make([]byte, 8)
			binary.BigEndian.PutUint64(heightBytes, uint64(h.blockHeight)) // BigEndian used so bolt will keep things sorted automatically.
			existingItems := bai.Get(heightBytes)

			// From the existing items, pull out a storage obligation.
			knownActionItems := make(map[types.FileContractID]struct{})
			obligationIDs := make([]types.FileContractID, len(existingItems)/crypto.HashSize)
			for i := 0; i < len(existingItems); i += crypto.HashSize {
				copy(obligationIDs[i/crypto.HashSize][:], existingItems[i:i+crypto.HashSize])
			}
			bso := tx.Bucket(bucketStorageObligations)
			for _, soid := range obligationIDs {
				soBytes := bso.Get(soid[:])
				if soBytes == nil {
					// Obligation may have been cleared out by a previous
					// action item.
					continue
				}
				var so storageObligation
				err := json.Unmarshal(soBytes, &so)
				if err != nil {
					build.Critical("json unmarshalling failed for storage obligation")
					continue
				}
				_, exists := knownActionItems[soid]
				if !exists {
					actionItems = append(actionItems, &so)
					knownActionItems[soid] = struct{}{}
				}
			}
		}
		return nil
	})
	if err != nil {
		h.log.Println(err)
	}

	// Handle the list of action items.
	for _, ai := range actionItems {
		h.handleActionItem(ai)
	}

	// Update the host's recent change pointer to point to the most recent
	// change.
	h.recentChange = cc.ID

	// Save the host.
	err = h.save()
	if err != nil {
		h.log.Println("ERROR: could not save during ProcessConsensusChange:", err)
	}
}
<|endoftext|>


<commit_before>package datastore

import (
	"bytes"
	"cluster"
	log "code.google.com/p/log4go"
	"configuration"
	"fmt"
	"github.com/jmhodges/levigo"
	"math"
	"os"
	"path/filepath"
	"protocol"
	"sync"
	"time"
)

type LevelDbShardDatastore struct {
	baseDbDir      string
	config         *configuration.Configuration
	shards         map[uint32]*LevelDbShard
	lastAccess     map[uint32]int64
	shardRefCounts map[uint32]int
	shardsToClose  map[uint32]bool
	shardsLock     sync.RWMutex
	levelDbOptions *levigo.Options
	writeBuffer    *cluster.WriteBuffer
	maxOpenShards  int
}

const (
	ONE_KILOBYTE                    = 1024
	ONE_MEGABYTE                    = 1024 * 1024
	ONE_GIGABYTE                    = ONE_MEGABYTE * 1024
	TWO_FIFTY_SIX_KILOBYTES         = 256 * 1024
	MAX_SERIES_SIZE                 = ONE_MEGABYTE
	DATABASE_DIR                    = "db"
	SHARD_BLOOM_FILTER_BITS_PER_KEY = 10
	SHARD_DATABASE_DIR              = "shard_db"
)

var (

	// This datastore implements the PersistentAtomicInteger interface. All of the persistent
	// integers start with this prefix, followed by their name
	ATOMIC_INCREMENT_PREFIX = []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFD}
	// NEXT_ID_KEY holds the next id. ids are used to "intern" timeseries and column names
	NEXT_ID_KEY = []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
	// SERIES_COLUMN_INDEX_PREFIX is the prefix of the series to column names index
	SERIES_COLUMN_INDEX_PREFIX = []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE}
	// DATABASE_SERIES_INDEX_PREFIX is the prefix of the database to series names index
	DATABASE_SERIES_INDEX_PREFIX = []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}
	MAX_SEQUENCE                 = []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}

	// replicateWrite = protocol.Request_REPLICATION_WRITE

	TRUE = true
)

type Field struct {
	Id   []byte
	Name string
}

type rawColumnValue struct {
	time     []byte
	sequence []byte
	value    []byte
}

func NewLevelDbShardDatastore(config *configuration.Configuration) (*LevelDbShardDatastore, error) {
	baseDbDir := filepath.Join(config.DataDir, SHARD_DATABASE_DIR)
	err := os.MkdirAll(baseDbDir, 0744)
	if err != nil {
		return nil, err
	}
	opts := levigo.NewOptions()
	opts.SetCache(levigo.NewLRUCache(config.LevelDbLruCacheSize))
	opts.SetCreateIfMissing(true)
	opts.SetBlockSize(64 * ONE_KILOBYTE)
	filter := levigo.NewBloomFilter(SHARD_BLOOM_FILTER_BITS_PER_KEY)
	opts.SetFilterPolicy(filter)
	opts.SetMaxOpenFiles(config.LevelDbMaxOpenFiles)

	return &LevelDbShardDatastore{
		baseDbDir:      baseDbDir,
		config:         config,
		shards:         make(map[uint32]*LevelDbShard),
		levelDbOptions: opts,
		maxOpenShards:  config.LevelDbMaxOpenShards,
		lastAccess:     make(map[uint32]int64),
		shardRefCounts: make(map[uint32]int),
		shardsToClose:  make(map[uint32]bool),
	}, nil
}

func (self *LevelDbShardDatastore) Close() {
	self.shardsLock.Lock()
	defer self.shardsLock.Unlock()
	for _, shard := range self.shards {
		shard.close()
	}
}

func (self *LevelDbShardDatastore) GetOrCreateShard(id uint32) (cluster.LocalShardDb, error) {
	now := time.Now().Unix()
	self.shardsLock.Lock()
	defer self.shardsLock.Unlock()
	db := self.shards[id]
	self.lastAccess[id] = now

	if db != nil {
		self.incrementShardRefCountAndCloseOldestIfNeeded(id)
		return db, nil
	}

	dbDir := self.shardDir(id)

	log.Info("DATASTORE: opening or creating shard %s", dbDir)
	ldb, err := levigo.Open(dbDir, self.levelDbOptions)
	if err != nil {
		return nil, err
	}

	db, err = NewLevelDbShard(ldb)
	if err != nil {
		return nil, err
	}
	self.shards[id] = db
	self.incrementShardRefCountAndCloseOldestIfNeeded(id)
	return db, nil
}

func (self *LevelDbShardDatastore) incrementShardRefCountAndCloseOldestIfNeeded(id uint32) {
	self.shardRefCounts[id] += 1
	if self.maxOpenShards > 0 && len(self.shards) > self.maxOpenShards {
		self.closeOldestShard()
	}
}

func (self *LevelDbShardDatastore) ReturnShard(id uint32) {
	self.shardsLock.Lock()
	defer self.shardsLock.Unlock()
	self.shardRefCounts[id] -= 1
	if self.shardsToClose[id] && self.shardRefCounts[id] == 0 {
		self.closeShard(id)
	}
}

func (self *LevelDbShardDatastore) Write(request *protocol.Request) error {
	shardDb, err := self.GetOrCreateShard(*request.ShardId)
	if err != nil {
		return err
	}
	defer self.ReturnShard(*request.ShardId)
	return shardDb.Write(*request.Database, request.Series)
}

func (self *LevelDbShardDatastore) BufferWrite(request *protocol.Request) {
	self.writeBuffer.Write(request)
}

func (self *LevelDbShardDatastore) SetWriteBuffer(writeBuffer *cluster.WriteBuffer) {
	self.writeBuffer = writeBuffer
}

func (self *LevelDbShardDatastore) DeleteShard(shardId uint32) error {
	self.shardsLock.Lock()
	shardDb := self.shards[shardId]
	delete(self.shards, shardId)
	delete(self.lastAccess, shardId)
	self.shardsLock.Unlock()

	if shardDb != nil {
		shardDb.close()
	}

	dir := self.shardDir(shardId)
	log.Info("DATASTORE: dropping shard %s", dir)
	return os.RemoveAll(dir)
}

func (self *LevelDbShardDatastore) shardDir(id uint32) string {
	return filepath.Join(self.baseDbDir, fmt.Sprintf("%.5d", id))
}

func (self *LevelDbShardDatastore) closeOldestShard() {
	var oldestId uint32
	oldestAccess := int64(math.MaxInt64)
	for id, lastAccess := range self.lastAccess {
		if lastAccess < oldestAccess {
			oldestId = id
			oldestAccess = lastAccess
		}
	}
	if self.shardRefCounts[oldestId] == 0 {
		self.closeShard(oldestId)
	} else {
		self.shardsToClose[oldestId] = true
	}
}

func (self *LevelDbShardDatastore) closeShard(id uint32) {
	shard := self.shards[id]
	if shard != nil {
		shard.close()
	}
	delete(self.shardRefCounts, id)
	delete(self.shards, id)
	delete(self.lastAccess, id)
	delete(self.shardsToClose, id)
}

// // returns true if the point has the correct field id and is
// // in the given time range
func isPointInRange(fieldId, startTime, endTime, point []byte) bool {
	id := point[:8]
	time := point[8:16]
	return bytes.Equal(id, fieldId) && bytes.Compare(time, startTime) > -1 && bytes.Compare(time, endTime) < 1
}

type FieldLookupError struct {
	message string
}

func (self FieldLookupError) Error() string {
	return self.message
}

// depending on the query order (whether it's ascending or not) returns
// the min (or max in case of descending query) of the current
// [timestamp,sequence] and the self's [timestamp,sequence]
//
// This is used to determine what the next point's timestamp
// and sequence number should be.
func (self *rawColumnValue) updatePointTimeAndSequence(currentTimeRaw, currentSequenceRaw []byte, isAscendingQuery bool) ([]byte, []byte) {
	if currentTimeRaw == nil {
		return self.time, self.sequence
	}

	compareValue := 1
	if isAscendingQuery {
		compareValue = -1
	}

	timeCompare := bytes.Compare(self.time, currentTimeRaw)
	if timeCompare == compareValue {
		return self.time, self.sequence
	}

	if timeCompare != 0 {
		return currentTimeRaw, currentSequenceRaw
	}

	if bytes.Compare(self.sequence, currentSequenceRaw) == compareValue {
		return currentTimeRaw, self.sequence
	}

	return currentTimeRaw, currentSequenceRaw
}
<commit_msg>Updated ref counting to remove accessed shards from the list of ones to close. Updated the close to close as many as it takes to get below the leve<commit_after>package datastore

import (
	"bytes"
	"cluster"
	log "code.google.com/p/log4go"
	"configuration"
	"fmt"
	"github.com/jmhodges/levigo"
	"math"
	"os"
	"path/filepath"
	"protocol"
	"sync"
	"time"
)

type LevelDbShardDatastore struct {
	baseDbDir      string
	config         *configuration.Configuration
	shards         map[uint32]*LevelDbShard
	lastAccess     map[uint32]int64
	shardRefCounts map[uint32]int
	shardsToClose  map[uint32]bool
	shardsLock     sync.RWMutex
	levelDbOptions *levigo.Options
	writeBuffer    *cluster.WriteBuffer
	maxOpenShards  int
}

const (
	ONE_KILOBYTE                    = 1024
	ONE_MEGABYTE                    = 1024 * 1024
	ONE_GIGABYTE                    = ONE_MEGABYTE * 1024
	TWO_FIFTY_SIX_KILOBYTES         = 256 * 1024
	MAX_SERIES_SIZE                 = ONE_MEGABYTE
	DATABASE_DIR                    = "db"
	SHARD_BLOOM_FILTER_BITS_PER_KEY = 10
	SHARD_DATABASE_DIR              = "shard_db"
)

var (

	// This datastore implements the PersistentAtomicInteger interface. All of the persistent
	// integers start with this prefix, followed by their name
	ATOMIC_INCREMENT_PREFIX = []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFD}
	// NEXT_ID_KEY holds the next id. ids are used to "intern" timeseries and column names
	NEXT_ID_KEY = []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
	// SERIES_COLUMN_INDEX_PREFIX is the prefix of the series to column names index
	SERIES_COLUMN_INDEX_PREFIX = []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE}
	// DATABASE_SERIES_INDEX_PREFIX is the prefix of the database to series names index
	DATABASE_SERIES_INDEX_PREFIX = []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}
	MAX_SEQUENCE                 = []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}

	// replicateWrite = protocol.Request_REPLICATION_WRITE

	TRUE = true
)

type Field struct {
	Id   []byte
	Name string
}

type rawColumnValue struct {
	time     []byte
	sequence []byte
	value    []byte
}

func NewLevelDbShardDatastore(config *configuration.Configuration) (*LevelDbShardDatastore, error) {
	baseDbDir := filepath.Join(config.DataDir, SHARD_DATABASE_DIR)
	err := os.MkdirAll(baseDbDir, 0744)
	if err != nil {
		return nil, err
	}
	opts := levigo.NewOptions()
	opts.SetCache(levigo.NewLRUCache(config.LevelDbLruCacheSize))
	opts.SetCreateIfMissing(true)
	opts.SetBlockSize(64 * ONE_KILOBYTE)
	filter := levigo.NewBloomFilter(SHARD_BLOOM_FILTER_BITS_PER_KEY)
	opts.SetFilterPolicy(filter)
	opts.SetMaxOpenFiles(config.LevelDbMaxOpenFiles)

	return &LevelDbShardDatastore{
		baseDbDir:      baseDbDir,
		config:         config,
		shards:         make(map[uint32]*LevelDbShard),
		levelDbOptions: opts,
		maxOpenShards:  config.LevelDbMaxOpenShards,
		lastAccess:     make(map[uint32]int64),
		shardRefCounts: make(map[uint32]int),
		shardsToClose:  make(map[uint32]bool),
	}, nil
}

func (self *LevelDbShardDatastore) Close() {
	self.shardsLock.Lock()
	defer self.shardsLock.Unlock()
	for _, shard := range self.shards {
		shard.close()
	}
}

func (self *LevelDbShardDatastore) GetOrCreateShard(id uint32) (cluster.LocalShardDb, error) {
	now := time.Now().Unix()
	self.shardsLock.Lock()
	defer self.shardsLock.Unlock()
	db := self.shards[id]
	self.lastAccess[id] = now

	if db != nil {
		self.incrementShardRefCountAndCloseOldestIfNeeded(id)
		return db, nil
	}

	dbDir := self.shardDir(id)

	log.Info("DATASTORE: opening or creating shard %s", dbDir)
	ldb, err := levigo.Open(dbDir, self.levelDbOptions)
	if err != nil {
		return nil, err
	}

	db, err = NewLevelDbShard(ldb)
	if err != nil {
		return nil, err
	}
	self.shards[id] = db
	self.incrementShardRefCountAndCloseOldestIfNeeded(id)
	return db, nil
}

func (self *LevelDbShardDatastore) incrementShardRefCountAndCloseOldestIfNeeded(id uint32) {
	self.shardRefCounts[id] += 1
	delete(self.shardsToClose, id)
	if self.maxOpenShards > 0 && len(self.shards) > self.maxOpenShards {
		for i := len(self.shards) - self.maxOpenShards; i > 0; i-- {
			self.closeOldestShard()
		}
	}
}

func (self *LevelDbShardDatastore) ReturnShard(id uint32) {
	self.shardsLock.Lock()
	defer self.shardsLock.Unlock()
	self.shardRefCounts[id] -= 1
	if self.shardsToClose[id] && self.shardRefCounts[id] == 0 {
		self.closeShard(id)
	}
}

func (self *LevelDbShardDatastore) Write(request *protocol.Request) error {
	shardDb, err := self.GetOrCreateShard(*request.ShardId)
	if err != nil {
		return err
	}
	defer self.ReturnShard(*request.ShardId)
	return shardDb.Write(*request.Database, request.Series)
}

func (self *LevelDbShardDatastore) BufferWrite(request *protocol.Request) {
	self.writeBuffer.Write(request)
}

func (self *LevelDbShardDatastore) SetWriteBuffer(writeBuffer *cluster.WriteBuffer) {
	self.writeBuffer = writeBuffer
}

func (self *LevelDbShardDatastore) DeleteShard(shardId uint32) error {
	self.shardsLock.Lock()
	shardDb := self.shards[shardId]
	delete(self.shards, shardId)
	delete(self.lastAccess, shardId)
	self.shardsLock.Unlock()

	if shardDb != nil {
		shardDb.close()
	}

	dir := self.shardDir(shardId)
	log.Info("DATASTORE: dropping shard %s", dir)
	return os.RemoveAll(dir)
}

func (self *LevelDbShardDatastore) shardDir(id uint32) string {
	return filepath.Join(self.baseDbDir, fmt.Sprintf("%.5d", id))
}

func (self *LevelDbShardDatastore) closeOldestShard() {
	var oldestId uint32
	oldestAccess := int64(math.MaxInt64)
	for id, lastAccess := range self.lastAccess {
		if lastAccess < oldestAccess && self.shardsToClose[id] == false {
			oldestId = id
			oldestAccess = lastAccess
		}
	}
	if self.shardRefCounts[oldestId] == 0 {
		self.closeShard(oldestId)
	} else {
		self.shardsToClose[oldestId] = true
	}
}

func (self *LevelDbShardDatastore) closeShard(id uint32) {
	shard := self.shards[id]
	if shard != nil {
		shard.close()
	}
	delete(self.shardRefCounts, id)
	delete(self.shards, id)
	delete(self.lastAccess, id)
	delete(self.shardsToClose, id)
}

// // returns true if the point has the correct field id and is
// // in the given time range
func isPointInRange(fieldId, startTime, endTime, point []byte) bool {
	id := point[:8]
	time := point[8:16]
	return bytes.Equal(id, fieldId) && bytes.Compare(time, startTime) > -1 && bytes.Compare(time, endTime) < 1
}

type FieldLookupError struct {
	message string
}

func (self FieldLookupError) Error() string {
	return self.message
}

// depending on the query order (whether it's ascending or not) returns
// the min (or max in case of descending query) of the current
// [timestamp,sequence] and the self's [timestamp,sequence]
//
// This is used to determine what the next point's timestamp
// and sequence number should be.
func (self *rawColumnValue) updatePointTimeAndSequence(currentTimeRaw, currentSequenceRaw []byte, isAscendingQuery bool) ([]byte, []byte) {
	if currentTimeRaw == nil {
		return self.time, self.sequence
	}

	compareValue := 1
	if isAscendingQuery {
		compareValue = -1
	}

	timeCompare := bytes.Compare(self.time, currentTimeRaw)
	if timeCompare == compareValue {
		return self.time, self.sequence
	}

	if timeCompare != 0 {
		return currentTimeRaw, currentSequenceRaw
	}

	if bytes.Compare(self.sequence, currentSequenceRaw) == compareValue {
		return currentTimeRaw, self.sequence
	}

	return currentTimeRaw, currentSequenceRaw
}
<|endoftext|>