<commit_before>package testing

import (
	"fmt"
	"log"
	"os"
	"reflect"
	"testing"

	"github.com/hashicorp/vault/api"
	"github.com/hashicorp/vault/http"
	"github.com/hashicorp/vault/logical"
	"github.com/hashicorp/vault/physical"
	"github.com/hashicorp/vault/vault"
)

// TestEnvVar must be set to a non-empty value for acceptance tests to run.
const TestEnvVar = "TF_ACC"

// TestCase is a single set of tests to run for a backend. A TestCase
// should generally map 1:1 to each test method for your acceptance
// tests.
type TestCase struct {
	// Precheck, if non-nil, will be called once before the test case
	// runs at all. This can be used for some validation prior to the
	// test running.
	PreCheck func()

	// Backend is the backend that will be mounted.
	Backend logical.Backend

	// Steps are the set of operations that are run for this test case.
	Steps []TestStep

	// Teardown will be called before the test case is over regardless
	// of if the test succeeded or failed. This should return an error
	// in the case that the test can't guarantee all resources were
	// properly cleaned up.
	Teardown TestTeardownFunc
}

// TestStep is a single step within a TestCase.
type TestStep struct {
	// Operation is the operation to execute
	Operation logical.Operation

	// Path is the request path. The mount prefix will be automatically added.
	Path string

	// Arguments to pass in
	Data map[string]interface{}

	// Check is called after this step is executed in order to test that
	// the step executed successfully. If this is not set, then the next
	// step will be called
	Check TestCheckFunc

	// ErrorOk, if true, will let erroneous responses through to the check
	ErrorOk bool

	// Unauthenticated, if true, will make the request unauthenticated.
	Unauthenticated bool

	// RemoteAddr, if set, will set the remote addr on the request.
	RemoteAddr string
}

// TestCheckFunc is the callback used for Check in TestStep.
type TestCheckFunc func(*logical.Response) error

// TestTeardownFunc is the callback used for Teardown in TestCase.
type TestTeardownFunc func() error

// Test performs an acceptance test on a backend with the given test case.
//
// Tests are not run unless an environmental variable "TF_ACC" is
// set to some non-empty value. This is to avoid test cases surprising
// a user by creating real resources.
//
// Tests will fail unless the verbose flag (`go test -v`, or explicitly
// the "-test.v" flag) is set. Because some acceptance tests take quite
// long, we require the verbose flag so users are able to see progress
// output.
func Test(t TestT, c TestCase) {
	// We only run acceptance tests if an env var is set because they're
	// slow and generally require some outside configuration.
	if os.Getenv(TestEnvVar) == "" {
		t.Skip(fmt.Sprintf(
			"Acceptance tests skipped unless env '%s' set",
			TestEnvVar))
		return
	}

	// We require verbose mode so that the user knows what is going on.
	if !testTesting && !testing.Verbose() {
		t.Fatal("Acceptance tests must be run with the -v flag on tests")
		return
	}

	// Run the PreCheck if we have it
	if c.PreCheck != nil {
		c.PreCheck()
	}

	// Create an in-memory Vault core
	core, err := vault.NewCore(&vault.CoreConfig{
		Physical: physical.NewInmem(),
		LogicalBackends: map[string]logical.Factory{
			"test": func(map[string]string) (logical.Backend, error) {
				return c.Backend, nil
			},
		},
	})
	if err != nil {
		t.Fatal("error initializing core: ", err)
	}

	// Initialize the core
	init, err := core.Initialize(&vault.SealConfig{
		SecretShares:    1,
		SecretThreshold: 1,
	})
	if err != nil {
		t.Fatal("error initializing core: ", err)
	}

	// Unseal the core
	if unsealed, err := core.Unseal(init.SecretShares[0]); err != nil {
		t.Fatal("error unsealing core: ", err)
	} else if !unsealed {
		t.Fatal("vault shouldn't be sealed")
	}

	// Create an HTTP API server and client
	ln, addr := http.TestServer(nil, core)
	defer ln.Close()
	clientConfig := api.DefaultConfig()
	clientConfig.Address = addr
	client, err := api.NewClient(clientConfig)
	if err != nil {
		t.Fatal("error initializing HTTP client: ", err)
	}

	// Set the token so we're authenticated
	client.SetToken(init.RootToken)

	// Mount the backend
	prefix := "mnt"
	if err := client.Sys().Mount(prefix, "test", "acceptance test"); err != nil {
		t.Fatal("error mounting backend: ", err)
	}

	// Make requests
	var revoke []*logical.Request
	for i, s := range c.Steps {
		log.Printf("[WARN] Executing test step %d", i+1)

		// Make sure to prefix the path with where we mounted the thing
		path := fmt.Sprintf("%s/%s", prefix, s.Path)

		// Create the request
		req := &logical.Request{
			Operation: s.Operation,
			Path:      path,
			Data:      s.Data,
		}
		if !s.Unauthenticated {
			req.ClientToken = client.Token()
		}
		if s.RemoteAddr != "" {
			req.Connection = &logical.Connection{RemoteAddr: s.RemoteAddr}
		}

		// Make the request
		resp, err := core.HandleRequest(req)
		if resp != nil && resp.Secret != nil {
			// Revoke this secret later
			revoke = append(revoke, logical.RevokeRequest(
				req.Path,
				resp.Secret,
				resp.Data,
			))
		}
		if err == nil && resp.IsError() && !s.ErrorOk {
			err = fmt.Errorf("Erroneous response:\n\n%#v", resp)
		}
		if err == nil && s.Check != nil {
			// Call the test method
			err = s.Check(resp)
		}
		if err != nil {
			t.Error(fmt.Sprintf("Failed step %d: %s", i+1, err))
			break
		}
	}

	// Revoke any secrets we might have.
	var failedRevokes []*logical.Secret
	for _, req := range revoke {
		log.Printf("[WARN] Revoking secret: %#v", req.Secret)
		req.ClientToken = client.Token()
		resp, err := core.HandleRequest(req)
		if err == nil && resp.IsError() {
			err = fmt.Errorf("Erroneous response:\n\n%#v", resp)
		}
		if err != nil {
			failedRevokes = append(failedRevokes, req.Secret)
			t.Error(fmt.Sprintf("[ERR] Revoke error: %s", err))
		}
	}

	// Perform any rollbacks. This should no-op if there aren't any.
	// We set the "immediate" flag here that any backend can pick up on
	// to do all rollbacks immediately even if the WAL entries are new.
	log.Printf("[WARN] Requesting RollbackOperation")
	req := logical.RollbackRequest(prefix + "/")
	req.Data["immediate"] = true
	req.ClientToken = client.Token()
	resp, err := core.HandleRequest(req)
	if err == nil && resp.IsError() {
		err = fmt.Errorf("Erroneous response:\n\n%#v", resp)
	}
	if err != nil && err != logical.ErrUnsupportedOperation {
		t.Error(fmt.Sprintf("[ERR] Rollback error: %s", err))
	}

	// If we have any failed revokes, log it.
	if len(failedRevokes) > 0 {
		for _, s := range failedRevokes {
			t.Error(fmt.Sprintf(
				"WARNING: Revoking the following secret failed. It may\n"+
					"still exist. Please verify:\n\n%#v",
				s))
		}
	}

	// Cleanup
	if c.Teardown != nil {
		c.Teardown()
	}
}

// TestCheckMulti is a helper to have multiple checks.
func TestCheckMulti(fs ...TestCheckFunc) TestCheckFunc {
	return func(resp *logical.Response) error {
		for _, f := range fs {
			if err := f(resp); err != nil {
				return err
			}
		}

		return nil
	}
}

// TestCheckAuth is a helper to check that a request generated an
// auth token with the proper policies.
func TestCheckAuth(policies []string) TestCheckFunc {
	return func(resp *logical.Response) error {
		if resp.Auth == nil {
			return fmt.Errorf("no auth in response")
		}
		if !reflect.DeepEqual(resp.Auth.Policies, policies) {
			return fmt.Errorf("invalid policies: %#v", resp.Auth.Policies)
		}

		return nil
	}
}

// TestCheckAuthDisplayName is a helper to check that a request generated a
// valid display name.
func TestCheckAuthDisplayName(n string) TestCheckFunc {
	return func(resp *logical.Response) error {
		if resp.Auth == nil {
			return fmt.Errorf("no auth in response")
		}
		if n != "" && resp.Auth.DisplayName != "mnt-"+n {
			return fmt.Errorf("invalid display name: %#v", resp.Auth.DisplayName)
		}

		return nil
	}
}

// TestCheckError is a helper to check that a response is an error.
func TestCheckError() TestCheckFunc {
	return func(resp *logical.Response) error {
		if !resp.IsError() {
			return fmt.Errorf("response should be error")
		}

		return nil
	}
}

// TestT is the interface used to handle the test lifecycle of a test.
//
// Users should just use a *testing.T object, which implements this.
type TestT interface {
	Error(args ...interface{})
	Fatal(args ...interface{})
	Skip(args ...interface{})
}

var testTesting = false
<commit_msg>logical: allow specifying ConnState<commit_after>package testing

import (
	"crypto/tls"
	"fmt"
	"log"
	"os"
	"reflect"
	"testing"

	"github.com/hashicorp/vault/api"
	"github.com/hashicorp/vault/http"
	"github.com/hashicorp/vault/logical"
	"github.com/hashicorp/vault/physical"
	"github.com/hashicorp/vault/vault"
)

// TestEnvVar must be set to a non-empty value for acceptance tests to run.
const TestEnvVar = "TF_ACC"

// TestCase is a single set of tests to run for a backend. A TestCase
// should generally map 1:1 to each test method for your acceptance
// tests.
type TestCase struct {
	// Precheck, if non-nil, will be called once before the test case
	// runs at all. This can be used for some validation prior to the
	// test running.
	PreCheck func()

	// Backend is the backend that will be mounted.
	Backend logical.Backend

	// Steps are the set of operations that are run for this test case.
	Steps []TestStep

	// Teardown will be called before the test case is over regardless
	// of if the test succeeded or failed. This should return an error
	// in the case that the test can't guarantee all resources were
	// properly cleaned up.
	Teardown TestTeardownFunc
}

// TestStep is a single step within a TestCase.
type TestStep struct {
	// Operation is the operation to execute
	Operation logical.Operation

	// Path is the request path. The mount prefix will be automatically added.
	Path string

	// Arguments to pass in
	Data map[string]interface{}

	// Check is called after this step is executed in order to test that
	// the step executed successfully. If this is not set, then the next
	// step will be called
	Check TestCheckFunc

	// ErrorOk, if true, will let erroneous responses through to the check
	ErrorOk bool

	// Unauthenticated, if true, will make the request unauthenticated.
	Unauthenticated bool

	// RemoteAddr, if set, will set the remote addr on the request.
	RemoteAddr string

	// ConnState, if set, will set the tls conneciton state
	ConnState *tls.ConnectionState
}

// TestCheckFunc is the callback used for Check in TestStep.
type TestCheckFunc func(*logical.Response) error

// TestTeardownFunc is the callback used for Teardown in TestCase.
type TestTeardownFunc func() error

// Test performs an acceptance test on a backend with the given test case.
//
// Tests are not run unless an environmental variable "TF_ACC" is
// set to some non-empty value. This is to avoid test cases surprising
// a user by creating real resources.
//
// Tests will fail unless the verbose flag (`go test -v`, or explicitly
// the "-test.v" flag) is set. Because some acceptance tests take quite
// long, we require the verbose flag so users are able to see progress
// output.
func Test(t TestT, c TestCase) {
	// We only run acceptance tests if an env var is set because they're
	// slow and generally require some outside configuration.
	if os.Getenv(TestEnvVar) == "" {
		t.Skip(fmt.Sprintf(
			"Acceptance tests skipped unless env '%s' set",
			TestEnvVar))
		return
	}

	// We require verbose mode so that the user knows what is going on.
	if !testTesting && !testing.Verbose() {
		t.Fatal("Acceptance tests must be run with the -v flag on tests")
		return
	}

	// Run the PreCheck if we have it
	if c.PreCheck != nil {
		c.PreCheck()
	}

	// Create an in-memory Vault core
	core, err := vault.NewCore(&vault.CoreConfig{
		Physical: physical.NewInmem(),
		LogicalBackends: map[string]logical.Factory{
			"test": func(map[string]string) (logical.Backend, error) {
				return c.Backend, nil
			},
		},
	})
	if err != nil {
		t.Fatal("error initializing core: ", err)
	}

	// Initialize the core
	init, err := core.Initialize(&vault.SealConfig{
		SecretShares:    1,
		SecretThreshold: 1,
	})
	if err != nil {
		t.Fatal("error initializing core: ", err)
	}

	// Unseal the core
	if unsealed, err := core.Unseal(init.SecretShares[0]); err != nil {
		t.Fatal("error unsealing core: ", err)
	} else if !unsealed {
		t.Fatal("vault shouldn't be sealed")
	}

	// Create an HTTP API server and client
	ln, addr := http.TestServer(nil, core)
	defer ln.Close()
	clientConfig := api.DefaultConfig()
	clientConfig.Address = addr
	client, err := api.NewClient(clientConfig)
	if err != nil {
		t.Fatal("error initializing HTTP client: ", err)
	}

	// Set the token so we're authenticated
	client.SetToken(init.RootToken)

	// Mount the backend
	prefix := "mnt"
	if err := client.Sys().Mount(prefix, "test", "acceptance test"); err != nil {
		t.Fatal("error mounting backend: ", err)
	}

	// Make requests
	var revoke []*logical.Request
	for i, s := range c.Steps {
		log.Printf("[WARN] Executing test step %d", i+1)

		// Make sure to prefix the path with where we mounted the thing
		path := fmt.Sprintf("%s/%s", prefix, s.Path)

		// Create the request
		req := &logical.Request{
			Operation: s.Operation,
			Path:      path,
			Data:      s.Data,
		}
		if !s.Unauthenticated {
			req.ClientToken = client.Token()
		}
		if s.RemoteAddr != "" {
			req.Connection = &logical.Connection{RemoteAddr: s.RemoteAddr}
		}
		if s.ConnState != nil {
			req.Connection = &logical.Connection{ConnState: s.ConnState}
		}

		// Make the request
		resp, err := core.HandleRequest(req)
		if resp != nil && resp.Secret != nil {
			// Revoke this secret later
			revoke = append(revoke, logical.RevokeRequest(
				req.Path,
				resp.Secret,
				resp.Data,
			))
		}
		if err == nil && resp.IsError() && !s.ErrorOk {
			err = fmt.Errorf("Erroneous response:\n\n%#v", resp)
		}
		if err == nil && s.Check != nil {
			// Call the test method
			err = s.Check(resp)
		}
		if err != nil {
			t.Error(fmt.Sprintf("Failed step %d: %s", i+1, err))
			break
		}
	}

	// Revoke any secrets we might have.
	var failedRevokes []*logical.Secret
	for _, req := range revoke {
		log.Printf("[WARN] Revoking secret: %#v", req.Secret)
		req.ClientToken = client.Token()
		resp, err := core.HandleRequest(req)
		if err == nil && resp.IsError() {
			err = fmt.Errorf("Erroneous response:\n\n%#v", resp)
		}
		if err != nil {
			failedRevokes = append(failedRevokes, req.Secret)
			t.Error(fmt.Sprintf("[ERR] Revoke error: %s", err))
		}
	}

	// Perform any rollbacks. This should no-op if there aren't any.
	// We set the "immediate" flag here that any backend can pick up on
	// to do all rollbacks immediately even if the WAL entries are new.
	log.Printf("[WARN] Requesting RollbackOperation")
	req := logical.RollbackRequest(prefix + "/")
	req.Data["immediate"] = true
	req.ClientToken = client.Token()
	resp, err := core.HandleRequest(req)
	if err == nil && resp.IsError() {
		err = fmt.Errorf("Erroneous response:\n\n%#v", resp)
	}
	if err != nil && err != logical.ErrUnsupportedOperation {
		t.Error(fmt.Sprintf("[ERR] Rollback error: %s", err))
	}

	// If we have any failed revokes, log it.
	if len(failedRevokes) > 0 {
		for _, s := range failedRevokes {
			t.Error(fmt.Sprintf(
				"WARNING: Revoking the following secret failed. It may\n"+
					"still exist. Please verify:\n\n%#v",
				s))
		}
	}

	// Cleanup
	if c.Teardown != nil {
		c.Teardown()
	}
}

// TestCheckMulti is a helper to have multiple checks.
func TestCheckMulti(fs ...TestCheckFunc) TestCheckFunc {
	return func(resp *logical.Response) error {
		for _, f := range fs {
			if err := f(resp); err != nil {
				return err
			}
		}

		return nil
	}
}

// TestCheckAuth is a helper to check that a request generated an
// auth token with the proper policies.
func TestCheckAuth(policies []string) TestCheckFunc {
	return func(resp *logical.Response) error {
		if resp == nil || resp.Auth == nil {
			return fmt.Errorf("no auth in response")
		}
		if !reflect.DeepEqual(resp.Auth.Policies, policies) {
			return fmt.Errorf("invalid policies: %#v", resp.Auth.Policies)
		}

		return nil
	}
}

// TestCheckAuthDisplayName is a helper to check that a request generated a
// valid display name.
func TestCheckAuthDisplayName(n string) TestCheckFunc {
	return func(resp *logical.Response) error {
		if resp.Auth == nil {
			return fmt.Errorf("no auth in response")
		}
		if n != "" && resp.Auth.DisplayName != "mnt-"+n {
			return fmt.Errorf("invalid display name: %#v", resp.Auth.DisplayName)
		}

		return nil
	}
}

// TestCheckError is a helper to check that a response is an error.
func TestCheckError() TestCheckFunc {
	return func(resp *logical.Response) error {
		if !resp.IsError() {
			return fmt.Errorf("response should be error")
		}

		return nil
	}
}

// TestT is the interface used to handle the test lifecycle of a test.
//
// Users should just use a *testing.T object, which implements this.
type TestT interface {
	Error(args ...interface{})
	Fatal(args ...interface{})
	Skip(args ...interface{})
}

var testTesting = false
<|endoftext|>


<commit_before>// Copyright 2018 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Scrape `information_schema.innodb_sys_tablespaces`.

package collector

import (
	"context"
	"database/sql"

	"github.com/go-kit/kit/log"
	"github.com/prometheus/client_golang/prometheus"
)

const innodbTablespacesQuery = `
	SELECT
	    SPACE,
	    NAME,
	    ifnull(FILE_FORMAT, 'NONE') as FILE_FORMAT,
	    ifnull(ROW_FORMAT, 'NONE') as ROW_FORMAT,
	    ifnull(SPACE_TYPE, 'NONE') as SPACE_TYPE,
	    FILE_SIZE,
	    ALLOCATED_SIZE
	  FROM information_schema.innodb_sys_tablespaces
	`

// Metric descriptors.
var (
	infoSchemaInnodbTablesspaceInfoDesc = prometheus.NewDesc(
		prometheus.BuildFQName(namespace, informationSchema, "innodb_tablespace_space_info"),
		"The Tablespace information and Space ID.",
		[]string{"tablespace_name", "file_format", "row_format", "space_type"}, nil,
	)
	infoSchemaInnodbTablesspaceFileSizeDesc = prometheus.NewDesc(
		prometheus.BuildFQName(namespace, informationSchema, "innodb_tablespace_file_size_bytes"),
		"The apparent size of the file, which represents the maximum size of the file, uncompressed.",
		[]string{"tablespace_name"}, nil,
	)
	infoSchemaInnodbTablesspaceAllocatedSizeDesc = prometheus.NewDesc(
		prometheus.BuildFQName(namespace, informationSchema, "innodb_tablespace_allocated_size_bytes"),
		"The actual size of the file, which is the amount of space allocated on disk.",
		[]string{"tablespace_name"}, nil,
	)
)

// ScrapeInfoSchemaInnodbTablespaces collects from `information_schema.innodb_sys_tablespaces`.
type ScrapeInfoSchemaInnodbTablespaces struct{}

// Name of the Scraper. Should be unique.
func (ScrapeInfoSchemaInnodbTablespaces) Name() string {
	return informationSchema + ".innodb_tablespaces"
}

// Help describes the role of the Scraper.
func (ScrapeInfoSchemaInnodbTablespaces) Help() string {
	return "Collect metrics from information_schema.innodb_sys_tablespaces"
}

// Version of MySQL from which scraper is available.
func (ScrapeInfoSchemaInnodbTablespaces) Version() float64 {
	return 5.7
}

// Scrape collects data from database connection and sends it over channel as prometheus metric.
func (ScrapeInfoSchemaInnodbTablespaces) Scrape(ctx context.Context, db *sql.DB, ch chan<- prometheus.Metric, logger log.Logger) error {
	tablespacesRows, err := db.QueryContext(ctx, innodbTablespacesQuery)
	if err != nil {
		return err
	}
	defer tablespacesRows.Close()

	var (
		tableSpace    uint32
		tableName     string
		fileFormat    string
		rowFormat     string
		spaceType     string
		fileSize      uint64
		allocatedSize uint64
	)

	for tablespacesRows.Next() {
		err = tablespacesRows.Scan(
			&tableSpace,
			&tableName,
			&fileFormat,
			&rowFormat,
			&spaceType,
			&fileSize,
			&allocatedSize,
		)
		if err != nil {
			return err
		}
		ch <- prometheus.MustNewConstMetric(
			infoSchemaInnodbTablesspaceInfoDesc, prometheus.GaugeValue, float64(tableSpace),
			tableName, fileFormat, rowFormat, spaceType,
		)
		ch <- prometheus.MustNewConstMetric(
			infoSchemaInnodbTablesspaceFileSizeDesc, prometheus.GaugeValue, float64(fileSize),
			tableName,
		)
		ch <- prometheus.MustNewConstMetric(
			infoSchemaInnodbTablesspaceAllocatedSizeDesc, prometheus.GaugeValue, float64(allocatedSize),
			tableName,
		)
	}

	return nil
}

// check interface
var _ Scraper = ScrapeInfoSchemaInnodbTablespaces{}
<commit_msg>fix collect.info_schema.innodb_tablespaces for mariadb 10.3+ and mysql 8.0+<commit_after>// Copyright 2018 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Scrape `information_schema.innodb_sys_tablespaces`.

package collector

import (
	"context"
	"database/sql"
	"errors"
	"fmt"

	"github.com/go-kit/kit/log"
	"github.com/prometheus/client_golang/prometheus"
)

const innodbTablespacesTablenameQuery = `
	SELECT
	    table_name
	  FROM information_schema.tables
	  WHERE table_name = 'INNODB_SYS_TABLESPACES'
	    OR table_name = 'INNODB_TABLESPACES'
	`
const innodbTablespacesQuery = `
	SELECT
	    SPACE,
	    NAME,
	    ifnull((SELECT column_name
			FROM information_schema.COLUMNS
			WHERE TABLE_SCHEMA = 'information_schema'
			  AND TABLE_NAME = ` + "'%s'" + `
			  AND COLUMN_NAME = 'FILE_FORMAT' LIMIT 1), 'NONE') as FILE_FORMAT,
	    ifnull(ROW_FORMAT, 'NONE') as ROW_FORMAT,
	    ifnull(SPACE_TYPE, 'NONE') as SPACE_TYPE,
	    FILE_SIZE,
	    ALLOCATED_SIZE
	  FROM information_schema.` + "`%s`"

// Metric descriptors.
var (
	infoSchemaInnodbTablesspaceInfoDesc = prometheus.NewDesc(
		prometheus.BuildFQName(namespace, informationSchema, "innodb_tablespace_space_info"),
		"The Tablespace information and Space ID.",
		[]string{"tablespace_name", "file_format", "row_format", "space_type"}, nil,
	)
	infoSchemaInnodbTablesspaceFileSizeDesc = prometheus.NewDesc(
		prometheus.BuildFQName(namespace, informationSchema, "innodb_tablespace_file_size_bytes"),
		"The apparent size of the file, which represents the maximum size of the file, uncompressed.",
		[]string{"tablespace_name"}, nil,
	)
	infoSchemaInnodbTablesspaceAllocatedSizeDesc = prometheus.NewDesc(
		prometheus.BuildFQName(namespace, informationSchema, "innodb_tablespace_allocated_size_bytes"),
		"The actual size of the file, which is the amount of space allocated on disk.",
		[]string{"tablespace_name"}, nil,
	)
)

// ScrapeInfoSchemaInnodbTablespaces collects from `information_schema.innodb_sys_tablespaces`.
type ScrapeInfoSchemaInnodbTablespaces struct{}

// Name of the Scraper. Should be unique.
func (ScrapeInfoSchemaInnodbTablespaces) Name() string {
	return informationSchema + ".innodb_tablespaces"
}

// Help describes the role of the Scraper.
func (ScrapeInfoSchemaInnodbTablespaces) Help() string {
	return "Collect metrics from information_schema.innodb_sys_tablespaces"
}

// Version of MySQL from which scraper is available.
func (ScrapeInfoSchemaInnodbTablespaces) Version() float64 {
	return 5.7
}

// Scrape collects data from database connection and sends it over channel as prometheus metric.
func (ScrapeInfoSchemaInnodbTablespaces) Scrape(ctx context.Context, db *sql.DB, ch chan<- prometheus.Metric, logger log.Logger) error {
	var tablespacesTablename string
	var query string
	err := db.QueryRowContext(ctx, innodbTablespacesTablenameQuery).Scan(&tablespacesTablename)
	if err != nil {
		return err
	}

	switch tablespacesTablename {
	case "INNODB_SYS_TABLESPACES", "INNODB_TABLESPACES":
		query = fmt.Sprintf(innodbTablespacesQuery, tablespacesTablename, tablespacesTablename)
	default:
		return errors.New("Couldn't find INNODB_SYS_TABLESPACES or INNODB_TABLESPACES in information_schema.")
	}

	tablespacesRows, err := db.QueryContext(ctx, query)
	if err != nil {
		return err
	}
	defer tablespacesRows.Close()

	var (
		tableSpace    uint32
		tableName     string
		fileFormat    string
		rowFormat     string
		spaceType     string
		fileSize      uint64
		allocatedSize uint64
	)

	for tablespacesRows.Next() {
		err = tablespacesRows.Scan(
			&tableSpace,
			&tableName,
			&fileFormat,
			&rowFormat,
			&spaceType,
			&fileSize,
			&allocatedSize,
		)
		if err != nil {
			return err
		}
		ch <- prometheus.MustNewConstMetric(
			infoSchemaInnodbTablesspaceInfoDesc, prometheus.GaugeValue, float64(tableSpace),
			tableName, fileFormat, rowFormat, spaceType,
		)
		ch <- prometheus.MustNewConstMetric(
			infoSchemaInnodbTablesspaceFileSizeDesc, prometheus.GaugeValue, float64(fileSize),
			tableName,
		)
		ch <- prometheus.MustNewConstMetric(
			infoSchemaInnodbTablesspaceAllocatedSizeDesc, prometheus.GaugeValue, float64(allocatedSize),
			tableName,
		)
	}

	return nil
}

// check interface
var _ Scraper = ScrapeInfoSchemaInnodbTablespaces{}
<|endoftext|>


<commit_before>package tskip

import (
	"fmt"
	"os"
	"runtime"
	"strconv"
	"strings"
)

type ti interface {
	Error(i ...interface{})
}

func Errorf(t ti, skip int, m string, args ...interface{}) {
	doError(t, 0, fmt.Sprintf(m, args...))
}

func Error(t ti, skip int, m string) {
	doError(t, 0, m)
}

func doError(t ti, skip int, m string) {
	_, file, line, _ := runtime.Caller(3 + skip)
	if tr := os.Getenv("TEST_ROOT"); tr != "" {
		file = strings.TrimPrefix(file, tr+"/")
	}
	t.Error("\r        " + file + ":" + strconv.Itoa(line) + ": " + m)
}
<commit_msg>actually use skip steps<commit_after>package tskip

import (
	"fmt"
	"os"
	"runtime"
	"strconv"
	"strings"
)

type ti interface {
	Error(i ...interface{})
}

func Errorf(t ti, skip int, m string, args ...interface{}) {
	doError(t, 0+skip, fmt.Sprintf(m, args...))
}

func Error(t ti, skip int, m string) {
	doError(t, 0+skip, m)
}

func doError(t ti, skip int, m string) {
	_, file, line, _ := runtime.Caller(1 + skip)
	if tr := os.Getenv("TEST_ROOT"); tr != "" {
		file = strings.TrimPrefix(file, tr+"/")
	}
	t.Error("\r        " + file + ":" + strconv.Itoa(line) + ": " + m)
}
<|endoftext|>


<commit_before>// Copyright 2016 tsuru-client authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package admin

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strconv"

	"github.com/tsuru/tsuru/cmd"
	"github.com/tsuru/tsuru/types/quota"
)

type UserQuotaView struct{}

func (*UserQuotaView) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "user-quota-view",
		MinArgs: 1,
		Usage:   "user-quota-view <user-email>",
		Desc:    "Displays the current usage and limit of the user.",
	}
}

func (*UserQuotaView) Run(context *cmd.Context, client *cmd.Client) error {
	url, err := cmd.GetURL("/users/" + context.Args[0] + "/quota")
	if err != nil {
		return err
	}
	request, _ := http.NewRequest("GET", url, nil)
	resp, err := client.Do(request)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	var quota quota.Quota
	err = json.NewDecoder(resp.Body).Decode(&quota)
	if err != nil {
		return err
	}
	fmt.Fprintf(context.Stdout, "User: %s\n", context.Args[0])
	fmt.Fprintf(context.Stdout, "Apps usage: %d/%d\n", quota.InUse, quota.Limit)
	return nil
}

type UserChangeQuota struct{}

func (*UserChangeQuota) Info() *cmd.Info {
	desc := `Changes the limit of apps that a user can create.

The new limit must be an integer, it may also be "unlimited".`
	return &cmd.Info{
		Name:    "user-quota-change",
		MinArgs: 2,
		Usage:   "user-quota-change <user-email> <new-limit>",
		Desc:    desc,
	}
}

func (*UserChangeQuota) Run(context *cmd.Context, client *cmd.Client) error {
	u, err := cmd.GetURL("/users/" + context.Args[0] + "/quota")
	if err != nil {
		return err
	}
	limit, err := parseLimit(context.Args[1])
	if err != nil {
		return err
	}
	v := url.Values{}
	v.Set("limit", limit)
	request, _ := http.NewRequest("PUT", u, bytes.NewBufferString(v.Encode()))
	request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	_, err = client.Do(request)
	if err != nil {
		return err
	}
	fmt.Fprintln(context.Stdout, "Quota successfully updated.")
	return nil
}

type AppQuotaView struct {
	cmd.AppNameMixIn
}

func (*AppQuotaView) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "app-quota-view",
		MinArgs: 0,
		Usage:   "app-quota-view [-a/--app appname]",
		Desc:    "Displays the current usage and limit of the given app.",
	}
}

func (c *AppQuotaView) Run(context *cmd.Context, client *cmd.Client) error {
	context.RawOutput()
	appName, err := c.AppName()
	if err != nil {
		return err
	}
	url, err := cmd.GetURL(fmt.Sprintf("/apps/%s/quota", appName))
	if err != nil {
		return err
	}
	request, _ := http.NewRequest("GET", url, nil)
	resp, err := client.Do(request)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	var quota quota.Quota
	err = json.NewDecoder(resp.Body).Decode(&quota)
	if err != nil {
		return err
	}
	fmt.Fprintf(context.Stdout, "App: %s\n", appName)
	fmt.Fprintf(context.Stdout, "Units usage: %d/%d\n", quota.InUse, quota.Limit)
	return nil
}

type AppQuotaChange struct {
	cmd.AppNameMixIn
}

func (*AppQuotaChange) Info() *cmd.Info {
	desc := `Changes the limit of units that an app can have.

The new limit must be an integer, it may also be "unlimited".`
	return &cmd.Info{
		Name:    "app-quota-change",
		MinArgs: 1,
		Usage:   "app-quota-change [-a/--app appname] <new-limit>",
		Desc:    desc,
	}
}

func (c *AppQuotaChange) Run(context *cmd.Context, client *cmd.Client) error {
	context.RawOutput()
	appName, err := c.AppName()
	if err != nil {
		return err
	}
	u, err := cmd.GetURL(fmt.Sprintf("/apps/%s/quota", appName))
	if err != nil {
		return err
	}
	limit, err := parseLimit(c.Flags().Arg(0))
	if err != nil {
		return err
	}
	v := url.Values{}
	v.Set("limit", limit)
	request, _ := http.NewRequest("PUT", u, bytes.NewBufferString(v.Encode()))
	request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	_, err = client.Do(request)
	if err != nil {
		return err
	}
	fmt.Fprintln(context.Stdout, "Quota successfully updated.")
	return nil
}

type TeamQuotaView struct{}

func (*TeamQuotaView) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "team-quota-view",
		MinArgs: 1,
		Usage:   "team-quota-view <team-name>",
		Desc:    "Displays the current usage and limit of the team.",
	}
}

func (*TeamQuotaView) Run(context *cmd.Context, client *cmd.Client) error {
	url, err := cmd.GetURL("teams/" + context.Args[0] + "/quota")
	if err != nil {
		return err
	}
	request, _ := http.NewRequest("GET", url, nil)
	resp, err := client.Do(request)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	var quota quota.Quota
	err = json.NewDecoder(resp.Body).Decode(&quota)
	if err != nil {
		return err
	}
	fmt.Fprintf(context.Stdout, "Team: %s\n", context.Args[0])
	fmt.Fprintf(context.Stdout, "Apps usage: %d/%d\n", quota.InUse, quota.Limit)
	return nil
}

type TeamChangeQuota struct{}

func (*TeamChangeQuota) Info() *cmd.Info {
	desc := `Changes the limit of apps that a team can create.

The new limit must be an integer, it may also be "unlimited".`
	return &cmd.Info{
		Name:    "team-quota-change",
		MinArgs: 2,
		Usage:   "team-quota-change <team-name> <new-limit>",
		Desc:    desc,
	}
}

func (*TeamChangeQuota) Run(context *cmd.Context, client *cmd.Client) error {
	u, err := cmd.GetURL("/teams/" + context.Args[0] + "/quota")
	if err != nil {
		return err
	}
	limit, err := parseLimit(context.Args[1])
	if err != nil {
		return err
	}
	v := url.Values{}
	v.Set("limit", limit)
	request, _ := http.NewRequest("PUT", u, bytes.NewBufferString(v.Encode()))
	request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	_, err = client.Do(request)
	if err != nil {
		return err
	}
	fmt.Fprintln(context.Stdout, "Quota successfully updated.")
	return nil
}

func parseLimit(value string) (string, error) {
	if value == "unlimited" {
		return "-1", nil
	}
	n, err := strconv.Atoi(value)
	if err != nil {
		return "", errors.New(`invalid limit. It must be either an integer or "unlimited"`)
	}
	return strconv.Itoa(n), nil
}
<commit_msg>Fix TemQuotaView tests<commit_after>// Copyright 2016 tsuru-client authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package admin

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strconv"

	"github.com/tsuru/tsuru/cmd"
	"github.com/tsuru/tsuru/types/quota"
)

type UserQuotaView struct{}

func (*UserQuotaView) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "user-quota-view",
		MinArgs: 1,
		Usage:   "user-quota-view <user-email>",
		Desc:    "Displays the current usage and limit of the user.",
	}
}

func (*UserQuotaView) Run(context *cmd.Context, client *cmd.Client) error {
	url, err := cmd.GetURL("/users/" + context.Args[0] + "/quota")
	if err != nil {
		return err
	}
	request, _ := http.NewRequest("GET", url, nil)
	resp, err := client.Do(request)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	var quota quota.Quota
	err = json.NewDecoder(resp.Body).Decode(&quota)
	if err != nil {
		return err
	}
	fmt.Fprintf(context.Stdout, "User: %s\n", context.Args[0])
	fmt.Fprintf(context.Stdout, "Apps usage: %d/%d\n", quota.InUse, quota.Limit)
	return nil
}

type UserChangeQuota struct{}

func (*UserChangeQuota) Info() *cmd.Info {
	desc := `Changes the limit of apps that a user can create.

The new limit must be an integer, it may also be "unlimited".`
	return &cmd.Info{
		Name:    "user-quota-change",
		MinArgs: 2,
		Usage:   "user-quota-change <user-email> <new-limit>",
		Desc:    desc,
	}
}

func (*UserChangeQuota) Run(context *cmd.Context, client *cmd.Client) error {
	u, err := cmd.GetURL("/users/" + context.Args[0] + "/quota")
	if err != nil {
		return err
	}
	limit, err := parseLimit(context.Args[1])
	if err != nil {
		return err
	}
	v := url.Values{}
	v.Set("limit", limit)
	request, _ := http.NewRequest("PUT", u, bytes.NewBufferString(v.Encode()))
	request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	_, err = client.Do(request)
	if err != nil {
		return err
	}
	fmt.Fprintln(context.Stdout, "Quota successfully updated.")
	return nil
}

type AppQuotaView struct {
	cmd.AppNameMixIn
}

func (*AppQuotaView) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "app-quota-view",
		MinArgs: 0,
		Usage:   "app-quota-view [-a/--app appname]",
		Desc:    "Displays the current usage and limit of the given app.",
	}
}

func (c *AppQuotaView) Run(context *cmd.Context, client *cmd.Client) error {
	context.RawOutput()
	appName, err := c.AppName()
	if err != nil {
		return err
	}
	url, err := cmd.GetURL(fmt.Sprintf("/apps/%s/quota", appName))
	if err != nil {
		return err
	}
	request, _ := http.NewRequest("GET", url, nil)
	resp, err := client.Do(request)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	var quota quota.Quota
	err = json.NewDecoder(resp.Body).Decode(&quota)
	if err != nil {
		return err
	}
	fmt.Fprintf(context.Stdout, "App: %s\n", appName)
	fmt.Fprintf(context.Stdout, "Units usage: %d/%d\n", quota.InUse, quota.Limit)
	return nil
}

type AppQuotaChange struct {
	cmd.AppNameMixIn
}

func (*AppQuotaChange) Info() *cmd.Info {
	desc := `Changes the limit of units that an app can have.

The new limit must be an integer, it may also be "unlimited".`
	return &cmd.Info{
		Name:    "app-quota-change",
		MinArgs: 1,
		Usage:   "app-quota-change [-a/--app appname] <new-limit>",
		Desc:    desc,
	}
}

func (c *AppQuotaChange) Run(context *cmd.Context, client *cmd.Client) error {
	context.RawOutput()
	appName, err := c.AppName()
	if err != nil {
		return err
	}
	u, err := cmd.GetURL(fmt.Sprintf("/apps/%s/quota", appName))
	if err != nil {
		return err
	}
	limit, err := parseLimit(c.Flags().Arg(0))
	if err != nil {
		return err
	}
	v := url.Values{}
	v.Set("limit", limit)
	request, _ := http.NewRequest("PUT", u, bytes.NewBufferString(v.Encode()))
	request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	_, err = client.Do(request)
	if err != nil {
		return err
	}
	fmt.Fprintln(context.Stdout, "Quota successfully updated.")
	return nil
}

type TeamQuotaView struct{}

func (*TeamQuotaView) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "team-quota-view",
		MinArgs: 1,
		Usage:   "team-quota-view <team-name>",
		Desc:    "Displays the current usage and limit of the team.",
	}
}

func (*TeamQuotaView) Run(context *cmd.Context, client *cmd.Client) error {
	url, err := cmd.GetURL("/teams/" + context.Args[0] + "/quota")
	if err != nil {
		return err
	}
	request, _ := http.NewRequest("GET", url, nil)
	resp, err := client.Do(request)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	var quota quota.Quota
	err = json.NewDecoder(resp.Body).Decode(&quota)
	if err != nil {
		return err
	}
	fmt.Fprintf(context.Stdout, "Team: %s\n", context.Args[0])
	fmt.Fprintf(context.Stdout, "Apps usage: %d/%d\n", quota.InUse, quota.Limit)
	return nil
}

type TeamChangeQuota struct{}

func (*TeamChangeQuota) Info() *cmd.Info {
	desc := `Changes the limit of apps that a team can create.

The new limit must be an integer, it may also be "unlimited".`
	return &cmd.Info{
		Name:    "team-quota-change",
		MinArgs: 2,
		Usage:   "team-quota-change <team-name> <new-limit>",
		Desc:    desc,
	}
}

func (*TeamChangeQuota) Run(context *cmd.Context, client *cmd.Client) error {
	u, err := cmd.GetURL("/teams/" + context.Args[0] + "/quota")
	if err != nil {
		return err
	}
	limit, err := parseLimit(context.Args[1])
	if err != nil {
		return err
	}
	v := url.Values{}
	v.Set("limit", limit)
	request, _ := http.NewRequest("PUT", u, bytes.NewBufferString(v.Encode()))
	request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	_, err = client.Do(request)
	if err != nil {
		return err
	}
	fmt.Fprintln(context.Stdout, "Quota successfully updated.")
	return nil
}

func parseLimit(value string) (string, error) {
	if value == "unlimited" {
		return "-1", nil
	}
	n, err := strconv.Atoi(value)
	if err != nil {
		return "", errors.New(`invalid limit. It must be either an integer or "unlimited"`)
	}
	return strconv.Itoa(n), nil
}
<|endoftext|>


<commit_before>// Copyright 2016 tsuru-client authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package admin

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strconv"

	"github.com/tsuru/tsuru/cmd"
	"github.com/tsuru/tsuru/types/quota"
)

type UserQuotaView struct{}

func (*UserQuotaView) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "user-quota-view",
		MinArgs: 1,
		Usage:   "user-quota-view <user-email>",
		Desc:    "Displays the current usage and limit of the user.",
	}
}

func (*UserQuotaView) Run(context *cmd.Context, client *cmd.Client) error {
	url, err := cmd.GetURL("/users/" + context.Args[0] + "/quota")
	if err != nil {
		return err
	}
	request, _ := http.NewRequest("GET", url, nil)
	resp, err := client.Do(request)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	var quota quota.Quota
	err = json.NewDecoder(resp.Body).Decode(&quota)
	if err != nil {
		return err
	}
	fmt.Fprintf(context.Stdout, "User: %s\n", context.Args[0])
	fmt.Fprintf(context.Stdout, "Apps usage: %d/%d\n", quota.InUse, quota.Limit)
	return nil
}

type UserChangeQuota struct{}

func (*UserChangeQuota) Info() *cmd.Info {
	desc := `Changes the limit of apps that a user can create.

The new limit must be an integer, it may also be "unlimited".`
	return &cmd.Info{
		Name:    "user-quota-change",
		MinArgs: 2,
		Usage:   "user-quota-change <user-email> <new-limit>",
		Desc:    desc,
	}
}

func (*UserChangeQuota) Run(context *cmd.Context, client *cmd.Client) error {
	u, err := cmd.GetURL("/users/" + context.Args[0] + "/quota")
	if err != nil {
		return err
	}
	limit, err := parseLimit(context.Args[1])
	if err != nil {
		return err
	}
	v := url.Values{}
	v.Set("limit", limit)
	request, _ := http.NewRequest("PUT", u, bytes.NewBufferString(v.Encode()))
	request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	_, err = client.Do(request)
	if err != nil {
		return err
	}
	fmt.Fprintln(context.Stdout, "Quota successfully updated.")
	return nil
}

type AppQuotaView struct {
	cmd.AppNameMixIn
}

func (*AppQuotaView) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "app-quota-view",
		MinArgs: 0,
		Usage:   "app-quota-view [-a/--app appname]",
		Desc:    "Displays the current usage and limit of the given app.",
	}
}

func (c *AppQuotaView) Run(context *cmd.Context, client *cmd.Client) error {
	context.RawOutput()
	appName, err := c.AppName()
	if err != nil {
		return err
	}
	url, err := cmd.GetURL(fmt.Sprintf("/apps/%s/quota", appName))
	if err != nil {
		return err
	}
	request, _ := http.NewRequest("GET", url, nil)
	resp, err := client.Do(request)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	var quota quota.Quota
	err = json.NewDecoder(resp.Body).Decode(&quota)
	if err != nil {
		return err
	}
	fmt.Fprintf(context.Stdout, "App: %s\n", appName)
	fmt.Fprintf(context.Stdout, "Units usage: %d/%d\n", quota.InUse, quota.Limit)
	return nil
}

type AppQuotaChange struct {
	cmd.AppNameMixIn
}

func (*AppQuotaChange) Info() *cmd.Info {
	desc := `Changes the limit of units that an app can have.

The new limit must be an integer, it may also be "unlimited".`
	return &cmd.Info{
		Name:    "app-quota-change",
		MinArgs: 1,
		Usage:   "app-quota-change [-a/--app appname] <new-limit>",
		Desc:    desc,
	}
}

func (c *AppQuotaChange) Run(context *cmd.Context, client *cmd.Client) error {
	context.RawOutput()
	appName, err := c.AppName()
	if err != nil {
		return err
	}
	u, err := cmd.GetURL(fmt.Sprintf("/apps/%s/quota", appName))
	if err != nil {
		return err
	}
	limit, err := parseLimit(c.Flags().Arg(0))
	if err != nil {
		return err
	}
	v := url.Values{}
	v.Set("limit", limit)
	request, _ := http.NewRequest("PUT", u, bytes.NewBufferString(v.Encode()))
	request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	_, err = client.Do(request)
	if err != nil {
		return err
	}
	fmt.Fprintln(context.Stdout, "Quota successfully updated.")
	return nil
}

func parseLimit(value string) (string, error) {
	if value == "unlimited" {
		return "-1", nil
	}
	n, err := strconv.Atoi(value)
	if err != nil {
		return "", errors.New(`invalid limit. It must be either an integer or "unlimited"`)
	}
	return strconv.Itoa(n), nil
}
<commit_msg>add team-quota-(view|change)<commit_after>// Copyright 2016 tsuru-client authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package admin

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strconv"

	"github.com/tsuru/tsuru/cmd"
	"github.com/tsuru/tsuru/types/quota"
)

type UserQuotaView struct{}

func (*UserQuotaView) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "user-quota-view",
		MinArgs: 1,
		Usage:   "user-quota-view <user-email>",
		Desc:    "Displays the current usage and limit of the user.",
	}
}

func (*UserQuotaView) Run(context *cmd.Context, client *cmd.Client) error {
	url, err := cmd.GetURL("/users/" + context.Args[0] + "/quota")
	if err != nil {
		return err
	}
	request, _ := http.NewRequest("GET", url, nil)
	resp, err := client.Do(request)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	var quota quota.Quota
	err = json.NewDecoder(resp.Body).Decode(&quota)
	if err != nil {
		return err
	}
	fmt.Fprintf(context.Stdout, "User: %s\n", context.Args[0])
	fmt.Fprintf(context.Stdout, "Apps usage: %d/%d\n", quota.InUse, quota.Limit)
	return nil
}

type UserChangeQuota struct{}

func (*UserChangeQuota) Info() *cmd.Info {
	desc := `Changes the limit of apps that a user can create.

The new limit must be an integer, it may also be "unlimited".`
	return &cmd.Info{
		Name:    "user-quota-change",
		MinArgs: 2,
		Usage:   "user-quota-change <user-email> <new-limit>",
		Desc:    desc,
	}
}

func (*UserChangeQuota) Run(context *cmd.Context, client *cmd.Client) error {
	u, err := cmd.GetURL("/users/" + context.Args[0] + "/quota")
	if err != nil {
		return err
	}
	limit, err := parseLimit(context.Args[1])
	if err != nil {
		return err
	}
	v := url.Values{}
	v.Set("limit", limit)
	request, _ := http.NewRequest("PUT", u, bytes.NewBufferString(v.Encode()))
	request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	_, err = client.Do(request)
	if err != nil {
		return err
	}
	fmt.Fprintln(context.Stdout, "Quota successfully updated.")
	return nil
}

type AppQuotaView struct {
	cmd.AppNameMixIn
}

func (*AppQuotaView) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "app-quota-view",
		MinArgs: 0,
		Usage:   "app-quota-view [-a/--app appname]",
		Desc:    "Displays the current usage and limit of the given app.",
	}
}

func (c *AppQuotaView) Run(context *cmd.Context, client *cmd.Client) error {
	context.RawOutput()
	appName, err := c.AppName()
	if err != nil {
		return err
	}
	url, err := cmd.GetURL(fmt.Sprintf("/apps/%s/quota", appName))
	if err != nil {
		return err
	}
	request, _ := http.NewRequest("GET", url, nil)
	resp, err := client.Do(request)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	var quota quota.Quota
	err = json.NewDecoder(resp.Body).Decode(&quota)
	if err != nil {
		return err
	}
	fmt.Fprintf(context.Stdout, "App: %s\n", appName)
	fmt.Fprintf(context.Stdout, "Units usage: %d/%d\n", quota.InUse, quota.Limit)
	return nil
}

type AppQuotaChange struct {
	cmd.AppNameMixIn
}

func (*AppQuotaChange) Info() *cmd.Info {
	desc := `Changes the limit of units that an app can have.

The new limit must be an integer, it may also be "unlimited".`
	return &cmd.Info{
		Name:    "app-quota-change",
		MinArgs: 1,
		Usage:   "app-quota-change [-a/--app appname] <new-limit>",
		Desc:    desc,
	}
}

func (c *AppQuotaChange) Run(context *cmd.Context, client *cmd.Client) error {
	context.RawOutput()
	appName, err := c.AppName()
	if err != nil {
		return err
	}
	u, err := cmd.GetURL(fmt.Sprintf("/apps/%s/quota", appName))
	if err != nil {
		return err
	}
	limit, err := parseLimit(c.Flags().Arg(0))
	if err != nil {
		return err
	}
	v := url.Values{}
	v.Set("limit", limit)
	request, _ := http.NewRequest("PUT", u, bytes.NewBufferString(v.Encode()))
	request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	_, err = client.Do(request)
	if err != nil {
		return err
	}
	fmt.Fprintln(context.Stdout, "Quota successfully updated.")
	return nil
}

type TeamQuotaView struct{}

func (*TeamQuotaView) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "team-quota-view",
		MinArgs: 1,
		Usage:   "team-quota-view <team-name>",
		Desc:    "Displays the current usage and limit of the team.",
	}
}

func (*TeamQuotaView) Run(context *cmd.Context, client *cmd.Client) error {
	url, err := cmd.GetURL("teams/" + context.Args[0] + "/quota")
	if err != nil {
		return err
	}
	request, _ := http.NewRequest("GET", url, nil)
	resp, err := client.Do(request)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	var quota quota.Quota
	err = json.NewDecoder(resp.Body).Decode(&quota)
	if err != nil {
		return err
	}
	fmt.Fprintf(context.Stdout, "Team: %s\n", context.Args[0])
	fmt.Fprintf(context.Stdout, "Apps usage: %d/%d\n", quota.InUse, quota.Limit)
	return nil
}

type TeamChangeQuota struct{}

func (*TeamChangeQuota) Info() *cmd.Info {
	desc := `Changes the limit of apps that a team can create.

The new limit must be an integer, it may also be "unlimited".`
	return &cmd.Info{
		Name:    "team-quota-change",
		MinArgs: 2,
		Usage:   "team-quota-change <team-name> <new-limit>",
		Desc:    desc,
	}
}

func (*TeamChangeQuota) Run(context *cmd.Context, client *cmd.Client) error {
	u, err := cmd.GetURL("/teams/" + context.Args[0] + "/quota")
	if err != nil {
		return err
	}
	limit, err := parseLimit(context.Args[1])
	if err != nil {
		return err
	}
	v := url.Values{}
	v.Set("limit", limit)
	request, _ := http.NewRequest("PUT", u, bytes.NewBufferString(v.Encode()))
	request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	_, err = client.Do(request)
	if err != nil {
		return err
	}
	fmt.Fprintln(context.Stdout, "Quota successfully updated.")
	return nil
}

func parseLimit(value string) (string, error) {
	if value == "unlimited" {
		return "-1", nil
	}
	n, err := strconv.Atoi(value)
	if err != nil {
		return "", errors.New(`invalid limit. It must be either an integer or "unlimited"`)
	}
	return strconv.Itoa(n), nil
}
<|endoftext|>


<commit_before>package sms

import "encoding/json"

type NexmoSms struct {
	ApiKey    string `json:"api_key"`
	ApiSecret string `json:"api_secret"`
	To        string `json:"to"`
	From      string `json:"from"`
	Text      string `json:"text"`
}

func (sms *NexmoSms) EncodeNexmoSms(apiKey, apiSecret string) ([]byte, error) {
	sms.ApiKey = apiKey
	sms.ApiSecret = apiSecret

	d, err := json.Marshal(&sms)
	if err != nil {
		logger.WithField("error", err.Error()).Error("Could not encode sms as json")
		return nil, err
	}
	return d, nil
}
<commit_msg>Omit json fields if empty<commit_after>package sms

import "encoding/json"

type NexmoSms struct {
	ApiKey    string `json:"api_key,omitempty"`
	ApiSecret string `json:"api_secret,omitempty"`
	To        string `json:"to"`
	From      string `json:"from"`
	Text      string `json:"text"`
}

func (sms *NexmoSms) EncodeNexmoSms(apiKey, apiSecret string) ([]byte, error) {
	sms.ApiKey = apiKey
	sms.ApiSecret = apiSecret

	d, err := json.Marshal(&sms)
	if err != nil {
		logger.WithField("error", err.Error()).Error("Could not encode sms as json")
		return nil, err
	}
	return d, nil
}
<|endoftext|>


<commit_before>package main

import (
    "encoding/json"
    "github.com/garyburd/redigo/redis"
)

type User struct {
    Name     string `json:"name"`
    Password string `json:"password"`
    Method   string `json:"method"`
    Limit    int64  `json:"limit"`
}

type Storage struct {
    pool *redis.Pool
}

func NewStorage(server string) *Storage {
    pool := redis.NewPool(func() (conn redis.Conn, err error) {
        conn, err = redis.Dial("tcp", server)
        return
    }, 3)
    return &Storage{pool}
}

func (s *Storage) Get(key string) (user User, err error) {
    var data []byte
    var conn = s.pool.Get()
    data, err = redis.Bytes(conn.Do("GET", key))
    if err != nil {
        return
    }
    err = json.Unmarshal(data, &user)
    return
}

func (s *Storage) Set(key string, user User) (err error) {
    data, err := json.Marshal(user)
    if err != nil {
        return err
    }
    conn := s.pool.Get()
    _, err = conn.Do("SET", key, data)
    return
}

func (s *Storage) IncrSize(key string, incr int) (score int64, err error) {
    conn := s.pool.Get()
    score, err = redis.Int64(conn.Do("INCRBY", key, incr))
    return
}

func (s *Storage) GetSize(key string) (score int64, err error) {
    conn := s.pool.Get()
    score, err = redis.Int64(conn.Do("GET", key))
    return
}
<commit_msg>close the redis poolConnection<commit_after>package main

import (
    "encoding/json"
    "github.com/garyburd/redigo/redis"
)

type User struct {
    Name     string `json:"name"`
    Password string `json:"password"`
    Method   string `json:"method"`
    Limit    int64  `json:"limit"`
}

type Storage struct {
    pool *redis.Pool
}

func NewStorage(server string) *Storage {
    pool := redis.NewPool(func() (conn redis.Conn, err error) {
        conn, err = redis.Dial("tcp", server)
        return
    }, 3)
    return &Storage{pool}
}

func (s *Storage) Get(key string) (user User, err error) {
    var data []byte
    var conn = s.pool.Get()
    defer conn.Close()
    data, err = redis.Bytes(conn.Do("GET", key))
    if err != nil {
        return
    }
    err = json.Unmarshal(data, &user)
    return
}

func (s *Storage) Set(key string, user User) (err error) {
    data, err := json.Marshal(user)
    if err != nil {
        return err
    }
    conn := s.pool.Get()
    defer conn.Close()
    _, err = conn.Do("SET", key, data)
    return
}

func (s *Storage) IncrSize(key string, incr int) (score int64, err error) {
    var conn = s.pool.Get()
    defer conn.Close()
    score, err = redis.Int64(conn.Do("INCRBY", key, incr))
    return
}

func (s *Storage) GetSize(key string) (score int64, err error) {
    var conn = s.pool.Get()
    defer conn.Close()
    score, err = redis.Int64(conn.Do("GET", key))
    return
}
<|endoftext|>


<commit_before>package main

// finds types.. O(n^2)
// looks for 0.01% (1 in 10,000) frequency or more
import (
	"bufio"
	"compress/gzip"
	"flag"
	"fmt"
	"log"
	"os"
	"sort"
	"strconv"
	"strings"

	"github.com/xrash/smetrics"
)

type pair struct {
	word  string
	count int
}

type counts []pair

func (s counts) Len() int      { return len(s) }
func (s counts) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s counts) Less(i, j int) bool {
	if s[i].count < s[j].count {
		return true
	}
	if s[i].count > s[j].count {
		return false
	}
	return s[i].word < s[j].word
}

// LoadCSV loads in a CSV in form of WORD,COUNT
func loadCSV(fname string) (counts, error) {
	fi, err := os.Open(fname)
	if err != nil {
		return nil, err
	}
	defer fi.Close()
	fizip, err := gzip.NewReader(fi)
	if err != nil {
		return nil, err
	}
	defer fizip.Close()
	words := make(counts, 0, 5000000)
	scanner := bufio.NewScanner(fizip)
	for scanner.Scan() {
		line := scanner.Text()
		parts := strings.SplitN(line, ",", 2)
		if len(parts) != 2 {
			return nil, fmt.Errorf("Got extra junk in line: %q", line)
		}
		c, err := strconv.Atoi(parts[1])
		if err != nil {
			return nil, fmt.Errorf("Number conversion failed: %q", line)
		}
		words = append(words, pair{parts[0], c})
	}
	if err := scanner.Err(); err != nil {
		return nil, err
	}
	sort.Sort(sort.Reverse(words))
	return words, nil
}

// LoadWordList loads in a list of known-good words
func LoadWordList(fname string) (map[string]bool, error) {
	fi, err := os.Open(fname)
	if err != nil {
		return nil, err
	}
	defer fi.Close()
	out := make(map[string]bool)
	intro := true
	scanner := bufio.NewScanner(fi)
	for scanner.Scan() {
		line := scanner.Text()
		out[strings.ToLower(line)] = true
	}
	if err := scanner.Err(); err != nil {
		return nil, err
	}
	return out, nil
}

func main() {
	dictfile := flag.String("d", "dict.txt", "aspell wordlist")

	// not compressing output file since this is a single go-routine
	// and two lazy to fire off a separate routine to compress...
	// i.e. it's not parallelized and gzip will just slow things down
	// Output is only like 2M anyways.
	outfile := flag.String("o", "RC-score.csv", "outfile")
	infile := flag.String("i", "RC-total.csv.gz", "infile")
	minScore := flag.Float64("minscore", 0.96, "min Jaro-Winkler score")
	minRatio := flag.Float64("minratio", 0.01, "error ratio")
	flag.Parse()

	// load known-good words`
	truewords, err := LoadWordList(*dictfile)
	if err != nil {
		log.Fatalf("Unable to read wordlist")
	}
	log.Printf("Got %d real words from dictionary", len(truewords))

	// load frequency counts
	words, err := loadCSV(*infile)
	if err != nil {
		log.Fatalf("Unable to freq counts: %s", err)
	}

	// make total count
	total := 0
	sum := 0
	for _, kv := range words {
		total += kv.count
	}
	log.Printf("Got %d uniques, %d total", len(words), total)

	fo, err := os.Create(*outfile)
	if err != nil {
		log.Fatalf("Unable to create outfile: %s", err)
	}
	fobuf := bufio.NewWriter(fo)

	for top := 0; top < len(words); top++ {
		a := words[top]
		sum += a.count
		cdf := 100.0 * float64(sum) / float64(total)
		// must have at least this many occurances to
		// have an entry

		// exit if we got 90% of words covered
		if cdf > 90.0 {
			break
		}

		aword := a.word
		for bottom := top + 1; bottom < len(words); bottom++ {
			b := words[bottom]
			// misspelling must occur twice
			// TODO: again this is a fixed point in the list
			if b.count < 2 {
				break
			}
			ratio := 100.0 * float64(b.count) / float64(a.count)

			// great than one percent probably means
			// bword is just a different work that is similar
			if ratio > 1.0 {
				continue
			}
			// if less than 0.01% ignore, too rare
			if ratio < *minRatio {
				break
			}

			bword := b.word

			// handle "foobar"/"foobars"
			if strings.HasPrefix(aword, bword) || strings.HasPrefix(bword, aword) {
				continue
			}

			// handle "foobars", "foobarr"
			if aword[:len(aword)-1] == bword[:len(bword)-1] {
				continue
			}
			val := smetrics.JaroWinkler(aword, bword, 0.7, 4)
			if val >= *minScore {
				fobuf.WriteString(fmt.Sprintf("%s,%s,%d,%.2f,%d,%d,%d,%.2f,%.3f\n",
					aword, bword,
					top, cdf, a.count,
					bottom, b.count, ratio,
					val))
			}
		}
	}

	fobuf.Flush()
	fo.Close()
}
<commit_msg>make word list generic<commit_after>package main

// finds types.. O(n^2)
// looks for 0.01% (1 in 10,000) frequency or more
import (
	"bufio"
	"compress/gzip"
	"flag"
	"fmt"
	"log"
	"os"
	"sort"
	"strconv"
	"strings"

	"github.com/xrash/smetrics"
)

type pair struct {
	word  string
	count int
}

type counts []pair

func (s counts) Len() int      { return len(s) }
func (s counts) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s counts) Less(i, j int) bool {
	if s[i].count < s[j].count {
		return true
	}
	if s[i].count > s[j].count {
		return false
	}
	return s[i].word < s[j].word
}

// LoadCSV loads in a CSV in form of WORD,COUNT
func loadCSV(fname string) (counts, error) {
	fi, err := os.Open(fname)
	if err != nil {
		return nil, err
	}
	defer fi.Close()
	fizip, err := gzip.NewReader(fi)
	if err != nil {
		return nil, err
	}
	defer fizip.Close()
	words := make(counts, 0, 5000000)
	scanner := bufio.NewScanner(fizip)
	for scanner.Scan() {
		line := scanner.Text()
		parts := strings.SplitN(line, ",", 2)
		if len(parts) != 2 {
			return nil, fmt.Errorf("Got extra junk in line: %q", line)
		}
		c, err := strconv.Atoi(parts[1])
		if err != nil {
			return nil, fmt.Errorf("Number conversion failed: %q", line)
		}
		words = append(words, pair{parts[0], c})
	}
	if err := scanner.Err(); err != nil {
		return nil, err
	}
	sort.Sort(sort.Reverse(words))
	return words, nil
}

// LoadWordList loads in a list of known-good words
func LoadWordList(fname string) (map[string]bool, error) {
	fi, err := os.Open(fname)
	if err != nil {
		return nil, err
	}
	defer fi.Close()
	out := make(map[string]bool)
	scanner := bufio.NewScanner(fi)
	for scanner.Scan() {
		line := scanner.Text()
		out[strings.ToLower(line)] = true
	}
	if err := scanner.Err(); err != nil {
		return nil, err
	}
	return out, nil
}

func main() {
	dictfile := flag.String("d", "dict.txt", "aspell wordlist")

	// not compressing output file since this is a single go-routine
	// and two lazy to fire off a separate routine to compress...
	// i.e. it's not parallelized and gzip will just slow things down
	// Output is only like 2M anyways.
	outfile := flag.String("o", "RC-score.csv", "outfile")
	infile := flag.String("i", "RC-total.csv.gz", "infile")
	minScore := flag.Float64("minscore", 0.96, "min Jaro-Winkler score")
	minRatio := flag.Float64("minratio", 0.01, "error ratio")
	flag.Parse()

	// load known-good words`
	truewords, err := LoadWordList(*dictfile)
	if err != nil {
		log.Fatalf("Unable to read wordlist")
	}
	log.Printf("Got %d real words from dictionary", len(truewords))

	// load frequency counts
	words, err := loadCSV(*infile)
	if err != nil {
		log.Fatalf("Unable to freq counts: %s", err)
	}

	// make total count
	total := 0
	sum := 0
	for _, kv := range words {
		total += kv.count
	}
	log.Printf("Got %d uniques, %d total", len(words), total)

	fo, err := os.Create(*outfile)
	if err != nil {
		log.Fatalf("Unable to create outfile: %s", err)
	}
	fobuf := bufio.NewWriter(fo)

	for top := 0; top < len(words); top++ {
		a := words[top]
		sum += a.count
		cdf := 100.0 * float64(sum) / float64(total)
		// must have at least this many occurances to
		// have an entry

		// exit if we got 90% of words covered
		if cdf > 90.0 {
			break
		}

		aword := a.word
		for bottom := top + 1; bottom < len(words); bottom++ {
			b := words[bottom]
			// misspelling must occur twice
			// TODO: again this is a fixed point in the list
			if b.count < 2 {
				break
			}
			ratio := 100.0 * float64(b.count) / float64(a.count)

			// great than one percent probably means
			// bword is just a different work that is similar
			if ratio > 1.0 {
				continue
			}
			// if less than 0.01% ignore, too rare
			if ratio < *minRatio {
				break
			}

			bword := b.word

			// handle "foobar"/"foobars"
			if strings.HasPrefix(aword, bword) || strings.HasPrefix(bword, aword) {
				continue
			}

			// handle "foobars", "foobarr"
			if aword[:len(aword)-1] == bword[:len(bword)-1] {
				continue
			}
			val := smetrics.JaroWinkler(aword, bword, 0.7, 4)
			if val >= *minScore {
				fobuf.WriteString(fmt.Sprintf("%s,%s,%d,%.2f,%d,%d,%d,%.2f,%.3f\n",
					aword, bword,
					top, cdf, a.count,
					bottom, b.count, ratio,
					val))
			}
		}
	}

	fobuf.Flush()
	fo.Close()
}
<|endoftext|>


<commit_before>package director

import (
	"bitbucket.org/engineerbetter/concourse-up/config"
	"bitbucket.org/engineerbetter/concourse-up/terraform"
	"bitbucket.org/engineerbetter/concourse-up/util"
)

type awsCloudConfigParams struct {
	AvailabilityZone            string
	VMsSecurityGroupID          string
	LoadBalancerSecurityGroupID string
	LoadBalancerID              string
	DirectorSubnetID            string
	ConcourseSubnetID           string
}

func generateCloudConfig(conf *config.Config, metadata *terraform.Metadata) ([]byte, error) {
	templateParams := awsCloudConfigParams{
		AvailabilityZone:            conf.AvailabilityZone,
		VMsSecurityGroupID:          metadata.VMsSecurityGroupID.Value,
		LoadBalancerSecurityGroupID: metadata.ELBSecurityGroupID.Value,
		LoadBalancerID:              metadata.ELBName.Value,
		DirectorSubnetID:            metadata.DirectorSubnetID.Value,
		ConcourseSubnetID:           metadata.ConcourseSubnetID.Value,
	}

	return util.RenderTemplate(awsCloudConfigtemplate, templateParams)
}

var awsCloudConfigtemplate = `---
azs:
- name: z1
  cloud_properties:
    availability_zone: <% .AvailabilityZone %>

vm_types:
- name: concourse-medium
  cloud_properties:
    instance_type: m3.medium
    spot_bid_price: 0.09 # on-demand price: 0.073
    spot_ondemand_fallback: true
    ephemeral_disk:
      size: 200_000
      type: gp2
    security_groups:
    - <% .VMsSecurityGroupID %>

- name: concourse-large
  cloud_properties:
    instance_type: m3.xlarge
    spot_bid_price: 0.320 # on-demand price: 0.266
    spot_ondemand_fallback: true
    ephemeral_disk:
      size: 200_000
      type: gp2
    security_groups:
    - <% .VMsSecurityGroupID %>

- name: compilation
  cloud_properties:
    instance_type: m3.xlarge
    spot_bid_price: 0.320 # on-demand price: 0.266
    spot_ondemand_fallback: true
    ephemeral_disk:
      size: 5_000
      type: gp2
    security_groups:
    - <% .VMsSecurityGroupID %>

disk_types:
- name: default
  disk_size: 50_000
  cloud_properties:
    type: gp2
- name: large
  disk_size: 200_000
  cloud_properties:
    type: gp2

networks:
- name: default
  type: manual
  subnets:
  - range: 10.0.0.0/24
    gateway: 10.0.0.1
    az: z1
    static:
    - 10.0.0.6
    reserved:
    - 10.0.0.1-10.0.0.5
    dns:
    - 10.0.0.2
    cloud_properties:
      subnet: <% .DirectorSubnetID %>

- name: concourse
  type: manual
  subnets:
  - range: 10.0.1.0/24
    gateway: 10.0.1.1
    az: z1
    reserved:
    - 10.0.1.1
    dns:
    - 10.0.0.2
    cloud_properties:
      subnet: <% .ConcourseSubnetID %>

- name: vip
  type: vip

vm_extensions:
- name: elb
  cloud_properties:
    elbs:
    - <% .LoadBalancerID %>
    security_groups:
    - <% .LoadBalancerSecurityGroupID %>
    - <% .VMsSecurityGroupID %>

compilation:
  workers: 5
  reuse_compilation_vms: true
  az: z1
  vm_type: compilation
  network: default
`
<commit_msg>fix subnet reserved range<commit_after>package director

import (
	"bitbucket.org/engineerbetter/concourse-up/config"
	"bitbucket.org/engineerbetter/concourse-up/terraform"
	"bitbucket.org/engineerbetter/concourse-up/util"
)

type awsCloudConfigParams struct {
	AvailabilityZone            string
	VMsSecurityGroupID          string
	LoadBalancerSecurityGroupID string
	LoadBalancerID              string
	DirectorSubnetID            string
	ConcourseSubnetID           string
}

func generateCloudConfig(conf *config.Config, metadata *terraform.Metadata) ([]byte, error) {
	templateParams := awsCloudConfigParams{
		AvailabilityZone:            conf.AvailabilityZone,
		VMsSecurityGroupID:          metadata.VMsSecurityGroupID.Value,
		LoadBalancerSecurityGroupID: metadata.ELBSecurityGroupID.Value,
		LoadBalancerID:              metadata.ELBName.Value,
		DirectorSubnetID:            metadata.DirectorSubnetID.Value,
		ConcourseSubnetID:           metadata.ConcourseSubnetID.Value,
	}

	return util.RenderTemplate(awsCloudConfigtemplate, templateParams)
}

var awsCloudConfigtemplate = `---
azs:
- name: z1
  cloud_properties:
    availability_zone: <% .AvailabilityZone %>

vm_types:
- name: concourse-medium
  cloud_properties:
    instance_type: m3.medium
    spot_bid_price: 0.09 # on-demand price: 0.073
    spot_ondemand_fallback: true
    ephemeral_disk:
      size: 200_000
      type: gp2
    security_groups:
    - <% .VMsSecurityGroupID %>

- name: concourse-large
  cloud_properties:
    instance_type: m3.xlarge
    spot_bid_price: 0.320 # on-demand price: 0.266
    spot_ondemand_fallback: true
    ephemeral_disk:
      size: 200_000
      type: gp2
    security_groups:
    - <% .VMsSecurityGroupID %>

- name: compilation
  cloud_properties:
    instance_type: m3.xlarge
    spot_bid_price: 0.320 # on-demand price: 0.266
    spot_ondemand_fallback: true
    ephemeral_disk:
      size: 5_000
      type: gp2
    security_groups:
    - <% .VMsSecurityGroupID %>

disk_types:
- name: default
  disk_size: 50_000
  cloud_properties:
    type: gp2
- name: large
  disk_size: 200_000
  cloud_properties:
    type: gp2

networks:
- name: default
  type: manual
  subnets:
  - range: 10.0.0.0/24
    gateway: 10.0.0.1
    az: z1
    static:
    - 10.0.0.6
    reserved:
    - 10.0.0.1-10.0.0.5
    dns:
    - 10.0.0.2
    cloud_properties:
      subnet: <% .DirectorSubnetID %>

- name: concourse
  type: manual
  subnets:
  - range: 10.0.1.0/24
    gateway: 10.0.1.1
    az: z1
    reserved:
    - 10.0.1.1-10.0.1.5
    dns:
    - 10.0.0.2
    cloud_properties:
      subnet: <% .ConcourseSubnetID %>

- name: vip
  type: vip

vm_extensions:
- name: elb
  cloud_properties:
    elbs:
    - <% .LoadBalancerID %>
    security_groups:
    - <% .LoadBalancerSecurityGroupID %>
    - <% .VMsSecurityGroupID %>

compilation:
  workers: 5
  reuse_compilation_vms: true
  az: z1
  vm_type: compilation
  network: default
`
<|endoftext|>


<commit_before>package main

import "sort"

type DataInput struct {
	ID             int64
	TrueState      bool
	ProbGivenTrue  float64
	ProbGivenFalse float64

	State bool
}

type Observation struct {
	ProbTrue  float64
	ProbFalse float64
}

type Bayesian struct {
	Name      string
	Prior     float64
	Threshold float64

	Inputs []DataInput

	Observations       []*Observation
	ObservationsSorted []int
	ID2Observation     map[int64]int

	Probability float64
}

func (b *Bayesian) ReadState() bool {
	return b.Probability >= b.Threshold
}

func (b *Bayesian) OnChange() {
	b.ProcessState()
	b.Update()
}

func (b *Bayesian) ProcessState() {
	for _, input := range b.Inputs {
		// Add entity to current observations if state conditions are met
		should_trigger := input.State
		if should_trigger {
			probtrue := input.ProbGivenTrue
			probfalse := input.ProbGivenFalse

			var obs *Observation
			obsi, exists := b.ID2Observation[input.ID]
			if !exists {
				obs = &Observation{ProbTrue: probtrue, ProbFalse: probfalse}
				obsi = len(b.Observations)

				b.Observations = append(b.Observations, obs)

				b.ObservationsSorted = append(b.ObservationsSorted, obsi)
				sort.Ints(b.ObservationsSorted)

				b.ID2Observation[input.ID] = obsi
			}
			obs.ProbTrue = probtrue
			obs.ProbFalse = probfalse
		} else {
			obsi, exists := b.ID2Observation[input.ID]
			if exists {
				sobsi := sort.SearchInts(b.ObservationsSorted, obsi)
				b.ObservationsSorted = append(b.ObservationsSorted[:sobsi], b.ObservationsSorted[sobsi+1:]...)
				delete(b.ID2Observation, input.ID)
			}
		}
	}
}

func (b *Bayesian) Update() {
	prior := b.Prior
	for _, obs := range b.Observations {
		prior = b.UpdateProbability(prior, obs.ProbTrue, obs.ProbFalse)
	}
	b.Probability = prior
}

func (b *Bayesian) UpdateProbability(prior float64, prob_true float64, prob_false float64) float64 {
	// Update probability using Bayes' rule.
	numerator := prob_true * prior
	denominator := numerator + prob_false*(1-prior)
	probability := numerator / denominator
	return probability
}

func main() {

}
<commit_msg>* Bayesian can be used now<commit_after>package bayesian

import "sort"

type dataInput struct {
	id             int64
	trueState      bool
	probGivenTrue  float64
	probGivenFalse float64
	state          bool
}

type observation struct {
	probTrue  float64
	probFalse float64
}

// Instance is the Bayesian object
type Instance struct {
	name               string
	inputs             map[int64]*dataInput
	observations       []*observation
	observationsSorted []int
	id2Observation     map[int64]int

	prior       float64
	threshold   float64
	probability float64
}

// New creates a new instance of a Bayesian object
func New() *Instance {
	inst := &Instance{}

	return inst
}

// AddInput is there to add an input to the bayesian observation
func (b *Instance) AddInput(id int64, truestate bool, probGivenTrue float64, probGivenFalse float64) {
	input := &dataInput{id: id, trueState: truestate, probGivenTrue: probGivenTrue, probGivenFalse: probGivenFalse, state: false}
	b.inputs[id] = input
}

// SetInputState will set the input state of one of the inputs that is identified by ID 'id'
func (b *Instance) SetInputState(id int64, state bool) {
	input, contains := b.inputs[id]
	if contains {
		input.state = state
	}
}

// ReadState will return true/false according to the bayesian computation
func (b *Instance) ReadState() bool {
	b.processState()
	b.updateState()
	return b.probability >= b.threshold
}

// Example will test all possible combinations
// presence := Bayesian.New()
// presence.Add("livingroom sensor", 0.95, 0.7)
func Example() {
	ishome := New()
	ishome.AddInput(0, true, 0.9, 0.2) // iPhoneX is home/away
	ishome.AddInput(1, true, 0.9, 0.2) // Macbook is home/away
	ishome.AddInput(2, true, 0.9, 0.2) // Kindle is home/away

	// Possibility A
	ishome.SetInputState(0, false) // iPhoneX is home
	ishome.SetInputState(0, false) // Macbook is away
	ishome.SetInputState(0, false) // Kindle is away
	ishome.ReadState()

	// Possibility B
	ishome.SetInputState(0, true)  // iPhoneX is home
	ishome.SetInputState(0, false) // Macbook is away
	ishome.SetInputState(0, false) // Kindle is away
	ishome.ReadState()

	// Possibility C
	ishome.SetInputState(0, false) // iPhoneX is home
	ishome.SetInputState(0, true)  // Macbook is away
	ishome.SetInputState(0, false) // Kindle is away
	ishome.ReadState()

	// Possibility D
	ishome.SetInputState(0, false) // iPhoneX is home
	ishome.SetInputState(0, false) // Macbook is away
	ishome.SetInputState(0, true)  // Kindle is away
	ishome.ReadState()

	// Possibility E
	ishome.SetInputState(0, true)  // iPhoneX is home
	ishome.SetInputState(0, true)  // Macbook is away
	ishome.SetInputState(0, false) // Kindle is away
	ishome.ReadState()
}

func (b *Instance) processState() {
	for _, input := range b.inputs {
		// Add entity to current observations if state conditions are met
		if input.state {
			probtrue := input.probGivenTrue
			probfalse := input.probGivenFalse

			var obs *observation
			obsi, exists := b.id2Observation[input.id]
			if !exists {
				obs = &observation{probTrue: probtrue, probFalse: probfalse}
				obsi = len(b.observations)

				b.observations = append(b.observations, obs)

				b.observationsSorted = append(b.observationsSorted, obsi)
				sort.Ints(b.observationsSorted)

				b.id2Observation[input.id] = obsi
			}
			obs.probTrue = probtrue
			obs.probFalse = probfalse
		} else {
			obsi, exists := b.id2Observation[input.id]
			if exists {
				sobsi := sort.SearchInts(b.observationsSorted, obsi)
				b.observationsSorted = append(b.observationsSorted[:sobsi], b.observationsSorted[sobsi+1:]...)
				delete(b.id2Observation, input.id)
			}
		}
	}
}

func (b *Instance) updateState() {
	prior := b.prior
	for _, obs := range b.observations {
		prior = b.computeProbability(prior, obs.probTrue, obs.probFalse)
	}
	b.probability = prior
}

func (b *Instance) computeProbability(prior float64, probTrue float64, probFalse float64) float64 {
	// Update probability using Bayes' rule.
	numerator := probTrue * prior
	denominator := numerator + probFalse*(1-prior)
	probability := numerator / denominator
	return probability
}
<|endoftext|>


<commit_before>package markdown

import (
	"io/ioutil"
	"testing"

	"github.com/russross/blackfriday"
)

func BenchmarkRenderSpecNoHTML(b *testing.B) {
	b.StopTimer()
	data, err := ioutil.ReadFile("spec/spec-0.20.txt")
	if err != nil {
		b.Fatal(err)
	}

	md := New(HTML(false), XHTMLOutput(true))
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		md.RenderToString(data)
	}
}

func BenchmarkRenderSpec(b *testing.B) {
	b.StopTimer()
	data, err := ioutil.ReadFile("spec/spec-0.20.txt")
	if err != nil {
		b.Fatal(err)
	}

	md := New(HTML(true), XHTMLOutput(true))
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		md.RenderToString(data)
	}
}

func BenchmarkRenderSpecBlackFriday(b *testing.B) {
	b.StopTimer()
	data, err := ioutil.ReadFile("spec/spec-0.20.txt")
	if err != nil {
		panic(err)
	}

	b.StartTimer()

	for i := 0; i < b.N; i++ {
		renderer := blackfriday.HtmlRenderer(blackfriday.HTML_USE_XHTML|blackfriday.HTML_USE_SMARTYPANTS|blackfriday.HTML_SMARTYPANTS_LATEX_DASHES, "", "")
		blackfriday.Markdown(data, renderer,
			blackfriday.EXTENSION_NO_INTRA_EMPHASIS|
				blackfriday.EXTENSION_TABLES|
				blackfriday.EXTENSION_FENCED_CODE|
				blackfriday.EXTENSION_AUTOLINK|
				blackfriday.EXTENSION_STRIKETHROUGH,
		)
	}
}
<commit_msg>Move blackfriday renderer instantiation outside of the benchmark loop<commit_after>package markdown

import (
	"io/ioutil"
	"testing"

	"github.com/russross/blackfriday"
)

func BenchmarkRenderSpecNoHTML(b *testing.B) {
	b.StopTimer()
	data, err := ioutil.ReadFile("spec/spec-0.20.txt")
	if err != nil {
		b.Fatal(err)
	}

	md := New(HTML(false), XHTMLOutput(true))
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		md.RenderToString(data)
	}
}

func BenchmarkRenderSpec(b *testing.B) {
	b.StopTimer()
	data, err := ioutil.ReadFile("spec/spec-0.20.txt")
	if err != nil {
		b.Fatal(err)
	}

	md := New(HTML(true), XHTMLOutput(true))
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		md.RenderToString(data)
	}
}

func BenchmarkRenderSpecBlackFriday(b *testing.B) {
	b.StopTimer()
	data, err := ioutil.ReadFile("spec/spec-0.20.txt")
	if err != nil {
		panic(err)
	}

	renderer := blackfriday.HtmlRenderer(blackfriday.HTML_USE_XHTML|blackfriday.HTML_USE_SMARTYPANTS|blackfriday.HTML_SMARTYPANTS_LATEX_DASHES, "", "")
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		blackfriday.Markdown(data, renderer,
			blackfriday.EXTENSION_NO_INTRA_EMPHASIS|
				blackfriday.EXTENSION_TABLES|
				blackfriday.EXTENSION_FENCED_CODE|
				blackfriday.EXTENSION_AUTOLINK|
				blackfriday.EXTENSION_STRIKETHROUGH,
		)
	}
}
<|endoftext|>


<commit_before>package index

import (
	"fmt"
	"net/url"
	"strings"
	"time"

	storage "code.google.com/p/google-api-go-client/storage/v1"
)

type Directory struct {
	Bucket  string
	Prefix  string
	SubDirs map[string]*Directory
	Objects map[string]*storage.Object
	Updated time.Time
}

func NewDirectory(rawURL string) (*Directory, error) {
	gsURL, err := url.Parse(rawURL)
	if err != nil {
		return nil, err
	}

	if gsURL.Scheme != "gs" {
		return nil, fmt.Errorf("URL missing gs:// scheme prefix: %q", rawURL)
	}
	if gsURL.Host == "" {
		return nil, fmt.Errorf("URL missing bucket name: %q", rawURL)
	}

	// Object name prefix must never start with / but always end with /
	gsURL.Path = strings.TrimLeft(gsURL.Path, "/")
	if gsURL.Path != "" && !strings.HasSuffix(gsURL.Path, "/") {
		gsURL.Path += "/"
	}

	return &Directory{
		Bucket:  gsURL.Host,
		Prefix:  gsURL.Path,
		SubDirs: make(map[string]*Directory),
		Objects: make(map[string]*storage.Object),
	}, nil
}

func (d *Directory) AddObject(obj *storage.Object) error {
	name := strings.TrimPrefix(obj.Name, d.Prefix)
	split := strings.SplitAfterN(name, "/", 2)

	// Propagate update time to parent directories, excluding indexes.
	// Used to detect when indexes should be regenerated.
	if split[len(split)-1] != "index.html" {
		objUpdated, err := time.Parse(time.RFC3339, obj.Updated)
		if err != nil {
			return err
		}
		if d.Updated.Before(objUpdated) {
			objUpdated = objUpdated
		}
	}

	// Save object locally if it has no slash or only ends in slash
	if len(split) == 1 || len(split[1]) == 0 {
		d.Objects[name] = obj
		return nil
	}

	sub, ok := d.SubDirs[split[0]]
	if !ok {
		sub = &Directory{
			Bucket:  d.Bucket,
			Prefix:  d.Prefix + split[0],
			SubDirs: make(map[string]*Directory),
			Objects: make(map[string]*storage.Object),
		}
		d.SubDirs[split[0]] = sub
	}

	return sub.AddObject(obj)
}

func (d *Directory) NeedsIndex() bool {
	if index, ok := d.Objects["index.html"]; ok {
		indexUpdated, err := time.Parse(time.RFC3339, index.Updated)
		return err != nil || d.Updated.After(indexUpdated)
	}
	return true
}
<commit_msg>Fix updating directory timestamp<commit_after>package index

import (
	"fmt"
	"net/url"
	"strings"
	"time"

	storage "code.google.com/p/google-api-go-client/storage/v1"
)

type Directory struct {
	Bucket  string
	Prefix  string
	SubDirs map[string]*Directory
	Objects map[string]*storage.Object
	Updated time.Time
}

func NewDirectory(rawURL string) (*Directory, error) {
	gsURL, err := url.Parse(rawURL)
	if err != nil {
		return nil, err
	}

	if gsURL.Scheme != "gs" {
		return nil, fmt.Errorf("URL missing gs:// scheme prefix: %q", rawURL)
	}
	if gsURL.Host == "" {
		return nil, fmt.Errorf("URL missing bucket name: %q", rawURL)
	}

	// Object name prefix must never start with / but always end with /
	gsURL.Path = strings.TrimLeft(gsURL.Path, "/")
	if gsURL.Path != "" && !strings.HasSuffix(gsURL.Path, "/") {
		gsURL.Path += "/"
	}

	return &Directory{
		Bucket:  gsURL.Host,
		Prefix:  gsURL.Path,
		SubDirs: make(map[string]*Directory),
		Objects: make(map[string]*storage.Object),
	}, nil
}

func (d *Directory) AddObject(obj *storage.Object) error {
	name := strings.TrimPrefix(obj.Name, d.Prefix)
	split := strings.SplitAfterN(name, "/", 2)

	// Propagate update time to parent directories, excluding indexes.
	// Used to detect when indexes should be regenerated.
	if split[len(split)-1] != "index.html" {
		objUpdated, err := time.Parse(time.RFC3339, obj.Updated)
		if err != nil {
			return err
		}
		if d.Updated.Before(objUpdated) {
			d.Updated = objUpdated
		}
	}

	// Save object locally if it has no slash or only ends in slash
	if len(split) == 1 || len(split[1]) == 0 {
		d.Objects[name] = obj
		return nil
	}

	sub, ok := d.SubDirs[split[0]]
	if !ok {
		sub = &Directory{
			Bucket:  d.Bucket,
			Prefix:  d.Prefix + split[0],
			SubDirs: make(map[string]*Directory),
			Objects: make(map[string]*storage.Object),
		}
		d.SubDirs[split[0]] = sub
	}

	return sub.AddObject(obj)
}

func (d *Directory) NeedsIndex() bool {
	if index, ok := d.Objects["index.html"]; ok {
		indexUpdated, err := time.Parse(time.RFC3339, index.Updated)
		return err != nil || d.Updated.After(indexUpdated)
	}
	return true
}
<|endoftext|>


<commit_before>package indicators

import (
	"container/list"
	"errors"
	"github.com/thetruetrade/gotrade"
)

// A Rate of Change Ratio Indicator (RocR), no storage, for use in other indicators
type RocRWithoutStorage struct {
	*baseIndicator
	*baseFloatBounds

	// private variables
	valueAvailableAction ValueAvailableActionFloat
	periodCounter        int
	periodHistory        *list.List
	timePeriod           int
}

// NewRocRWithoutStorage creates a Rate of Change Ratio Indicator (RocR) without storage
func NewRocRWithoutStorage(timePeriod int, valueAvailableAction ValueAvailableActionFloat) (indicator *RocRWithoutStorage, err error) {

	// an indicator without storage MUST have a value available action
	if valueAvailableAction == nil {
		return nil, ErrValueAvailableActionIsNil
	}

	// the minimum timeperiod for this indicator is 1
	if timePeriod < 1 {
		return nil, errors.New("timePeriod is less than the minimum (1)")
	}

	// check the maximum timeperiod
	if timePeriod > MaximumLookbackPeriod {
		return nil, errors.New("timePeriod is greater than the maximum (100000)")
	}

	lookback := timePeriod
	ind := RocRWithoutStorage{
		baseIndicator:        newBaseIndicator(lookback),
		baseFloatBounds:      newBaseFloatBounds(),
		periodCounter:        (timePeriod * -1),
		periodHistory:        list.New(),
		valueAvailableAction: valueAvailableAction,
		timePeriod:           timePeriod,
	}

	return &ind, err
}

// A Rate of Change Ratio Indicator (RocR)
type RocR struct {
	*RocRWithoutStorage
	selectData gotrade.DataSelectionFunc

	// public variables
	Data []float64
}

// NewRocR creates a Rate of Change Ratio Indicator (RocR) for online usage
func NewRocR(timePeriod int, selectData gotrade.DataSelectionFunc) (indicator *RocR, err error) {
	ind := RocR{selectData: selectData}
	ind.RocRWithoutStorage, err = NewRocRWithoutStorage(timePeriod,
		func(dataItem float64, streamBarIndex int) {
			ind.Data = append(ind.Data, dataItem)
		})

	return &ind, err
}

// NewDefaultRocR creates a Rate of Change Ratio Indicator (RocR) for online usage with default parameters
//	- timePeriod: 10
func NewDefaultRocR() (indicator *RocR, err error) {
	timePeriod := 10
	return NewRocR(timePeriod, gotrade.UseClosePrice)
}

// NewRocRWithSrcLen creates a Rate of Change Ratio Indicator (RocR) for offline usage
func NewRocRWithSrcLen(sourceLength uint, timePeriod int, selectData gotrade.DataSelectionFunc) (indicator *RocR, err error) {
	ind, err := NewRocR(timePeriod, selectData)

	// only initialise the storage if there is enough source data to require it
	if sourceLength-uint(ind.GetLookbackPeriod()) > 1 {
		ind.Data = make([]float64, 0, sourceLength-uint(ind.GetLookbackPeriod()))
	}

	return ind, err
}

// NewDefaultRocRWithSrcLen creates a Rate of Change Ratio Indicator (RocR) for offline usage with default parameters
func NewDefaultRocRWithSrcLen(sourceLength uint) (indicator *RocR, err error) {
	ind, err := NewDefaultRocR()

	// only initialise the storage if there is enough source data to require it
	if sourceLength-uint(ind.GetLookbackPeriod()) > 1 {
		ind.Data = make([]float64, 0, sourceLength-uint(ind.GetLookbackPeriod()))
	}

	return ind, err
}

// NewRocRForStream creates a Rate of Change Ratio Indicator (RocR) for online usage with a source data stream
func NewRocRForStream(priceStream gotrade.DOHLCVStreamSubscriber, timePeriod int, selectData gotrade.DataSelectionFunc) (indicator *RocR, err error) {
	ind, err := NewRocR(timePeriod, selectData)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewDefaultRocRForStream creates a Rate of Change Ratio Indicator (RocR) for online usage with a source data stream
func NewDefaultRocRForStream(priceStream gotrade.DOHLCVStreamSubscriber) (indicator *RocR, err error) {
	ind, err := NewDefaultRocR()
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewRocRForStreamWithSrcLen creates a Rate of Change Ratio Indicator (RocR) for offline usage with a source data stream
func NewRocRForStreamWithSrcLen(sourceLength uint, priceStream gotrade.DOHLCVStreamSubscriber, timePeriod int, selectData gotrade.DataSelectionFunc) (indicator *RocR, err error) {
	ind, err := NewRocRWithSrcLen(sourceLength, timePeriod, selectData)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewDefaultRocRForStreamWithSrcLen creates a Rate of Change Ratio Indicator (RocR) for offline usage with a source data stream
func NewDefaultRocRForStreamWithSrcLen(sourceLength uint, priceStream gotrade.DOHLCVStreamSubscriber) (indicator *RocR, err error) {
	ind, err := NewDefaultRocRWithSrcLen(sourceLength)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// ReceiveDOHLCVTick consumes a source data DOHLCV price tick
func (ind *RocR) ReceiveDOHLCVTick(tickData gotrade.DOHLCV, streamBarIndex int) {
	var selectedData = ind.selectData(tickData)
	ind.ReceiveTick(selectedData, streamBarIndex)
}

func (ind *RocRWithoutStorage) ReceiveTick(tickData float64, streamBarIndex int) {
	ind.periodCounter += 1
	ind.periodHistory.PushBack(tickData)

	if ind.periodCounter > 0 {

		//    RocR = (price/previousPrice - 1) * 100
		previousPrice := ind.periodHistory.Front().Value.(float64)

		// increment the number of results this indicator can be expected to return
		ind.dataLength += 1
		if ind.validFromBar == -1 {
			// set the streamBarIndex from which this indicator returns valid results
			ind.validFromBar = streamBarIndex
		}
		var result float64
		if previousPrice != 0 {
			result = (tickData / previousPrice)
		} else {
			result = 0.0
		}

		// update the maximum result value
		if result > ind.maxValue {
			ind.maxValue = result
		}

		// update the minimum result value
		if result < ind.minValue {
			ind.minValue = result
		}

		// notify of a new result value though the value available action
		ind.valueAvailableAction(result, streamBarIndex)
	}

	if ind.periodHistory.Len() > ind.timePeriod {
		first := ind.periodHistory.Front()
		ind.periodHistory.Remove(first)
	}
}
<commit_msg>#76 Remove duplication - rocr<commit_after>package indicators

import (
	"container/list"
	"errors"
	"github.com/thetruetrade/gotrade"
)

// A Rate of Change Ratio Indicator (RocR), no storage, for use in other indicators
type RocRWithoutStorage struct {
	*baseIndicatorWithFloatBounds

	// private variables
	periodCounter int
	periodHistory *list.List
	timePeriod    int
}

// NewRocRWithoutStorage creates a Rate of Change Ratio Indicator (RocR) without storage
func NewRocRWithoutStorage(timePeriod int, valueAvailableAction ValueAvailableActionFloat) (indicator *RocRWithoutStorage, err error) {

	// an indicator without storage MUST have a value available action
	if valueAvailableAction == nil {
		return nil, ErrValueAvailableActionIsNil
	}

	// the minimum timeperiod for this indicator is 1
	if timePeriod < 1 {
		return nil, errors.New("timePeriod is less than the minimum (1)")
	}

	// check the maximum timeperiod
	if timePeriod > MaximumLookbackPeriod {
		return nil, errors.New("timePeriod is greater than the maximum (100000)")
	}

	lookback := timePeriod
	ind := RocRWithoutStorage{
		baseIndicatorWithFloatBounds: newBaseIndicatorWithFloatBounds(lookback, valueAvailableAction),
		periodCounter:                (timePeriod * -1),
		periodHistory:                list.New(),
		timePeriod:                   timePeriod,
	}

	return &ind, err
}

// A Rate of Change Ratio Indicator (RocR)
type RocR struct {
	*RocRWithoutStorage
	selectData gotrade.DataSelectionFunc

	// public variables
	Data []float64
}

// NewRocR creates a Rate of Change Ratio Indicator (RocR) for online usage
func NewRocR(timePeriod int, selectData gotrade.DataSelectionFunc) (indicator *RocR, err error) {
	ind := RocR{selectData: selectData}
	ind.RocRWithoutStorage, err = NewRocRWithoutStorage(timePeriod,
		func(dataItem float64, streamBarIndex int) {
			ind.Data = append(ind.Data, dataItem)
		})

	return &ind, err
}

// NewDefaultRocR creates a Rate of Change Ratio Indicator (RocR) for online usage with default parameters
//	- timePeriod: 10
func NewDefaultRocR() (indicator *RocR, err error) {
	timePeriod := 10
	return NewRocR(timePeriod, gotrade.UseClosePrice)
}

// NewRocRWithSrcLen creates a Rate of Change Ratio Indicator (RocR) for offline usage
func NewRocRWithSrcLen(sourceLength uint, timePeriod int, selectData gotrade.DataSelectionFunc) (indicator *RocR, err error) {
	ind, err := NewRocR(timePeriod, selectData)

	// only initialise the storage if there is enough source data to require it
	if sourceLength-uint(ind.GetLookbackPeriod()) > 1 {
		ind.Data = make([]float64, 0, sourceLength-uint(ind.GetLookbackPeriod()))
	}

	return ind, err
}

// NewDefaultRocRWithSrcLen creates a Rate of Change Ratio Indicator (RocR) for offline usage with default parameters
func NewDefaultRocRWithSrcLen(sourceLength uint) (indicator *RocR, err error) {
	ind, err := NewDefaultRocR()

	// only initialise the storage if there is enough source data to require it
	if sourceLength-uint(ind.GetLookbackPeriod()) > 1 {
		ind.Data = make([]float64, 0, sourceLength-uint(ind.GetLookbackPeriod()))
	}

	return ind, err
}

// NewRocRForStream creates a Rate of Change Ratio Indicator (RocR) for online usage with a source data stream
func NewRocRForStream(priceStream gotrade.DOHLCVStreamSubscriber, timePeriod int, selectData gotrade.DataSelectionFunc) (indicator *RocR, err error) {
	ind, err := NewRocR(timePeriod, selectData)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewDefaultRocRForStream creates a Rate of Change Ratio Indicator (RocR) for online usage with a source data stream
func NewDefaultRocRForStream(priceStream gotrade.DOHLCVStreamSubscriber) (indicator *RocR, err error) {
	ind, err := NewDefaultRocR()
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewRocRForStreamWithSrcLen creates a Rate of Change Ratio Indicator (RocR) for offline usage with a source data stream
func NewRocRForStreamWithSrcLen(sourceLength uint, priceStream gotrade.DOHLCVStreamSubscriber, timePeriod int, selectData gotrade.DataSelectionFunc) (indicator *RocR, err error) {
	ind, err := NewRocRWithSrcLen(sourceLength, timePeriod, selectData)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewDefaultRocRForStreamWithSrcLen creates a Rate of Change Ratio Indicator (RocR) for offline usage with a source data stream
func NewDefaultRocRForStreamWithSrcLen(sourceLength uint, priceStream gotrade.DOHLCVStreamSubscriber) (indicator *RocR, err error) {
	ind, err := NewDefaultRocRWithSrcLen(sourceLength)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// ReceiveDOHLCVTick consumes a source data DOHLCV price tick
func (ind *RocR) ReceiveDOHLCVTick(tickData gotrade.DOHLCV, streamBarIndex int) {
	var selectedData = ind.selectData(tickData)
	ind.ReceiveTick(selectedData, streamBarIndex)
}

func (ind *RocRWithoutStorage) ReceiveTick(tickData float64, streamBarIndex int) {
	ind.periodCounter += 1
	ind.periodHistory.PushBack(tickData)

	if ind.periodCounter > 0 {

		//    RocR = (price/previousPrice - 1) * 100
		previousPrice := ind.periodHistory.Front().Value.(float64)

		var result float64
		if previousPrice != 0 {
			result = (tickData / previousPrice)
		} else {
			result = 0.0
		}

		ind.UpdateIndicatorWithNewValue(result, streamBarIndex)
	}

	if ind.periodHistory.Len() > ind.timePeriod {
		first := ind.periodHistory.Front()
		ind.periodHistory.Remove(first)
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package indices

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/mattbaird/elastigo/api"
)

// AnalyzeIndices performs the analysis process on a text and return the tokens breakdown of the text.
// http://www.elasticsearch.org/guide/reference/api/admin-indices-analyze/
func AnalyzeIndices(index string, args map[string]interface{}) (AnalyzeResponse, error) {
	var retval AnalyzeResponse
	if len(args["text"].(string)) == 0 {
		return retval, errors.New("text to analyze must not be blank")
	}
	var analyzeUrl string = "/_analyze"
	if len(index) > 0 {
		analyzeUrl = fmt.Sprintf("/%s/%s", index, analyzeUrl)
	}

	body, err := api.DoCommand("GET", analyzeUrl, args, nil)
	if err != nil {
		return retval, err
	}
	if err == nil {
		// marshall into json
		jsonErr := json.Unmarshal(body, &retval)
		if jsonErr != nil {
			return retval, jsonErr
		}
	}
	return retval, err
}

type AnalyzeResponse struct {
	Tokens []Token `json:"tokens"`
}
type Token struct {
	Name        string `json:"token"`
	StartOffset string `json:"start_offset"`
	EndOffset   string `json:"end_offset"`
	Type        string `json:"type"`
	Position    string `json:"position"`
}
<commit_msg>fix json formart<commit_after>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package indices

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/mattbaird/elastigo/api"
)

// AnalyzeIndices performs the analysis process on a text and return the tokens breakdown of the text.
// http://www.elasticsearch.org/guide/reference/api/admin-indices-analyze/
func AnalyzeIndices(index string, args map[string]interface{}) (AnalyzeResponse, error) {
	var retval AnalyzeResponse
	if len(args["text"].(string)) == 0 {
		return retval, errors.New("text to analyze must not be blank")
	}
	var analyzeUrl string = "/_analyze"
	if len(index) > 0 {
		analyzeUrl = fmt.Sprintf("/%s/%s", index, analyzeUrl)
	}

	body, err := api.DoCommand("GET", analyzeUrl, args, nil)
	if err != nil {
		return retval, err
	}
	if err == nil {
		// marshall into json
		jsonErr := json.Unmarshal(body, &retval)
		if jsonErr != nil {
			return retval, jsonErr
		}
	}
	return retval, err
}

type AnalyzeResponse struct {
	Tokens []Token `json:"tokens"`
}
type Token struct {
	Name        string `json:"token"`
	StartOffset int    `json:"start_offset"`
	EndOffset   int    `json:"end_offset"`
	Type        string `json:"type"`
	Position    int    `json:"position"`
}
<|endoftext|>