<commit_before>package client

import (
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"strconv"
	"sync"
	"time"

	"github.com/hashicorp/nomad/client/driver"
	"github.com/hashicorp/nomad/client/fingerprint"
	"github.com/hashicorp/nomad/nomad"
	"github.com/hashicorp/nomad/nomad/structs"
)

const (
	// clientRPCCache controls how long we keep an idle connection
	// open to a server
	clientRPCCache = 30 * time.Second

	// clientMaxStreams controsl how many idle streams we keep
	// open to a server
	clientMaxStreams = 2

	// registerRetryIntv is minimum interval on which we retry
	// registration. We pick a value between this and 2x this.
	registerRetryIntv = 30 * time.Second

	// getAllocRetryIntv is minimum interval on which we retry
	// to fetch allocations. We pick a value between this and 2x this.
	getAllocRetryIntv = 30 * time.Second
)

// RPCHandler can be provided to the Client if there is a local server
// to avoid going over the network. If not provided, the Client will
// maintain a connection pool to the servers
type RPCHandler interface {
	RPC(method string, args interface{}, reply interface{}) error
}

// Config is used to parameterize and configure the behavior of the client
type Config struct {
	// LogOutput is the destination for logs
	LogOutput io.Writer

	// Region is the clients region
	Region string

	// Servers is a list of known server addresses. These are as "host:port"
	Servers []string

	// RPCHandler can be provided to avoid network traffic if the
	// server is running locally.
	RPCHandler RPCHandler

	// Node provides the base node
	Node *structs.Node
}

// DefaultConfig returns the default configuration
func DefaultConfig() *Config {
	return &Config{
		LogOutput: os.Stderr,
		Region:    "region1",
	}
}

// Client is used to implement the client interaction with Nomad. Clients
// are expected to register as a schedulable node to the servers, and to
// run allocations as determined by the servers.
type Client struct {
	config *Config

	logger *log.Logger

	lastServer     net.Addr
	lastRPCTime    time.Time
	lastServerLock sync.Mutex

	connPool *nomad.ConnPool

	lastHeartbeat time.Time
	heartbeatTTL  time.Duration

	shutdown     bool
	shutdownCh   chan struct{}
	shutdownLock sync.Mutex
}

// NewClient is used to create a new client from the given configuration
func NewClient(config *Config) (*Client, error) {
	// Create a logger
	logger := log.New(config.LogOutput, "", log.LstdFlags)

	// Create the client
	c := &Client{
		config:     config,
		connPool:   nomad.NewPool(config.LogOutput, clientRPCCache, clientMaxStreams, nil),
		logger:     logger,
		shutdownCh: make(chan struct{}),
	}

	// Setup the node
	if err := c.setupNode(); err != nil {
		return nil, fmt.Errorf("node setup failed: %v", err)
	}

	// Fingerprint the node
	if err := c.fingerprint(); err != nil {
		return nil, fmt.Errorf("fingerprinting failed: %v", err)
	}

	// Scan for drivers
	if err := c.setupDrivers(); err != nil {
		return nil, fmt.Errorf("driver setup failed: %v", err)
	}

	// Start the client!
	go c.run()
	return c, nil
}

// Shutdown is used to tear down the client
func (c *Client) Shutdown() error {
	c.logger.Printf("[INFO] client: shutting down")
	c.shutdownLock.Lock()
	defer c.shutdownLock.Unlock()

	if c.shutdown {
		return nil
	}
	c.shutdown = true
	close(c.shutdownCh)
	c.connPool.Shutdown()
	return nil
}

// RPC is used to forward an RPC call to a nomad server, or fail if no servers
func (c *Client) RPC(method string, args interface{}, reply interface{}) error {
	// Invoke the RPCHandle if it exists
	if c.config.RPCHandler != nil {
		return c.config.RPCHandler.RPC(method, args, reply)
	}

	// Pick a server to request from
	addr, err := c.pickServer()
	if err != nil {
		return err
	}

	// Make the RPC request
	err = c.connPool.RPC(c.config.Region, addr, 1, method, args, reply)

	// Update the last server information
	c.lastServerLock.Lock()
	if err != nil {
		c.lastServer = nil
		c.lastRPCTime = time.Time{}
	} else {
		c.lastServer = addr
		c.lastRPCTime = time.Now()
	}
	c.lastServerLock.Unlock()
	return err
}

// pickServer is used to pick a target RPC server
func (c *Client) pickServer() (net.Addr, error) {
	c.lastServerLock.Lock()
	defer c.lastServerLock.Unlock()

	// Check for a valid last-used server
	if c.lastServer != nil && time.Now().Sub(c.lastRPCTime) < clientRPCCache {
		return c.lastServer, nil
	}

	// Bail if we can't find any servers
	if len(c.config.Servers) == 0 {
		return nil, fmt.Errorf("no known servers")
	}

	// Copy the list of servers and shuffle
	servers := make([]string, len(c.config.Servers))
	copy(servers, c.config.Servers)
	shuffleStrings(servers)

	// Try to resolve each server
	for i := 0; i < len(servers); i++ {
		addr, err := net.ResolveTCPAddr("tcp", servers[i])
		if err == nil {
			c.lastServer = addr
			c.lastRPCTime = time.Now()
			return addr, nil
		}
		c.logger.Printf("[WARN] client: failed to resolve '%s': %v", err)
	}

	// Bail if we reach this point
	return nil, fmt.Errorf("failed to resolve any servers")
}

// Stats is used to return statistics for debugging and insight
// for various sub-systems
func (c *Client) Stats() map[string]map[string]string {
	toString := func(v uint64) string {
		return strconv.FormatUint(v, 10)
	}
	stats := map[string]map[string]string{
		"nomad": map[string]string{
			"server":        "false",
			"known_servers": toString(uint64(len(c.config.Servers))),
		},
		"runtime": nomad.RuntimeStats(),
	}
	return stats
}

// Node returns the locally registered node
func (c *Client) Node() *structs.Node {
	return c.config.Node
}

// setupNode is used to setup the initial node
func (c *Client) setupNode() error {
	node := c.config.Node
	if node == nil {
		node = &structs.Node{}
		c.config.Node = node
	}
	if node.Attributes == nil {
		node.Attributes = make(map[string]string)
	}
	if node.Links == nil {
		node.Links = make(map[string]string)
	}
	if node.Meta == nil {
		node.Meta = make(map[string]string)
	}
	if node.Resources == nil {
		node.Resources = &structs.Resources{}
	}
	if node.ID == "" {
		node.ID = generateUUID()
	}
	if node.Datacenter == "" {
		node.Datacenter = "dc1"
	}
	if node.Name == "" {
		node.Name, _ = os.Hostname()
	}
	if node.Name == "" {
		node.Name = node.ID
	}
	node.Status = structs.NodeStatusInit
	return nil
}

// fingerprint is used to fingerprint the client and setup the node
func (c *Client) fingerprint() error {
	var applied []string
	for name := range fingerprint.BuiltinFingerprints {
		f, err := fingerprint.NewFingerprint(name, c.logger)
		if err != nil {
			return err
		}
		applies, err := f.Fingerprint(c.config.Node)
		if err != nil {
			return err
		}
		if applies {
			applied = append(applied, name)
		}
	}
	c.logger.Printf("[DEBUG] client: applied fingerprints %v", applied)
	return nil
}

// setupDrivers is used to find the available drivers
func (c *Client) setupDrivers() error {
	var avail []string
	for name := range driver.BuiltinDrivers {
		d, err := driver.NewDriver(name, c.logger)
		if err != nil {
			return err
		}
		applies, err := d.Fingerprint(c.config.Node)
		if err != nil {
			return err
		}
		if applies {
			avail = append(avail, name)
		}
	}
	c.logger.Printf("[DEBUG] client: available drivers %v", avail)
	return nil
}

// run is a long lived goroutine used to run the client
func (c *Client) run() {
	// Register the client
	for {
		if err := c.registerNode(); err == nil {
			break
		}
		select {
		case <-time.After(registerRetryIntv + randomStagger(registerRetryIntv)):
		case <-c.shutdownCh:
			return
		}
	}

	// Setup the heartbeat timer
	heartbeat := time.After(c.heartbeatTTL)

	// Watch for changes in allocations
	allocUpdates := make(chan []*structs.Allocation, 1)
	go c.watchAllocations(allocUpdates)

	// Periodically update our status and wait for termination
	select {
	case allocs := <-allocUpdates:
		c.runAllocs(allocs)

	case <-heartbeat:
		if err := c.updateNodeStatus(); err != nil {
			heartbeat = time.After(registerRetryIntv + randomStagger(registerRetryIntv))
		} else {
			heartbeat = time.After(c.heartbeatTTL)
		}

	case <-c.shutdownCh:
		return
	}
}

// registerNode is used to register the node or update the registration
func (c *Client) registerNode() error {
	node := c.Node()
	req := structs.NodeRegisterRequest{
		Node:         node,
		WriteRequest: structs.WriteRequest{Region: c.config.Region},
	}
	var resp structs.NodeUpdateResponse
	err := c.RPC("Client.Register", &req, &resp)
	if err != nil {
		c.logger.Printf("[ERR] client: failed to register node: %v", err)
		return err
	}
	c.logger.Printf("[DEBUG] client: node registration complete")
	if len(resp.EvalIDs) != 0 {
		c.logger.Printf("[DEBUG] client: %d evaluations triggered by node registration", len(resp.EvalIDs))
	}
	c.lastHeartbeat = time.Now()
	c.heartbeatTTL = resp.HeartbeatTTL
	return nil
}

// updateNodeStatus is used to heartbeat and update the status of the node
func (c *Client) updateNodeStatus() error {
	node := c.Node()
	req := structs.NodeUpdateStatusRequest{
		NodeID:       node.ID,
		Status:       structs.NodeStatusReady,
		WriteRequest: structs.WriteRequest{Region: c.config.Region},
	}
	var resp structs.NodeUpdateResponse
	err := c.RPC("Client.UpdateStatus", &req, &resp)
	if err != nil {
		c.logger.Printf("[ERR] client: failed to update status: %v", err)
		return err
	}
	if len(resp.EvalIDs) != 0 {
		c.logger.Printf("[DEBUG] client: %d evaluations triggered by node update", len(resp.EvalIDs))
	}
	if resp.Index != 0 {
		c.logger.Printf("[DEBUG] client: client state updated")
	}
	c.lastHeartbeat = time.Now()
	c.heartbeatTTL = resp.HeartbeatTTL
	return nil
}

// watchAllocations is used to scan for updates to allocations
func (c *Client) watchAllocations(allocUpdates chan []*structs.Allocation) {
	var lastIndex uint64
	req := structs.NodeSpecificRequest{
		NodeID: c.Node().ID,
		QueryOptions: structs.QueryOptions{
			Region:        c.config.Region,
			MinQueryIndex: lastIndex,
			AllowStale:    true,
		},
	}
	var resp structs.NodeAllocsResponse

	for {
		// Get the allocations, blocking for updates
		err := c.RPC("Client.GetAllocs", &req, &resp)
		if err != nil {
			c.logger.Printf("[ERR] client: failed to query for node allocations: %v", err)
			retry := getAllocRetryIntv + randomStagger(getAllocRetryIntv)
			select {
			case <-time.After(retry):
				continue
			case <-c.shutdownCh:
				return
			}
		}

		// Check for shutdown
		select {
		case <-c.shutdownCh:
			return
		default:
		}

		// Check for updates
		if resp.Index == lastIndex {
			continue
		}
		lastIndex = resp.Index
		c.logger.Printf("[DEBUG] client: updated allocations at index %d (%d allocs)", lastIndex, len(resp.Allocs))

		// Push the updates
		select {
		case allocUpdates <- resp.Allocs:
		case <-c.shutdownCh:
			return
		}
	}
}

// runAllocs is invoked when we get an updated set of allocations
func (c *Client) runAllocs([]*structs.Allocation) {
	// TODO
}
<commit_msg>client: state stubs<commit_after>package client

import (
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"strconv"
	"sync"
	"time"

	"github.com/hashicorp/nomad/client/driver"
	"github.com/hashicorp/nomad/client/fingerprint"
	"github.com/hashicorp/nomad/nomad"
	"github.com/hashicorp/nomad/nomad/structs"
)

const (
	// clientRPCCache controls how long we keep an idle connection
	// open to a server
	clientRPCCache = 30 * time.Second

	// clientMaxStreams controsl how many idle streams we keep
	// open to a server
	clientMaxStreams = 2

	// registerRetryIntv is minimum interval on which we retry
	// registration. We pick a value between this and 2x this.
	registerRetryIntv = 30 * time.Second

	// getAllocRetryIntv is minimum interval on which we retry
	// to fetch allocations. We pick a value between this and 2x this.
	getAllocRetryIntv = 30 * time.Second
)

// RPCHandler can be provided to the Client if there is a local server
// to avoid going over the network. If not provided, the Client will
// maintain a connection pool to the servers
type RPCHandler interface {
	RPC(method string, args interface{}, reply interface{}) error
}

// Config is used to parameterize and configure the behavior of the client
type Config struct {
	// DevMode controls if we are in a development mode which
	// avoids persistent storage.
	DevMode bool

	// DataDir is where we store our state
	DataDir string

	// LogOutput is the destination for logs
	LogOutput io.Writer

	// Region is the clients region
	Region string

	// Servers is a list of known server addresses. These are as "host:port"
	Servers []string

	// RPCHandler can be provided to avoid network traffic if the
	// server is running locally.
	RPCHandler RPCHandler

	// Node provides the base node
	Node *structs.Node
}

// DefaultConfig returns the default configuration
func DefaultConfig() *Config {
	return &Config{
		LogOutput: os.Stderr,
		Region:    "region1",
	}
}

// Client is used to implement the client interaction with Nomad. Clients
// are expected to register as a schedulable node to the servers, and to
// run allocations as determined by the servers.
type Client struct {
	config *Config

	logger *log.Logger

	lastServer     net.Addr
	lastRPCTime    time.Time
	lastServerLock sync.Mutex

	connPool *nomad.ConnPool

	lastHeartbeat time.Time
	heartbeatTTL  time.Duration

	shutdown     bool
	shutdownCh   chan struct{}
	shutdownLock sync.Mutex
}

// NewClient is used to create a new client from the given configuration
func NewClient(config *Config) (*Client, error) {
	// Create a logger
	logger := log.New(config.LogOutput, "", log.LstdFlags)

	// Create the client
	c := &Client{
		config:     config,
		connPool:   nomad.NewPool(config.LogOutput, clientRPCCache, clientMaxStreams, nil),
		logger:     logger,
		shutdownCh: make(chan struct{}),
	}

	// Restore the state
	if err := c.restoreState(); err != nil {
		return nil, fmt.Errorf("failed to restore state: %v", err)
	}

	// Setup the node
	if err := c.setupNode(); err != nil {
		return nil, fmt.Errorf("node setup failed: %v", err)
	}

	// Fingerprint the node
	if err := c.fingerprint(); err != nil {
		return nil, fmt.Errorf("fingerprinting failed: %v", err)
	}

	// Scan for drivers
	if err := c.setupDrivers(); err != nil {
		return nil, fmt.Errorf("driver setup failed: %v", err)
	}

	// Start the client!
	go c.run()
	return c, nil
}

// Shutdown is used to tear down the client
func (c *Client) Shutdown() error {
	c.logger.Printf("[INFO] client: shutting down")
	c.shutdownLock.Lock()
	defer c.shutdownLock.Unlock()

	if c.shutdown {
		return nil
	}
	c.shutdown = true
	close(c.shutdownCh)
	c.connPool.Shutdown()
	return nil
}

// RPC is used to forward an RPC call to a nomad server, or fail if no servers
func (c *Client) RPC(method string, args interface{}, reply interface{}) error {
	// Invoke the RPCHandle if it exists
	if c.config.RPCHandler != nil {
		return c.config.RPCHandler.RPC(method, args, reply)
	}

	// Pick a server to request from
	addr, err := c.pickServer()
	if err != nil {
		return err
	}

	// Make the RPC request
	err = c.connPool.RPC(c.config.Region, addr, 1, method, args, reply)

	// Update the last server information
	c.lastServerLock.Lock()
	if err != nil {
		c.lastServer = nil
		c.lastRPCTime = time.Time{}
	} else {
		c.lastServer = addr
		c.lastRPCTime = time.Now()
	}
	c.lastServerLock.Unlock()
	return err
}

// pickServer is used to pick a target RPC server
func (c *Client) pickServer() (net.Addr, error) {
	c.lastServerLock.Lock()
	defer c.lastServerLock.Unlock()

	// Check for a valid last-used server
	if c.lastServer != nil && time.Now().Sub(c.lastRPCTime) < clientRPCCache {
		return c.lastServer, nil
	}

	// Bail if we can't find any servers
	if len(c.config.Servers) == 0 {
		return nil, fmt.Errorf("no known servers")
	}

	// Copy the list of servers and shuffle
	servers := make([]string, len(c.config.Servers))
	copy(servers, c.config.Servers)
	shuffleStrings(servers)

	// Try to resolve each server
	for i := 0; i < len(servers); i++ {
		addr, err := net.ResolveTCPAddr("tcp", servers[i])
		if err == nil {
			c.lastServer = addr
			c.lastRPCTime = time.Now()
			return addr, nil
		}
		c.logger.Printf("[WARN] client: failed to resolve '%s': %v", err)
	}

	// Bail if we reach this point
	return nil, fmt.Errorf("failed to resolve any servers")
}

// Stats is used to return statistics for debugging and insight
// for various sub-systems
func (c *Client) Stats() map[string]map[string]string {
	toString := func(v uint64) string {
		return strconv.FormatUint(v, 10)
	}
	stats := map[string]map[string]string{
		"nomad": map[string]string{
			"server":        "false",
			"known_servers": toString(uint64(len(c.config.Servers))),
		},
		"runtime": nomad.RuntimeStats(),
	}
	return stats
}

// Node returns the locally registered node
func (c *Client) Node() *structs.Node {
	return c.config.Node
}

// restoreState is used to restore our state from the data dir
func (c *Client) restoreState() error {
	if c.config.DevMode {
		return nil
	}

	// TODO
	return nil
}

// saveState is used to snapshot our state into the data dir
func (c *Client) saveState() error {
	if c.config.DevMode {
		return nil
	}

	// TODO
	return nil
}

// setupNode is used to setup the initial node
func (c *Client) setupNode() error {
	node := c.config.Node
	if node == nil {
		node = &structs.Node{}
		c.config.Node = node
	}
	if node.Attributes == nil {
		node.Attributes = make(map[string]string)
	}
	if node.Links == nil {
		node.Links = make(map[string]string)
	}
	if node.Meta == nil {
		node.Meta = make(map[string]string)
	}
	if node.Resources == nil {
		node.Resources = &structs.Resources{}
	}
	if node.ID == "" {
		node.ID = generateUUID()
	}
	if node.Datacenter == "" {
		node.Datacenter = "dc1"
	}
	if node.Name == "" {
		node.Name, _ = os.Hostname()
	}
	if node.Name == "" {
		node.Name = node.ID
	}
	node.Status = structs.NodeStatusInit
	return nil
}

// fingerprint is used to fingerprint the client and setup the node
func (c *Client) fingerprint() error {
	var applied []string
	for name := range fingerprint.BuiltinFingerprints {
		f, err := fingerprint.NewFingerprint(name, c.logger)
		if err != nil {
			return err
		}
		applies, err := f.Fingerprint(c.config.Node)
		if err != nil {
			return err
		}
		if applies {
			applied = append(applied, name)
		}
	}
	c.logger.Printf("[DEBUG] client: applied fingerprints %v", applied)
	return nil
}

// setupDrivers is used to find the available drivers
func (c *Client) setupDrivers() error {
	var avail []string
	for name := range driver.BuiltinDrivers {
		d, err := driver.NewDriver(name, c.logger)
		if err != nil {
			return err
		}
		applies, err := d.Fingerprint(c.config.Node)
		if err != nil {
			return err
		}
		if applies {
			avail = append(avail, name)
		}
	}
	c.logger.Printf("[DEBUG] client: available drivers %v", avail)
	return nil
}

// run is a long lived goroutine used to run the client
func (c *Client) run() {
	// Register the client
	for {
		if err := c.registerNode(); err == nil {
			break
		}
		select {
		case <-time.After(registerRetryIntv + randomStagger(registerRetryIntv)):
		case <-c.shutdownCh:
			return
		}
	}

	// Setup the heartbeat timer
	heartbeat := time.After(c.heartbeatTTL)

	// Watch for changes in allocations
	allocUpdates := make(chan []*structs.Allocation, 1)
	go c.watchAllocations(allocUpdates)

	// Periodically update our status and wait for termination
	select {
	case allocs := <-allocUpdates:
		c.runAllocs(allocs)

	case <-heartbeat:
		if err := c.updateNodeStatus(); err != nil {
			heartbeat = time.After(registerRetryIntv + randomStagger(registerRetryIntv))
		} else {
			heartbeat = time.After(c.heartbeatTTL)
		}

	case <-c.shutdownCh:
		return
	}
}

// registerNode is used to register the node or update the registration
func (c *Client) registerNode() error {
	node := c.Node()
	req := structs.NodeRegisterRequest{
		Node:         node,
		WriteRequest: structs.WriteRequest{Region: c.config.Region},
	}
	var resp structs.NodeUpdateResponse
	err := c.RPC("Client.Register", &req, &resp)
	if err != nil {
		c.logger.Printf("[ERR] client: failed to register node: %v", err)
		return err
	}
	c.logger.Printf("[DEBUG] client: node registration complete")
	if len(resp.EvalIDs) != 0 {
		c.logger.Printf("[DEBUG] client: %d evaluations triggered by node registration", len(resp.EvalIDs))
	}
	c.lastHeartbeat = time.Now()
	c.heartbeatTTL = resp.HeartbeatTTL
	return nil
}

// updateNodeStatus is used to heartbeat and update the status of the node
func (c *Client) updateNodeStatus() error {
	node := c.Node()
	req := structs.NodeUpdateStatusRequest{
		NodeID:       node.ID,
		Status:       structs.NodeStatusReady,
		WriteRequest: structs.WriteRequest{Region: c.config.Region},
	}
	var resp structs.NodeUpdateResponse
	err := c.RPC("Client.UpdateStatus", &req, &resp)
	if err != nil {
		c.logger.Printf("[ERR] client: failed to update status: %v", err)
		return err
	}
	if len(resp.EvalIDs) != 0 {
		c.logger.Printf("[DEBUG] client: %d evaluations triggered by node update", len(resp.EvalIDs))
	}
	if resp.Index != 0 {
		c.logger.Printf("[DEBUG] client: client state updated")
	}
	c.lastHeartbeat = time.Now()
	c.heartbeatTTL = resp.HeartbeatTTL
	return nil
}

// watchAllocations is used to scan for updates to allocations
func (c *Client) watchAllocations(allocUpdates chan []*structs.Allocation) {
	var lastIndex uint64
	req := structs.NodeSpecificRequest{
		NodeID: c.Node().ID,
		QueryOptions: structs.QueryOptions{
			Region:        c.config.Region,
			MinQueryIndex: lastIndex,
			AllowStale:    true,
		},
	}
	var resp structs.NodeAllocsResponse

	for {
		// Get the allocations, blocking for updates
		err := c.RPC("Client.GetAllocs", &req, &resp)
		if err != nil {
			c.logger.Printf("[ERR] client: failed to query for node allocations: %v", err)
			retry := getAllocRetryIntv + randomStagger(getAllocRetryIntv)
			select {
			case <-time.After(retry):
				continue
			case <-c.shutdownCh:
				return
			}
		}

		// Check for shutdown
		select {
		case <-c.shutdownCh:
			return
		default:
		}

		// Check for updates
		if resp.Index == lastIndex {
			continue
		}
		lastIndex = resp.Index
		c.logger.Printf("[DEBUG] client: updated allocations at index %d (%d allocs)", lastIndex, len(resp.Allocs))

		// Push the updates
		select {
		case allocUpdates <- resp.Allocs:
		case <-c.shutdownCh:
			return
		}
	}
}

// runAllocs is invoked when we get an updated set of allocations
func (c *Client) runAllocs([]*structs.Allocation) {
	// TODO
}
<|endoftext|>


<commit_before>package rocserv

import (
	"fmt"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/shawnfeng/sutil/slog"
	"net/http"
	"strings"
	"sync"
	"time"

	"regexp"

	"github.com/prometheus/client_golang/prometheus"
)

var (
	DefaultMetricsOpts = &MetricsOpts{
		DefBuckets:  []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10},
		DefQuantile: map[float64]float64{.25: .01, .5: .01, .75: .01, .9: .01, .95: .001, .99: .001},
	}
	DefaultMetrics = newMetrics()
)

const (
	Name_space_palfish          = "palfish"
	Name_server_req_total       = "server_request_total"
	Name_server_duration_second = "server_duration_second"
	Label_instance              = "instance"
	Label_servname              = "servname"
	Label_servid                = "servid"
	Label_api                   = "api"
	Label_type                  = "type"
	Label_source                = "source"
	Label_status                = "status"
	Status_succ                 = 1
	Status_fail                 = 0
)

// MetricsOpts is used to configure the Metrics
type MetricsOpts struct {
	DefBuckets  []float64
	DefQuantile map[float64]float64
}

type Metrics struct {
	mu          sync.Mutex
	counters    map[string]prometheus.Counter
	gauges      map[string]prometheus.Gauge
	historams   map[string]prometheus.Histogram
	summaries   map[string]prometheus.Summary
	defBuckets  []float64
	defQuantile map[float64]float64
}

// NewMetrics creates a new Metrics using the default options.
func newMetrics() *Metrics {
	return newMetricsFrom(DefaultMetricsOpts)
}

// NewMetricsFrom creates a new Metrics using the passed options.
func newMetricsFrom(opts *MetricsOpts) *Metrics {
	metrics := &Metrics{
		counters:    make(map[string]prometheus.Counter),
		gauges:      make(map[string]prometheus.Gauge),
		historams:   make(map[string]prometheus.Histogram),
		summaries:   make(map[string]prometheus.Summary),
		defBuckets:  opts.DefBuckets,
		defQuantile: opts.DefQuantile,
	}
	return metrics
}

//no use only for register
func (p *Metrics) Describe(c chan<- *prometheus.Desc) {
	prometheus.NewGauge(prometheus.GaugeOpts{Name: "Dummy", Help: "Dummy"}).Describe(c)
}

// Collect meets the collection interface and allows us to enforce our expiration
// logic to clean up ephemeral metrics if their value haven't been set for a
// duration exceeding our allowed expiration time.
func (p *Metrics) Collect(c chan<- prometheus.Metric) {
	p.mu.Lock()
	defer p.mu.Unlock()
	for _, v := range p.counters {
		v.Collect(c)
	}
	for _, v := range p.gauges {
		v.Collect(c)
	}
	for _, v := range p.historams {
		v.Collect(c)
	}
	for _, v := range p.summaries {
		v.Collect(c)
	}

}

type Label struct {
	Name  string
	Value string
}

var forbiddenChars = regexp.MustCompile("[ .=\\-/]")

func flattenKey(namekeys []string, labels []Label) (string, string) {
	key := strings.Join(namekeys, "_")
	key = SafePromethuesValue(key)

	hash := key
	for _, label := range labels {
		label.Name = SafePromethuesValue(label.Name)
		label.Value = SafePromethuesValue(label.Value)
		hash += fmt.Sprintf(";%s=%s", label.Name, label.Value)
	}

	return key, hash
}
func SafePromethuesValue(v string) string {
	return forbiddenChars.ReplaceAllString(v, "_")
}

func prometheusLabels(labels []Label) prometheus.Labels {
	l := make(prometheus.Labels)
	for _, label := range labels {
		l[label.Name] = label.Value
	}
	return l
}

func (p *Metrics) CreateCounter(namekeys []string, labels []Label) prometheus.Counter {
	key, hash := flattenKey(namekeys, labels)
	var m prometheus.Counter
	var ok bool
	p.putMetricIfAbsent(func() bool {
		m, ok = p.counters[hash]
		return ok
	}, func() {
		m = prometheus.NewCounter(prometheus.CounterOpts{
			Name:        key,
			Help:        key,
			ConstLabels: prometheusLabels(labels),
		})
		p.counters[hash] = m
	})
	return m
}
func (p *Metrics) CreateGauge(namekeys []string, labels []Label) prometheus.Gauge {
	key, hash := flattenKey(namekeys, labels)
	var m prometheus.Gauge
	var ok bool
	p.putMetricIfAbsent(func() bool {
		m, ok = p.gauges[hash]
		return ok
	}, func() {
		m = prometheus.NewGauge(prometheus.GaugeOpts{
			Name:        key,
			Help:        key,
			ConstLabels: prometheusLabels(labels),
		})
		p.gauges[hash] = m
	})
	return m
}
func (p *Metrics) CreateHistogram(namekeys []string, labels []Label, buckets []float64) prometheus.Histogram {
	key, hash := flattenKey(namekeys, labels)
	var m prometheus.Histogram
	var ok bool
	p.putMetricIfAbsent(func() bool {
		m, ok = p.historams[hash]
		return ok
	}, func() {
		if buckets == nil || len(buckets) == 0 {
			buckets = DefaultMetricsOpts.DefBuckets
		}
		m = prometheus.NewHistogram(prometheus.HistogramOpts{
			Name:        key,
			Help:        key,
			ConstLabels: prometheusLabels(labels),
			Buckets:     buckets,
		})
		p.historams[hash] = m
	})
	return m
}
func (p *Metrics) CreateSummary(namekeys []string, labels []Label, quantile map[float64]float64) prometheus.Summary {
	key, hash := flattenKey(namekeys, labels)
	var m prometheus.Summary
	var ok bool
	p.putMetricIfAbsent(func() bool {
		m, ok = p.summaries[hash]
		return ok
	}, func() {
		if quantile == nil || len(quantile) == 0 {
			quantile = DefaultMetricsOpts.DefQuantile
		}
		m = prometheus.NewSummary(prometheus.SummaryOpts{
			Name:        key,
			Help:        key,
			MaxAge:      10 * time.Second,
			ConstLabels: prometheusLabels(labels),
			Objectives:  quantile,
		})
		p.summaries[hash] = m
	})
	return m
}
func (p *Metrics) putMetricIfAbsent(isExist func() bool, put func()) {
	ok := isExist()
	if !ok {
		p.mu.Lock()
		defer p.mu.Unlock()
		ok := isExist()
		if !ok {
			put()
		}
	}
}

func (p *Metrics) SetGaugeCreateIfAbsent(namekeys []string, val float64, labels []Label) {
	key, hash := flattenKey(namekeys, labels)
	g, ok := p.gauges[hash]
	if !ok {
		g = p.CreateGauge(namekeys, labels)
	}
	if g != nil {
		g.Set(val)
	} else {
		slog.Warnf("set gauge fail of no metric:%s,%s,%v", key, hash, val)
	}
}

func (p *Metrics) AddHistoramSampleCreateIfAbsent(namekeys []string, val float64, labels []Label, buckets []float64) {
	key, hash := flattenKey(namekeys, labels)
	g, ok := p.historams[hash]
	if !ok {
		g = p.CreateHistogram(namekeys, labels, buckets)
	}
	if g != nil {
		g.Observe(val)
	} else {
		slog.Warnf("set historam fail of no metric:%s,%s,%v", key, hash, val)
	}
}

func (p *Metrics) AddSummarySampleCreateIfAbsent(namekeys []string, val float64, labels []Label, quantile map[float64]float64) {

	key, hash := flattenKey(namekeys, labels)
	g, ok := p.summaries[hash]
	if !ok {
		g = p.CreateSummary(namekeys, labels, quantile)
	}
	if g != nil {
		g.Observe(val)
	} else {
		slog.Warnf("set summary fail of no metric:%s,%s,%v", key, hash, val)
	}
}

func (p *Metrics) IncrCounterCreateIfAbsent(namekeys []string, val float64, labels []Label) {
	key, hash := flattenKey(namekeys, labels)
	g, ok := p.counters[hash]
	if !ok {
		g = p.CreateCounter(namekeys, labels)
	}
	if g != nil {
		g.Add(val)
	} else {
		slog.Warnf("set counter fail of no metric:%s,%s,%v", key, hash, val)
	}
}
func (p *Metrics) Exportor() http.Handler {
	registry := prometheus.NewRegistry()
	registry.MustRegister(p)

	handlerFor := promhttp.HandlerFor(registry, promhttp.HandlerOpts{})
	return handlerFor
}
<commit_msg>edit map read write panic<commit_after>package rocserv

import (
	"fmt"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/shawnfeng/sutil/slog"
	"net/http"
	"strings"
	"sync"
	"time"

	"regexp"

	"github.com/prometheus/client_golang/prometheus"
)

var (
	DefaultMetricsOpts = &MetricsOpts{
		DefBuckets:  []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10},
		DefQuantile: map[float64]float64{.25: .01, .5: .01, .75: .01, .9: .01, .95: .001, .99: .001},
	}
	DefaultMetrics = newMetrics()
)

const (
	Name_space_palfish          = "palfish"
	Name_server_req_total       = "server_request_total"
	Name_server_duration_second = "server_duration_second"
	Label_instance              = "instance"
	Label_servname              = "servname"
	Label_servid                = "servid"
	Label_api                   = "api"
	Label_type                  = "type"
	Label_source                = "source"
	Label_status                = "status"
	Status_succ                 = 1
	Status_fail                 = 0
)

// MetricsOpts is used to configure the Metrics
type MetricsOpts struct {
	DefBuckets  []float64
	DefQuantile map[float64]float64
}

type Metrics struct {
	mucout      sync.RWMutex
	mugau       sync.RWMutex
	muhist      sync.RWMutex
	musumm      sync.RWMutex
	counters    map[string]prometheus.Counter
	gauges      map[string]prometheus.Gauge
	historams   map[string]prometheus.Histogram
	summaries   map[string]prometheus.Summary
	defBuckets  []float64
	defQuantile map[float64]float64
}

// NewMetrics creates a new Metrics using the default options.
func newMetrics() *Metrics {
	return newMetricsFrom(DefaultMetricsOpts)
}

// NewMetricsFrom creates a new Metrics using the passed options.
func newMetricsFrom(opts *MetricsOpts) *Metrics {
	metrics := &Metrics{
		counters:    make(map[string]prometheus.Counter, 512),
		gauges:      make(map[string]prometheus.Gauge, 512),
		historams:   make(map[string]prometheus.Histogram, 512),
		summaries:   make(map[string]prometheus.Summary, 512),
		defBuckets:  opts.DefBuckets,
		defQuantile: opts.DefQuantile,
	}
	return metrics
}

//no use only for register
func (p *Metrics) Describe(c chan<- *prometheus.Desc) {
	prometheus.NewGauge(prometheus.GaugeOpts{Name: "Dummy", Help: "Dummy"}).Describe(c)
}

// Collect meets the collection interface and allows us to enforce our expiration
// logic to clean up ephemeral metrics if their value haven't been set for a
// duration exceeding our allowed expiration time.
func (p *Metrics) Collect(c chan<- prometheus.Metric) {
	//rlockCollect(c,&p.mucout,p.counters)
	//rlockCollect(c,&p.mugau,p.gauges)
	//rlockCollect(c,&p.muhist,p.historams)
	//rlockCollect(c,&p.musumm,p.summaries)
	p.rlockCollectCounter(c)
	p.rlockCollectGauge(c)
	p.rlockCollectHistorams(c)
	p.rlockCollectSummaries(c)
}
func rlockCollect(c chan<- prometheus.Metric, rw *sync.RWMutex, collmap map[string]prometheus.Collector) {
	rw.RLock()
	defer rw.RUnlock()
	for _, v := range collmap {
		v.Collect(c)
	}
}

type Label struct {
	Name  string
	Value string
}

var forbiddenChars = regexp.MustCompile("[ .=\\-/]")

func flattenKey(namekeys []string, labels []Label) (string, string) {
	key := strings.Join(namekeys, "_")
	key = SafePromethuesValue(key)

	hash := key
	for _, label := range labels {
		label.Name = SafePromethuesValue(label.Name)
		label.Value = SafePromethuesValue(label.Value)
		hash += fmt.Sprintf(";%s=%s", label.Name, label.Value)
	}

	return key, hash
}
func SafePromethuesValue(v string) string {
	return forbiddenChars.ReplaceAllString(v, "_")
}

func prometheusLabels(labels []Label) prometheus.Labels {
	l := make(prometheus.Labels)
	for _, label := range labels {
		l[label.Name] = label.Value
	}
	return l
}

func (p *Metrics) CreateCounter(namekeys []string, labels []Label) prometheus.Counter {
	key, hash := flattenKey(namekeys, labels)
	var m prometheus.Counter
	var ok bool
	p.putMetricIfAbsentWithWLock(&p.mucout, func() bool {
		m, ok = p.counters[hash]
		return ok
	}, func() {
		m = prometheus.NewCounter(prometheus.CounterOpts{
			Name:        key,
			Help:        key,
			ConstLabels: prometheusLabels(labels),
		})
		p.counters[hash] = m
	})
	return m
}
func (p *Metrics) CreateGauge(namekeys []string, labels []Label) prometheus.Gauge {
	key, hash := flattenKey(namekeys, labels)
	var m prometheus.Gauge
	var ok bool
	p.putMetricIfAbsentWithWLock(&p.mugau, func() bool {
		m, ok = p.gauges[hash]
		return ok
	}, func() {
		m = prometheus.NewGauge(prometheus.GaugeOpts{
			Name:        key,
			Help:        key,
			ConstLabels: prometheusLabels(labels),
		})
		p.gauges[hash] = m
	})
	return m
}
func (p *Metrics) CreateHistogram(namekeys []string, labels []Label, buckets []float64) prometheus.Histogram {
	key, hash := flattenKey(namekeys, labels)
	var m prometheus.Histogram
	var ok bool
	p.putMetricIfAbsentWithWLock(&p.muhist, func() bool {
		m, ok = p.historams[hash]
		return ok
	}, func() {
		if buckets == nil || len(buckets) == 0 {
			buckets = DefaultMetricsOpts.DefBuckets
		}
		m = prometheus.NewHistogram(prometheus.HistogramOpts{
			Name:        key,
			Help:        key,
			ConstLabels: prometheusLabels(labels),
			Buckets:     buckets,
		})
		p.historams[hash] = m
	})
	return m
}
func (p *Metrics) CreateSummary(namekeys []string, labels []Label, quantile map[float64]float64) prometheus.Summary {
	key, hash := flattenKey(namekeys, labels)
	var m prometheus.Summary
	var ok bool
	p.putMetricIfAbsentWithWLock(&p.musumm, func() bool {
		m, ok = p.summaries[hash]
		return ok
	}, func() {
		if quantile == nil || len(quantile) == 0 {
			quantile = DefaultMetricsOpts.DefQuantile
		}
		m = prometheus.NewSummary(prometheus.SummaryOpts{
			Name:        key,
			Help:        key,
			MaxAge:      10 * time.Second,
			ConstLabels: prometheusLabels(labels),
			Objectives:  quantile,
		})
		p.summaries[hash] = m
	})
	return m
}
func (p *Metrics) putMetricIfAbsentWithWLock(rw *sync.RWMutex, isExist func() bool, put func()) {
	rw.Lock()
	defer rw.Unlock()
	ok := isExist()
	if !ok {
		put()
	}
}

func (p *Metrics) SetGaugeCreateIfAbsent(namekeys []string, val float64, labels []Label) {
	key, hash := flattenKey(namekeys, labels)
	g, ok := p.rlockGetGauge(hash)
	if !ok {
		g = p.CreateGauge(namekeys, labels)
	}
	if g != nil {
		g.Set(val)
	} else {
		slog.Warnf("set gauge fail of no metric:%s,%s,%v", key, hash, val)
	}
}

func (p *Metrics) AddHistoramSampleCreateIfAbsent(namekeys []string, val float64, labels []Label, buckets []float64) {
	key, hash := flattenKey(namekeys, labels)
	g, ok := p.rlockGetHistogram(hash)
	if !ok {
		g = p.CreateHistogram(namekeys, labels, buckets)
	}
	if g != nil {
		g.Observe(val)
	} else {
		slog.Warnf("set historam fail of no metric:%s,%s,%v", key, hash, val)
	}
}

func (p *Metrics) AddSummarySampleCreateIfAbsent(namekeys []string, val float64, labels []Label, quantile map[float64]float64) {

	key, hash := flattenKey(namekeys, labels)
	g, ok := p.rlockGetSummary(hash)
	if !ok {
		g = p.CreateSummary(namekeys, labels, quantile)
	}
	if g != nil {
		g.Observe(val)
	} else {
		slog.Warnf("set summary fail of no metric:%s,%s,%v", key, hash, val)
	}
}

func (p *Metrics) IncrCounterCreateIfAbsent(namekeys []string, val float64, labels []Label) {
	key, hash := flattenKey(namekeys, labels)
	g, ok := p.rlockGetCounter(hash)
	if !ok {
		g = p.CreateCounter(namekeys, labels)
	}
	if g != nil {
		g.Add(val)
	} else {
		slog.Warnf("set counter fail of no metric:%s,%s,%v", key, hash, val)
	}
}
func (p *Metrics) rlockGetCounter(hash string) (prometheus.Counter, bool) {
	p.mucout.RLock()
	defer p.mucout.RUnlock()
	g, ok := p.counters[hash]
	return g, ok
}
func (p *Metrics) rlockGetGauge(hash string) (prometheus.Gauge, bool) {
	p.mugau.RLock()
	defer p.mugau.RUnlock()
	g, ok := p.gauges[hash]
	return g, ok
}
func (p *Metrics) rlockGetHistogram(hash string) (prometheus.Histogram, bool) {
	p.muhist.RLock()
	defer p.muhist.RUnlock()
	g, ok := p.historams[hash]
	return g, ok
}
func (p *Metrics) rlockGetSummary(hash string) (prometheus.Summary, bool) {
	p.musumm.RLock()
	defer p.musumm.RUnlock()
	g, ok := p.historams[hash]
	return g, ok
}
func (p *Metrics) Exportor() http.Handler {
	registry := prometheus.NewRegistry()
	registry.MustRegister(p)

	handlerFor := promhttp.HandlerFor(registry, promhttp.HandlerOpts{})
	return handlerFor
}

func (p *Metrics) rlockCollectCounter(c chan<- prometheus.Metric) {
	p.mucout.RLock()
	defer p.mucout.RUnlock()
	for _, v := range p.counters {
		v.Collect(c)
	}
}

func (p *Metrics) rlockCollectGauge(c chan<- prometheus.Metric) {
	p.mugau.RLock()
	defer p.mugau.RUnlock()
	for _, v := range p.gauges {
		v.Collect(c)
	}
}

func (p *Metrics) rlockCollectHistorams(c chan<- prometheus.Metric) {
	p.muhist.RLock()
	defer p.muhist.RUnlock()
	for _, v := range p.historams {
		v.Collect(c)
	}
}

func (p *Metrics) rlockCollectSummaries(c chan<- prometheus.Metric) {
	p.musumm.RLock()
	defer p.musumm.RUnlock()
	for _, v := range p.summaries {
		v.Collect(c)
	}
}
<|endoftext|>


<commit_before>package set_test

import (
	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/utils/set"
	"sort"
	"testing"
)

func Test(t *testing.T) {
	TestingT(t)
}

type stringSetSuite struct{}

var _ = Suite(stringSetSuite{})

// Helper methods for the tests.
func AssertEmpty(c *C, s set.StringSet) {
	c.Assert(len(s.Values()), Equals, 0)
}

func AssertValues(c *C, s set.StringSet, expected ...string) {
	values := s.Values()
	sort.Strings(expected)
	sort.Strings(values)
	c.Assert(values, DeepEquals, expected)
	c.Assert(s.Size(), Equals, len(expected))
}

func AssertSortedValues(c *C, s set.StringSet, expected ...string) {
	values := s.SortedValues()
	sort.Strings(expected)
	c.Assert(values, DeepEquals, expected)
	c.Assert(s.Size(), Equals, len(expected))
}

// Actual tests start here.
func (stringSetSuite) TestEmpty(c *C) {
	s := set.MakeStringSet()
	AssertEmpty(c, s)
}

func (stringSetSuite) TestInitialValues(c *C) {
	values := []string{"foo", "bar", "baz"}
	s := set.MakeStringSet(values...)
	AssertValues(c, s, values...)
	AssertSortedValues(c, s, values...)
}

func (stringSetSuite) TestAdd(c *C) {
	s := set.MakeStringSet()
	s.Add("foo")
	s.Add("foo")
	s.Add("bar")
	AssertValues(c, s, "foo", "bar")
}

func (stringSetSuite) TestRemove(c *C) {
	s := set.MakeStringSet("foo", "bar")
	s.Remove("foo")
	AssertValues(c, s, "bar")
}

func (stringSetSuite) TestContains(c *C) {
	s := set.MakeStringSet("foo", "bar")
	c.Assert(s.Contains("foo"), Equals, true)
	c.Assert(s.Contains("bar"), Equals, true)
	c.Assert(s.Contains("baz"), Equals, false)
}

func (stringSetSuite) TestRemoveNonExistant(c *C) {
	s := set.MakeStringSet()
	s.Remove("foo")
	AssertEmpty(c, s)
}

func (stringSetSuite) TestUnion(c *C) {
	s1 := set.MakeStringSet("foo", "bar")
	s2 := set.MakeStringSet("foo", "baz", "bang")
	union1 := s1.Union(s2)
	union2 := s2.Union(s1)

	AssertValues(c, union1, "foo", "bar", "baz", "bang")
	AssertValues(c, union2, "foo", "bar", "baz", "bang")
}

func (stringSetSuite) TestIntersection(c *C) {
	s1 := set.MakeStringSet("foo", "bar")
	s2 := set.MakeStringSet("foo", "baz", "bang")
	int1 := s1.Intersection(s2)
	int2 := s2.Intersection(s1)

	AssertValues(c, int1, "foo")
	AssertValues(c, int2, "foo")
}

func (stringSetSuite) TestDifference(c *C) {
	s1 := set.MakeStringSet("foo", "bar")
	s2 := set.MakeStringSet("foo", "baz", "bang")
	diff1 := s1.Difference(s2)
	diff2 := s2.Difference(s1)

	AssertValues(c, diff1, "bar")
	AssertValues(c, diff2, "baz", "bang")
}
<commit_msg>Adding a test for size of an unitialized set.<commit_after>package set_test

import (
	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/utils/set"
	"sort"
	"testing"
)

func Test(t *testing.T) {
	TestingT(t)
}

type stringSetSuite struct{}

var _ = Suite(stringSetSuite{})

// Helper methods for the tests.
func AssertEmpty(c *C, s set.StringSet) {
	c.Assert(len(s.Values()), Equals, 0)
}

func AssertValues(c *C, s set.StringSet, expected ...string) {
	values := s.Values()
	sort.Strings(expected)
	sort.Strings(values)
	c.Assert(values, DeepEquals, expected)
	c.Assert(s.Size(), Equals, len(expected))
}

func AssertSortedValues(c *C, s set.StringSet, expected ...string) {
	values := s.SortedValues()
	sort.Strings(expected)
	c.Assert(values, DeepEquals, expected)
	c.Assert(s.Size(), Equals, len(expected))
}

// Actual tests start here.
func (stringSetSuite) TestEmpty(c *C) {
	s := set.MakeStringSet()
	AssertEmpty(c, s)
}

func (stringSetSuite) TestInitialValues(c *C) {
	values := []string{"foo", "bar", "baz"}
	s := set.MakeStringSet(values...)
	AssertValues(c, s, values...)
	AssertSortedValues(c, s, values...)
}

func (stringSetSuite) TestSize(c *C) {
	// Works on an uninitialized StringSet
	var uninitialized set.StringSet
	c.Assert(uninitialized.Size(), Equals, 0)

	// Empty sets are empty.
	s := set.MakeStringSet()
	c.Assert(s.Size(), Equals, 0)

	// Size returns number of unique values.
	s = set.MakeStringSet("foo", "foo", "bar")
	c.Assert(s.Size(), Equals, 2)
}

func (stringSetSuite) TestAdd(c *C) {
	s := set.MakeStringSet()
	s.Add("foo")
	s.Add("foo")
	s.Add("bar")
	AssertValues(c, s, "foo", "bar")
}

func (stringSetSuite) TestRemove(c *C) {
	s := set.MakeStringSet("foo", "bar")
	s.Remove("foo")
	AssertValues(c, s, "bar")
}

func (stringSetSuite) TestContains(c *C) {
	s := set.MakeStringSet("foo", "bar")
	c.Assert(s.Contains("foo"), Equals, true)
	c.Assert(s.Contains("bar"), Equals, true)
	c.Assert(s.Contains("baz"), Equals, false)
}

func (stringSetSuite) TestRemoveNonExistant(c *C) {
	s := set.MakeStringSet()
	s.Remove("foo")
	AssertEmpty(c, s)
}

func (stringSetSuite) TestUnion(c *C) {
	s1 := set.MakeStringSet("foo", "bar")
	s2 := set.MakeStringSet("foo", "baz", "bang")
	union1 := s1.Union(s2)
	union2 := s2.Union(s1)

	AssertValues(c, union1, "foo", "bar", "baz", "bang")
	AssertValues(c, union2, "foo", "bar", "baz", "bang")
}

func (stringSetSuite) TestIntersection(c *C) {
	s1 := set.MakeStringSet("foo", "bar")
	s2 := set.MakeStringSet("foo", "baz", "bang")
	int1 := s1.Intersection(s2)
	int2 := s2.Intersection(s1)

	AssertValues(c, int1, "foo")
	AssertValues(c, int2, "foo")
}

func (stringSetSuite) TestDifference(c *C) {
	s1 := set.MakeStringSet("foo", "bar")
	s2 := set.MakeStringSet("foo", "baz", "bang")
	diff1 := s1.Difference(s2)
	diff2 := s2.Difference(s1)

	AssertValues(c, diff1, "bar")
	AssertValues(c, diff2, "baz", "bang")
}
<|endoftext|>


<commit_before>package client

import (
	"bufio"
	"crypto/sha256"
	"encoding/hex"
	"io"
	"io/ioutil"
	"math/rand"
	"os"
	"path"
	"strings"
	"time"
)

func randomString(l int) string {
	rand.Seed(time.Now().UTC().UnixNano())
	bytes := make([]byte, l)
	for i := 0; i < l; i++ {
		bytes[i] = byte(65 + rand.Intn(90-65))
	}
	return string(bytes)
}

//Split: Splits a 'file' into blocks of size 'block_size' and outputs them into
//The directory given by 'filedir'
func Split(file io.Reader, block_size int, filedir string) []string {
	final_files := make([]string, 0)
	buf := make([]byte, block_size)
	//Open the file
	file = bufio.NewReader(file)
	for {
		// read a chunk
		n, err := file.Read(buf)
		if err != nil && err != io.EOF {
			panic(err)
		}
		if n == 0 {
			break
		}

		//Hash the file with SHA256 to determine a unique filename
		h := sha256.New()
		s := string(buf[:n]) + filedir + randomString(8)
		io.WriteString(h, s)
		//Creating the filename by appending filedir with the SHA256 Hash
		filename := path.Join(filedir, hex.EncodeToString(h.Sum(nil)))

		//Write the file and add it's relative path to the list
		fo, err := os.Create(filename)
		// fo, err := os.Create(filename)
		if err != nil {
			panic(err)
		}
		// close fo on exit and check for its returned error
		defer func() {
			if err := fo.Close(); err != nil {
				panic(err)
			}
		}()
		// make a write buffer
		w := bufio.NewWriter(fo)
		if _, err := w.Write(buf[:n]); err != nil {
			panic(err)
		}
		if err = w.Flush(); err != nil {
			panic(err)
		}
		final_files = append(final_files, filename)
	}
	return final_files
}

//Join: Takes a list of 'files' and joins them back together according to it's order
func Join(files []string) io.Reader {
	result := make([]byte, 0)
	for _, file := range files {
		//read the entire file
		buf, err := ioutil.ReadFile(file)
		if err != nil {
			panic(err)
		}

		//combine the bytes together
		for _, bite := range buf {
			result = append(result, bite)
		}
	}

	//create an io.Reader and return it
	sresult := string(result)
	return strings.NewReader(sresult)
}
<commit_msg>Split and Join uses filedir now<commit_after>package client

import (
	"bufio"
	"crypto/sha256"
	"encoding/hex"
	"io"
	"io/ioutil"
	"math/rand"
	"os"
	"path"
	"strconv"
	"strings"
	"time"
)

func randomString(l int) string {
	rand.Seed(time.Now().UTC().UnixNano())
	bytes := make([]byte, l)
	for i := 0; i < l; i++ {
		bytes[i] = byte(65 + rand.Intn(90-65))
	}
	return string(bytes)
}

//Split: Splits a 'file' into blocks of size 'block_size' and outputs them into
//The directory given by 'filedir'
func Split(file io.Reader, block_size int, filename string) string {
	final_files := make([]string, 0)
	buf := make([]byte, block_size)
	//Open the file
	file = bufio.NewReader(file)
	filedir := path.Join(os.TempDir(), filename)
	os.Mkdir(filedir, 0775)
	i := 0
	for {
		i++
		// read a chunk
		n, err := file.Read(buf)
		if err != nil && err != io.EOF {
			panic(err)
		}
		if n == 0 {
			break
		}

		//Hash the file with SHA256 to determine a unique filename
		h := sha256.New()
		s := string(buf[:n]) + filedir + randomString(8)
		io.WriteString(h, s)
		//Creating the filename by appending filedir with the SHA256 Hash
		filename := path.Join(filedir, hex.EncodeToString(h.Sum(nil))+"."+strconv.Itoa(i))

		//Write the file and add it's relative path to the list
		fo, err := os.Create(filename)
		// fo, err := os.Create(filename)
		if err != nil {
			panic(err)
		}
		// close fo on exit and check for its returned error
		defer func() {
			if err := fo.Close(); err != nil {
				panic(err)
			}
		}()
		// make a write buffer
		w := bufio.NewWriter(fo)
		if _, err := w.Write(buf[:n]); err != nil {
			panic(err)
		}
		if err = w.Flush(); err != nil {
			panic(err)
		}
		final_files = append(final_files, filename)
	}
	return filedir
}

//Join: Takes a list of 'files' and joins them back together according to it's order
func Join(filedir string) io.Reader {
	dir, _ := os.Open(filedir)
	defer dir.Close()
	files, _ := dir.Readdir(0)
	result := make([]byte, 0)
	for _, file := range files {
		//read the entire file
		buf, err := ioutil.ReadFile(path.Join(filedir, file.Name()))
		if err != nil {
			panic(err)
		}

		//combine the bytes together
		for _, bite := range buf {
			result = append(result, bite)
		}
	}

	//create an io.Reader and return it
	sresult := string(result)
	return strings.NewReader(sresult)
}
<|endoftext|>


<commit_before>package vault

import (
	"bytes"
	"context"
	"errors"
	"reflect"
	"strings"
	"testing"
	"time"

	"github.com/armon/go-metrics"
	"github.com/hashicorp/vault/helper/metricsutil"

	proto "github.com/golang/protobuf/proto"
	wrapping "github.com/hashicorp/go-kms-wrapping"
	"github.com/hashicorp/vault/sdk/physical"
	"github.com/hashicorp/vault/vault/seal"
)

// phy implements physical.Backend. It maps keys to a slice of entries.
// Each call to Put appends the entry to the slice of entries for that
// key. No deduplication is done. This allows the test for UpgradeKeys to
// verify entries are only being updated when the underlying encryption key
// has been updated.
type phy struct {
	t       *testing.T
	entries map[string][]*physical.Entry
}

var _ physical.Backend = (*phy)(nil)

func newTestBackend(t *testing.T) *phy {
	return &phy{
		t:       t,
		entries: make(map[string][]*physical.Entry),
	}
}

func (p *phy) Put(_ context.Context, entry *physical.Entry) error {
	p.entries[entry.Key] = append(p.entries[entry.Key], entry)
	return nil
}

func (p *phy) Get(_ context.Context, key string) (*physical.Entry, error) {
	entries := p.entries[key]
	if entries == nil {
		return nil, nil
	}
	return entries[len(entries)-1], nil
}

func (p *phy) Delete(_ context.Context, key string) error {
	p.t.Errorf("Delete called on phy: key: %v", key)
	return nil
}

func (p *phy) List(_ context.Context, prefix string) ([]string, error) {
	p.t.Errorf("List called on phy: prefix: %v", prefix)
	return []string{}, nil
}

func (p *phy) Len() int {
	return len(p.entries)
}

func TestAutoSeal_UpgradeKeys(t *testing.T) {
	core, _, _ := TestCoreUnsealed(t)
	testSeal := seal.NewTestSeal(nil)

	var encKeys []string
	changeKey := func(key string) {
		encKeys = append(encKeys, key)
		testSeal.Wrapper.(*wrapping.TestWrapper).SetKeyID(key)
	}

	// Set initial encryption key.
	changeKey("kaz")

	autoSeal := NewAutoSeal(testSeal)
	autoSeal.SetCore(core)
	pBackend := newTestBackend(t)
	core.physical = pBackend

	ctx := context.Background()

	inkeys := [][]byte{[]byte("grist"), []byte("house")}
	if err := autoSeal.SetStoredKeys(ctx, inkeys); err != nil {
		t.Fatalf("SetStoredKeys: want no error, got %v", err)
	}

	inRecoveryKey := []byte("falernum")
	if err := autoSeal.SetRecoveryKey(ctx, inRecoveryKey); err != nil {
		t.Fatalf("SetRecoveryKey: want no error, got %v", err)
	}

	check := func() {
		// The values of the stored keys should never change.
		outkeys, err := autoSeal.GetStoredKeys(ctx)
		if err != nil {
			t.Fatalf("GetStoredKeys: want no error, got %v", err)
		}
		if !reflect.DeepEqual(inkeys, outkeys) {
			t.Errorf("incorrect stored keys: want %v, got %v", inkeys, outkeys)
		}

		// The value of the recovery key should also never change.
		outRecoveryKey, err := autoSeal.RecoveryKey(ctx)
		if err != nil {
			t.Fatalf("RecoveryKey: want no error, got %v", err)
		}
		if !bytes.Equal(inRecoveryKey, outRecoveryKey) {
			t.Errorf("incorrect recovery key: want %q, got %q", inRecoveryKey, outRecoveryKey)
		}

		// There should only be 2 entries in the physical backend. One for
		// the stored keys and one for the recovery key.
		if want, got := 2, pBackend.Len(); want != got {
			t.Errorf("backend unexpected Len: want %d, got %d", want, got)
		}

		for phyKey, phyEntries := range pBackend.entries {
			// Calling UpgradeKeys should only add an entry if the key has
			// changed.
			if keyCount, entryCount := len(encKeys), len(phyEntries); keyCount != entryCount {
				t.Errorf("phyKey = %s: encryption key count not equal to entry count: keys=%d, entries=%d", phyKey, keyCount, entryCount)
			}

			// Each phyEntry should correspond to a key at the same index
			// in encKeys. Iterate over each phyEntry and verify it was
			// encrypted with its corresponding key in encKeys.
			for i, phyEntry := range phyEntries {
				blobInfo := &wrapping.EncryptedBlobInfo{}
				if err := proto.Unmarshal(phyEntry.Value, blobInfo); err != nil {
					t.Errorf("phyKey = %s: failed to proto decode stored keys: %s", phyKey, err)
				}
				if blobInfo.KeyInfo == nil {
					t.Errorf("phyKey = %s: KeyInfo missing: %+v", phyKey, blobInfo)
				}
				if want, got := encKeys[i], blobInfo.KeyInfo.KeyID; want != got {
					t.Errorf("phyKey = %s: Incorrect encryption key: want %s, got %s", phyKey, want, got)
				}
			}
		}
	}

	// Verify the current state is correct before calling UpgradeKeys.
	check()

	// Call UpgradeKeys before changing the encryption key and verify
	// nothing has changed.
	if err := autoSeal.UpgradeKeys(ctx); err != nil {
		t.Fatalf("UpgradeKeys: want no error, got %v", err)
	}
	check()

	// Change the encryption key, call UpgradeKeys, then verify the stored
	// keys and recovery key has been re-encrypted with the new encryption
	// key.
	changeKey("primanti")
	if err := autoSeal.UpgradeKeys(ctx); err != nil {
		t.Fatalf("UpgradeKeys: want no error, got %v", err)
	}
	check()
}

func TestAutoSeal_HealthCheck(t *testing.T) {
	inmemSink := metrics.NewInmemSink(
		1000000*time.Hour,
		2000000*time.Hour)

	metricsConf := metrics.DefaultConfig("")
	metricsConf.EnableHostname = false
	metricsConf.EnableHostnameLabel = false
	metricsConf.EnableServiceLabel = false
	metricsConf.EnableTypePrefix = false

	metrics.NewGlobal(metricsConf, inmemSink)

	core, _, _ := TestCoreUnsealed(t)
	testSeal, setErr := seal.NewToggleableTestSeal(nil)

	var encKeys []string
	changeKey := func(key string) {
		encKeys = append(encKeys, key)
		testSeal.Wrapper.(*seal.ToggleableWrapper).Wrapper.(*wrapping.TestWrapper).SetKeyID(key)
	}

	// Set initial encryption key.
	changeKey("kaz")

	autoSeal := NewAutoSeal(testSeal)
	autoSeal.SetCore(core)
	pBackend := newTestBackend(t)
	core.physical = pBackend
	core.metricSink = metricsutil.NewClusterMetricSink("", inmemSink)

	sealHealthTestIntervalNominal = 10 * time.Millisecond
	sealHealthTestIntervalUnhealthy = 10 * time.Millisecond
	setErr(errors.New("disconnected"))
	autoSeal.StartHealthCheck()
	defer autoSeal.StopHealthCheck()

	time.Sleep(50 * time.Millisecond)

	asu := strings.Join(autoSealUnavailableDuration, ".") + ";cluster="
	intervals := inmemSink.Data()
	if len(intervals) == 1 {
		interval := inmemSink.Data()[0]

		if _, ok := interval.Gauges[asu]; !ok {
			t.Fatalf("Expected metrics to include a value for gauge %s", asu)
		}
		if interval.Gauges[asu].Value == 0 {
			t.Fatalf("Expected value metric %s to be non-zero", asu)
		}
	}
	setErr(nil)
	time.Sleep(50 * time.Millisecond)
	intervals = inmemSink.Data()
	if len(intervals) == 1 {
		interval := inmemSink.Data()[0]

		if _, ok := interval.Gauges[asu]; !ok {
			t.Fatalf("Expected metrics to include a value for gauge %s", asu)
		}
		if interval.Gauges[asu].Value != 0 {
			t.Fatalf("Expected value metric %s to be non-zero", asu)
		}
	}
}
<commit_msg>Fix autoseal health check race by passing metrics sink in CoreConfig (#14196)<commit_after>package vault

import (
	"bytes"
	"context"
	"errors"
	"reflect"
	"strings"
	"testing"
	"time"

	"github.com/armon/go-metrics"
	"github.com/hashicorp/vault/helper/metricsutil"

	proto "github.com/golang/protobuf/proto"
	wrapping "github.com/hashicorp/go-kms-wrapping"
	"github.com/hashicorp/vault/sdk/physical"
	"github.com/hashicorp/vault/vault/seal"
)

// phy implements physical.Backend. It maps keys to a slice of entries.
// Each call to Put appends the entry to the slice of entries for that
// key. No deduplication is done. This allows the test for UpgradeKeys to
// verify entries are only being updated when the underlying encryption key
// has been updated.
type phy struct {
	t       *testing.T
	entries map[string][]*physical.Entry
}

var _ physical.Backend = (*phy)(nil)

func newTestBackend(t *testing.T) *phy {
	return &phy{
		t:       t,
		entries: make(map[string][]*physical.Entry),
	}
}

func (p *phy) Put(_ context.Context, entry *physical.Entry) error {
	p.entries[entry.Key] = append(p.entries[entry.Key], entry)
	return nil
}

func (p *phy) Get(_ context.Context, key string) (*physical.Entry, error) {
	entries := p.entries[key]
	if entries == nil {
		return nil, nil
	}
	return entries[len(entries)-1], nil
}

func (p *phy) Delete(_ context.Context, key string) error {
	p.t.Errorf("Delete called on phy: key: %v", key)
	return nil
}

func (p *phy) List(_ context.Context, prefix string) ([]string, error) {
	p.t.Errorf("List called on phy: prefix: %v", prefix)
	return []string{}, nil
}

func (p *phy) Len() int {
	return len(p.entries)
}

func TestAutoSeal_UpgradeKeys(t *testing.T) {
	core, _, _ := TestCoreUnsealed(t)
	testSeal := seal.NewTestSeal(nil)

	var encKeys []string
	changeKey := func(key string) {
		encKeys = append(encKeys, key)
		testSeal.Wrapper.(*wrapping.TestWrapper).SetKeyID(key)
	}

	// Set initial encryption key.
	changeKey("kaz")

	autoSeal := NewAutoSeal(testSeal)
	autoSeal.SetCore(core)
	pBackend := newTestBackend(t)
	core.physical = pBackend

	ctx := context.Background()

	inkeys := [][]byte{[]byte("grist"), []byte("house")}
	if err := autoSeal.SetStoredKeys(ctx, inkeys); err != nil {
		t.Fatalf("SetStoredKeys: want no error, got %v", err)
	}

	inRecoveryKey := []byte("falernum")
	if err := autoSeal.SetRecoveryKey(ctx, inRecoveryKey); err != nil {
		t.Fatalf("SetRecoveryKey: want no error, got %v", err)
	}

	check := func() {
		// The values of the stored keys should never change.
		outkeys, err := autoSeal.GetStoredKeys(ctx)
		if err != nil {
			t.Fatalf("GetStoredKeys: want no error, got %v", err)
		}
		if !reflect.DeepEqual(inkeys, outkeys) {
			t.Errorf("incorrect stored keys: want %v, got %v", inkeys, outkeys)
		}

		// The value of the recovery key should also never change.
		outRecoveryKey, err := autoSeal.RecoveryKey(ctx)
		if err != nil {
			t.Fatalf("RecoveryKey: want no error, got %v", err)
		}
		if !bytes.Equal(inRecoveryKey, outRecoveryKey) {
			t.Errorf("incorrect recovery key: want %q, got %q", inRecoveryKey, outRecoveryKey)
		}

		// There should only be 2 entries in the physical backend. One for
		// the stored keys and one for the recovery key.
		if want, got := 2, pBackend.Len(); want != got {
			t.Errorf("backend unexpected Len: want %d, got %d", want, got)
		}

		for phyKey, phyEntries := range pBackend.entries {
			// Calling UpgradeKeys should only add an entry if the key has
			// changed.
			if keyCount, entryCount := len(encKeys), len(phyEntries); keyCount != entryCount {
				t.Errorf("phyKey = %s: encryption key count not equal to entry count: keys=%d, entries=%d", phyKey, keyCount, entryCount)
			}

			// Each phyEntry should correspond to a key at the same index
			// in encKeys. Iterate over each phyEntry and verify it was
			// encrypted with its corresponding key in encKeys.
			for i, phyEntry := range phyEntries {
				blobInfo := &wrapping.EncryptedBlobInfo{}
				if err := proto.Unmarshal(phyEntry.Value, blobInfo); err != nil {
					t.Errorf("phyKey = %s: failed to proto decode stored keys: %s", phyKey, err)
				}
				if blobInfo.KeyInfo == nil {
					t.Errorf("phyKey = %s: KeyInfo missing: %+v", phyKey, blobInfo)
				}
				if want, got := encKeys[i], blobInfo.KeyInfo.KeyID; want != got {
					t.Errorf("phyKey = %s: Incorrect encryption key: want %s, got %s", phyKey, want, got)
				}
			}
		}
	}

	// Verify the current state is correct before calling UpgradeKeys.
	check()

	// Call UpgradeKeys before changing the encryption key and verify
	// nothing has changed.
	if err := autoSeal.UpgradeKeys(ctx); err != nil {
		t.Fatalf("UpgradeKeys: want no error, got %v", err)
	}
	check()

	// Change the encryption key, call UpgradeKeys, then verify the stored
	// keys and recovery key has been re-encrypted with the new encryption
	// key.
	changeKey("primanti")
	if err := autoSeal.UpgradeKeys(ctx); err != nil {
		t.Fatalf("UpgradeKeys: want no error, got %v", err)
	}
	check()
}

func TestAutoSeal_HealthCheck(t *testing.T) {
	inmemSink := metrics.NewInmemSink(
		1000000*time.Hour,
		2000000*time.Hour)

	metricsConf := metrics.DefaultConfig("")
	metricsConf.EnableHostname = false
	metricsConf.EnableHostnameLabel = false
	metricsConf.EnableServiceLabel = false
	metricsConf.EnableTypePrefix = false

	metrics.NewGlobal(metricsConf, inmemSink)

	pBackend := newTestBackend(t)
	testSealAccess, setErr := seal.NewToggleableTestSeal(nil)
	core, _, _ := TestCoreUnsealedWithConfig(t, &CoreConfig{
		MetricSink: metricsutil.NewClusterMetricSink("", inmemSink),
		Physical:   pBackend,
	})
	sealHealthTestIntervalNominal = 10 * time.Millisecond
	sealHealthTestIntervalUnhealthy = 10 * time.Millisecond
	autoSeal := NewAutoSeal(testSealAccess)
	autoSeal.SetCore(core)
	core.seal = autoSeal
	autoSeal.StartHealthCheck()
	defer autoSeal.StopHealthCheck()
	setErr(errors.New("disconnected"))

	asu := strings.Join(autoSealUnavailableDuration, ".") + ";cluster=" + core.clusterName
	tries := 10
	for tries = 10; tries > 0; tries-- {
		intervals := inmemSink.Data()
		if len(intervals) == 1 {
			interval := inmemSink.Data()[0]

			if _, ok := interval.Gauges[asu]; ok {
				if interval.Gauges[asu].Value > 0 {
					break
				}
			}
		}
		time.Sleep(100 * time.Millisecond)
	}
	if tries == 0 {
		t.Fatalf("Expected value metric %s to be non-zero", asu)
	}

	setErr(nil)
	time.Sleep(50 * time.Millisecond)
	intervals := inmemSink.Data()
	if len(intervals) == 1 {
		interval := inmemSink.Data()[0]

		if _, ok := interval.Gauges[asu]; !ok {
			t.Fatalf("Expected metrics to include a value for gauge %s", asu)
		}
		if interval.Gauges[asu].Value != 0 {
			t.Fatalf("Expected value metric %s to be zero", asu)
		}
	}
}
<|endoftext|>


<commit_before>package acceptance_test

import (
	"os"
	"testing"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

func TestAcceptance(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "bbl")
}

var (
	pathToBBL             string
	bblPlanTimeout        time.Duration
	bblDownTimeout        time.Duration
	bblUpTimeout          time.Duration
	bblRotateTimeout      time.Duration
	bblLatestErrorTimeout time.Duration
	bblLeftoversTimeout   time.Duration
)

var _ = BeforeSuite(func() {
	var err error

	pathToBBL, err = gexec.Build("github.com/cloudfoundry/bosh-bootloader/bbl")
	Expect(err).NotTo(HaveOccurred())

	bblPlanTimeout = getTimeout("BBL_PLAN_TIMEOUT", 5*time.Minute)
	bblDownTimeout = getTimeout("BBL_DOWN_TIMEOUT", 10*time.Minute)
	bblUpTimeout = getTimeout("BBL_UP_TIMEOUT", 40*time.Minute)
	bblRotateTimeout = getTimeout("BBL_ROTATE_TIMEOUT", 40*time.Minute)
	bblLatestErrorTimeout = getTimeout("BBL_LATEST_ERROR_TIMEOUT", 10*time.Second)
	bblLeftoversTimeout = getTimeout("BBL_LEFTOVERS_TIMEOUT", 10*time.Minute)
})

var _ = AfterSuite(func() {
	gexec.CleanupBuildArtifacts()
})

func getTimeout(envVar string, defaultTimeout time.Duration) time.Duration {
	envVal := os.Getenv(envVar)
	if envVal == "" {
		return defaultTimeout
	}
	timeout, err := time.ParseDuration(os.Getenv(envVal))
	Expect(err).NotTo(HaveOccurred())
	return timeout
}
<commit_msg>correctly parse timeout environment variable<commit_after>package acceptance_test

import (
	"os"
	"testing"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

func TestAcceptance(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "bbl")
}

var (
	pathToBBL             string
	bblPlanTimeout        time.Duration
	bblDownTimeout        time.Duration
	bblUpTimeout          time.Duration
	bblRotateTimeout      time.Duration
	bblLatestErrorTimeout time.Duration
	bblLeftoversTimeout   time.Duration
)

var _ = BeforeSuite(func() {
	var err error

	pathToBBL, err = gexec.Build("github.com/cloudfoundry/bosh-bootloader/bbl")
	Expect(err).NotTo(HaveOccurred())

	bblPlanTimeout = getTimeout("BBL_PLAN_TIMEOUT", 5*time.Minute)
	bblDownTimeout = getTimeout("BBL_DOWN_TIMEOUT", 10*time.Minute)
	bblUpTimeout = getTimeout("BBL_UP_TIMEOUT", 40*time.Minute)
	bblRotateTimeout = getTimeout("BBL_ROTATE_TIMEOUT", 40*time.Minute)
	bblLatestErrorTimeout = getTimeout("BBL_LATEST_ERROR_TIMEOUT", 10*time.Second)
	bblLeftoversTimeout = getTimeout("BBL_LEFTOVERS_TIMEOUT", 10*time.Minute)
})

var _ = AfterSuite(func() {
	gexec.CleanupBuildArtifacts()
})

func getTimeout(envVar string, defaultTimeout time.Duration) time.Duration {
	envVal := os.Getenv(envVar)
	if envVal == "" {
		return defaultTimeout
	}
	timeout, err := time.ParseDuration(envVal)
	Expect(err).NotTo(HaveOccurred())
	return timeout
}
<|endoftext|>


<commit_before>package object

import (
	"fmt"
	"github.com/qiniu/qshell/v2/iqshell/common/data"
	"github.com/qiniu/qshell/v2/iqshell/common/flow"
	"github.com/qiniu/qshell/v2/iqshell/common/log"
	"github.com/qiniu/qshell/v2/iqshell/common/utils"
	"os"
)

type MatchApiInfo struct {
	Bucket    string // 文件所在七牛云的空间名，必选
	Key       string // 文件所在七牛云的 Key， 必选
	FileHash  string // 文件 Etag，可以是 etagV1, 也可以是 etagV2；可选，没有会从服务获取
	LocalFile string // 本地文件路径；必选
}

func (m *MatchApiInfo) WorkId() string {
	return utils.Md5Hex(fmt.Sprintf("%s:%s:%s:%s", m.Bucket, m.Key, m.LocalFile, m.FileHash))
}

type MatchResult struct {
	Match bool
}

var _ flow.Result = (*MatchResult)(nil)

func (m *MatchResult) IsValid() bool {
	return m != nil
}

func Match(info MatchApiInfo) (match *MatchResult, err *data.CodeError) {
	if len(info.LocalFile) == 0 {
		return nil, data.NewEmptyError().AppendDesc("Match Check: file is empty")
	}

	hashFile, oErr := os.Open(info.LocalFile)
	if oErr != nil {
		return nil, data.NewEmptyError().AppendDescF("Match Check: get local file error:%v", oErr)
	}

	var serverObjectStat *StatusResult
	if len(info.FileHash) == 0 {
		if stat, sErr := Status(StatusApiInfo{
			Bucket:   info.Bucket,
			Key:      info.Key,
			NeedPart: true,
		}); sErr != nil {
			return nil, data.NewEmptyError().AppendDescF("Match Check: get file status error:%v", sErr)
		} else {
			info.FileHash = stat.Hash
			serverObjectStat = &stat
		}
	}

	// 计算本地文件 hash
	var hash string
	if utils.IsSignByEtagV2(info.FileHash) {
		log.DebugF("Match Check: get etag by v2 for key:%s", info.Key)
		if serverObjectStat == nil {
			if stat, sErr := Status(StatusApiInfo{
				Bucket:   info.Bucket,
				Key:      info.Key,
				NeedPart: true,
			}); sErr != nil {
				return nil, data.NewEmptyError().AppendDescF("Match Check: etag v2, get file status error:%v", sErr)
			} else {
				serverObjectStat = &stat
			}
		}
		if h, eErr := utils.EtagV2(hashFile, serverObjectStat.Parts); eErr != nil {
			return nil, data.NewEmptyError().AppendDescF("Match Check: get file etag v2 error:%v", eErr)
		} else {
			hash = h
		}
		log.DebugF("Match Check: get etag by v2 for key:%s hash:%s", info.Key, hash)
	} else {
		log.DebugF("Match Check: get etag by v1 for key:%s", info.Key)
		if h, eErr := utils.EtagV1(hashFile); eErr != nil {
			return nil, data.NewEmptyError().AppendDescF("Match Check: get file etag v1 error:%v", eErr)
		} else {
			hash = h
		}
		log.DebugF("Match Check: get etag by v1 for key:%s hash:%s", info.Key, hash)
	}
	log.DebugF("Match Check:       server hash, key:%s hash:%s", info.Key, hash)
	if hash != info.FileHash {
		return nil, data.NewEmptyError().AppendDescF("Match Check: file hash doesn't match for key:%s, local file hash:%s server file hash:%s", info.Key, hash, info.FileHash)
	}

	return &MatchResult{
		Match: true,
	}, nil
}
<commit_msg>change match valid conf<commit_after>package object

import (
	"fmt"
	"github.com/qiniu/qshell/v2/iqshell/common/data"
	"github.com/qiniu/qshell/v2/iqshell/common/flow"
	"github.com/qiniu/qshell/v2/iqshell/common/log"
	"github.com/qiniu/qshell/v2/iqshell/common/utils"
	"os"
)

type MatchApiInfo struct {
	Bucket    string // 文件所在七牛云的空间名，必选
	Key       string // 文件所在七牛云的 Key， 必选
	FileHash  string // 文件 Etag，可以是 etagV1, 也可以是 etagV2；可选，没有会从服务获取
	LocalFile string // 本地文件路径；必选
}

func (m *MatchApiInfo) WorkId() string {
	return utils.Md5Hex(fmt.Sprintf("%s:%s:%s:%s", m.Bucket, m.Key, m.LocalFile, m.FileHash))
}

type MatchResult struct {
	Match bool
}

var _ flow.Result = (*MatchResult)(nil)

func (m *MatchResult) IsValid() bool {
	return m != nil && m.Match
}

func Match(info MatchApiInfo) (match *MatchResult, err *data.CodeError) {
	if len(info.LocalFile) == 0 {
		return nil, data.NewEmptyError().AppendDesc("Match Check: file is empty")
	}

	hashFile, oErr := os.Open(info.LocalFile)
	if oErr != nil {
		return nil, data.NewEmptyError().AppendDescF("Match Check: get local file error:%v", oErr)
	}

	var serverObjectStat *StatusResult
	if len(info.FileHash) == 0 {
		if stat, sErr := Status(StatusApiInfo{
			Bucket:   info.Bucket,
			Key:      info.Key,
			NeedPart: true,
		}); sErr != nil {
			return nil, data.NewEmptyError().AppendDescF("Match Check: get file status error:%v", sErr)
		} else {
			info.FileHash = stat.Hash
			serverObjectStat = &stat
		}
	}

	// 计算本地文件 hash
	var hash string
	if utils.IsSignByEtagV2(info.FileHash) {
		log.DebugF("Match Check: get etag by v2 for key:%s", info.Key)
		if serverObjectStat == nil {
			if stat, sErr := Status(StatusApiInfo{
				Bucket:   info.Bucket,
				Key:      info.Key,
				NeedPart: true,
			}); sErr != nil {
				return nil, data.NewEmptyError().AppendDescF("Match Check: etag v2, get file status error:%v", sErr)
			} else {
				serverObjectStat = &stat
			}
		}
		if h, eErr := utils.EtagV2(hashFile, serverObjectStat.Parts); eErr != nil {
			return nil, data.NewEmptyError().AppendDescF("Match Check: get file etag v2 error:%v", eErr)
		} else {
			hash = h
		}
		log.DebugF("Match Check: get etag by v2 for key:%s hash:%s", info.Key, hash)
	} else {
		log.DebugF("Match Check: get etag by v1 for key:%s", info.Key)
		if h, eErr := utils.EtagV1(hashFile); eErr != nil {
			return nil, data.NewEmptyError().AppendDescF("Match Check: get file etag v1 error:%v", eErr)
		} else {
			hash = h
		}
		log.DebugF("Match Check: get etag by v1 for key:%s hash:%s", info.Key, hash)
	}
	log.DebugF("Match Check:       server hash, key:%s hash:%s", info.Key, hash)
	if hash != info.FileHash {
		return nil, data.NewEmptyError().AppendDescF("Match Check: file hash doesn't match for key:%s, local file hash:%s server file hash:%s", info.Key, hash, info.FileHash)
	}

	return &MatchResult{
		Match: true,
	}, nil
}
<|endoftext|>


<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package apiserver

import (
	"crypto/tls"
	"fmt"
	"net"
	"net/http"
	"sync"
	"time"

	"code.google.com/p/go.net/websocket"
	"launchpad.net/loggo"
	"launchpad.net/tomb"

	"launchpad.net/juju-core/rpc"
	"launchpad.net/juju-core/rpc/jsoncodec"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/state/apiserver/common"
)

var logger = loggo.GetLogger("juju.state.apiserver")

// Server holds the server side of the API.
type Server struct {
	tomb  tomb.Tomb
	wg    sync.WaitGroup
	state *state.State
	addr  net.Addr
}

// Serve serves the given state by accepting requests on the given
// listener, using the given certificate and key (in PEM format) for
// authentication.
func NewServer(s *state.State, addr string, cert, key []byte) (*Server, error) {
	lis, err := net.Listen("tcp", addr)
	if err != nil {
		return nil, err
	}
	logger.Infof("listening on %q", lis.Addr())
	tlsCert, err := tls.X509KeyPair(cert, key)
	if err != nil {
		return nil, err
	}
	srv := &Server{
		state: s,
		addr:  lis.Addr(),
	}
	// TODO(rog) check that *srvRoot is a valid type for using
	// as an RPC server.
	lis = tls.NewListener(lis, &tls.Config{
		Certificates: []tls.Certificate{tlsCert},
	})
	go srv.run(lis)
	return srv, nil
}

// Dead returns a channel that signals when the server has exited.
func (srv *Server) Dead() <-chan struct{} {
	return srv.tomb.Dead()
}

// Stop stops the server and returns when all running requests
// have completed.
func (srv *Server) Stop() error {
	srv.tomb.Kill(nil)
	return srv.tomb.Wait()
}

// Kill implements worker.Worker.Kill.
func (srv *Server) Kill() {
	srv.tomb.Kill(nil)
}

// Wait implements worker.Worker.Wait.
func (srv *Server) Wait() error {
	return srv.tomb.Wait()
}

type requestNotifier struct {
	remoteAddr net.Addr
	identifier string
}

func (n *requestNotifier) SetIdentifier(identifier string) {
	n.identifier = identifier
}

func (n *requestNotifier) getIdentifier() string {
	if n.identifier != "" {
		return n.identifier
	}
	return fmt.Sprintf("%s", n.remoteAddr.Network())
}

func (n *requestNotifier) ServerRequest(hdr *rpc.Header, body interface{}) {
	if hdr.Request.Type == "Pinger" && hdr.Request.Action == "Ping" {
		return
	}
	// TODO(rog) 2013-10-11 remove secrets from some requests.
	logger.Debugf("<- %p %s %s", n, n.getIdentifier(), jsoncodec.DumpRequest(hdr, body))
}

func (n *requestNotifier) ServerReply(req rpc.Request, hdr *rpc.Header, body interface{}, timeSpent time.Duration) {
	if req.Type == "Pinger" && req.Action == "Ping" {
		return
	}
	logger.Debugf("<- %p %s %s %s %s[%q].%s", n, n.getIdentifier(), timeSpent, jsoncodec.DumpRequest(hdr, body), req.Type, req.Id, req.Action)
}

func (n requestNotifier) ClientRequest(hdr *rpc.Header, body interface{}) {
}

func (n requestNotifier) ClientReply(req rpc.Request, hdr *rpc.Header, body interface{}) {
}

func (srv *Server) run(lis net.Listener) {
	defer srv.tomb.Done()
	defer srv.wg.Wait() // wait for any outstanding requests to complete.
	srv.wg.Add(1)
	go func() {
		<-srv.tomb.Dying()
		lis.Close()
		srv.wg.Done()
	}()
	handler := websocket.Handler(func(conn *websocket.Conn) {
		srv.wg.Add(1)
		defer srv.wg.Done()
		// If we've got to this stage and the tomb is still
		// alive, we know that any tomb.Kill must occur after we
		// have called wg.Add, so we avoid the possibility of a
		// handler goroutine running after Stop has returned.
		if srv.tomb.Err() != tomb.ErrStillAlive {
			return
		}
		if err := srv.serveConn(conn); err != nil {
			logger.Errorf("error serving RPCs: %v", err)
		}
	})
	// The error from http.Serve is not interesting.
	http.Serve(lis, handler)
}

// Addr returns the address that the server is listening on.
func (srv *Server) Addr() string {
	return srv.addr.String()
}

func (srv *Server) serveConn(wsConn *websocket.Conn) error {
	codec := jsoncodec.NewWebsocket(wsConn)
	if loggo.GetLogger("juju.rpc.jsoncodec").EffectiveLogLevel() <= loggo.TRACE {
		codec.SetLogging(true)
	}
	var notifier rpc.RequestNotifier
	var loginCallback func(string)
	if logger.EffectiveLogLevel() <= loggo.DEBUG {
		reqNotifier := &requestNotifier{wsConn.RemoteAddr(), ""}
		loginCallback = reqNotifier.SetIdentifier
		notifier = reqNotifier
	}
	conn := rpc.NewConn(codec, notifier)
	conn.Serve(newStateServer(srv, conn, loginCallback), serverError)
	conn.Start()
	select {
	case <-conn.Dead():
	case <-srv.tomb.Dying():
	}
	return conn.Close()
}

func serverError(err error) error {
	if err := common.ServerError(err); err != nil {
		return err
	}
	return nil
}

var logRequests = true
<commit_msg>Switch to using a counter per connection.<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package apiserver

import (
	"crypto/tls"
	"net"
	"net/http"
	"sync"
	"time"

	"code.google.com/p/go.net/websocket"
	"launchpad.net/loggo"
	"launchpad.net/tomb"

	"launchpad.net/juju-core/rpc"
	"launchpad.net/juju-core/rpc/jsoncodec"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/state/apiserver/common"
)

var logger = loggo.GetLogger("juju.state.apiserver")

// Server holds the server side of the API.
type Server struct {
	tomb  tomb.Tomb
	wg    sync.WaitGroup
	state *state.State
	addr  net.Addr
}

// Serve serves the given state by accepting requests on the given
// listener, using the given certificate and key (in PEM format) for
// authentication.
func NewServer(s *state.State, addr string, cert, key []byte) (*Server, error) {
	lis, err := net.Listen("tcp", addr)
	if err != nil {
		return nil, err
	}
	logger.Infof("listening on %q", lis.Addr())
	tlsCert, err := tls.X509KeyPair(cert, key)
	if err != nil {
		return nil, err
	}
	srv := &Server{
		state: s,
		addr:  lis.Addr(),
	}
	// TODO(rog) check that *srvRoot is a valid type for using
	// as an RPC server.
	lis = tls.NewListener(lis, &tls.Config{
		Certificates: []tls.Certificate{tlsCert},
	})
	go srv.run(lis)
	return srv, nil
}

// Dead returns a channel that signals when the server has exited.
func (srv *Server) Dead() <-chan struct{} {
	return srv.tomb.Dead()
}

// Stop stops the server and returns when all running requests
// have completed.
func (srv *Server) Stop() error {
	srv.tomb.Kill(nil)
	return srv.tomb.Wait()
}

// Kill implements worker.Worker.Kill.
func (srv *Server) Kill() {
	srv.tomb.Kill(nil)
}

// Wait implements worker.Worker.Wait.
func (srv *Server) Wait() error {
	return srv.tomb.Wait()
}

type requestNotifier struct {
	connCounter int64
	identifier string
}

var globalCounter int64
var globalCounterMu = sync.Mutex{}

func nextCounter() int64 {
	globalCounterMu.Lock()
	globalCounter++
	val := globalCounter
	globalCounterMu.Unlock()
	return val
}

func (n *requestNotifier) SetIdentifier(identifier string) {
	n.identifier = identifier
}

func (n requestNotifier) ServerRequest(hdr *rpc.Header, body interface{}) {
	if hdr.Request.Type == "Pinger" && hdr.Request.Action == "Ping" {
		return
	}
	// TODO(rog) 2013-10-11 remove secrets from some requests.
	logger.Debugf("<- [%X] %s %s", n.connCounter, n.identifier, jsoncodec.DumpRequest(hdr, body))
}

func (n requestNotifier) ServerReply(req rpc.Request, hdr *rpc.Header, body interface{}, timeSpent time.Duration) {
	if req.Type == "Pinger" && req.Action == "Ping" {
		return
	}
	logger.Debugf("<- [%X] %s %s %s %s[%q].%s", n.connCounter, n.identifier, timeSpent, jsoncodec.DumpRequest(hdr, body), req.Type, req.Id, req.Action)
}

func (n requestNotifier) ClientRequest(hdr *rpc.Header, body interface{}) {
}

func (n requestNotifier) ClientReply(req rpc.Request, hdr *rpc.Header, body interface{}) {
}

func (srv *Server) run(lis net.Listener) {
	defer srv.tomb.Done()
	defer srv.wg.Wait() // wait for any outstanding requests to complete.
	srv.wg.Add(1)
	go func() {
		<-srv.tomb.Dying()
		lis.Close()
		srv.wg.Done()
	}()
	handler := websocket.Handler(func(conn *websocket.Conn) {
		srv.wg.Add(1)
		defer srv.wg.Done()
		// If we've got to this stage and the tomb is still
		// alive, we know that any tomb.Kill must occur after we
		// have called wg.Add, so we avoid the possibility of a
		// handler goroutine running after Stop has returned.
		if srv.tomb.Err() != tomb.ErrStillAlive {
			return
		}
		if err := srv.serveConn(conn); err != nil {
			logger.Errorf("error serving RPCs: %v", err)
		}
	})
	// The error from http.Serve is not interesting.
	http.Serve(lis, handler)
}

// Addr returns the address that the server is listening on.
func (srv *Server) Addr() string {
	return srv.addr.String()
}

func (srv *Server) serveConn(wsConn *websocket.Conn) error {
	codec := jsoncodec.NewWebsocket(wsConn)
	if loggo.GetLogger("juju.rpc.jsoncodec").EffectiveLogLevel() <= loggo.TRACE {
		codec.SetLogging(true)
	}
	var notifier rpc.RequestNotifier
	var loginCallback func(string)
	if logger.EffectiveLogLevel() <= loggo.DEBUG {
		reqNotifier := &requestNotifier{nextCounter(), ""}
		loginCallback = reqNotifier.SetIdentifier
		notifier = reqNotifier
	}
	conn := rpc.NewConn(codec, notifier)
	conn.Serve(newStateServer(srv, conn, loginCallback), serverError)
	conn.Start()
	select {
	case <-conn.Dead():
	case <-srv.tomb.Dying():
	}
	return conn.Close()
}

func serverError(err error) error {
	if err := common.ServerError(err); err != nil {
		return err
	}
	return nil
}

var logRequests = true
<|endoftext|>


<commit_before>package hwaflib

func (ctx *Context) Version() string {
	version := "20131109"
	return version
}

func (ctx *Context) Revision() string {
	revision := "41aa849"
	return revision
}

// EOF

<commit_msg>version: 20131112<commit_after>package hwaflib

func (ctx *Context) Version() string {
	version := "20131112"
	return version
}

func (ctx *Context) Revision() string {
	revision := "37e6685"
	return revision
}

// EOF

<|endoftext|>


<commit_before>package scope

import "strings"

const (
	// Scope prefix which indicates initiation of a cross-client authentication flow.
	// See https://developers.google.com/identity/protocols/CrossClientAuth
	ScopeGoogleCrossClient = "audience:server:client_id:"
)

type Scopes []string

func (s Scopes) OfflineAccess() bool {
	return s.HasScope("offline_access")
}

func (s Scopes) HasScope(scope string) bool {
	for _, curScope := range s {
		if curScope == scope {
			return true
		}
	}
	return false
}

func (s Scopes) CrossClientIDs() []string {
	clients := []string{}
	for _, scope := range s {
		if strings.HasPrefix(scope, ScopeGoogleCrossClient) {
			clients = append(clients, scope[len(ScopeGoogleCrossClient):])
		}
	}
	return clients
}

func (s Scopes) Contains(other Scopes) bool {
	rScopes := map[string]struct{}{}
	for _, scope := range s {
		rScopes[scope] = struct{}{}
	}

	for _, scope := range other {
		if _, ok := rScopes[scope]; !ok {
			return false
		}
	}
	return true
}
<commit_msg>scope: don't validate empty scopes<commit_after>package scope

import "strings"

const (
	// Scope prefix which indicates initiation of a cross-client authentication flow.
	// See https://developers.google.com/identity/protocols/CrossClientAuth
	ScopeGoogleCrossClient = "audience:server:client_id:"
)

type Scopes []string

func (s Scopes) OfflineAccess() bool {
	return s.HasScope("offline_access")
}

func (s Scopes) HasScope(scope string) bool {
	for _, curScope := range s {
		if curScope == scope {
			return true
		}
	}
	return false
}

func (s Scopes) CrossClientIDs() []string {
	clients := []string{}
	for _, scope := range s {
		if strings.HasPrefix(scope, ScopeGoogleCrossClient) {
			clients = append(clients, scope[len(ScopeGoogleCrossClient):])
		}
	}
	return clients
}

func (s Scopes) Contains(other Scopes) bool {
	rScopes := map[string]struct{}{}
	for _, scope := range s {
		rScopes[scope] = struct{}{}
	}

	for _, scope := range other {
		if _, ok := rScopes[scope]; !ok {
			if scope == "" {
				continue
			}
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package score

import "fmt"
import "bytes"
import "os"
import "bufio"

type Board []Score

//Load saved scores from the file 'scores'
func Load() Board {
    f, _ := os.Open("scores")
    buf := bufio.NewReader(f)
    b := make(Board, 0, 10)
    for i := 0; ; i++ {
        if ln, e := buf.ReadString('\n'); e == nil {
            s := unmarshal(ln)
            if len(b) == cap(b) {
                tmp := make(Board, len(b), cap(b)+10)
                copy(tmp, b)
                b = tmp
            }
            b = b[:i+1]
            b[i] = s
        } else {
            break
        }
    }
    b.bsort()
    return b
}

//Format Board object into a scoreboard suitable for output
func (b Board) String() string {
    var buf bytes.Buffer
    buf.WriteString("Diff.\tScore\tPlayer\n")
    buf.WriteString("-----\t-----\t------\n")
    for i, s := range b {
        //Limit scores to Top 10...
        if i > 9 {
            break
        }
        buf.WriteString(fmt.Sprintln(s))
    }
    return buf.String()
}

//Rough implementation of a bubble sort...
func (b Board) bsort() {
    for done := false; !done; done = true {
        for x := len(b)-1; x > 0; x-- {
            s1, _ := b[x].c.Score(b[x].t)
            s2, _ := b[x-1].c.Score(b[x].t)
            if s1 > s2 {
                b[x], b[x-1] = b[x-1], b[x]
                done = false
            }
        }
    }
}
<commit_msg>Replaced the usage of the hacky bsort() bubble sort method in favour of compliance with the sort package interface & sort.Sort()<commit_after>package score

import "fmt"
import "bytes"
import "os"
import "bufio"
import "sort"

type Board []Score

//Load saved scores from the file 'scores'
func Load() Board {
    f, _ := os.Open("scores")
    buf := bufio.NewReader(f)
    b := make(Board, 0, 10)
    for i := 0; ; i++ {
        if ln, e := buf.ReadString('\n'); e == nil {
            s := unmarshal(ln)
            if len(b) == cap(b) {
                tmp := make(Board, len(b), cap(b)+10)
                copy(tmp, b)
                b = tmp
            }
            b = b[:i+1]
            b[i] = s
        } else {
            break
        }
    }
    sort.Sort(b)
    return b
}

//Format Board object into a scoreboard suitable for output
func (b Board) String() string {
    var buf bytes.Buffer
    buf.WriteString("Diff.\tScore\tPlayer\n")
    buf.WriteString("-----\t-----\t------\n")
    for i, s := range b {
        //Limit scores to Top 10...
        if i > 9 {
            break
        }
        buf.WriteString(fmt.Sprintln(s))
    }
    return buf.String()
}

//Rough implementation of a bubble sort...
func (b Board) bsort() {
    for done := false; !done; done = true {
        for x := len(b)-1; x > 0; x-- {
            s1, _ := b[x].c.Score(b[x].t)
            s2, _ := b[x-1].c.Score(b[x].t)
            if s1 > s2 {
                b[x], b[x-1] = b[x-1], b[x]
                done = false
            }
        }
    }
}

//sort.Sort() compliance..
func (b Board) Len() int {
    return len(b)
}
func (b Board) Less(x, y int) bool {
    i, _ := b[x].c.Score(b[x].t)
    j, _ := b[y].c.Score(b[y].t)
    return i > j
}
func (b Board) Swap(x, y int) {
    b[x], b[y] = b[y], b[x]
}
<|endoftext|>


<commit_before>package score

import (
	"encoding/json"
	"errors"
	"strings"
)

// hiddenSecret is used to replace the global secret when parsing.
var hiddenSecret = "hidden"

// ErrScoreNotFound is returned if the parse string did not contain a
// JSON score string.
var ErrScoreNotFound = errors.New("score not found in string")

// Parse returns a score object for the provided JSON string s
// which contains secret.
func Parse(s, secret string) (*Score, error) {
	if strings.Contains(s, secret) {
		var sc Score
		err := json.Unmarshal([]byte(s), &sc)
		if err == nil {
			if sc.Secret == secret {
				sc.Secret = hiddenSecret // overwrite secret
			}
			return &sc, nil
		}
		if strings.Contains(err.Error(), secret) {
			// this is probably not necessary, but to be safe
			return nil, errors.New("error suppressed to avoid revealing secret")
		}
		return nil, err
	}
	return nil, ErrScoreNotFound
}

// HasPrefix returns true if the provided string s has a parsable prefix string.
func HasPrefix(s string) bool {
	prefixes := []string{
		`{"Secret":`,
		`{"TestName":`,
		`{"Score":`,
		`{"MaxScore":`,
		`{"Weight":`,
	}
	for _, prefix := range prefixes {
		if strings.HasPrefix(s, prefix) {
			return true
		}
	}
	return false
}
<commit_msg>made HasPrefix robust against whitespace<commit_after>package score

import (
	"encoding/json"
	"errors"
	"strings"
)

// hiddenSecret is used to replace the global secret when parsing.
var hiddenSecret = "hidden"

// ErrScoreNotFound is returned if the parse string did not contain a
// JSON score string.
var ErrScoreNotFound = errors.New("score not found in string")

// Parse returns a score object for the provided JSON string s
// which contains secret.
func Parse(s, secret string) (*Score, error) {
	if strings.Contains(s, secret) {
		var sc Score
		err := json.Unmarshal([]byte(s), &sc)
		if err == nil {
			if sc.Secret == secret {
				sc.Secret = hiddenSecret // overwrite secret
			}
			return &sc, nil
		}
		if strings.Contains(err.Error(), secret) {
			// this is probably not necessary, but to be safe
			return nil, errors.New("error suppressed to avoid revealing secret")
		}
		return nil, err
	}
	return nil, ErrScoreNotFound
}

// HasPrefix returns true if the provided string s has a parsable prefix string.
func HasPrefix(s string) bool {
	prefixes := []string{
		`{"Secret":`,
		`{"TestName":`,
		`{"Score":`,
		`{"MaxScore":`,
		`{"Weight":`,
	}
	trimmed := strings.TrimSpace(s)
	for _, prefix := range prefixes {
		if strings.HasPrefix(trimmed, prefix) {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package htm

import (
	// 	"fmt"
	// 	"github.com/cznic/mathutil"
	// 	"github.com/zacg/floats"
	// 	"github.com/zacg/go.matrix"
	//"github.com/zacg/htm/utils"
	"github.com/zacg/ints"
	// 	//"math"
	// 	"math/rand"
	// 	//"sort"
)

type TmSynapse struct {
	Segment    int
	SourceCell int
	Permanence float64
}

/*
 Structure holds data representing the connectivity of a layer of cells,
that the TM operates on.
*/
type TemporalMemoryConnections struct {
	ColumnDimensions []int
	CellsPerColumn   int

	segments []int
	synapses []*TmSynapse

	synapsesForSegment    [][]int
	synapsesForSourceCell [][]int

	segmentsForCell [][]int

	segmentIndex int
	synIndex     int

	maxSynapseCount int
}

//Create a new temporal memory
func NewTemporalMemoryConnections(maxSynCount int, cellsPerColumn int, colDimensions []int) *TemporalMemoryConnections {
	if len(colDimensions) < 1 {
		panic("Column dimensions must be greater than 0")
	}

	if cellsPerColumn < 1 {
		panic("Number of cells per column must be greater than 0")
	}

	c := new(TemporalMemoryConnections)
	c.maxSynapseCount = maxSynCount
	c.CellsPerColumn = cellsPerColumn
	c.ColumnDimensions = colDimensions

	c.synapses = make([]*TmSynapse, 0, c.maxSynapseCount)
	//TODO: init segments

	return c
}

// func (tmc *TemporalMemoryConnections) nextSegmentIndex() int {
// 	idx := tmc.segmentIndex
// 	tmc.segmentIndex++
// 	return idx
// }

// func (tmc *TemporalMemoryConnections) nextSynapseIndex() int {
// 	idx := tmc.synIndex
// 	tmc.synIndex++
// 	return idx
// }

func (tmc *TemporalMemoryConnections) CreateSynapse(segment int, sourceCell int, permanence float64) *TmSynapse {
	syn := len(tmc.synapses)
	data := new(TmSynapse)
	data.Segment = segment
	data.SourceCell = sourceCell
	data.Permanence = permanence
	tmc.synapses = append(tmc.synapses, data)

	//Update indexes
	tmc.synapsesForSegment[segment] = append(tmc.synapsesForSegment[segment], syn)
	tmc.synapsesForSourceCell[sourceCell] = append(tmc.synapsesForSourceCell[sourceCell], syn)

	return data
}

//Creates a new segment on specified cell, returns segment index
func (tmc *TemporalMemoryConnections) CreateSegment(cell int) int {
	idx := len(tmc.segments)
	// Add data
	tmc.segments = append(tmc.segments, cell)
	tmc.segmentsForCell[cell] = append(tmc.segmentsForCell[idx], idx)
	return idx
}

//Updates the permanence for a synapse.
func (tmc *TemporalMemoryConnections) UpdateSynapsePermanence(synapse int, permanence float64) {
	tmc.validatePermanence(permanence)
	tmc.synapses[synapse].Permanence = permanence
}

//Returns the index of the column that a cell belongs to.
func (tmc *TemporalMemoryConnections) ColumnForCell(cell int) int {
	return int(cell / tmc.CellsPerColumn)
}

//Returns the indices of cells that belong to a column.
func (tmc *TemporalMemoryConnections) CellsForColumn(column int) []int {
	start := tmc.CellsPerColumn * column
	result := make([]int, tmc.CellsPerColumn)
	for idx, _ := range result {
		result[idx] = start + idx
	}
	return result
}

//Returns the cell that a segment belongs to.
func (tmc *TemporalMemoryConnections) CellForSegment(segment int) int {
	return tmc.segments[segment]
}

//Returns the segments that belong to a cell.
func (tmc *TemporalMemoryConnections) SegmentsForCell(cell int) []int {
	return tmc.segmentsForCell[cell]
}

//Returns synapse data for specified index
func (tmc *TemporalMemoryConnections) DataForSynapse(synapse int) *TmSynapse {
	return tmc.synapses[synapse]
}

//Returns the synapses on a segment.
func (tmc *TemporalMemoryConnections) SynapsesForSegment(segment int) []int {
	return tmc.synapsesForSegment[segment]
}

//Returns the synapses for the source cell that they synapse on.
func (tmc *TemporalMemoryConnections) SynapsesForSourceCell(sourceCell int) []int {
	return tmc.synapsesForSourceCell[sourceCell]
}

// Helpers

//Returns the number of columns in this layer.
func (tmc *TemporalMemoryConnections) NumberOfColumns() int {
	return ints.Prod(tmc.ColumnDimensions)
}

//Returns the number of cells in this layer.
func (tmc *TemporalMemoryConnections) NumberOfcells() int {
	return tmc.NumberOfColumns() * tmc.CellsPerColumn
}

//Validation

func (tmc *TemporalMemoryConnections) validatePermanence(permanence float64) {
	if permanence < 0 || permanence > 1 {
		panic("invalid permanence value")
	}
}
<commit_msg>intialize connection indexes<commit_after>package htm

import (
	// 	"fmt"
	// 	"github.com/cznic/mathutil"
	// 	"github.com/zacg/floats"
	// 	"github.com/zacg/go.matrix"
	//"github.com/zacg/htm/utils"
	"github.com/zacg/ints"
	// 	//"math"
	// 	"math/rand"
	// 	//"sort"
)

type TmSynapse struct {
	Segment    int
	SourceCell int
	Permanence float64
}

/*
 Structure holds data representing the connectivity of a layer of cells,
that the TM operates on.
*/
type TemporalMemoryConnections struct {
	ColumnDimensions []int
	CellsPerColumn   int

	segments []int
	synapses []*TmSynapse

	synapsesForSegment    [][]int
	synapsesForSourceCell [][]int

	segmentsForCell [][]int

	segmentIndex int
	synIndex     int

	maxSynapseCount int
}

//Create a new temporal memory
func NewTemporalMemoryConnections(maxSynCount int, cellsPerColumn int, colDimensions []int) *TemporalMemoryConnections {
	if len(colDimensions) < 1 {
		panic("Column dimensions must be greater than 0")
	}

	if cellsPerColumn < 1 {
		panic("Number of cells per column must be greater than 0")
	}

	c := new(TemporalMemoryConnections)
	c.maxSynapseCount = maxSynCount
	c.CellsPerColumn = cellsPerColumn
	c.ColumnDimensions = colDimensions

	c.synapses = make([]*TmSynapse, 0, c.maxSynapseCount)
	//TODO: calc better size
	c.segments = make([]int, 0, 1000)
	c.segmentsForCell = make([][]int, cap(c.segments))
	c.synapsesForSegment = make([][]int, cap(c.segments))
	c.synapsesForSourceCell = make([][]int, cap(c.segments))

	return c
}

// func (tmc *TemporalMemoryConnections) nextSegmentIndex() int {
// 	idx := tmc.segmentIndex
// 	tmc.segmentIndex++
// 	return idx
// }

// func (tmc *TemporalMemoryConnections) nextSynapseIndex() int {
// 	idx := tmc.synIndex
// 	tmc.synIndex++
// 	return idx
// }

func (tmc *TemporalMemoryConnections) CreateSynapse(segment int, sourceCell int, permanence float64) *TmSynapse {
	syn := len(tmc.synapses)
	data := new(TmSynapse)
	data.Segment = segment
	data.SourceCell = sourceCell
	data.Permanence = permanence
	tmc.synapses = append(tmc.synapses, data)

	//Update indexes
	tmc.synapsesForSegment[segment] = append(tmc.synapsesForSegment[segment], syn)
	tmc.synapsesForSourceCell[sourceCell] = append(tmc.synapsesForSourceCell[sourceCell], syn)

	return data
}

//Creates a new segment on specified cell, returns segment index
func (tmc *TemporalMemoryConnections) CreateSegment(cell int) int {
	idx := len(tmc.segments)
	// Add data
	tmc.segments = append(tmc.segments, cell)
	tmc.segmentsForCell[cell] = append(tmc.segmentsForCell[cell], idx)
	return idx
}

//Updates the permanence for a synapse.
func (tmc *TemporalMemoryConnections) UpdateSynapsePermanence(synapse int, permanence float64) {
	tmc.validatePermanence(permanence)
	tmc.synapses[synapse].Permanence = permanence
}

//Returns the index of the column that a cell belongs to.
func (tmc *TemporalMemoryConnections) ColumnForCell(cell int) int {
	return int(cell / tmc.CellsPerColumn)
}

//Returns the indices of cells that belong to a column.
func (tmc *TemporalMemoryConnections) CellsForColumn(column int) []int {
	start := tmc.CellsPerColumn * column
	result := make([]int, tmc.CellsPerColumn)
	for idx, _ := range result {
		result[idx] = start + idx
	}
	return result
}

//Returns the cell that a segment belongs to.
func (tmc *TemporalMemoryConnections) CellForSegment(segment int) int {
	return tmc.segments[segment]
}

//Returns the segments that belong to a cell.
func (tmc *TemporalMemoryConnections) SegmentsForCell(cell int) []int {
	return tmc.segmentsForCell[cell]
}

//Returns synapse data for specified index
func (tmc *TemporalMemoryConnections) DataForSynapse(synapse int) *TmSynapse {
	return tmc.synapses[synapse]
}

//Returns the synapses on a segment.
func (tmc *TemporalMemoryConnections) SynapsesForSegment(segment int) []int {
	return tmc.synapsesForSegment[segment]
}

//Returns the synapses for the source cell that they synapse on.
func (tmc *TemporalMemoryConnections) SynapsesForSourceCell(sourceCell int) []int {
	return tmc.synapsesForSourceCell[sourceCell]
}

// Helpers

//Returns the number of columns in this layer.
func (tmc *TemporalMemoryConnections) NumberOfColumns() int {
	return ints.Prod(tmc.ColumnDimensions)
}

//Returns the number of cells in this layer.
func (tmc *TemporalMemoryConnections) NumberOfcells() int {
	return tmc.NumberOfColumns() * tmc.CellsPerColumn
}

//Validation

func (tmc *TemporalMemoryConnections) validatePermanence(permanence float64) {
	if permanence < 0 || permanence > 1 {
		panic("invalid permanence value")
	}
}
<|endoftext|>