<commit_before>package libcarina

import (
	"archive/zip"
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"path"
	"reflect"
	"strconv"
	"strings"

	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/rackspace"
)

// BetaEndpoint reflects the default endpoint for this library
const BetaEndpoint = "https://app.getcarina.com"
const mimetypeJSON = "application/json"
const authHeaderKey = "X-Auth-Token"
const userAgent = "rackerlabs/libcarina"

// ZipURLResponse is the response that comes back from the zip endpoint
type ZipURLResponse struct {
	URL string `json:"zip_url"`
}

// ClusterClient accesses Carina directly
type ClusterClient struct {
	client   *http.Client
	Username string
	Token    string
	Endpoint string
}

// ErrorResponse is the JSON formatted error response from Carina
type ErrorResponse struct {
	Error string `json:"error"`
}

// Cluster is a cluster
type Cluster struct {
	ClusterName string `json:"cluster_name"`
	Username    string `json:"username"`

	// Flavor of compute to use for cluster, should be a default value currently
	Flavor string `json:"flavor,omitempty"`

	// UUID of image to use for cluster, should be a default value currently
	Image string `json:"image,omitempty"`

	// Node is optional, but allowed on create
	// Sadly it comes back as string instead of int in all cases
	// with the API
	Nodes Number `json:"nodes,omitempty"`

	AutoScale bool   `json:"autoscale,omitempty"`
	Status    string `json:"status,omitempty"`
	TaskID    string `json:"task_id,omitempty"`
	Token     string `json:"token,omitempty"`
}

// Credentials holds the keys to the kingdom
type Credentials struct {
	README     []byte
	Cert       []byte
	Key        []byte
	CA         []byte
	CAKey      []byte
	DockerEnv  []byte
	DockerCmd  []byte
	DockerPS1  []byte
	DockerHost string
	Files      map[string][]byte
}

// Number - specify this type for any struct fields that
// might be unmarshaled from JSON numbers of the following
// types: floats, integers, scientific notation, or strings
type Number float64

// Int64 return the Int64 version of this
func (n Number) Int64() int64 {
	return int64(n)
}

// Int return the Int version of this
func (n Number) Int() int {
	return int(n)
}

// Float64 return the Float64 version of this
func (n Number) Float64() float64 {
	return float64(n)
}

// UnmarshalJSON required to enforce that string values are attempted to be parsed as numbers
func (n *Number) UnmarshalJSON(data []byte) error {
	var f float64
	var err error
	if data[0] == '"' {
		f, err = strconv.ParseFloat(string(data[1:len(data)-1]), 64)
		if err != nil {
			return &json.UnmarshalTypeError{
				Value: string(data),
				Type:  reflect.TypeOf(*n),
			}
		}
	} else {
		if err := json.Unmarshal(data, &f); err != nil {
			return &json.UnmarshalTypeError{
				Value: string(data),
				Type:  reflect.TypeOf(*n),
			}
		}
	}
	*n = Number(f)
	return nil
}

func newClusterClient(endpoint string, ao gophercloud.AuthOptions) (*ClusterClient, error) {
	provider, err := rackspace.AuthenticatedClient(ao)
	if err != nil {
		return nil, err
	}

	return &ClusterClient{
		client:   &http.Client{},
		Username: ao.Username,
		Token:    provider.TokenID,
		Endpoint: endpoint,
	}, nil
}

// NewClusterClient create a new clusterclient by API Key
func NewClusterClient(endpoint, username, apikey string) (*ClusterClient, error) {
	ao := gophercloud.AuthOptions{
		Username:         username,
		APIKey:           apikey,
		IdentityEndpoint: rackspace.RackspaceUSIdentity,
	}

	return newClusterClient(endpoint, ao)
}

// NewRequest handles a request using auth used by Carina
func (c *ClusterClient) NewRequest(method string, uri string, body io.Reader) (*http.Response, error) {
	req, err := http.NewRequest(method, c.Endpoint+uri, body)
	if err != nil {
		return nil, err
	}
	req.Header.Set("User-Agent", userAgent)
	req.Header.Add("Content-Type", mimetypeJSON)
	req.Header.Add("Accept", mimetypeJSON)
	req.Header.Add(authHeaderKey, c.Token)
	resp, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode >= 400 {
		b, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, errors.New(resp.Status)
		}
		return nil, errors.New(string(b))
	}

	return resp, nil
}

// List the current clusters
func (c *ClusterClient) List() ([]Cluster, error) {
	clusters := []Cluster{}

	resp, err := c.NewRequest("GET", "/clusters/"+c.Username, nil)
	if err != nil {
		return nil, err
	}

	err = json.NewDecoder(resp.Body).Decode(&clusters)
	if err != nil {
		return nil, err
	}
	return clusters, nil
}

func clusterFromResponse(resp *http.Response, err error) (*Cluster, error) {
	if err != nil {
		return nil, err
	}
	cluster := new(Cluster)
	err = json.NewDecoder(resp.Body).Decode(&cluster)
	if err != nil {
		return nil, err
	}
	return cluster, nil
}

// Get a cluster by cluster name
func (c *ClusterClient) Get(clusterName string) (*Cluster, error) {
	uri := path.Join("/clusters", c.Username, clusterName)
	resp, err := c.NewRequest("GET", uri, nil)
	return clusterFromResponse(resp, err)
}

// Create a new cluster with cluster options
func (c *ClusterClient) Create(clusterOpts Cluster) (*Cluster, error) {
	// Even though username is in the URI path, the API expects the username
	// inside the body
	if clusterOpts.Username == "" {
		clusterOpts.Username = c.Username
	}
	clusterOptsJSON, err := json.Marshal(clusterOpts)
	if err != nil {
		return nil, err
	}

	body := bytes.NewReader(clusterOptsJSON)
	uri := path.Join("/clusters", c.Username)
	resp, err := c.NewRequest("POST", uri, body)
	return clusterFromResponse(resp, err)
}

// GetZipURL returns the URL for downloading credentials
func (c *ClusterClient) GetZipURL(clusterName string) (string, error) {
	uri := path.Join("/clusters", c.Username, clusterName, "zip")
	resp, err := c.NewRequest("GET", uri, nil)
	if err != nil {
		return "", err
	}

	var zipURLResp ZipURLResponse

	err = json.NewDecoder(resp.Body).Decode(&zipURLResp)

	if err != nil {
		return "", err
	}

	return zipURLResp.URL, nil
}

// GetCredentials returns a Credentials struct for the given cluster name
func (c *ClusterClient) GetCredentials(clusterName string) (*Credentials, error) {
	url, err := c.GetZipURL(clusterName)
	if err != nil {
		return nil, err
	}
	zr, err := fetchZip(url)
	if err != nil || len(zr.File) < 6 {
		return nil, err
	}

	// fetch the contents for each credential/note
	creds := new(Credentials)
	creds.Files = make(map[string][]byte)
	for _, zf := range zr.File {
		_, fname := path.Split(zf.Name)
		fi := zf.FileInfo()

		if fi.IsDir() {
			// Explicitly skip past directories (the UUID directory from a previous release)
			continue
		}

		rc, err := zf.Open()
		if err != nil {
			return nil, err
		}

		b, err := ioutil.ReadAll(rc)
		if err != nil {
			return nil, err
		}
		creds.Files[fname] = b

		switch fname {
		case "ca.pem":
			creds.CA = b
		case "README.md":
			creds.README = b
		case "ca-key.pem":
			creds.CAKey = b
		case "docker.env":
			creds.DockerEnv = b
		case "cert.pem":
			creds.Cert = b
		case "key.pem":
			creds.Key = b
		case "docker.ps1":
			creds.DockerPS1 = b
		case "docker.cmd":
			creds.DockerCmd = b
		}

	}

	sourceLines := strings.Split(string(creds.DockerEnv), "\n")
	for _, line := range sourceLines {
		if strings.Index(line, "export ") == 0 {
			varDecl := strings.TrimRight(line[7:], "\n")
			eqLocation := strings.Index(varDecl, "=")

			varName := varDecl[:eqLocation]
			varValue := varDecl[eqLocation+1:]

			switch varName {
			case "DOCKER_HOST":
				creds.DockerHost = varValue
			}

		}
	}

	return creds, nil
}

// GetDockerConfig returns the hostname and tls.Config for a given clustername
func (c *ClusterClient) GetDockerConfig(clusterName string) (hostname string, tlsConfig *tls.Config, err error) {
	creds, err := c.GetCredentials(clusterName)
	if err != nil {
		return "", nil, err
	}
	tlsConfig, err = creds.GetTLSConfig()
	return creds.DockerHost, tlsConfig, err
}

// GetTLSConfig returns a tls.Config for a credential set
func (creds *Credentials) GetTLSConfig() (*tls.Config, error) {
	// TLS config
	var tlsConfig tls.Config
	tlsConfig.InsecureSkipVerify = true
	certPool := x509.NewCertPool()

	certPool.AppendCertsFromPEM(creds.CA)
	tlsConfig.RootCAs = certPool
	keypair, err := tls.X509KeyPair(creds.Cert, creds.Key)
	if err != nil {
		return &tlsConfig, err
	}
	tlsConfig.Certificates = []tls.Certificate{keypair}

	return &tlsConfig, nil
}

func fetchZip(zipurl string) (*zip.Reader, error) {
	req, err := http.NewRequest("GET", zipurl, nil)
	if err != nil {
		return nil, err
	}

	req.Header.Set("User-Agent", userAgent)

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode >= 400 {
		b, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, errors.New(resp.Status)
		}
		return nil, errors.New(string(b))
	}

	buf := &bytes.Buffer{}

	_, err = io.Copy(buf, resp.Body)
	if err != nil {
		return nil, err
	}

	b := bytes.NewReader(buf.Bytes())
	return zip.NewReader(b, int64(b.Len()))
}

// Grow increases a cluster by the provided number of nodes
func (c *ClusterClient) Grow(clusterName string, nodes int) (*Cluster, error) {
	incr := map[string]int{
		"nodes": nodes,
	}

	growthRequest, err := json.Marshal(incr)
	if err != nil {
		return nil, err
	}
	r := bytes.NewReader(growthRequest)

	uri := path.Join("/clusters", c.Username, clusterName, "grow")
	resp, err := c.NewRequest("POST", uri, r)
	return clusterFromResponse(resp, err)
}

const rebuildSwarmAction = "rebuild-swarm"

type actionRequest struct {
	Action string `json:"action"`
}

func (c *ClusterClient) doAction(clusterName, action string) (*Cluster, error) {
	actionReq, err := json.Marshal(actionRequest{Action: action})
	if err != nil {
		return nil, err
	}
	r := bytes.NewReader(actionReq)
	uri := path.Join("/clusters", c.Username, clusterName, "action")
	resp, err := c.NewRequest("POST", uri, r)
	return clusterFromResponse(resp, err)
}

// Rebuild creates a wholly new Swarm cluster
func (c *ClusterClient) Rebuild(clusterName string) (*Cluster, error) {
	return c.doAction(clusterName, rebuildSwarmAction)
}

// Delete nukes a cluster out of existence
func (c *ClusterClient) Delete(clusterName string) (*Cluster, error) {
	uri := path.Join("/clusters", c.Username, clusterName)
	resp, err := c.NewRequest("DELETE", uri, nil)
	return clusterFromResponse(resp, err)
}
<commit_msg>Expose the http.Client<commit_after>package libcarina

import (
	"archive/zip"
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"path"
	"reflect"
	"strconv"
	"strings"

	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/rackspace"
)

// BetaEndpoint reflects the default endpoint for this library
const BetaEndpoint = "https://app.getcarina.com"
const mimetypeJSON = "application/json"
const authHeaderKey = "X-Auth-Token"
const userAgent = "rackerlabs/libcarina"

// ZipURLResponse is the response that comes back from the zip endpoint
type ZipURLResponse struct {
	URL string `json:"zip_url"`
}

// ClusterClient accesses Carina directly
type ClusterClient struct {
	Client   *http.Client
	Username string
	Token    string
	Endpoint string
}

// ErrorResponse is the JSON formatted error response from Carina
type ErrorResponse struct {
	Error string `json:"error"`
}

// Cluster is a cluster
type Cluster struct {
	ClusterName string `json:"cluster_name"`
	Username    string `json:"username"`

	// Flavor of compute to use for cluster, should be a default value currently
	Flavor string `json:"flavor,omitempty"`

	// UUID of image to use for cluster, should be a default value currently
	Image string `json:"image,omitempty"`

	// Node is optional, but allowed on create
	// Sadly it comes back as string instead of int in all cases
	// with the API
	Nodes Number `json:"nodes,omitempty"`

	AutoScale bool   `json:"autoscale,omitempty"`
	Status    string `json:"status,omitempty"`
	TaskID    string `json:"task_id,omitempty"`
	Token     string `json:"token,omitempty"`
}

// Credentials holds the keys to the kingdom
type Credentials struct {
	README     []byte
	Cert       []byte
	Key        []byte
	CA         []byte
	CAKey      []byte
	DockerEnv  []byte
	DockerCmd  []byte
	DockerPS1  []byte
	DockerHost string
	Files      map[string][]byte
}

// Number - specify this type for any struct fields that
// might be unmarshaled from JSON numbers of the following
// types: floats, integers, scientific notation, or strings
type Number float64

// Int64 return the Int64 version of this
func (n Number) Int64() int64 {
	return int64(n)
}

// Int return the Int version of this
func (n Number) Int() int {
	return int(n)
}

// Float64 return the Float64 version of this
func (n Number) Float64() float64 {
	return float64(n)
}

// UnmarshalJSON required to enforce that string values are attempted to be parsed as numbers
func (n *Number) UnmarshalJSON(data []byte) error {
	var f float64
	var err error
	if data[0] == '"' {
		f, err = strconv.ParseFloat(string(data[1:len(data)-1]), 64)
		if err != nil {
			return &json.UnmarshalTypeError{
				Value: string(data),
				Type:  reflect.TypeOf(*n),
			}
		}
	} else {
		if err := json.Unmarshal(data, &f); err != nil {
			return &json.UnmarshalTypeError{
				Value: string(data),
				Type:  reflect.TypeOf(*n),
			}
		}
	}
	*n = Number(f)
	return nil
}

func newClusterClient(endpoint string, ao gophercloud.AuthOptions) (*ClusterClient, error) {
	provider, err := rackspace.AuthenticatedClient(ao)
	if err != nil {
		return nil, err
	}

	return &ClusterClient{
		Client:   &http.Client{},
		Username: ao.Username,
		Token:    provider.TokenID,
		Endpoint: endpoint,
	}, nil
}

// NewClusterClient create a new clusterclient by API Key
func NewClusterClient(endpoint, username, apikey string) (*ClusterClient, error) {
	ao := gophercloud.AuthOptions{
		Username:         username,
		APIKey:           apikey,
		IdentityEndpoint: rackspace.RackspaceUSIdentity,
	}

	return newClusterClient(endpoint, ao)
}

// NewRequest handles a request using auth used by Carina
func (c *ClusterClient) NewRequest(method string, uri string, body io.Reader) (*http.Response, error) {
	req, err := http.NewRequest(method, c.Endpoint+uri, body)
	if err != nil {
		return nil, err
	}
	req.Header.Set("User-Agent", userAgent)
	req.Header.Add("Content-Type", mimetypeJSON)
	req.Header.Add("Accept", mimetypeJSON)
	req.Header.Add(authHeaderKey, c.Token)
	resp, err := c.Client.Do(req)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode >= 400 {
		b, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, errors.New(resp.Status)
		}
		return nil, errors.New(string(b))
	}

	return resp, nil
}

// List the current clusters
func (c *ClusterClient) List() ([]Cluster, error) {
	clusters := []Cluster{}

	resp, err := c.NewRequest("GET", "/clusters/"+c.Username, nil)
	if err != nil {
		return nil, err
	}

	err = json.NewDecoder(resp.Body).Decode(&clusters)
	if err != nil {
		return nil, err
	}
	return clusters, nil
}

func clusterFromResponse(resp *http.Response, err error) (*Cluster, error) {
	if err != nil {
		return nil, err
	}
	cluster := new(Cluster)
	err = json.NewDecoder(resp.Body).Decode(&cluster)
	if err != nil {
		return nil, err
	}
	return cluster, nil
}

// Get a cluster by cluster name
func (c *ClusterClient) Get(clusterName string) (*Cluster, error) {
	uri := path.Join("/clusters", c.Username, clusterName)
	resp, err := c.NewRequest("GET", uri, nil)
	return clusterFromResponse(resp, err)
}

// Create a new cluster with cluster options
func (c *ClusterClient) Create(clusterOpts Cluster) (*Cluster, error) {
	// Even though username is in the URI path, the API expects the username
	// inside the body
	if clusterOpts.Username == "" {
		clusterOpts.Username = c.Username
	}
	clusterOptsJSON, err := json.Marshal(clusterOpts)
	if err != nil {
		return nil, err
	}

	body := bytes.NewReader(clusterOptsJSON)
	uri := path.Join("/clusters", c.Username)
	resp, err := c.NewRequest("POST", uri, body)
	return clusterFromResponse(resp, err)
}

// GetZipURL returns the URL for downloading credentials
func (c *ClusterClient) GetZipURL(clusterName string) (string, error) {
	uri := path.Join("/clusters", c.Username, clusterName, "zip")
	resp, err := c.NewRequest("GET", uri, nil)
	if err != nil {
		return "", err
	}

	var zipURLResp ZipURLResponse

	err = json.NewDecoder(resp.Body).Decode(&zipURLResp)

	if err != nil {
		return "", err
	}

	return zipURLResp.URL, nil
}

// GetCredentials returns a Credentials struct for the given cluster name
func (c *ClusterClient) GetCredentials(clusterName string) (*Credentials, error) {
	url, err := c.GetZipURL(clusterName)
	if err != nil {
		return nil, err
	}
	zr, err := fetchZip(url)
	if err != nil || len(zr.File) < 6 {
		return nil, err
	}

	// fetch the contents for each credential/note
	creds := new(Credentials)
	creds.Files = make(map[string][]byte)
	for _, zf := range zr.File {
		_, fname := path.Split(zf.Name)
		fi := zf.FileInfo()

		if fi.IsDir() {
			// Explicitly skip past directories (the UUID directory from a previous release)
			continue
		}

		rc, err := zf.Open()
		if err != nil {
			return nil, err
		}

		b, err := ioutil.ReadAll(rc)
		if err != nil {
			return nil, err
		}
		creds.Files[fname] = b

		switch fname {
		case "ca.pem":
			creds.CA = b
		case "README.md":
			creds.README = b
		case "ca-key.pem":
			creds.CAKey = b
		case "docker.env":
			creds.DockerEnv = b
		case "cert.pem":
			creds.Cert = b
		case "key.pem":
			creds.Key = b
		case "docker.ps1":
			creds.DockerPS1 = b
		case "docker.cmd":
			creds.DockerCmd = b
		}

	}

	sourceLines := strings.Split(string(creds.DockerEnv), "\n")
	for _, line := range sourceLines {
		if strings.Index(line, "export ") == 0 {
			varDecl := strings.TrimRight(line[7:], "\n")
			eqLocation := strings.Index(varDecl, "=")

			varName := varDecl[:eqLocation]
			varValue := varDecl[eqLocation+1:]

			switch varName {
			case "DOCKER_HOST":
				creds.DockerHost = varValue
			}

		}
	}

	return creds, nil
}

// GetDockerConfig returns the hostname and tls.Config for a given clustername
func (c *ClusterClient) GetDockerConfig(clusterName string) (hostname string, tlsConfig *tls.Config, err error) {
	creds, err := c.GetCredentials(clusterName)
	if err != nil {
		return "", nil, err
	}
	tlsConfig, err = creds.GetTLSConfig()
	return creds.DockerHost, tlsConfig, err
}

// GetTLSConfig returns a tls.Config for a credential set
func (creds *Credentials) GetTLSConfig() (*tls.Config, error) {
	// TLS config
	var tlsConfig tls.Config
	tlsConfig.InsecureSkipVerify = true
	certPool := x509.NewCertPool()

	certPool.AppendCertsFromPEM(creds.CA)
	tlsConfig.RootCAs = certPool
	keypair, err := tls.X509KeyPair(creds.Cert, creds.Key)
	if err != nil {
		return &tlsConfig, err
	}
	tlsConfig.Certificates = []tls.Certificate{keypair}

	return &tlsConfig, nil
}

func fetchZip(zipurl string) (*zip.Reader, error) {
	req, err := http.NewRequest("GET", zipurl, nil)
	if err != nil {
		return nil, err
	}

	req.Header.Set("User-Agent", userAgent)

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode >= 400 {
		b, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, errors.New(resp.Status)
		}
		return nil, errors.New(string(b))
	}

	buf := &bytes.Buffer{}

	_, err = io.Copy(buf, resp.Body)
	if err != nil {
		return nil, err
	}

	b := bytes.NewReader(buf.Bytes())
	return zip.NewReader(b, int64(b.Len()))
}

// Grow increases a cluster by the provided number of nodes
func (c *ClusterClient) Grow(clusterName string, nodes int) (*Cluster, error) {
	incr := map[string]int{
		"nodes": nodes,
	}

	growthRequest, err := json.Marshal(incr)
	if err != nil {
		return nil, err
	}
	r := bytes.NewReader(growthRequest)

	uri := path.Join("/clusters", c.Username, clusterName, "grow")
	resp, err := c.NewRequest("POST", uri, r)
	return clusterFromResponse(resp, err)
}

const rebuildSwarmAction = "rebuild-swarm"

type actionRequest struct {
	Action string `json:"action"`
}

func (c *ClusterClient) doAction(clusterName, action string) (*Cluster, error) {
	actionReq, err := json.Marshal(actionRequest{Action: action})
	if err != nil {
		return nil, err
	}
	r := bytes.NewReader(actionReq)
	uri := path.Join("/clusters", c.Username, clusterName, "action")
	resp, err := c.NewRequest("POST", uri, r)
	return clusterFromResponse(resp, err)
}

// Rebuild creates a wholly new Swarm cluster
func (c *ClusterClient) Rebuild(clusterName string) (*Cluster, error) {
	return c.doAction(clusterName, rebuildSwarmAction)
}

// Delete nukes a cluster out of existence
func (c *ClusterClient) Delete(clusterName string) (*Cluster, error) {
	uri := path.Join("/clusters", c.Username, clusterName)
	resp, err := c.NewRequest("DELETE", uri, nil)
	return clusterFromResponse(resp, err)
}
<|endoftext|>


<commit_before>package main

import (
	"compress/gzip"
	"context"
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	"strings"
	"time"

	"github.com/google/subcommands"
	"github.com/pkg/profile"
	"github.com/uluyol/fabbench/bench"
	"github.com/uluyol/fabbench/internal/ranges"
	"github.com/uluyol/fabbench/recorders"
	"github.com/uluyol/hdrhist"
)

const formatsDoc = `
CONFIG FORMAT
	The config file is a simple JSON file with the following schema:
		{
			"db": {
				"name": NAME_STR,
				"options": DB_SPECIFIC_OPTIONS
			},
			"workload": {
				"recordCount": RECORDS_INT,
				"keySize": KEY_BYTES_INT,
				"valSize": VAL_BYTES_INT,
			}
		}

	Because fabbench is meant to handle multiple databases, db.options is
	db-specific.

	Currently, fabbench supports the following databases:
		eckv:      ECKV
		dummy:     A dummy database useful for testing
		cassandra: Apache Cassandra via gocql

	For eckv, there is no schema for db.options. Use {}

	For dummy, the schema for db.options is
		{
			"maxQPS": INT // optional
		}

	For cassandra, the schema for db.options is
		{
			"clientRetries":        INT,    // optional
			"numRetries":           INT,    // optional
			"numConns":             INT,    // optional
			"readConsistency":      STRING, // default: ONE
			"writeConsistency":     STRING, // default: ONE
			"keyspace":             STRING, // optional
			"table":                STRING, // optional
			"replicationFactor":    INT,
			"keyCaching":           STRING,
			"compactionStrategy":   STRING,
			"leveledSSTableSizeMB": INT,    // default: 160
			"connectTimeout":       STRING, // default: 5s, time.Duration
			"timeout":              STRING, // default: 600ms, time.Duration

			"traceData": STRING, // path to store traces
			"traceRate": INT     // freq for tracing i.e. every N requests
		}

TRACE FORMAT
	The trace consists of a series of lines with property values set.
	Each line is a step of the trace and inherits values in previous lines.
	Empty lines are ignored.

	The properties are listed below:
		d		duration of trace step (e.g. 5m3s)
		rw		frac of requests that are reads (e.g. 0.8 -> 80% are reads)
		qps		avg requests per second during trace step
		ad		request interarrival distribution
		rkd		key distribution for reads
		wkd		key distribution for writes

	Valid values for these properties are below
		d		any valid time.Duration in Go
		rw		any float in [0, 1]
		qps		any non-negative integer
		ad		poisson:    poisson dist with avg qps
				closed-N:   closed-loop workload of qps*d ops with N workers
				uniform-W:  uniform dist with vals in [avg-avg*W, avg+avg*W]
		rkd		zipfian-θ:  zipfian with param of θ in (0, 1)
				linstep-K:  PDF linearly dec in K steps
				linear:     linearly dec PDF
				uniform:    uniform
		wkd		same options as rkd

	For example, a valid trace line might be
		d=10m rw=0.5 qps=500 ad=poisson rkd=zipfian-0.99999 wkd=uniform
`

type nopStop struct{}

func (nopStop) Stop() {}

type baseFlags struct {
	profPath string
	prof     string
}

func (f *baseFlags) setupProfiling() interface {
	Stop()
} {
	if f.profPath != "" {
		opts := []func(*profile.Profile){profile.ProfilePath(f.profPath)}
		switch f.prof {
		case "cpu":
			opts = append(opts, profile.CPUProfile)
		case "mutex":
			opts = append(opts, profile.MutexProfile)
		case "block":
			opts = append(opts, profile.BlockProfile)
		default:
			// ignore
		}
		return profile.Start(opts...)
	}
	return nopStop{}
}

func (f *baseFlags) SetFlags(fs *flag.FlagSet) {
	fs.StringVar(&f.profPath, "profiledir", "", "turn profiling on and write profiles to this directory")
	fs.StringVar(&f.prof, "profile", "cpu", "resource to profile (possible values: cpu, mutex, block)")
}

type formatsCmd struct{}

func (formatsCmd) Name() string           { return "formats" }
func (formatsCmd) Synopsis() string       { return "describes the config and trace formats" }
func (formatsCmd) Usage() string          { return "" }
func (formatsCmd) SetFlags(*flag.FlagSet) {}

func (formatsCmd) Execute(_ context.Context, _ *flag.FlagSet, args ...interface{}) subcommands.ExitStatus {
	fmt.Fprintln(os.Stderr, formatsDoc)
	return subcommands.ExitSuccess
}

type mkTableCmd struct {
	configPath string
	hostsCSV   string
	baseFlags
}

func (*mkTableCmd) Name() string     { return "mktable" }
func (*mkTableCmd) Synopsis() string { return "create the database table" }
func (*mkTableCmd) Usage() string    { return "" }

func (c *mkTableCmd) SetFlags(fs *flag.FlagSet) {
	fs.StringVar(&c.configPath, "config", "", "config file path")
	fs.StringVar(&c.hostsCSV, "hosts", "", "host addresses (comma separated)")
	c.baseFlags.SetFlags(fs)
}

func (c *mkTableCmd) Execute(ctx context.Context, fs *flag.FlagSet, args ...interface{}) subcommands.ExitStatus {
	defer c.setupProfiling().Stop()
	hosts := strings.Split(c.hostsCSV, ",")
	db, _, err := loadConfig(hosts, c.configPath)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()
	if err := db.Init(ctx); err != nil {
		log.Fatal(err)
	}

	return subcommands.ExitSuccess
}

type loadCmd struct {
	configPath  string
	hostsCSV    string
	workers     int
	maxFailFrac float64

	loadStart int64
	loadCount int64

	nshard int64
	shardi int64

	baseFlags
}

func (*loadCmd) Name() string     { return "load" }
func (*loadCmd) Synopsis() string { return "load data into the database" }
func (*loadCmd) Usage() string {
	return `fabbench load populates the database.

For loading from multiple processes in parallel (potentially across machines),
use either the -start and -count flags, or the -nshard and -shardi flags.
If both pairs of flags are passed, one set will be used arbitrarily.

`
}

func (c *loadCmd) SetFlags(fs *flag.FlagSet) {
	fs.StringVar(&c.configPath, "config", "", "config file path")
	fs.StringVar(&c.hostsCSV, "hosts", "", "host addresses (comma separated)")
	fs.IntVar(&c.workers, "workers", 20, "number of concurrent workers")
	fs.Float64Var(&c.maxFailFrac, "maxfailfrac", 0.01, "fraction of records that are allowed to fail to load")

	fs.Int64Var(&c.loadStart, "start", 0, "index to start loading from (use either start+count, or nshard+shardi)")
	fs.Int64Var(&c.loadCount, "count", -1, "number of records to load (use either start+count, or nshard+shardi)")

	fs.Int64Var(&c.nshard, "nshard", 0, "number of parallel worker processes (use either start+count, or nshard+shardi)")
	fs.Int64Var(&c.shardi, "shardi", 0, "parallel worker process index (use either start+count, or nshard+shardi)")
	c.baseFlags.SetFlags(fs)
}

func (c *loadCmd) Execute(ctx context.Context, fs *flag.FlagSet, args ...interface{}) subcommands.ExitStatus {
	defer c.setupProfiling().Stop()
	hosts := strings.Split(c.hostsCSV, ",")
	db, bcfg, err := loadConfig(hosts, c.configPath)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	loadStart := c.loadStart
	loadCount := c.loadCount
	if c.nshard > 0 {
		shards := ranges.SplitRecords(bcfg.RecordCount, c.nshard)
		if c.shardi < 0 || c.shardi > int64(len(shards)) {
			log.Fatalf("invalid value for -shardi: %d, number of shards: %d", c.shardi, len(shards))
		}
		loadStart = shards[c.shardi].Start
		loadCount = shards[c.shardi].Count
	}

	l := bench.Loader{
		Log:             log.New(os.Stderr, "fabbench: load: ", log.LstdFlags),
		DB:              db,
		Config:          *bcfg,
		Rand:            rand.New(rand.NewSource(rand.Int63())),
		NumWorkers:      c.workers,
		AllowedFailFrac: c.maxFailFrac,
		LoadStart:       loadStart,
		LoadCount:       loadCount,
	}

	if err := l.Run(ctx); err != nil {
		log.Fatal(err)
	}

	return subcommands.ExitSuccess
}

type runCmd struct {
	configPath string
	tracePath  string
	outPre     string
	hostsCSV   string
	baseFlags
}

func (*runCmd) Name() string     { return "run" }
func (*runCmd) Synopsis() string { return "run the workload" }
func (*runCmd) Usage() string    { return "" }

func (c *runCmd) SetFlags(fs *flag.FlagSet) {
	fs.StringVar(&c.configPath, "config", "", "config file path")
	fs.StringVar(&c.tracePath, "trace", "", "trace file path")
	fs.StringVar(&c.outPre, "out", "", "output path prefix (will add -ro.gz and -wo.gz)")
	fs.StringVar(&c.hostsCSV, "hosts", "", "host addresses (comma separated)")
	c.baseFlags.SetFlags(fs)
}

func (c *runCmd) Execute(ctx context.Context, fs *flag.FlagSet, args ...interface{}) subcommands.ExitStatus {
	defer c.setupProfiling().Stop()
	hosts := strings.Split(c.hostsCSV, ",")
	db, bcfg, err := loadConfig(hosts, c.configPath)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	trace, err := loadTrace(c.tracePath)
	if err != nil {
		log.Fatalf("unable to load trace: %v", err)
	}

	benchStart := time.Now()
	readw := recorders.NewMultiLogWriter(c.outPre+"-ro", benchStart, gzip.BestSpeed)
	writew := recorders.NewMultiLogWriter(c.outPre+"-wo", benchStart, gzip.BestSpeed)

	hdrCfg := hdrhist.Config{
		LowestDiscernible: int64(10 * time.Microsecond),
		HighestTrackable:  int64(100 * time.Second),
		SigFigs:           3,
		AutoResize:        true,
	}

	traceDescs := make([]string, len(trace))
	for i := range trace {
		traceDescs[i] = trace[i].String()
	}

	readRec := recorders.NewMultiLatency(hdrCfg, traceDescs)
	writeRec := recorders.NewMultiLatency(hdrCfg, traceDescs)

	r := bench.Runner{
		Log:           log.New(os.Stderr, "fabbench: run: ", log.LstdFlags),
		DB:            db,
		Config:        *bcfg,
		Rand:          rand.New(rand.NewSource(rand.Int63())),
		Trace:         trace,
		ReadRecorder:  readRec,
		ReadWriter:    readw,
		WriteRecorder: writeRec,
		WriteWriter:   writew,
	}

	if err := r.Run(ctx); err != nil {
		log.Print(err)
		return subcommands.ExitFailure
	}

	return subcommands.ExitSuccess
}

func loadTrace(p string) ([]bench.TraceStep, error) {
	f, err := os.Open(p)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	t, err := bench.ParseTrace(f)
	if err != nil {
		return nil, fmt.Errorf("%s:%v", p, err)
	}

	return t, nil
}

func main() {
	log.SetPrefix("fabbench: ")
	log.SetFlags(0)
	subcommands.Register(subcommands.HelpCommand(), "")
	subcommands.Register(subcommands.FlagsCommand(), "")
	subcommands.Register(subcommands.CommandsCommand(), "")
	subcommands.Register(formatsCmd{}, "")
	subcommands.Register(new(mkTableCmd), "")
	subcommands.Register(new(loadCmd), "")
	subcommands.Register(new(runCmd), "")

	flag.Parse()
	os.Exit(int(subcommands.Execute(context.Background())))
}
<commit_msg>cmd/fabbench: support non-deterministic execution<commit_after>package main

import (
	"compress/gzip"
	"context"
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	"strings"
	"time"

	"github.com/google/subcommands"
	"github.com/pkg/profile"
	"github.com/uluyol/fabbench/bench"
	"github.com/uluyol/fabbench/internal/ranges"
	"github.com/uluyol/fabbench/recorders"
	"github.com/uluyol/hdrhist"
)

const formatsDoc = `
CONFIG FORMAT
	The config file is a simple JSON file with the following schema:
		{
			"db": {
				"name": NAME_STR,
				"options": DB_SPECIFIC_OPTIONS
			},
			"workload": {
				"recordCount": RECORDS_INT,
				"keySize": KEY_BYTES_INT,
				"valSize": VAL_BYTES_INT,
			}
		}

	Because fabbench is meant to handle multiple databases, db.options is
	db-specific.

	Currently, fabbench supports the following databases:
		eckv:      ECKV
		dummy:     A dummy database useful for testing
		cassandra: Apache Cassandra via gocql

	For eckv, there is no schema for db.options. Use {}

	For dummy, the schema for db.options is
		{
			"maxQPS": INT // optional
		}

	For cassandra, the schema for db.options is
		{
			"clientRetries":        INT,    // optional
			"numRetries":           INT,    // optional
			"numConns":             INT,    // optional
			"readConsistency":      STRING, // default: ONE
			"writeConsistency":     STRING, // default: ONE
			"keyspace":             STRING, // optional
			"table":                STRING, // optional
			"replicationFactor":    INT,
			"keyCaching":           STRING,
			"compactionStrategy":   STRING,
			"leveledSSTableSizeMB": INT,    // default: 160
			"connectTimeout":       STRING, // default: 5s, time.Duration
			"timeout":              STRING, // default: 600ms, time.Duration

			"traceData": STRING, // path to store traces
			"traceRate": INT     // freq for tracing i.e. every N requests
		}

TRACE FORMAT
	The trace consists of a series of lines with property values set.
	Each line is a step of the trace and inherits values in previous lines.
	Empty lines are ignored.

	The properties are listed below:
		d		duration of trace step (e.g. 5m3s)
		rw		frac of requests that are reads (e.g. 0.8 -> 80% are reads)
		qps		avg requests per second during trace step
		ad		request interarrival distribution
		rkd		key distribution for reads
		wkd		key distribution for writes

	Valid values for these properties are below
		d		any valid time.Duration in Go
		rw		any float in [0, 1]
		qps		any non-negative integer
		ad		poisson:    poisson dist with avg qps
				closed-N:   closed-loop workload of qps*d ops with N workers
				uniform-W:  uniform dist with vals in [avg-avg*W, avg+avg*W]
		rkd		zipfian-θ:  zipfian with param of θ in (0, 1)
				linstep-K:  PDF linearly dec in K steps
				linear:     linearly dec PDF
				uniform:    uniform
		wkd		same options as rkd

	For example, a valid trace line might be
		d=10m rw=0.5 qps=500 ad=poisson rkd=zipfian-0.99999 wkd=uniform
`

type nopStop struct{}

func (nopStop) Stop() {}

type baseFlags struct {
	profPath string
	prof     string
}

func (f *baseFlags) setupProfiling() interface {
	Stop()
} {
	if f.profPath != "" {
		opts := []func(*profile.Profile){profile.ProfilePath(f.profPath)}
		switch f.prof {
		case "cpu":
			opts = append(opts, profile.CPUProfile)
		case "mutex":
			opts = append(opts, profile.MutexProfile)
		case "block":
			opts = append(opts, profile.BlockProfile)
		default:
			// ignore
		}
		return profile.Start(opts...)
	}
	return nopStop{}
}

func (f *baseFlags) SetFlags(fs *flag.FlagSet) {
	fs.StringVar(&f.profPath, "profiledir", "", "turn profiling on and write profiles to this directory")
	fs.StringVar(&f.prof, "profile", "cpu", "resource to profile (possible values: cpu, mutex, block)")
}

type formatsCmd struct{}

func (formatsCmd) Name() string           { return "formats" }
func (formatsCmd) Synopsis() string       { return "describes the config and trace formats" }
func (formatsCmd) Usage() string          { return "" }
func (formatsCmd) SetFlags(*flag.FlagSet) {}

func (formatsCmd) Execute(_ context.Context, _ *flag.FlagSet, args ...interface{}) subcommands.ExitStatus {
	fmt.Fprintln(os.Stderr, formatsDoc)
	return subcommands.ExitSuccess
}

type mkTableCmd struct {
	configPath string
	hostsCSV   string
	baseFlags
}

func (*mkTableCmd) Name() string     { return "mktable" }
func (*mkTableCmd) Synopsis() string { return "create the database table" }
func (*mkTableCmd) Usage() string    { return "" }

func (c *mkTableCmd) SetFlags(fs *flag.FlagSet) {
	fs.StringVar(&c.configPath, "config", "", "config file path")
	fs.StringVar(&c.hostsCSV, "hosts", "", "host addresses (comma separated)")
	c.baseFlags.SetFlags(fs)
}

func (c *mkTableCmd) Execute(ctx context.Context, fs *flag.FlagSet, args ...interface{}) subcommands.ExitStatus {
	defer c.setupProfiling().Stop()
	hosts := strings.Split(c.hostsCSV, ",")
	db, _, err := loadConfig(hosts, c.configPath)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()
	if err := db.Init(ctx); err != nil {
		log.Fatal(err)
	}

	return subcommands.ExitSuccess
}

type loadCmd struct {
	configPath    string
	hostsCSV      string
	workers       int
	maxFailFrac   float64
	randSeedIndex int64

	loadStart int64
	loadCount int64

	nshard int64
	shardi int64

	baseFlags
}

func (*loadCmd) Name() string     { return "load" }
func (*loadCmd) Synopsis() string { return "load data into the database" }
func (*loadCmd) Usage() string {
	return `fabbench load populates the database.

For loading from multiple processes in parallel (potentially across machines),
use either the -start and -count flags, or the -nshard and -shardi flags.
If both pairs of flags are passed, one set will be used arbitrarily.

`
}

func (c *loadCmd) SetFlags(fs *flag.FlagSet) {
	fs.StringVar(&c.configPath, "config", "", "config file path")
	fs.StringVar(&c.hostsCSV, "hosts", "", "host addresses (comma separated)")
	fs.IntVar(&c.workers, "workers", 20, "number of concurrent workers")
	fs.Float64Var(&c.maxFailFrac, "maxfailfrac", 0.01, "fraction of records that are allowed to fail to load")
	fs.Int64Var(&c.randSeedIndex, "rand-seed-index", 0, "partial see to initialize rng (deterministic if 0)")

	fs.Int64Var(&c.loadStart, "start", 0, "index to start loading from (use either start+count, or nshard+shardi)")
	fs.Int64Var(&c.loadCount, "count", -1, "number of records to load (use either start+count, or nshard+shardi)")

	fs.Int64Var(&c.nshard, "nshard", 0, "number of parallel worker processes (use either start+count, or nshard+shardi)")
	fs.Int64Var(&c.shardi, "shardi", 0, "parallel worker process index (use either start+count, or nshard+shardi)")
	c.baseFlags.SetFlags(fs)
}

func (c *loadCmd) Execute(ctx context.Context, fs *flag.FlagSet, args ...interface{}) subcommands.ExitStatus {
	defer c.setupProfiling().Stop()
	hosts := strings.Split(c.hostsCSV, ",")
	db, bcfg, err := loadConfig(hosts, c.configPath)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	loadStart := c.loadStart
	loadCount := c.loadCount
	if c.nshard > 0 {
		shards := ranges.SplitRecords(bcfg.RecordCount, c.nshard)
		if c.shardi < 0 || c.shardi > int64(len(shards)) {
			log.Fatalf("invalid value for -shardi: %d, number of shards: %d", c.shardi, len(shards))
		}
		loadStart = shards[c.shardi].Start
		loadCount = shards[c.shardi].Count
	}

	var seed int64
	if c.randSeedIndex > 0 {
		seed = time.Now().UnixNano() ^ c.randSeedIndex
	}

	l := bench.Loader{
		Log:             log.New(os.Stderr, "fabbench: load: ", log.LstdFlags),
		DB:              db,
		Config:          *bcfg,
		Rand:            rand.New(rand.NewSource(seed)),
		NumWorkers:      c.workers,
		AllowedFailFrac: c.maxFailFrac,
		LoadStart:       loadStart,
		LoadCount:       loadCount,
	}

	if err := l.Run(ctx); err != nil {
		log.Fatal(err)
	}

	return subcommands.ExitSuccess
}

type runCmd struct {
	configPath    string
	tracePath     string
	outPre        string
	hostsCSV      string
	randSeedIndex int64
	baseFlags
}

func (*runCmd) Name() string     { return "run" }
func (*runCmd) Synopsis() string { return "run the workload" }
func (*runCmd) Usage() string    { return "" }

func (c *runCmd) SetFlags(fs *flag.FlagSet) {
	fs.StringVar(&c.configPath, "config", "", "config file path")
	fs.StringVar(&c.tracePath, "trace", "", "trace file path")
	fs.StringVar(&c.outPre, "out", "", "output path prefix (will add -ro.gz and -wo.gz)")
	fs.StringVar(&c.hostsCSV, "hosts", "", "host addresses (comma separated)")
	fs.Int64Var(&c.randSeedIndex, "rand-seed-index", 0, "partial see to initialize rng (deterministic if 0)")
	c.baseFlags.SetFlags(fs)
}

func (c *runCmd) Execute(ctx context.Context, fs *flag.FlagSet, args ...interface{}) subcommands.ExitStatus {
	defer c.setupProfiling().Stop()
	hosts := strings.Split(c.hostsCSV, ",")
	db, bcfg, err := loadConfig(hosts, c.configPath)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	trace, err := loadTrace(c.tracePath)
	if err != nil {
		log.Fatalf("unable to load trace: %v", err)
	}

	benchStart := time.Now()
	readw := recorders.NewMultiLogWriter(c.outPre+"-ro", benchStart, gzip.BestSpeed)
	writew := recorders.NewMultiLogWriter(c.outPre+"-wo", benchStart, gzip.BestSpeed)

	hdrCfg := hdrhist.Config{
		LowestDiscernible: int64(10 * time.Microsecond),
		HighestTrackable:  int64(100 * time.Second),
		SigFigs:           3,
		AutoResize:        true,
	}

	traceDescs := make([]string, len(trace))
	for i := range trace {
		traceDescs[i] = trace[i].String()
	}

	readRec := recorders.NewMultiLatency(hdrCfg, traceDescs)
	writeRec := recorders.NewMultiLatency(hdrCfg, traceDescs)

	var seed int64
	if c.randSeedIndex > 0 {
		seed = time.Now().UnixNano() ^ c.randSeedIndex
	}

	r := bench.Runner{
		Log:           log.New(os.Stderr, "fabbench: run: ", log.LstdFlags),
		DB:            db,
		Config:        *bcfg,
		Rand:          rand.New(rand.NewSource(seed)),
		Trace:         trace,
		ReadRecorder:  readRec,
		ReadWriter:    readw,
		WriteRecorder: writeRec,
		WriteWriter:   writew,
	}

	if err := r.Run(ctx); err != nil {
		log.Print(err)
		return subcommands.ExitFailure
	}

	return subcommands.ExitSuccess
}

func loadTrace(p string) ([]bench.TraceStep, error) {
	f, err := os.Open(p)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	t, err := bench.ParseTrace(f)
	if err != nil {
		return nil, fmt.Errorf("%s:%v", p, err)
	}

	return t, nil
}

func main() {
	log.SetPrefix("fabbench: ")
	log.SetFlags(0)
	subcommands.Register(subcommands.HelpCommand(), "")
	subcommands.Register(subcommands.FlagsCommand(), "")
	subcommands.Register(subcommands.CommandsCommand(), "")
	subcommands.Register(formatsCmd{}, "")
	subcommands.Register(new(mkTableCmd), "")
	subcommands.Register(new(loadCmd), "")
	subcommands.Register(new(runCmd), "")

	flag.Parse()
	os.Exit(int(subcommands.Execute(context.Background())))
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build appengine

package main

// This file replaces main.go when running godoc under app-engine.
// See README.godoc-app for details.

import (
	"archive/zip"
	"log"
	"path"

	"code.google.com/p/go.tools/godoc"
	"code.google.com/p/go.tools/godoc/vfs"
	"code.google.com/p/go.tools/godoc/vfs/zipfs"
)

func init() {
	log.Println("initializing godoc ...")
	log.Printf(".zip file   = %s", zipFilename)
	log.Printf(".zip GOROOT = %s", zipGoroot)
	log.Printf("index files = %s", indexFilenames)

	goroot := path.Join("/", zipGoroot) // fsHttp paths are relative to '/'

	// read .zip file and set up file systems
	const zipfile = zipFilename
	rc, err := zip.OpenReader(zipfile)
	if err != nil {
		log.Fatalf("%s: %s\n", zipfile, err)
	}
	// rc is never closed (app running forever)
	fs.Bind("/", zipfs.New(rc, zipFilename), goroot, vfs.BindReplace)

	corpus := godoc.NewCorpus(fs)
	corpus.Verbose = false
	corpus.IndexEnabled = true
	corpus.IndexFiles = indexFilenames

	if err := corpus.Init(); err != nil {
		log.Fatal(err)
	}

	pres = godoc.NewPresentation(corpus)
	pres.TabWidth = 8
	pres.ShowPlayground = true
	pres.ShowExamples = true
	pres.DeclLinks = true

	readTemplates(pres)
	registerHandlers(pres)

	log.Println("godoc initialization complete")
}
<commit_msg>go.tools/cmd/godoc: start indexer in app engine mode<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build appengine

package main

// This file replaces main.go when running godoc under app-engine.
// See README.godoc-app for details.

import (
	"archive/zip"
	"log"
	"path"

	"code.google.com/p/go.tools/godoc"
	"code.google.com/p/go.tools/godoc/vfs"
	"code.google.com/p/go.tools/godoc/vfs/zipfs"
)

func init() {
	log.Println("initializing godoc ...")
	log.Printf(".zip file   = %s", zipFilename)
	log.Printf(".zip GOROOT = %s", zipGoroot)
	log.Printf("index files = %s", indexFilenames)

	goroot := path.Join("/", zipGoroot) // fsHttp paths are relative to '/'

	// read .zip file and set up file systems
	const zipfile = zipFilename
	rc, err := zip.OpenReader(zipfile)
	if err != nil {
		log.Fatalf("%s: %s\n", zipfile, err)
	}
	// rc is never closed (app running forever)
	fs.Bind("/", zipfs.New(rc, zipFilename), goroot, vfs.BindReplace)

	corpus := godoc.NewCorpus(fs)
	corpus.Verbose = false
	corpus.IndexEnabled = true
	corpus.IndexFiles = indexFilenames
	if err := corpus.Init(); err != nil {
		log.Fatal(err)
	}
	go corups.RunIndexer()

	pres = godoc.NewPresentation(corpus)
	pres.TabWidth = 8
	pres.ShowPlayground = true
	pres.ShowExamples = true
	pres.DeclLinks = true

	readTemplates(pres)
	registerHandlers(pres)

	log.Println("godoc initialization complete")
}
<|endoftext|>


<commit_before>package main

import (
	"os"
)

func shortUsage() {
	os.Stderr.WriteString(`
Usage: monico [OPTION]... COMMAND [ARGS]...
Try 'grep --help' for more information.
`[1:])
}

func usage() {
	os.Stderr.WriteString(`
Usage: monico [OPTION]... COMMAND [ARGS]...
Run COMMAND if there is modified current directory.

Options:
	-d, --dir=PATH   cd PATH before monitering
	-h, --help       show this help message
`[1:])
}

func main() {
}
<commit_msg>Implement flag parse<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
)

func shortUsage() {
	os.Stderr.WriteString(`
Usage: monico [OPTION]... COMMAND [ARGS]...
Try 'grep --help' for more information.
`[1:])
}

func usage() {
	os.Stderr.WriteString(`
Usage: monico [OPTION]... COMMAND [ARGS]...
Run COMMAND if there is modified current directory.

Options:
	-d, --dir=PATH   cd PATH before monitering
	-h, --help       show this help message
`[1:])
}

func main() {
	var moniterPath string
	flag.StringVar(&moniterPath, "d", "", "")
	flag.StringVar(&moniterPath, "dir", "", "")

	var isHelp bool
	flag.BoolVar(&isHelp, "h", false, "")
	flag.BoolVar(&isHelp, "help", false, "")
	flag.Usage = shortUsage
	flag.Parse()
	switch {
	case isHelp:
		usage()
		os.Exit(0)
	case flag.NArg() < 1:
		fmt.Fprintln(os.Stderr, "monico:", "no specify COMMAND")
		shortUsage()
		os.Exit(2)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The prometheus-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"net"
	"net/http"
	"net/http/pprof"
	"os"
	"os/signal"
	"strings"
	"syscall"

	alertmanagercontroller "github.com/coreos/prometheus-operator/pkg/alertmanager"
	"github.com/coreos/prometheus-operator/pkg/api"
	monitoring "github.com/coreos/prometheus-operator/pkg/apis/monitoring"
	monitoringv1 "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1"
	prometheuscontroller "github.com/coreos/prometheus-operator/pkg/prometheus"
	"github.com/coreos/prometheus-operator/pkg/version"

	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"golang.org/x/sync/errgroup"
	v1 "k8s.io/api/core/v1"
	"k8s.io/klog"
)

const (
	logLevelAll   = "all"
	logLevelDebug = "debug"
	logLevelInfo  = "info"
	logLevelWarn  = "warn"
	logLevelError = "error"
	logLevelNone  = "none"
)

const (
	logFormatLogfmt = "logfmt"
	logFormatJson   = "json"
)

var (
	ns = namespaces{}
)

type namespaces map[string]struct{}

// Set implements the flagset.Value interface.
func (n namespaces) Set(value string) error {
	if n == nil {
		return errors.New("expected n of type namespaces to be initialized")
	}
	ns := strings.Split(value, ",")
	for i := range ns {
		n[ns[i]] = struct{}{}
	}
	return nil
}

// String implements the flagset.Value interface.
func (n namespaces) String() string {
	return strings.Join(n.asSlice(), ",")
}

func (n namespaces) asSlice() []string {
	var ns []string
	for k := range n {
		ns = append(ns, k)
	}
	if len(ns) == 0 {
		ns = append(ns, v1.NamespaceAll)
	}
	return ns
}

var (
	cfg                prometheuscontroller.Config
	availableLogLevels = []string{
		logLevelAll,
		logLevelDebug,
		logLevelInfo,
		logLevelWarn,
		logLevelError,
		logLevelNone,
	}
	availableLogFormats = []string{
		logFormatLogfmt,
		logFormatJson,
	}
)

func init() {
	cfg.CrdKinds = monitoringv1.DefaultCrdKinds
	flagset := flag.CommandLine
	klog.InitFlags(flagset)
	flagset.StringVar(&cfg.Host, "apiserver", "", "API Server addr, e.g. ' - NOT RECOMMENDED FOR PRODUCTION - http://127.0.0.1:8080'. Omit parameter to run in on-cluster mode and utilize the service account token.")
	flagset.StringVar(&cfg.TLSConfig.CertFile, "cert-file", "", " - NOT RECOMMENDED FOR PRODUCTION - Path to public TLS certificate file.")
	flagset.StringVar(&cfg.TLSConfig.KeyFile, "key-file", "", "- NOT RECOMMENDED FOR PRODUCTION - Path to private TLS certificate file.")
	flagset.StringVar(&cfg.TLSConfig.CAFile, "ca-file", "", "- NOT RECOMMENDED FOR PRODUCTION - Path to TLS CA file.")
	flagset.StringVar(&cfg.KubeletObject, "kubelet-service", "", "Service/Endpoints object to write kubelets into in format \"namespace/name\"")
	flagset.BoolVar(&cfg.TLSInsecure, "tls-insecure", false, "- NOT RECOMMENDED FOR PRODUCTION - Don't verify API server's CA certificate.")
	// The Prometheus config reloader image is released along with the
	// Prometheus Operator image, tagged with the same semver version. Default to
	// the Prometheus Operator version if no Prometheus config reloader image is
	// specified.
	flagset.StringVar(&cfg.PrometheusConfigReloader, "prometheus-config-reloader", fmt.Sprintf("quay.io/coreos/prometheus-config-reloader:v%v", version.Version), "Prometheus config reloader image")
	flagset.StringVar(&cfg.ConfigReloaderImage, "config-reloader-image", "quay.io/coreos/configmap-reload:v0.0.1", "Reload Image")
	flagset.StringVar(&cfg.ConfigReloaderCPU, "config-reloader-cpu", "100m", "Config Reloader CPU")
	flagset.StringVar(&cfg.ConfigReloaderMemory, "config-reloader-memory", "15Mi", "Config Reloader Memory")
	flagset.StringVar(&cfg.AlertmanagerDefaultBaseImage, "alertmanager-default-base-image", "quay.io/prometheus/alertmanager", "Alertmanager default base image")
	flagset.StringVar(&cfg.PrometheusDefaultBaseImage, "prometheus-default-base-image", "quay.io/prometheus/prometheus", "Prometheus default base image")
	flagset.StringVar(&cfg.ThanosDefaultBaseImage, "thanos-default-base-image", "improbable/thanos", "Thanos default base image")
	flagset.Var(ns, "namespaces", "Namespaces to scope the interaction of the Prometheus Operator and the apiserver.")
	flagset.Var(&cfg.Labels, "labels", "Labels to be add to all resources created by the operator")
	flagset.StringVar(&cfg.CrdGroup, "crd-apigroup", monitoring.GroupName, "prometheus CRD  API group name")
	flagset.Var(&cfg.CrdKinds, "crd-kinds", " - EXPERIMENTAL (could be removed in future releases) - customize CRD kind names")
	flagset.BoolVar(&cfg.EnableValidation, "with-validation", true, "Include the validation spec in the CRD")
	flagset.StringVar(&cfg.LocalHost, "localhost", "localhost", "EXPERIMENTAL (could be removed in future releases) - Host used to communicate between local services on a pod. Fixes issues where localhost resolves incorrectly.")
	flagset.StringVar(&cfg.LogLevel, "log-level", logLevelInfo, fmt.Sprintf("Log level to use. Possible values: %s", strings.Join(availableLogLevels, ", ")))
	flagset.StringVar(&cfg.LogFormat, "log-format", logFormatLogfmt, fmt.Sprintf("Log format to use. Possible values: %s", strings.Join(availableLogFormats, ", ")))
	flagset.BoolVar(&cfg.ManageCRDs, "manage-crds", true, "Manage all CRDs with the Prometheus Operator.")
	flagset.Parse(os.Args[1:])
	cfg.Namespaces = ns.asSlice()
}

func Main() int {
	logger := log.NewLogfmtLogger(log.NewSyncWriter(os.Stdout))
	if cfg.LogFormat == logFormatJson {
		logger = log.NewJSONLogger(log.NewSyncWriter(os.Stdout))
	}
	switch cfg.LogLevel {
	case logLevelAll:
		logger = level.NewFilter(logger, level.AllowAll())
	case logLevelDebug:
		logger = level.NewFilter(logger, level.AllowDebug())
	case logLevelInfo:
		logger = level.NewFilter(logger, level.AllowInfo())
	case logLevelWarn:
		logger = level.NewFilter(logger, level.AllowWarn())
	case logLevelError:
		logger = level.NewFilter(logger, level.AllowError())
	case logLevelNone:
		logger = level.NewFilter(logger, level.AllowNone())
	default:
		fmt.Fprintf(os.Stderr, "log level %v unknown, %v are possible values", cfg.LogLevel, availableLogLevels)
		return 1
	}
	logger = log.With(logger, "ts", log.DefaultTimestampUTC)
	logger = log.With(logger, "caller", log.DefaultCaller)

	logger.Log("msg", fmt.Sprintf("Starting Prometheus Operator version '%v'.", version.Version))

	po, err := prometheuscontroller.New(cfg, log.With(logger, "component", "prometheusoperator"))
	if err != nil {
		fmt.Fprint(os.Stderr, "instantiating prometheus controller failed: ", err)
		return 1
	}

	ao, err := alertmanagercontroller.New(cfg, log.With(logger, "component", "alertmanageroperator"))
	if err != nil {
		fmt.Fprint(os.Stderr, "instantiating alertmanager controller failed: ", err)
		return 1
	}

	mux := http.NewServeMux()
	web, err := api.New(cfg, log.With(logger, "component", "api"))
	if err != nil {
		fmt.Fprint(os.Stderr, "instantiating api failed: ", err)
		return 1
	}

	web.Register(mux)
	l, err := net.Listen("tcp", ":8080")
	if err != nil {
		fmt.Fprint(os.Stderr, "listening port 8080 failed", err)
		return 1
	}

	reconcileErrorsCounter := prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "prometheus_operator_reconcile_errors_total",
		Help: "Number of errors that occurred while reconciling the alertmanager statefulset",
	}, []string{"controller"})

	triggerByCounter := prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "prometheus_operator_triggered_total",
		Help: "Number of times a Kubernetes object add, delete or update event" +
			" triggered the Prometheus Operator to reconcile an object",
	}, []string{"controller", "triggered_by", "action"})

	r := prometheus.NewRegistry()
	r.MustRegister(
		prometheus.NewGoCollector(),
		prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}),
		reconcileErrorsCounter,
		triggerByCounter,
	)

	prometheusLabels := prometheus.Labels{"controller": "prometheus"}
	po.RegisterMetrics(
		prometheus.WrapRegistererWith(prometheusLabels, r),
		reconcileErrorsCounter.MustCurryWith(prometheusLabels),
		triggerByCounter.MustCurryWith(prometheusLabels),
	)

	alertmanagerLabels := prometheus.Labels{"controller": "alertmanager"}
	ao.RegisterMetrics(
		prometheus.WrapRegistererWith(alertmanagerLabels, r),
		reconcileErrorsCounter.MustCurryWith(alertmanagerLabels),
		triggerByCounter.MustCurryWith(alertmanagerLabels),
	)

	mux.Handle("/metrics", promhttp.HandlerFor(r, promhttp.HandlerOpts{}))
	mux.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
	mux.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
	mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
	mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
	mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))

	ctx, cancel := context.WithCancel(context.Background())
	wg, ctx := errgroup.WithContext(ctx)

	wg.Go(func() error { return po.Run(ctx.Done()) })
	wg.Go(func() error { return ao.Run(ctx.Done()) })

	srv := &http.Server{Handler: mux}
	go srv.Serve(l)

	term := make(chan os.Signal)
	signal.Notify(term, os.Interrupt, syscall.SIGTERM)

	select {
	case <-term:
		logger.Log("msg", "Received SIGTERM, exiting gracefully...")
	case <-ctx.Done():
	}

	cancel()
	if err := wg.Wait(); err != nil {
		logger.Log("msg", "Unhandled error received. Exiting...", "err", err)
		return 1
	}

	return 0
}

func main() {
	os.Exit(Main())
}
<commit_msg>Increase memory to 25Mi for runc<commit_after>// Copyright 2016 The prometheus-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"net"
	"net/http"
	"net/http/pprof"
	"os"
	"os/signal"
	"strings"
	"syscall"

	alertmanagercontroller "github.com/coreos/prometheus-operator/pkg/alertmanager"
	"github.com/coreos/prometheus-operator/pkg/api"
	monitoring "github.com/coreos/prometheus-operator/pkg/apis/monitoring"
	monitoringv1 "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1"
	prometheuscontroller "github.com/coreos/prometheus-operator/pkg/prometheus"
	"github.com/coreos/prometheus-operator/pkg/version"

	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"golang.org/x/sync/errgroup"
	v1 "k8s.io/api/core/v1"
	"k8s.io/klog"
)

const (
	logLevelAll   = "all"
	logLevelDebug = "debug"
	logLevelInfo  = "info"
	logLevelWarn  = "warn"
	logLevelError = "error"
	logLevelNone  = "none"
)

const (
	logFormatLogfmt = "logfmt"
	logFormatJson   = "json"
)

var (
	ns = namespaces{}
)

type namespaces map[string]struct{}

// Set implements the flagset.Value interface.
func (n namespaces) Set(value string) error {
	if n == nil {
		return errors.New("expected n of type namespaces to be initialized")
	}
	ns := strings.Split(value, ",")
	for i := range ns {
		n[ns[i]] = struct{}{}
	}
	return nil
}

// String implements the flagset.Value interface.
func (n namespaces) String() string {
	return strings.Join(n.asSlice(), ",")
}

func (n namespaces) asSlice() []string {
	var ns []string
	for k := range n {
		ns = append(ns, k)
	}
	if len(ns) == 0 {
		ns = append(ns, v1.NamespaceAll)
	}
	return ns
}

var (
	cfg                prometheuscontroller.Config
	availableLogLevels = []string{
		logLevelAll,
		logLevelDebug,
		logLevelInfo,
		logLevelWarn,
		logLevelError,
		logLevelNone,
	}
	availableLogFormats = []string{
		logFormatLogfmt,
		logFormatJson,
	}
)

func init() {
	cfg.CrdKinds = monitoringv1.DefaultCrdKinds
	flagset := flag.CommandLine
	klog.InitFlags(flagset)
	flagset.StringVar(&cfg.Host, "apiserver", "", "API Server addr, e.g. ' - NOT RECOMMENDED FOR PRODUCTION - http://127.0.0.1:8080'. Omit parameter to run in on-cluster mode and utilize the service account token.")
	flagset.StringVar(&cfg.TLSConfig.CertFile, "cert-file", "", " - NOT RECOMMENDED FOR PRODUCTION - Path to public TLS certificate file.")
	flagset.StringVar(&cfg.TLSConfig.KeyFile, "key-file", "", "- NOT RECOMMENDED FOR PRODUCTION - Path to private TLS certificate file.")
	flagset.StringVar(&cfg.TLSConfig.CAFile, "ca-file", "", "- NOT RECOMMENDED FOR PRODUCTION - Path to TLS CA file.")
	flagset.StringVar(&cfg.KubeletObject, "kubelet-service", "", "Service/Endpoints object to write kubelets into in format \"namespace/name\"")
	flagset.BoolVar(&cfg.TLSInsecure, "tls-insecure", false, "- NOT RECOMMENDED FOR PRODUCTION - Don't verify API server's CA certificate.")
	// The Prometheus config reloader image is released along with the
	// Prometheus Operator image, tagged with the same semver version. Default to
	// the Prometheus Operator version if no Prometheus config reloader image is
	// specified.
	flagset.StringVar(&cfg.PrometheusConfigReloader, "prometheus-config-reloader", fmt.Sprintf("quay.io/coreos/prometheus-config-reloader:v%v", version.Version), "Prometheus config reloader image")
	flagset.StringVar(&cfg.ConfigReloaderImage, "config-reloader-image", "quay.io/coreos/configmap-reload:v0.0.1", "Reload Image")
	flagset.StringVar(&cfg.ConfigReloaderCPU, "config-reloader-cpu", "100m", "Config Reloader CPU")
	flagset.StringVar(&cfg.ConfigReloaderMemory, "config-reloader-memory", "25Mi", "Config Reloader Memory")
	flagset.StringVar(&cfg.AlertmanagerDefaultBaseImage, "alertmanager-default-base-image", "quay.io/prometheus/alertmanager", "Alertmanager default base image")
	flagset.StringVar(&cfg.PrometheusDefaultBaseImage, "prometheus-default-base-image", "quay.io/prometheus/prometheus", "Prometheus default base image")
	flagset.StringVar(&cfg.ThanosDefaultBaseImage, "thanos-default-base-image", "improbable/thanos", "Thanos default base image")
	flagset.Var(ns, "namespaces", "Namespaces to scope the interaction of the Prometheus Operator and the apiserver.")
	flagset.Var(&cfg.Labels, "labels", "Labels to be add to all resources created by the operator")
	flagset.StringVar(&cfg.CrdGroup, "crd-apigroup", monitoring.GroupName, "prometheus CRD  API group name")
	flagset.Var(&cfg.CrdKinds, "crd-kinds", " - EXPERIMENTAL (could be removed in future releases) - customize CRD kind names")
	flagset.BoolVar(&cfg.EnableValidation, "with-validation", true, "Include the validation spec in the CRD")
	flagset.StringVar(&cfg.LocalHost, "localhost", "localhost", "EXPERIMENTAL (could be removed in future releases) - Host used to communicate between local services on a pod. Fixes issues where localhost resolves incorrectly.")
	flagset.StringVar(&cfg.LogLevel, "log-level", logLevelInfo, fmt.Sprintf("Log level to use. Possible values: %s", strings.Join(availableLogLevels, ", ")))
	flagset.StringVar(&cfg.LogFormat, "log-format", logFormatLogfmt, fmt.Sprintf("Log format to use. Possible values: %s", strings.Join(availableLogFormats, ", ")))
	flagset.BoolVar(&cfg.ManageCRDs, "manage-crds", true, "Manage all CRDs with the Prometheus Operator.")
	flagset.Parse(os.Args[1:])
	cfg.Namespaces = ns.asSlice()
}

func Main() int {
	logger := log.NewLogfmtLogger(log.NewSyncWriter(os.Stdout))
	if cfg.LogFormat == logFormatJson {
		logger = log.NewJSONLogger(log.NewSyncWriter(os.Stdout))
	}
	switch cfg.LogLevel {
	case logLevelAll:
		logger = level.NewFilter(logger, level.AllowAll())
	case logLevelDebug:
		logger = level.NewFilter(logger, level.AllowDebug())
	case logLevelInfo:
		logger = level.NewFilter(logger, level.AllowInfo())
	case logLevelWarn:
		logger = level.NewFilter(logger, level.AllowWarn())
	case logLevelError:
		logger = level.NewFilter(logger, level.AllowError())
	case logLevelNone:
		logger = level.NewFilter(logger, level.AllowNone())
	default:
		fmt.Fprintf(os.Stderr, "log level %v unknown, %v are possible values", cfg.LogLevel, availableLogLevels)
		return 1
	}
	logger = log.With(logger, "ts", log.DefaultTimestampUTC)
	logger = log.With(logger, "caller", log.DefaultCaller)

	logger.Log("msg", fmt.Sprintf("Starting Prometheus Operator version '%v'.", version.Version))

	po, err := prometheuscontroller.New(cfg, log.With(logger, "component", "prometheusoperator"))
	if err != nil {
		fmt.Fprint(os.Stderr, "instantiating prometheus controller failed: ", err)
		return 1
	}

	ao, err := alertmanagercontroller.New(cfg, log.With(logger, "component", "alertmanageroperator"))
	if err != nil {
		fmt.Fprint(os.Stderr, "instantiating alertmanager controller failed: ", err)
		return 1
	}

	mux := http.NewServeMux()
	web, err := api.New(cfg, log.With(logger, "component", "api"))
	if err != nil {
		fmt.Fprint(os.Stderr, "instantiating api failed: ", err)
		return 1
	}

	web.Register(mux)
	l, err := net.Listen("tcp", ":8080")
	if err != nil {
		fmt.Fprint(os.Stderr, "listening port 8080 failed", err)
		return 1
	}

	reconcileErrorsCounter := prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "prometheus_operator_reconcile_errors_total",
		Help: "Number of errors that occurred while reconciling the alertmanager statefulset",
	}, []string{"controller"})

	triggerByCounter := prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: "prometheus_operator_triggered_total",
		Help: "Number of times a Kubernetes object add, delete or update event" +
			" triggered the Prometheus Operator to reconcile an object",
	}, []string{"controller", "triggered_by", "action"})

	r := prometheus.NewRegistry()
	r.MustRegister(
		prometheus.NewGoCollector(),
		prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}),
		reconcileErrorsCounter,
		triggerByCounter,
	)

	prometheusLabels := prometheus.Labels{"controller": "prometheus"}
	po.RegisterMetrics(
		prometheus.WrapRegistererWith(prometheusLabels, r),
		reconcileErrorsCounter.MustCurryWith(prometheusLabels),
		triggerByCounter.MustCurryWith(prometheusLabels),
	)

	alertmanagerLabels := prometheus.Labels{"controller": "alertmanager"}
	ao.RegisterMetrics(
		prometheus.WrapRegistererWith(alertmanagerLabels, r),
		reconcileErrorsCounter.MustCurryWith(alertmanagerLabels),
		triggerByCounter.MustCurryWith(alertmanagerLabels),
	)

	mux.Handle("/metrics", promhttp.HandlerFor(r, promhttp.HandlerOpts{}))
	mux.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
	mux.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
	mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
	mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
	mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))

	ctx, cancel := context.WithCancel(context.Background())
	wg, ctx := errgroup.WithContext(ctx)

	wg.Go(func() error { return po.Run(ctx.Done()) })
	wg.Go(func() error { return ao.Run(ctx.Done()) })

	srv := &http.Server{Handler: mux}
	go srv.Serve(l)

	term := make(chan os.Signal)
	signal.Notify(term, os.Interrupt, syscall.SIGTERM)

	select {
	case <-term:
		logger.Log("msg", "Received SIGTERM, exiting gracefully...")
	case <-ctx.Done():
	}

	cancel()
	if err := wg.Wait(); err != nil {
		logger.Log("msg", "Unhandled error received. Exiting...", "err", err)
		return 1
	}

	return 0
}

func main() {
	os.Exit(Main())
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

func newProjectInDir(path string) error {
	// set path to be nested inside $GOPATH/src
	gopath := os.Getenv("GOPATH")
	path = filepath.Join(gopath, "src", path)

	// check if anything exists at the path, ask if it should be overwritten
	if _, err := os.Stat(path); !os.IsNotExist(err) {
		fmt.Println("Path exists, overwrite contents? (y/N):")

		var answer string
		_, err := fmt.Scanf("%s\n", &answer)
		if err != nil {
			if err.Error() == "unexpected newline" {
				answer = ""
			} else {
				return err
			}
		}

		answer = strings.ToLower(answer)

		switch answer {
		case "n", "no", "\r\n", "\n", "":
			fmt.Println("")

		case "y", "yes":
			err := os.RemoveAll(path)
			if err != nil {
				return fmt.Errorf("Failed to overwrite %s. \n%s", path, err)
			}

			return createProjInDir(path)

		default:
			fmt.Println("Input not recognized. No files overwritten. Answer as 'y' or 'n' only.")
		}

		return nil
	}

	return createProjInDir(path)
}

var ponzuRepo = []string{"github.com", "bosssauce", "ponzu"}

func createProjInDir(path string) error {
	gopath := os.Getenv("GOPATH")
	repo := ponzuRepo
	local := filepath.Join(gopath, "src", filepath.Join(repo...))
	network := "https://" + strings.Join(repo, "/") + ".git"

	// create the directory or overwrite it
	err := os.MkdirAll(path, os.ModeDir|os.ModePerm)
	if err != nil {
		return err
	}

	if dev {
		if fork != "" {
			local = filepath.Join(gopath, "src", fork)
		}

		devClone := exec.Command("git", "clone", local, "--branch", "ponzu-dev", "--single-branch", path)
		devClone.Stdout = os.Stdout
		devClone.Stderr = os.Stderr

		err = devClone.Start()
		if err != nil {
			return err
		}

		err = devClone.Wait()
		if err != nil {
			return err
		}

		err = vendorCorePackages(path)
		if err != nil {
			return err
		}

		fmt.Println("Dev build cloned from " + local + ":ponzu-dev")
		return nil
	}

	// try to git clone the repository from the local machine's $GOPATH
	localClone := exec.Command("git", "clone", local, path)
	localClone.Stdout = os.Stdout
	localClone.Stderr = os.Stderr

	err = localClone.Start()
	if err != nil {
		return err
	}
	err = localClone.Wait()
	if err != nil {
		fmt.Println("Couldn't clone from", local, ". Trying network...")

		// try to git clone the repository over the network
		networkClone := exec.Command("git", "clone", network, path)
		networkClone.Stdout = os.Stdout
		networkClone.Stderr = os.Stderr

		err = networkClone.Start()
		if err != nil {
			fmt.Println("Network clone failed to start. Try again and make sure you have a network connection.")
			return err
		}
		err = networkClone.Wait()
		if err != nil {
			fmt.Println("Network clone failure.")
			// failed
			return fmt.Errorf("Failed to clone files from local machine [%s] and over the network [%s].\n%s", local, network, err)
		}
	}

	// create an internal vendor directory in ./cmd/ponzu and move content,
	// management and system packages into it
	err = vendorCorePackages(path)
	if err != nil {
		return err
	}

	gitDir := filepath.Join(path, ".git")
	err = os.RemoveAll(gitDir)
	if err != nil {
		fmt.Println("Failed to remove .git directory from your project path. Consider removing it manually.")
	}

	fmt.Println("New ponzu project created at", path)
	return nil
}

func vendorCorePackages(path string) error {
	vendorPath := filepath.Join(path, "cmd", "ponzu", "vendor", "github.com", "bosssauce", "ponzu")
	err := os.MkdirAll(vendorPath, os.ModeDir|os.ModePerm)
	if err != nil {
		return err
	}

	dirs := []string{"content", "management", "system"}
	for _, dir := range dirs {
		err = os.Rename(filepath.Join(path, dir), filepath.Join(vendorPath, dir))
		if err != nil {
			return err
		}
	}

	// create a user content directory
	contentPath := filepath.Join(path, "content")
	err = os.Mkdir(contentPath, os.ModeDir|os.ModePerm)
	if err != nil {
		return err
	}

	return nil
}

func copyFile(info os.FileInfo, src string, dst string) error {
	dstFile, err := os.Create(filepath.Join(dst, info.Name()))
	defer dstFile.Close()
	if err != nil {
		return err
	}

	srcFile, err := os.Open(filepath.Join(src, info.Name()))
	defer srcFile.Close()
	if err != nil {
		return err
	}

	_, err = io.Copy(dstFile, srcFile)
	if err != nil {
		return err
	}

	return nil
}

func copyFilesWarnConflicts(srcDir, dstDir string, conflicts []string) error {
	err := filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error {
		fmt.Println(path)
		if err != nil {
			return err
		}

		if info.IsDir() {
			fmt.Println(path, srcDir)
			if len(path) >= len(srcDir) {
				path = path[len(srcDir)+1:]
			}
			dir := filepath.Join(dstDir, path)
			err := os.MkdirAll(dir, os.ModeDir|os.ModePerm)
			if err != nil {
				return err
			}

			return nil
		}

		for _, conflict := range conflicts {
			if info.Name() == conflict {
				fmt.Println("Ponzu couldn't fully build your project:")
				fmt.Println("You must rename the following file, as it conflicts with Ponzu core:")
				fmt.Println(path)

				fmt.Println("Once the files above have been renamed, run '$ ponzu build' to retry.")
				return errors.New("Ponzu has very few internal conflicts, sorry for the inconvenience.")
			}
		}

		err = copyFile(info, srcDir, dstDir)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return err
	}

	return nil
}

func buildPonzuServer(args []string) error {
	pwd, err := os.Getwd()
	if err != nil {
		return err
	}

	// copy all ./content files to internal vendor directory
	src := "content"
	dst := filepath.Join("cmd", "ponzu", "vendor", "github.com", "bosssauce", "ponzu", "content")
	err = copyFilesWarnConflicts(src, dst, []string{"item.go", "types.go"})
	if err != nil {
		return err
	}

	// copy all ./addons files & dirs to internal vendor directory
	src = "addons"
	dst = filepath.Join("cmd", "ponzu", "vendor")
	err = copyFilesWarnConflicts(src, dst, nil)
	if err != nil {
		return err
	}

	// execute go build -o ponzu-cms cmd/ponzu/*.go
	mainPath := filepath.Join(pwd, "cmd", "ponzu", "main.go")
	optsPath := filepath.Join(pwd, "cmd", "ponzu", "options.go")
	genPath := filepath.Join(pwd, "cmd", "ponzu", "generate.go")
	build := exec.Command("go", "build", "-o", "ponzu-server", mainPath, optsPath, genPath)
	build.Stderr = os.Stderr
	build.Stdout = os.Stdout

	err = build.Start()
	if err != nil {
		return errors.New("Ponzu build step failed. Please try again. " + "\n" + err.Error())

	}
	err = build.Wait()
	if err != nil {
		return errors.New("Ponzu build step failed. Please try again. " + "\n" + err.Error())

	}

	return nil
}
<commit_msg>debug<commit_after>package main

import (
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

func newProjectInDir(path string) error {
	// set path to be nested inside $GOPATH/src
	gopath := os.Getenv("GOPATH")
	path = filepath.Join(gopath, "src", path)

	// check if anything exists at the path, ask if it should be overwritten
	if _, err := os.Stat(path); !os.IsNotExist(err) {
		fmt.Println("Path exists, overwrite contents? (y/N):")

		var answer string
		_, err := fmt.Scanf("%s\n", &answer)
		if err != nil {
			if err.Error() == "unexpected newline" {
				answer = ""
			} else {
				return err
			}
		}

		answer = strings.ToLower(answer)

		switch answer {
		case "n", "no", "\r\n", "\n", "":
			fmt.Println("")

		case "y", "yes":
			err := os.RemoveAll(path)
			if err != nil {
				return fmt.Errorf("Failed to overwrite %s. \n%s", path, err)
			}

			return createProjInDir(path)

		default:
			fmt.Println("Input not recognized. No files overwritten. Answer as 'y' or 'n' only.")
		}

		return nil
	}

	return createProjInDir(path)
}

var ponzuRepo = []string{"github.com", "bosssauce", "ponzu"}

func createProjInDir(path string) error {
	gopath := os.Getenv("GOPATH")
	repo := ponzuRepo
	local := filepath.Join(gopath, "src", filepath.Join(repo...))
	network := "https://" + strings.Join(repo, "/") + ".git"

	// create the directory or overwrite it
	err := os.MkdirAll(path, os.ModeDir|os.ModePerm)
	if err != nil {
		return err
	}

	if dev {
		if fork != "" {
			local = filepath.Join(gopath, "src", fork)
		}

		devClone := exec.Command("git", "clone", local, "--branch", "ponzu-dev", "--single-branch", path)
		devClone.Stdout = os.Stdout
		devClone.Stderr = os.Stderr

		err = devClone.Start()
		if err != nil {
			return err
		}

		err = devClone.Wait()
		if err != nil {
			return err
		}

		err = vendorCorePackages(path)
		if err != nil {
			return err
		}

		fmt.Println("Dev build cloned from " + local + ":ponzu-dev")
		return nil
	}

	// try to git clone the repository from the local machine's $GOPATH
	localClone := exec.Command("git", "clone", local, path)
	localClone.Stdout = os.Stdout
	localClone.Stderr = os.Stderr

	err = localClone.Start()
	if err != nil {
		return err
	}
	err = localClone.Wait()
	if err != nil {
		fmt.Println("Couldn't clone from", local, ". Trying network...")

		// try to git clone the repository over the network
		networkClone := exec.Command("git", "clone", network, path)
		networkClone.Stdout = os.Stdout
		networkClone.Stderr = os.Stderr

		err = networkClone.Start()
		if err != nil {
			fmt.Println("Network clone failed to start. Try again and make sure you have a network connection.")
			return err
		}
		err = networkClone.Wait()
		if err != nil {
			fmt.Println("Network clone failure.")
			// failed
			return fmt.Errorf("Failed to clone files from local machine [%s] and over the network [%s].\n%s", local, network, err)
		}
	}

	// create an internal vendor directory in ./cmd/ponzu and move content,
	// management and system packages into it
	err = vendorCorePackages(path)
	if err != nil {
		return err
	}

	gitDir := filepath.Join(path, ".git")
	err = os.RemoveAll(gitDir)
	if err != nil {
		fmt.Println("Failed to remove .git directory from your project path. Consider removing it manually.")
	}

	fmt.Println("New ponzu project created at", path)
	return nil
}

func vendorCorePackages(path string) error {
	vendorPath := filepath.Join(path, "cmd", "ponzu", "vendor", "github.com", "bosssauce", "ponzu")
	err := os.MkdirAll(vendorPath, os.ModeDir|os.ModePerm)
	if err != nil {
		return err
	}

	dirs := []string{"content", "management", "system"}
	for _, dir := range dirs {
		err = os.Rename(filepath.Join(path, dir), filepath.Join(vendorPath, dir))
		if err != nil {
			return err
		}
	}

	// create a user content directory
	contentPath := filepath.Join(path, "content")
	err = os.Mkdir(contentPath, os.ModeDir|os.ModePerm)
	if err != nil {
		return err
	}

	return nil
}

func copyFile(info os.FileInfo, src string, dst string) error {
	dstFile, err := os.Create(filepath.Join(dst, info.Name()))
	defer dstFile.Close()
	if err != nil {
		return err
	}

	srcFile, err := os.Open(filepath.Join(src, info.Name()))
	defer srcFile.Close()
	if err != nil {
		return err
	}

	_, err = io.Copy(dstFile, srcFile)
	if err != nil {
		return err
	}

	return nil
}

func copyFilesWarnConflicts(srcDir, dstDir string, conflicts []string) error {
	err := filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error {
		fmt.Println(path)
		if err != nil {
			return err
		}

		if info.IsDir() {
			fmt.Println(path, srcDir)
			if len(path) > len(srcDir) {
				path = path[len(srcDir)+1:]
			}
			dir := filepath.Join(dstDir, path)
			err := os.MkdirAll(dir, os.ModeDir|os.ModePerm)
			if err != nil {
				return err
			}

			return nil
		}

		for _, conflict := range conflicts {
			if info.Name() == conflict {
				fmt.Println("Ponzu couldn't fully build your project:")
				fmt.Println("You must rename the following file, as it conflicts with Ponzu core:")
				fmt.Println(path)

				fmt.Println("Once the files above have been renamed, run '$ ponzu build' to retry.")
				return errors.New("Ponzu has very few internal conflicts, sorry for the inconvenience.")
			}
		}

		err = copyFile(info, srcDir, dstDir)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return err
	}

	return nil
}

func buildPonzuServer(args []string) error {
	pwd, err := os.Getwd()
	if err != nil {
		return err
	}

	// copy all ./content files to internal vendor directory
	src := "content"
	dst := filepath.Join("cmd", "ponzu", "vendor", "github.com", "bosssauce", "ponzu", "content")
	err = copyFilesWarnConflicts(src, dst, []string{"item.go", "types.go"})
	if err != nil {
		return err
	}

	// copy all ./addons files & dirs to internal vendor directory
	src = "addons"
	dst = filepath.Join("cmd", "ponzu", "vendor")
	err = copyFilesWarnConflicts(src, dst, nil)
	if err != nil {
		return err
	}

	// execute go build -o ponzu-cms cmd/ponzu/*.go
	mainPath := filepath.Join(pwd, "cmd", "ponzu", "main.go")
	optsPath := filepath.Join(pwd, "cmd", "ponzu", "options.go")
	genPath := filepath.Join(pwd, "cmd", "ponzu", "generate.go")
	build := exec.Command("go", "build", "-o", "ponzu-server", mainPath, optsPath, genPath)
	build.Stderr = os.Stderr
	build.Stdout = os.Stdout

	err = build.Start()
	if err != nil {
		return errors.New("Ponzu build step failed. Please try again. " + "\n" + err.Error())

	}
	err = build.Wait()
	if err != nil {
		return errors.New("Ponzu build step failed. Please try again. " + "\n" + err.Error())

	}

	return nil
}
<|endoftext|>


<commit_before>// 擷取盤中即時資訊與大盤、上櫃、寶島指數.
//
/*

Install:

	go install github.com/toomore/gogrs/cmd/realtime

Usage:

	realtime [flags]

The flags are:

	-twse
		上市股票代碼，可使用 ',' 分隔多組代碼，例：2618,2329
	-otc
		上櫃股票代碼，可使用 ',' 分隔多組代碼，例：8446,2719
	-index
		顯示大盤、上櫃、寶島指數（default: false）

範例

	/realtime -twse=2618,2329 -otc=8446,2719 -index=true

回傳內容

	2015/04/29 23:43:13 櫃買指數(o00) $144.72(-1.15) 1271/23250 [2015-04-29 13:33:00 +0800 CST] [20150429 23:43:13]
	2015/04/29 23:43:13 發行量加權股價指數(t00) $9853.83(-80.99) 5742/113762 [2015-04-29 13:33:00 +0800 CST] [20150429 23:43:13]
	2015/04/29 23:43:13 寶島股價指數(FRMSA) $11416.04(-93.58) 0/0 [2015-04-29 13:33:00 +0800 CST] [20150429 23:43:13]
	2015/04/29 23:43:13 華研(8446) $124.00(1.00) 6/293 [2015-04-29 14:30:00 +0800 CST] [20150429 23:43:13]
	2015/04/29 23:43:13 長榮航(2618) $24.20(-0.55) 666/15618 [2015-04-29 14:30:00 +0800 CST] [20150429 23:43:13]
	2015/04/29 23:43:13 燦星旅(2719) $24.15(0.25) 6/83 [2015-04-29 14:30:00 +0800 CST] [20150429 23:43:13]
	2015/04/29 23:43:13 華泰(2329) $15.95(0.50) 648/19995 [2015-04-29 14:30:00 +0800 CST] [20150429 23:43:13]

*/
package main

import (
	"flag"
	"fmt"
	"log"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/toomore/gogrs/realtime"
	"github.com/toomore/gogrs/tradingdays"
	"github.com/toomore/gogrs/twse"
	"github.com/toomore/gogrs/utils"
)

var cacheTime time.Time

// TaipeiNow show Taipei Now time.
func TaipeiNow() time.Time {
	if cacheTime.Year() == 1 {
		d := time.Now().UTC()
		days := d.Day()
		for {
			if tradingdays.IsOpen(d.Year(), d.Month(), days) {
				break
			}
			days--
		}
		if d.Before(time.Date(d.Year(), d.Month(), days, 1, 0, 0, 0, time.FixedZone("UTC", 0))) {
			if d.Hour() == 0 && d.Minute() <= 59 {
				days--
			}
		}
		cacheTime = time.Date(d.Year(), d.Month(), days, 0, 0, 0, 0, utils.TaipeiTimeZone)
	}
	return cacheTime
}

func prettyprint(data realtime.Data) string {
	return fmt.Sprintf("%s(%s) $%.2f(%.2f) %.0f/%.0f [%s] [%s %s]",
		data.Info.Name, data.Info.No,
		data.Price, data.Price-data.Open, data.Volume, data.VolumeAcc,
		data.TradeTime, data.SysInfo["sysDate"], data.SysInfo["sysTime"])
}

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
}

var twseNo = flag.String("twse", "", "上市股票代碼，可使用 ',' 分隔多組代碼，例：2618,2329")
var twseCate = flag.String("twsecate", "", "上市股票類別，可使用 ',' 分隔多組代碼，例：11,15")
var otcNo = flag.String("otc", "", "上櫃股票代碼，可使用 ',' 分隔多組代碼，例：8446,2719")
var index = flag.Bool("index", false, "顯示大盤、上櫃、寶島指數（default: false）")

func main() {
	flag.Parse()
	queue := []*realtime.StockRealTime{}
	if *twseNo != "" {
		for _, no := range strings.Split(*twseNo, ",") {
			queue = append(queue, realtime.NewTWSE(no, TaipeiNow()))
		}
	}
	if *twseCate != "" {
		l := &twse.Lists{Date: TaipeiNow()}
		for _, no := range strings.Split(*twseCate, ",") {
			for _, s := range l.GetCategoryList(no) {
				queue = append(queue, realtime.NewTWSE(s.No, TaipeiNow()))
			}
		}
	}
	if *otcNo != "" {
		for _, no := range strings.Split(*otcNo, ",") {
			queue = append(queue, realtime.NewOTC(no, TaipeiNow()))
		}
	}
	if *index {
		queue = append(queue, []*realtime.StockRealTime{realtime.NewWeight(TaipeiNow()),
			realtime.NewOTCI(TaipeiNow()), realtime.NewFRMSA(TaipeiNow())}...)
	}
	result := make(chan string, runtime.NumCPU())
	var wg sync.WaitGroup
	wg.Add(len(queue))
	for _, r := range queue {
		go func(r *realtime.StockRealTime) {
			runtime.Gosched()
			data, _ := r.Get()
			result <- prettyprint(data)
		}(r)
	}
	go func() {
		for v := range result {
			wg.Done()
			log.Println(v)
		}
	}()
	wg.Wait()
	if len(queue) == 0 {
		flag.PrintDefaults()
	}
}
<commit_msg>Add `showtwsecatelist` into cmd realtime.<commit_after>// 擷取盤中即時資訊與大盤、上櫃、寶島指數.
//
/*

Install:

	go install github.com/toomore/gogrs/cmd/realtime

Usage:

	realtime [flags]

The flags are:

	-twse
		上市股票代碼，可使用 ',' 分隔多組代碼，例：2618,2329
	-otc
		上櫃股票代碼，可使用 ',' 分隔多組代碼，例：8446,2719
	-index
		顯示大盤、上櫃、寶島指數（default: false）
	-twsecate
		上市股票類別，可使用 ',' 分隔多組代碼，例：11,15

範例

	/realtime -twse=2618,2329 -otc=8446,2719 -index=true

回傳內容

	2015/04/29 23:43:13 櫃買指數(o00) $144.72(-1.15) 1271/23250 [2015-04-29 13:33:00 +0800 CST] [20150429 23:43:13]
	2015/04/29 23:43:13 發行量加權股價指數(t00) $9853.83(-80.99) 5742/113762 [2015-04-29 13:33:00 +0800 CST] [20150429 23:43:13]
	2015/04/29 23:43:13 寶島股價指數(FRMSA) $11416.04(-93.58) 0/0 [2015-04-29 13:33:00 +0800 CST] [20150429 23:43:13]
	2015/04/29 23:43:13 華研(8446) $124.00(1.00) 6/293 [2015-04-29 14:30:00 +0800 CST] [20150429 23:43:13]
	2015/04/29 23:43:13 長榮航(2618) $24.20(-0.55) 666/15618 [2015-04-29 14:30:00 +0800 CST] [20150429 23:43:13]
	2015/04/29 23:43:13 燦星旅(2719) $24.15(0.25) 6/83 [2015-04-29 14:30:00 +0800 CST] [20150429 23:43:13]
	2015/04/29 23:43:13 華泰(2329) $15.95(0.50) 648/19995 [2015-04-29 14:30:00 +0800 CST] [20150429 23:43:13]

*/
package main

import (
	"flag"
	"fmt"
	"log"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/toomore/gogrs/realtime"
	"github.com/toomore/gogrs/tradingdays"
	"github.com/toomore/gogrs/twse"
	"github.com/toomore/gogrs/utils"
)

var cacheTime time.Time

// TaipeiNow show Taipei Now time.
func TaipeiNow() time.Time {
	if cacheTime.Year() == 1 {
		d := time.Now().UTC()
		days := d.Day()
		for {
			if tradingdays.IsOpen(d.Year(), d.Month(), days) {
				break
			}
			days--
		}
		if d.Before(time.Date(d.Year(), d.Month(), days, 1, 0, 0, 0, time.FixedZone("UTC", 0))) {
			if d.Hour() == 0 && d.Minute() <= 59 {
				days--
			}
		}
		cacheTime = time.Date(d.Year(), d.Month(), days, 0, 0, 0, 0, utils.TaipeiTimeZone)
	}
	return cacheTime
}

func prettyprint(data realtime.Data) string {
	return fmt.Sprintf("%s(%s) $%.2f(%.2f) %.0f/%.0f [%s] [%s %s]",
		data.Info.Name, data.Info.No,
		data.Price, data.Price-data.Open, data.Volume, data.VolumeAcc,
		data.TradeTime, data.SysInfo["sysDate"], data.SysInfo["sysTime"])
}

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
}

var twseNo = flag.String("twse", "", "上市股票代碼，可使用 ',' 分隔多組代碼，例：2618,2329")
var twseCate = flag.String("twsecate", "", "上市股票類別，可使用 ',' 分隔多組代碼，例：11,15")
var showtwsecatelist = flag.Bool("showcatelist", false, "顯示上市分類表")
var otcNo = flag.String("otc", "", "上櫃股票代碼，可使用 ',' 分隔多組代碼，例：8446,2719")
var index = flag.Bool("index", false, "顯示大盤、上櫃、寶島指數（default: false）")

func main() {
	flag.Parse()
	queue := []*realtime.StockRealTime{}
	if *showtwsecatelist {
		var index = 1
		for cateNo, cateName := range twse.TWSECLASS {
			fmt.Printf("%s(%s)\t\t", cateName, cateNo)
			if index%3 == 0 {
				fmt.Println("")
			}
			index++
		}
		fmt.Println("")
	}
	if *twseNo != "" {
		for _, no := range strings.Split(*twseNo, ",") {
			queue = append(queue, realtime.NewTWSE(no, TaipeiNow()))
		}
	}
	if *twseCate != "" {
		l := &twse.Lists{Date: TaipeiNow()}
		for _, no := range strings.Split(*twseCate, ",") {
			for _, s := range l.GetCategoryList(no) {
				queue = append(queue, realtime.NewTWSE(s.No, TaipeiNow()))
			}
		}
	}
	if *otcNo != "" {
		for _, no := range strings.Split(*otcNo, ",") {
			queue = append(queue, realtime.NewOTC(no, TaipeiNow()))
		}
	}
	if *index {
		queue = append(queue, []*realtime.StockRealTime{realtime.NewWeight(TaipeiNow()),
			realtime.NewOTCI(TaipeiNow()), realtime.NewFRMSA(TaipeiNow())}...)
	}
	result := make(chan string, runtime.NumCPU())
	var wg sync.WaitGroup
	wg.Add(len(queue))
	for _, r := range queue {
		go func(r *realtime.StockRealTime) {
			runtime.Gosched()
			data, _ := r.Get()
			result <- prettyprint(data)
		}(r)
	}
	go func() {
		for v := range result {
			wg.Done()
			log.Println(v)
		}
	}()
	wg.Wait()
	if len(queue) == 0 {
		flag.PrintDefaults()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"path"

	"github.com/tsavola/wag"
	"github.com/tsavola/wag/dewag"
	"github.com/tsavola/wag/sections"

	"github.com/tsavola/gate/run"
	"github.com/tsavola/gate/service"
	_ "github.com/tsavola/gate/service/defaults"
	"github.com/tsavola/gate/service/echo"
	"github.com/tsavola/gate/service/origin"
)

type readWriteCloser struct {
	io.Reader
	io.WriteCloser
}

func init() {
	echo.Default.Log = log.New(os.Stderr, "echo service: ", 0)
}

var (
	executor      string
	loader        string
	loaderSymbols string

	stackSize = 16 * 1024 * 1024
	dumpText  = false
	dumpStack = false
)

func main() {
	dir, err := os.Getwd()
	if err != nil {
		log.Fatal(err)
	}

	executor = path.Join(dir, "bin/executor")
	loader = path.Join(dir, "bin/loader")
	loaderSymbols = loader + ".symbols"

	var (
		addr string
	)

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [options] wasm...\nOptions:\n", os.Args[0])
		flag.PrintDefaults()
	}

	flag.StringVar(&executor, "executor", executor, "filename")
	flag.StringVar(&loader, "loader", loader, "filename")
	flag.StringVar(&loaderSymbols, "loader-symbols", loaderSymbols, "filename")
	flag.IntVar(&stackSize, "stack-size", stackSize, "stack size")
	flag.BoolVar(&dumpText, "dump-text", dumpText, "disassemble before running")
	flag.BoolVar(&dumpStack, "dump-stack", dumpStack, "print stacktrace after running")
	flag.StringVar(&addr, "addr", addr, "I/O socket path (replaces stdio)")

	flag.Parse()

	args := flag.Args()
	if len(args) == 0 {
		flag.Usage()
		os.Exit(2)
	}

	if addr != "" {
		os.Remove(addr)
		l, err := net.Listen("unix", addr)
		if err != nil {
			log.Fatal(err)
		}
		conn, err := l.Accept()
		l.Close()
		if err != nil {
			log.Fatal(err)
		}
		defer conn.Close()

		origin.Default.R = conn
		origin.Default.W = conn
	} else {
		origin.Default.R = os.Stdin
		origin.Default.W = os.Stdout
	}

	done := make(chan struct{}, len(args))

	for i, arg := range args {
		var r run.ServiceRegistry

		if i == 0 {
			r = service.Defaults
		} else {
			r = origin.CloneRegistryWith(service.Defaults, nil, os.Stdout)
		}

		go execute(arg, r, done)
	}

	for range args {
		<-done
	}
}

func execute(filename string, services run.ServiceRegistry, done chan<- struct{}) {
	defer func() {
		done <- struct{}{}
	}()

	wasm, err := os.Open(filename)
	if err != nil {
		log.Fatal(err)
	}

	env, err := run.NewEnvironment(executor, loader, loaderSymbols)
	if err != nil {
		log.Fatalf("environment: %v", err)
	}

	var ns sections.NameSection

	m := wag.Module{
		MainSymbol:           "main",
		UnknownSectionLoader: sections.UnknownLoaders{"name": ns.Load}.Load,
	}

	err = m.Load(bufio.NewReader(wasm), env, new(bytes.Buffer), nil, run.RODataAddr, nil)
	if err != nil {
		log.Fatalf("module: %v", err)
	}

	_, memorySize := m.MemoryLimits()

	payload, err := run.NewPayload(&m, memorySize, int32(stackSize))
	if err != nil {
		log.Fatalf("payload: %v", err)
	}
	defer payload.Close()

	if dumpText {
		dewag.PrintTo(os.Stderr, m.Text(), m.FunctionMap(), &ns)
	}

	exit, trap, err := run.Run(env, payload, services, os.Stderr)
	if err != nil {
		log.Fatal(err)
	} else if trap != 0 {
		log.Printf("trap: %s", trap)
	} else if exit != 0 {
		log.Printf("exit: %d", exit)
	}

	if dumpStack {
		err := payload.DumpStacktrace(os.Stderr, m.FunctionMap(), m.CallMap(), m.FunctionSignatures(), &ns)
		if err != nil {
			log.Printf("stacktrace: %v", err)
		}
	}
}
<commit_msg>runner: close file after loading<commit_after>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"path"

	"github.com/tsavola/wag"
	"github.com/tsavola/wag/dewag"
	"github.com/tsavola/wag/sections"

	"github.com/tsavola/gate/run"
	"github.com/tsavola/gate/service"
	_ "github.com/tsavola/gate/service/defaults"
	"github.com/tsavola/gate/service/echo"
	"github.com/tsavola/gate/service/origin"
)

type readWriteCloser struct {
	io.Reader
	io.WriteCloser
}

func init() {
	echo.Default.Log = log.New(os.Stderr, "echo service: ", 0)
}

var (
	executor      string
	loader        string
	loaderSymbols string

	stackSize = 16 * 1024 * 1024
	dumpText  = false
	dumpStack = false
)

func main() {
	dir, err := os.Getwd()
	if err != nil {
		log.Fatal(err)
	}

	executor = path.Join(dir, "bin/executor")
	loader = path.Join(dir, "bin/loader")
	loaderSymbols = loader + ".symbols"

	var (
		addr string
	)

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [options] wasm...\nOptions:\n", os.Args[0])
		flag.PrintDefaults()
	}

	flag.StringVar(&executor, "executor", executor, "filename")
	flag.StringVar(&loader, "loader", loader, "filename")
	flag.StringVar(&loaderSymbols, "loader-symbols", loaderSymbols, "filename")
	flag.IntVar(&stackSize, "stack-size", stackSize, "stack size")
	flag.BoolVar(&dumpText, "dump-text", dumpText, "disassemble before running")
	flag.BoolVar(&dumpStack, "dump-stack", dumpStack, "print stacktrace after running")
	flag.StringVar(&addr, "addr", addr, "I/O socket path (replaces stdio)")

	flag.Parse()

	args := flag.Args()
	if len(args) == 0 {
		flag.Usage()
		os.Exit(2)
	}

	if addr != "" {
		os.Remove(addr)
		l, err := net.Listen("unix", addr)
		if err != nil {
			log.Fatal(err)
		}
		conn, err := l.Accept()
		l.Close()
		if err != nil {
			log.Fatal(err)
		}
		defer conn.Close()

		origin.Default.R = conn
		origin.Default.W = conn
	} else {
		origin.Default.R = os.Stdin
		origin.Default.W = os.Stdout
	}

	done := make(chan struct{}, len(args))

	for i, arg := range args {
		var r run.ServiceRegistry

		if i == 0 {
			r = service.Defaults
		} else {
			r = origin.CloneRegistryWith(service.Defaults, nil, os.Stdout)
		}

		go execute(arg, r, done)
	}

	for range args {
		<-done
	}
}

func execute(filename string, services run.ServiceRegistry, done chan<- struct{}) {
	defer func() {
		done <- struct{}{}
	}()

	env, err := run.NewEnvironment(executor, loader, loaderSymbols)
	if err != nil {
		log.Fatalf("environment: %v", err)
	}

	var ns sections.NameSection

	m := wag.Module{
		MainSymbol:           "main",
		UnknownSectionLoader: sections.UnknownLoaders{"name": ns.Load}.Load,
	}

	err = load(&m, filename, env)
	if err != nil {
		log.Fatalf("module: %v", err)
	}

	_, memorySize := m.MemoryLimits()

	payload, err := run.NewPayload(&m, memorySize, int32(stackSize))
	if err != nil {
		log.Fatalf("payload: %v", err)
	}
	defer payload.Close()

	if dumpText {
		dewag.PrintTo(os.Stderr, m.Text(), m.FunctionMap(), &ns)
	}

	exit, trap, err := run.Run(env, payload, services, os.Stderr)
	if err != nil {
		log.Fatal(err)
	} else if trap != 0 {
		log.Printf("trap: %s", trap)
	} else if exit != 0 {
		log.Printf("exit: %d", exit)
	}

	if dumpStack {
		err := payload.DumpStacktrace(os.Stderr, m.FunctionMap(), m.CallMap(), m.FunctionSignatures(), &ns)
		if err != nil {
			log.Printf("stacktrace: %v", err)
		}
	}
}

func load(m *wag.Module, filename string, env *run.Environment) (err error) {
	f, err := os.Open(filename)
	if err != nil {
		return
	}
	defer f.Close()

	err = m.Load(bufio.NewReader(f), env, new(bytes.Buffer), nil, run.RODataAddr, nil)
	return
}
<|endoftext|>