<commit_before>package rcs

import (
	"archive/zip"
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"path"
	"reflect"
	"strconv"
	"strings"

	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/rackspace"
)

// BetaEndpoint reflects the default endpoint for this library
const BetaEndpoint = "https://mycluster.rackspacecloud.com"
const mimetypeJSON = "application/json"
const authHeaderKey = "X-Auth-Token"
const userAgent = "rgbkrk/gorcs"

// ZipURLResponse is the response that comes back from the zip endpoint
type ZipURLResponse struct {
	URL string `json:"zip_url"`
}

// ClusterClient accesses RCS
type ClusterClient struct {
	client   *http.Client
	Username string
	Token    string
	Endpoint string
}

// ErrorResponse is the JSON formatted error response from RCS
type ErrorResponse struct {
	Error string `json:"error"`
}

// Cluster is a cluster
type Cluster struct {
	ClusterName string `json:"cluster_name"`
	Username    string `json:"username"`

	// Flavor of compute to use for cluster, should be a default value currently
	Flavor string `json:"flavor,omitempty"`

	// UUID of image to use for cluster, should be a default value currently
	Image string `json:"image,omitempty"`

	// Node is optional, but allowed on create
	// Sadly it comes back as string instead of int in all cases
	// with the API
	Nodes number `json:"nodes,omitempty"`

	AutoScale bool   `json:"autoscale,omitempty"`
	Status    string `json:"status,omitempty"`
	TaskID    string `json:"task_id,omitempty"`
	Token     string `json:"token,omitempty"`
}

// Credentials holds the keys to the kingdom
type Credentials struct {
	README     []byte
	Cert       []byte
	Key        []byte
	CA         []byte
	CAKey      []byte
	DockerEnv  []byte
	DockerHost string
	Files      map[string][]byte
	UUID       UUID
}

// Specify this type for any struct fields that
// might be unmarshaled from JSON numbers of the following
// types: floats, integers, scientific notation, or strings
type number float64

func (n number) Int64() int64 {
	return int64(n)
}

func (n number) Int() int {
	return int(n)
}

func (n number) Float64() float64 {
	return float64(n)
}

// Required to enforce that string values are attempted to be parsed as numbers
func (n *number) UnmarshalJSON(data []byte) error {
	var f float64
	var err error
	if data[0] == '"' {
		f, err = strconv.ParseFloat(string(data[1:len(data)-1]), 64)
		if err != nil {
			return &json.UnmarshalTypeError{
				Value: string(data),
				Type:  reflect.TypeOf(*n),
			}
		}
	} else {
		if err := json.Unmarshal(data, &f); err != nil {
			return &json.UnmarshalTypeError{
				Value: string(data),
				Type:  reflect.TypeOf(*n),
			}
		}
	}
	*n = number(f)
	return nil
}

func newClusterClient(endpoint string, ao gophercloud.AuthOptions) (*ClusterClient, error) {
	provider, err := rackspace.AuthenticatedClient(ao)
	if err != nil {
		return nil, err
	}

	return &ClusterClient{
		client:   &http.Client{},
		Username: ao.Username,
		Token:    provider.TokenID,
		Endpoint: endpoint,
	}, nil
}

// NewClusterClientByAPIKey Auth using API Key
func NewClusterClientByAPIKey(endpoint, username, apikey string) (*ClusterClient, error) {
	ao := gophercloud.AuthOptions{
		Username:         username,
		APIKey:           apikey,
		IdentityEndpoint: rackspace.RackspaceUSIdentity,
	}

	return newClusterClient(endpoint, ao)
}

// NewClusterClient creates a new ClusterClient
func NewClusterClient(endpoint, username, password string) (*ClusterClient, error) {
	ao := gophercloud.AuthOptions{
		Username:         username,
		Password:         password,
		IdentityEndpoint: rackspace.RackspaceUSIdentity,
	}

	return newClusterClient(endpoint, ao)
}

// NewRequest handles a request using auth used by RCS
func (c *ClusterClient) NewRequest(method string, uri string, body io.Reader) (*http.Response, error) {
	req, err := http.NewRequest(method, c.Endpoint+uri, body)
	if err != nil {
		return nil, err
	}
	req.Header.Set("User-Agent", userAgent)
	req.Header.Add("Content-Type", mimetypeJSON)
	req.Header.Add(authHeaderKey, c.Token)
	resp, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode >= 400 {
		b, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, errors.New(resp.Status)
		}
		return nil, errors.New(string(b))
	}

	return resp, nil
}

// List the current clusters
func (c *ClusterClient) List() ([]Cluster, error) {
	clusters := []Cluster{}

	resp, err := c.NewRequest("GET", "/clusters/"+c.Username, nil)
	if err != nil {
		return nil, err
	}

	err = json.NewDecoder(resp.Body).Decode(&clusters)
	if err != nil {
		return nil, err
	}
	return clusters, nil
}

func clusterFromResponse(resp *http.Response, err error) (*Cluster, error) {
	if err != nil {
		return nil, err
	}
	cluster := new(Cluster)
	err = json.NewDecoder(resp.Body).Decode(&cluster)
	if err != nil {
		return nil, err
	}
	return cluster, nil
}

// Get a cluster by cluster name
func (c *ClusterClient) Get(clusterName string) (*Cluster, error) {
	uri := path.Join("/clusters", c.Username, clusterName)
	resp, err := c.NewRequest("GET", uri, nil)
	return clusterFromResponse(resp, err)
}

// Create a new cluster with cluster options
func (c *ClusterClient) Create(clusterOpts Cluster) (*Cluster, error) {
	// Even though username is in the URI path, the API expects the username
	// inside the body
	if clusterOpts.Username == "" {
		clusterOpts.Username = c.Username
	}
	clusterOptsJSON, err := json.Marshal(clusterOpts)
	if err != nil {
		return nil, err
	}

	body := bytes.NewReader(clusterOptsJSON)
	uri := path.Join("/clusters", c.Username)
	resp, err := c.NewRequest("POST", uri, body)
	return clusterFromResponse(resp, err)
}

// GetZipURL returns the URL for downloading credentials
func (c *ClusterClient) GetZipURL(clusterName string) (string, error) {
	uri := path.Join("/clusters", c.Username, clusterName, "zip")
	resp, err := c.NewRequest("GET", uri, nil)
	if err != nil {
		return "", err
	}

	var zipURLResp ZipURLResponse

	err = json.NewDecoder(resp.Body).Decode(&zipURLResp)

	if err != nil {
		return "", err
	}

	return zipURLResp.URL, nil
}

// UUID represents a UUID value. UUIDs can be compared and set to other values
// and accessed by byte.
type UUID [16]byte

func extractUUID(s string) (UUID, error) {
	s = strings.Trim(s, "/")
	var u UUID
	var err error

	if len(s) != 36 {
		return UUID{}, fmt.Errorf("Invalid UUID")
	}
	format := "%08x-%04x-%04x-%04x-%012x"

	// create stack addresses for each section of the uuid.
	p := make([][]byte, 5)

	if _, err := fmt.Sscanf(s, format, &p[0], &p[1], &p[2], &p[3], &p[4]); err != nil {
		return u, err
	}

	copy(u[0:4], p[0])
	copy(u[4:6], p[1])
	copy(u[6:8], p[2])
	copy(u[8:10], p[3])
	copy(u[10:16], p[4])

	return u, err
}

// GetCredentials returns a Credentials struct for the given cluster name
func (c *ClusterClient) GetCredentials(clusterName string) (*Credentials, error) {
	url, err := c.GetZipURL(clusterName)
	if err != nil {
		return nil, err
	}
	zr, err := fetchZip(url)
	if err != nil || len(zr.File) < 6 {
		return nil, err
	}

	// fetch the contents for each credential/note
	creds := new(Credentials)
	creds.Files = make(map[string][]byte)
	for _, zf := range zr.File {
		// dir should be the UUID that comes out in the bundle
		dir, fname := path.Split(zf.Name)
		fi := zf.FileInfo()

		if fi.IsDir() {
			// get uuid that's part of the zip dump
			creds.UUID, err = extractUUID(dir)
			if err != nil {
				return nil, errors.New("Unable to read UUID from directory name in zip file: " + err.Error())
			}
			continue
		}

		rc, err := zf.Open()
		if err != nil {
			return nil, err
		}

		b, err := ioutil.ReadAll(rc)
		if err != nil {
			return nil, err
		}
		creds.Files[fname] = b

		switch fname {
		case "ca.pem":
			creds.CA = b
		case "README.md":
			creds.README = b
		case "ca-key.pem":
			creds.CAKey = b
		case "docker.env":
			creds.DockerEnv = b
		case "cert.pem":
			creds.Cert = b
		case "key.pem":
			creds.Key = b
		}
	}

	sourceLines := strings.Split(string(creds.DockerEnv), "\n")
	for _, line := range sourceLines {
		if strings.Index(line, "export ") == 0 {
			varDecl := strings.TrimRight(line[7:], "\n")
			eqLocation := strings.Index(varDecl, "=")

			varName := varDecl[:eqLocation]
			varValue := varDecl[eqLocation+1:]

			switch varName {
			case "DOCKER_HOST":
				creds.DockerHost = varValue
			}

		}
	}

	return creds, nil
}

// GetDockerConfig returns the hostname and tls.Config for a given clustername
func (c *ClusterClient) GetDockerConfig(clusterName string) (hostname string, tlsConfig *tls.Config, err error) {
	creds, err := c.GetCredentials(clusterName)
	if err != nil {
		return "", nil, err
	}
	tlsConfig, err = creds.GetTLSConfig()
	return creds.DockerHost, tlsConfig, err
}

// GetTLSConfig returns a tls.Config for a credential set
func (creds *Credentials) GetTLSConfig() (*tls.Config, error) {
	// TLS config
	var tlsConfig tls.Config
	tlsConfig.InsecureSkipVerify = true
	certPool := x509.NewCertPool()

	certPool.AppendCertsFromPEM(creds.CA)
	tlsConfig.RootCAs = certPool
	keypair, err := tls.X509KeyPair(creds.Cert, creds.Key)
	if err != nil {
		return &tlsConfig, err
	}
	tlsConfig.Certificates = []tls.Certificate{keypair}

	return &tlsConfig, nil
}

func fetchZip(zipurl string) (*zip.Reader, error) {
	req, err := http.NewRequest("GET", zipurl, nil)
	if err != nil {
		return nil, err
	}

	req.Header.Set("User-Agent", userAgent)

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode >= 400 {
		b, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, errors.New(resp.Status)
		}
		return nil, errors.New(string(b))
	}

	buf := &bytes.Buffer{}

	_, err = io.Copy(buf, resp.Body)
	if err != nil {
		return nil, err
	}

	b := bytes.NewReader(buf.Bytes())
	return zip.NewReader(b, int64(b.Len()))
}

// Grow increases a cluster by the provided number of nodes
func (c *ClusterClient) Grow(clusterName string, nodes int) (*Cluster, error) {
	incr := make(map[string]json.Number)
	incr["nodes"] = json.Number(nodes)
	growthRequest, err := json.Marshal(incr)
	if err != nil {
		return nil, err
	}
	r := bytes.NewReader(growthRequest)

	uri := path.Join("/clusters", c.Username, clusterName, "grow")
	resp, err := c.NewRequest("POST", uri, r)
	return clusterFromResponse(resp, err)
}

// Delete nukes a cluster out of existence
func (c *ClusterClient) Delete(clusterName string) (*Cluster, error) {
	uri := path.Join("/clusters", c.Username, clusterName)
	resp, err := c.NewRequest("DELETE", uri, nil)
	return clusterFromResponse(resp, err)
}
<commit_msg>Only have NewClusterClient with API Key<commit_after>package rcs

import (
	"archive/zip"
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"path"
	"reflect"
	"strconv"
	"strings"

	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/rackspace"
)

// BetaEndpoint reflects the default endpoint for this library
const BetaEndpoint = "https://mycluster.rackspacecloud.com"
const mimetypeJSON = "application/json"
const authHeaderKey = "X-Auth-Token"
const userAgent = "rgbkrk/gorcs"

// ZipURLResponse is the response that comes back from the zip endpoint
type ZipURLResponse struct {
	URL string `json:"zip_url"`
}

// ClusterClient accesses RCS
type ClusterClient struct {
	client   *http.Client
	Username string
	Token    string
	Endpoint string
}

// ErrorResponse is the JSON formatted error response from RCS
type ErrorResponse struct {
	Error string `json:"error"`
}

// Cluster is a cluster
type Cluster struct {
	ClusterName string `json:"cluster_name"`
	Username    string `json:"username"`

	// Flavor of compute to use for cluster, should be a default value currently
	Flavor string `json:"flavor,omitempty"`

	// UUID of image to use for cluster, should be a default value currently
	Image string `json:"image,omitempty"`

	// Node is optional, but allowed on create
	// Sadly it comes back as string instead of int in all cases
	// with the API
	Nodes number `json:"nodes,omitempty"`

	AutoScale bool   `json:"autoscale,omitempty"`
	Status    string `json:"status,omitempty"`
	TaskID    string `json:"task_id,omitempty"`
	Token     string `json:"token,omitempty"`
}

// Credentials holds the keys to the kingdom
type Credentials struct {
	README     []byte
	Cert       []byte
	Key        []byte
	CA         []byte
	CAKey      []byte
	DockerEnv  []byte
	DockerHost string
	Files      map[string][]byte
	UUID       UUID
}

// Specify this type for any struct fields that
// might be unmarshaled from JSON numbers of the following
// types: floats, integers, scientific notation, or strings
type number float64

func (n number) Int64() int64 {
	return int64(n)
}

func (n number) Int() int {
	return int(n)
}

func (n number) Float64() float64 {
	return float64(n)
}

// Required to enforce that string values are attempted to be parsed as numbers
func (n *number) UnmarshalJSON(data []byte) error {
	var f float64
	var err error
	if data[0] == '"' {
		f, err = strconv.ParseFloat(string(data[1:len(data)-1]), 64)
		if err != nil {
			return &json.UnmarshalTypeError{
				Value: string(data),
				Type:  reflect.TypeOf(*n),
			}
		}
	} else {
		if err := json.Unmarshal(data, &f); err != nil {
			return &json.UnmarshalTypeError{
				Value: string(data),
				Type:  reflect.TypeOf(*n),
			}
		}
	}
	*n = number(f)
	return nil
}

func newClusterClient(endpoint string, ao gophercloud.AuthOptions) (*ClusterClient, error) {
	provider, err := rackspace.AuthenticatedClient(ao)
	if err != nil {
		return nil, err
	}

	return &ClusterClient{
		client:   &http.Client{},
		Username: ao.Username,
		Token:    provider.TokenID,
		Endpoint: endpoint,
	}, nil
}

// NewClusterClient create a new clusterclient by API Key
func NewClusterClient(endpoint, username, apikey string) (*ClusterClient, error) {
	ao := gophercloud.AuthOptions{
		Username:         username,
		APIKey:           apikey,
		IdentityEndpoint: rackspace.RackspaceUSIdentity,
	}

	return newClusterClient(endpoint, ao)
}

// NewRequest handles a request using auth used by RCS
func (c *ClusterClient) NewRequest(method string, uri string, body io.Reader) (*http.Response, error) {
	req, err := http.NewRequest(method, c.Endpoint+uri, body)
	if err != nil {
		return nil, err
	}
	req.Header.Set("User-Agent", userAgent)
	req.Header.Add("Content-Type", mimetypeJSON)
	req.Header.Add(authHeaderKey, c.Token)
	resp, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode >= 400 {
		b, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, errors.New(resp.Status)
		}
		return nil, errors.New(string(b))
	}

	return resp, nil
}

// List the current clusters
func (c *ClusterClient) List() ([]Cluster, error) {
	clusters := []Cluster{}

	resp, err := c.NewRequest("GET", "/clusters/"+c.Username, nil)
	if err != nil {
		return nil, err
	}

	err = json.NewDecoder(resp.Body).Decode(&clusters)
	if err != nil {
		return nil, err
	}
	return clusters, nil
}

func clusterFromResponse(resp *http.Response, err error) (*Cluster, error) {
	if err != nil {
		return nil, err
	}
	cluster := new(Cluster)
	err = json.NewDecoder(resp.Body).Decode(&cluster)
	if err != nil {
		return nil, err
	}
	return cluster, nil
}

// Get a cluster by cluster name
func (c *ClusterClient) Get(clusterName string) (*Cluster, error) {
	uri := path.Join("/clusters", c.Username, clusterName)
	resp, err := c.NewRequest("GET", uri, nil)
	return clusterFromResponse(resp, err)
}

// Create a new cluster with cluster options
func (c *ClusterClient) Create(clusterOpts Cluster) (*Cluster, error) {
	// Even though username is in the URI path, the API expects the username
	// inside the body
	if clusterOpts.Username == "" {
		clusterOpts.Username = c.Username
	}
	clusterOptsJSON, err := json.Marshal(clusterOpts)
	if err != nil {
		return nil, err
	}

	body := bytes.NewReader(clusterOptsJSON)
	uri := path.Join("/clusters", c.Username)
	resp, err := c.NewRequest("POST", uri, body)
	return clusterFromResponse(resp, err)
}

// GetZipURL returns the URL for downloading credentials
func (c *ClusterClient) GetZipURL(clusterName string) (string, error) {
	uri := path.Join("/clusters", c.Username, clusterName, "zip")
	resp, err := c.NewRequest("GET", uri, nil)
	if err != nil {
		return "", err
	}

	var zipURLResp ZipURLResponse

	err = json.NewDecoder(resp.Body).Decode(&zipURLResp)

	if err != nil {
		return "", err
	}

	return zipURLResp.URL, nil
}

// UUID represents a UUID value. UUIDs can be compared and set to other values
// and accessed by byte.
type UUID [16]byte

func extractUUID(s string) (UUID, error) {
	s = strings.Trim(s, "/")
	var u UUID
	var err error

	if len(s) != 36 {
		return UUID{}, fmt.Errorf("Invalid UUID")
	}
	format := "%08x-%04x-%04x-%04x-%012x"

	// create stack addresses for each section of the uuid.
	p := make([][]byte, 5)

	if _, err := fmt.Sscanf(s, format, &p[0], &p[1], &p[2], &p[3], &p[4]); err != nil {
		return u, err
	}

	copy(u[0:4], p[0])
	copy(u[4:6], p[1])
	copy(u[6:8], p[2])
	copy(u[8:10], p[3])
	copy(u[10:16], p[4])

	return u, err
}

// GetCredentials returns a Credentials struct for the given cluster name
func (c *ClusterClient) GetCredentials(clusterName string) (*Credentials, error) {
	url, err := c.GetZipURL(clusterName)
	if err != nil {
		return nil, err
	}
	zr, err := fetchZip(url)
	if err != nil || len(zr.File) < 6 {
		return nil, err
	}

	// fetch the contents for each credential/note
	creds := new(Credentials)
	creds.Files = make(map[string][]byte)
	for _, zf := range zr.File {
		// dir should be the UUID that comes out in the bundle
		dir, fname := path.Split(zf.Name)
		fi := zf.FileInfo()

		if fi.IsDir() {
			// get uuid that's part of the zip dump
			creds.UUID, err = extractUUID(dir)
			if err != nil {
				return nil, errors.New("Unable to read UUID from directory name in zip file: " + err.Error())
			}
			continue
		}

		rc, err := zf.Open()
		if err != nil {
			return nil, err
		}

		b, err := ioutil.ReadAll(rc)
		if err != nil {
			return nil, err
		}
		creds.Files[fname] = b

		switch fname {
		case "ca.pem":
			creds.CA = b
		case "README.md":
			creds.README = b
		case "ca-key.pem":
			creds.CAKey = b
		case "docker.env":
			creds.DockerEnv = b
		case "cert.pem":
			creds.Cert = b
		case "key.pem":
			creds.Key = b
		}
	}

	sourceLines := strings.Split(string(creds.DockerEnv), "\n")
	for _, line := range sourceLines {
		if strings.Index(line, "export ") == 0 {
			varDecl := strings.TrimRight(line[7:], "\n")
			eqLocation := strings.Index(varDecl, "=")

			varName := varDecl[:eqLocation]
			varValue := varDecl[eqLocation+1:]

			switch varName {
			case "DOCKER_HOST":
				creds.DockerHost = varValue
			}

		}
	}

	return creds, nil
}

// GetDockerConfig returns the hostname and tls.Config for a given clustername
func (c *ClusterClient) GetDockerConfig(clusterName string) (hostname string, tlsConfig *tls.Config, err error) {
	creds, err := c.GetCredentials(clusterName)
	if err != nil {
		return "", nil, err
	}
	tlsConfig, err = creds.GetTLSConfig()
	return creds.DockerHost, tlsConfig, err
}

// GetTLSConfig returns a tls.Config for a credential set
func (creds *Credentials) GetTLSConfig() (*tls.Config, error) {
	// TLS config
	var tlsConfig tls.Config
	tlsConfig.InsecureSkipVerify = true
	certPool := x509.NewCertPool()

	certPool.AppendCertsFromPEM(creds.CA)
	tlsConfig.RootCAs = certPool
	keypair, err := tls.X509KeyPair(creds.Cert, creds.Key)
	if err != nil {
		return &tlsConfig, err
	}
	tlsConfig.Certificates = []tls.Certificate{keypair}

	return &tlsConfig, nil
}

func fetchZip(zipurl string) (*zip.Reader, error) {
	req, err := http.NewRequest("GET", zipurl, nil)
	if err != nil {
		return nil, err
	}

	req.Header.Set("User-Agent", userAgent)

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode >= 400 {
		b, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, errors.New(resp.Status)
		}
		return nil, errors.New(string(b))
	}

	buf := &bytes.Buffer{}

	_, err = io.Copy(buf, resp.Body)
	if err != nil {
		return nil, err
	}

	b := bytes.NewReader(buf.Bytes())
	return zip.NewReader(b, int64(b.Len()))
}

// Grow increases a cluster by the provided number of nodes
func (c *ClusterClient) Grow(clusterName string, nodes int) (*Cluster, error) {
	incr := make(map[string]json.Number)
	incr["nodes"] = json.Number(nodes)
	growthRequest, err := json.Marshal(incr)
	if err != nil {
		return nil, err
	}
	r := bytes.NewReader(growthRequest)

	uri := path.Join("/clusters", c.Username, clusterName, "grow")
	resp, err := c.NewRequest("POST", uri, r)
	return clusterFromResponse(resp, err)
}

// Delete nukes a cluster out of existence
func (c *ClusterClient) Delete(clusterName string) (*Cluster, error) {
	uri := path.Join("/clusters", c.Username, clusterName)
	resp, err := c.NewRequest("DELETE", uri, nil)
	return clusterFromResponse(resp, err)
}
<|endoftext|>


<commit_before>package gorma

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/raphael/goa/design"
	"github.com/raphael/goa/goagen/codegen"
	"gopkg.in/alecthomas/kingpin.v2"
)

// Generator is the application code generator.
type Generator struct {
	genfiles []string
}

// Generate is the generator entry point called by the meta generator.
func Generate(api *design.APIDefinition) ([]string, error) {
	g, err := NewGenerator()
	if err != nil {
		return nil, err
	}
	return g.Generate(api)
}

// NewGenerator returns the application code generator.
func NewGenerator() (*Generator, error) {
	return new(Generator), nil
}

// Generate produces the skeleton main.
func (g *Generator) Generate(api *design.APIDefinition) ([]string, error) {

	os.RemoveAll(ModelDir())
	os.MkdirAll(ModelDir(), 0755)
	app := kingpin.New("Model generator", "model generator")
	codegen.RegisterFlags(app)
	_, err := app.Parse(os.Args[1:])
	if err != nil {
		panic(err)
	}
	imp, err := filepath.Rel(filepath.Join(os.Getenv("GOPATH"), "src"), codegen.OutputDir)
	if err != nil {
		return nil, err
	}
	imp = filepath.Join(imp, "app")
	imports := []*codegen.ImportSpec{
		codegen.SimpleImport(imp),
		codegen.SimpleImport("github.com/jinzhu/gorm"),
		codegen.SimpleImport("github.com/jinzhu/copier"),
		codegen.SimpleImport("database/sql"),
	}

	rbacimports := []*codegen.ImportSpec{
		codegen.SimpleImport(imp),
		codegen.SimpleImport("github.com/mikespook/gorbac"),
	}

	rbactitle := fmt.Sprintf("%s: RBAC", api.Name)
	_, dorbac := api.Metadata["github.com/bketelsen/gorma#rbac"]

	err = api.IterateUserTypes(func(res *design.UserTypeDefinition) error {
		if res.Type.IsObject() {
			title := fmt.Sprintf("%s: Models", api.Name)
			filename := filepath.Join(ModelDir(), res.Name()+"_model.go")
			mtw, err := NewModelWriter(filename)
			if err != nil {
				panic(err)
			}
			mtw.WriteHeader(title, "models", imports)
			if md, ok := res.Metadata["github.com/bketelsen/gorma"]; ok && md == "Model" {
				fmt.Println("Found Gorma Metadata:", md)
				err = mtw.Execute(res)
				if err != nil {
					g.Cleanup()
					return err
				}
			}
			if err := mtw.FormatCode(); err != nil {
				g.Cleanup()

			}
			if err != nil {
				g.genfiles = append(g.genfiles, filename)
			}
			return nil
		}

		return nil

	})
	if dorbac {
		rbacfilename := filepath.Join(ModelDir(), "rbac.go")
		rbacw, err := NewRbacWriter(rbacfilename)
		if err != nil {
			panic(err)
		}
		rbacw.WriteHeader(rbactitle, "models", rbacimports)
		err = rbacw.Execute(api)
		if err != nil {
			g.Cleanup()
			return g.genfiles, err
		}
		if err := rbacw.FormatCode(); err != nil {
			g.Cleanup()
			return nil, err
		}
		if err != nil {
			g.genfiles = append(g.genfiles, rbacfilename)
		}

	}

	return g.genfiles, err
}

// Cleanup removes all the files generated by this generator during the last invokation of Generate.
func (g *Generator) Cleanup() {
	for _, f := range g.genfiles {
		os.Remove(f)
	}
	g.genfiles = nil
}
<commit_msg>trying it<commit_after>package gorma

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/raphael/goa/design"
	"github.com/raphael/goa/goagen/codegen"
	"gopkg.in/alecthomas/kingpin.v2"
)

// Generator is the application code generator.
type Generator struct {
	genfiles []string
}

// Generate is the generator entry point called by the meta generator.
func Generate(api *design.APIDefinition) ([]string, error) {
	g, err := NewGenerator()
	if err != nil {
		return nil, err
	}
	return g.Generate(api)
}

// NewGenerator returns the application code generator.
func NewGenerator() (*Generator, error) {
	return new(Generator), nil
}

// Generate produces the skeleton main.
func (g *Generator) Generate(api *design.APIDefinition) ([]string, error) {

	os.RemoveAll(ModelDir())
	os.MkdirAll(ModelDir(), 0755)
	app := kingpin.New("Model generator", "model generator")
	codegen.RegisterFlags(app)
	_, err := app.Parse(os.Args[1:])
	if err != nil {
		panic(err)
	}
	imp, err := filepath.Rel(filepath.Join(os.Getenv("GOPATH"), "src"), codegen.OutputDir)
	if err != nil {
		return nil, err
	}
	imp = filepath.Join(imp, "app")
	imports := []*codegen.ImportSpec{
		codegen.SimpleImport(imp),
		codegen.SimpleImport("github.com/jinzhu/gorm"),
		codegen.SimpleImport("github.com/jinzhu/copier"),
		codegen.SimpleImport("database/sql"),
	}

	rbacimports := []*codegen.ImportSpec{
		codegen.SimpleImport(imp),
		codegen.SimpleImport("github.com/mikespook/gorbac"),
	}

	rbactitle := fmt.Sprintf("%s: RBAC", api.Name)
	_, dorbac := api.Metadata["github.com/bketelsen/gorma#rbac"]

	err = api.IterateUserTypes(func(res *design.UserTypeDefinition) error {
		if res.Type.IsObject() {
			title := fmt.Sprintf("%s: Models", api.Name)
			filename := filepath.Join(ModelDir(), res.Type.Name()+"_model.go")
			mtw, err := NewModelWriter(filename)
			if err != nil {
				panic(err)
			}
			mtw.WriteHeader(title, "models", imports)
			if md, ok := res.Metadata["github.com/bketelsen/gorma"]; ok && md == "Model" {
				fmt.Println("Found Gorma Metadata:", md)
				err = mtw.Execute(res)
				if err != nil {
					g.Cleanup()
					return err
				}
			}
			if err := mtw.FormatCode(); err != nil {
				g.Cleanup()

			}
			if err != nil {
				g.genfiles = append(g.genfiles, filename)
			}
			return nil
		}

		return nil

	})
	if dorbac {
		rbacfilename := filepath.Join(ModelDir(), "rbac.go")
		rbacw, err := NewRbacWriter(rbacfilename)
		if err != nil {
			panic(err)
		}
		rbacw.WriteHeader(rbactitle, "models", rbacimports)
		err = rbacw.Execute(api)
		if err != nil {
			g.Cleanup()
			return g.genfiles, err
		}
		if err := rbacw.FormatCode(); err != nil {
			g.Cleanup()
			return nil, err
		}
		if err != nil {
			g.genfiles = append(g.genfiles, rbacfilename)
		}

	}

	return g.genfiles, err
}

// Cleanup removes all the files generated by this generator during the last invokation of Generate.
func (g *Generator) Cleanup() {
	for _, f := range g.genfiles {
		os.Remove(f)
	}
	g.genfiles = nil
}
<|endoftext|>


<commit_before>// Copyright ©2014 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package graph

// All a node needs to do is identify itself. This allows the user to pass in nodes more
// interesting than an int, but also allow us to reap the benefits of having a map-storable,
// ==able type.
type Node interface {
	ID() int
}

// Allows edges to do something more interesting that just be a group of nodes. While the methods
// are called Head and Tail, they are not considered directed unless the given interface specifies
// otherwise.
type Edge interface {
	Head() Node
	Tail() Node
}

// A Graph implements the behavior of an undirected graph.
//
// All methods in Graph are implicitly undirected. Graph algorithms that care about directionality
// will intelligently choose the DirectedGraph behavior if that interface is also implemented,
// even if the function itself only takes in a Graph (or a super-interface of graph).
type Graph interface {
	// NodeExists returns true when node is currently in the graph.
	NodeExists(Node) bool

	// NodeList returns a list of all nodes in no particular order, useful for
	// determining things like if a graph is fully connected. The caller is
	// free to modify this list. Implementations should construct a new list
	// and not return internal representation.
	NodeList() []Node

	// Neighbors returns all nodes connected by any edge to this node.
	Neighbors(Node) []Node

	// EdgeBetween returns an edge between node and neighbor such that
	// Head is one argument and Tail is the other. If no
	// such edge exists, this function returns nil.
	EdgeBetween(node, neighbor Node) Edge
}

// Directed graphs are characterized by having seperable Heads and Tails in their edges.
// That is, if node1 goes to node2, that does not necessarily imply that node2 goes to node1.
//
// While it's possible for a directed graph to have fully reciprocal edges (i.e. the graph is
// symmetric) -- it is not required to be. The graph is also required to implement Graph
// because in many cases it can be useful to know all neighbors regardless of direction.
type DirectedGraph interface {
	Graph
	// Successors gives the nodes connected by OUTBOUND edges.
	// If the graph is an undirected graph, this set is equal to Predecessors.
	Successors(Node) []Node

	// EdgeTo returns an edge between node and successor such that
	// Head returns node and Tail returns successor, if no
	// such edge exists, this function returns nil.
	EdgeTo(node, successor Node) Edge

	// Predecessors gives the nodes connected by INBOUND edges.
	// If the graph is an undirected graph, this set is equal to Successors.
	Predecessors(Node) []Node
}

// Returns all undirected edges in the graph
type EdgeLister interface {
	EdgeList() []Edge
}

type EdgeListGraph interface {
	Graph
	EdgeLister
}

// Returns all directed edges in the graph.
type DirectedEdgeLister interface {
	DirectedEdgeList() []Edge
}

type DirectedEdgeListGraph interface {
	Graph
	DirectedEdgeLister
}

// A crunch graph forces a sparse graph to become a dense graph. That is, if the node IDs are
// [1,4,9,7] it would "crunch" the ids into the contiguous block [0,1,2,3]. Order is not
// required to be preserved between the non-cruched and crunched instances (that means in
// the example above 0 may correspond to 4 or 7 or 9, not necessarily 1).
//
// All dense graphs must have the first ID as 0.
type CrunchGraph interface {
	Graph
	Crunch()
}

// A Graph that implements Coster has an actual cost between adjacent nodes, also known as a
// weighted graph. If a graph implements coster and a function needs to read cost (e.g. A*),
// this function will take precedence over the Uniform Cost function (all weights are 1) if "nil"
// is passed in for the function argument.
//
// If the argument is nil, or the edge is invalid for some reason, this should return math.Inf(1)
type Coster interface {
	Cost(edge Edge) float64
}

type CostGraph interface {
	Coster
	Graph
}

type CostDirectedGraph interface {
	Coster
	DirectedGraph
}

// A graph that implements HeuristicCoster implements a heuristic between any two given nodes.
// Like Coster, if a graph implements this and a function needs a heuristic cost (e.g. A*), this
// function will take precedence over the Null Heuristic (always returns 0) if "nil" is passed in
// for the function argument. If HeuristicCost is not intended to be used, it can be implemented as
// the null heuristic (always returns 0.)
type HeuristicCoster interface {
	// HeuristicCost returns a heuristic cost between any two nodes.
	HeuristicCost(node1, node2 Node) float64
}

// A Mutable is a graph that can have arbitrary nodes and edges added or removed.
//
// Anything implementing Mutable is required to store the actual argument. So if AddNode(myNode) is
// called and later a user calls on the graph graph.NodeList(), the node added by AddNode must be
// an the exact node, not a new node with the same ID.
//
// In any case where conflict is possible (e.g. adding two nodes with the same ID), the later
// call always supercedes the earlier one.
//
// Functions will generally expect one of MutableGraph or MutableDirectedGraph and not Mutable
// itself. That said, any function that takes Mutable[x], the destination mutable should
// always be a different graph than the source.
type Mutable interface {
	// NewNode adds a node with an arbitrary ID and returns the new, unique ID
	// used.
	NewNode() Node

	// Adds a node to the graph. If this is called multiple times for the same ID, the newer node
	// overwrites the old one.
	AddNode(Node)

	// EmptyGraph clears the graph of all nodes and edges.
	EmptyGraph()

	// RemoveNode removes a node from the graph, as well as any edges
	// attached to it. If no such node exists, this is a no-op, not an error.
	RemoveNode(Node)
}

// MutableGraph is an interface ensuring the implementation of the ability to construct
// an arbitrary undirected graph. It is very important to note that any implementation
// of MutableGraph absolutely cannot safely implement the DirectedGraph interface.
//
// A MutableGraph is required to store any Edge argument in the same way Mutable must
// store a Node argument -- any retrieval call is required to return the exact supplied edge.
// This is what makes it incompatible with DirectedGraph.
//
// A call to AddEdgeBetween(Edge{head,tail}) make is so there is simply no way to safely
// return EdgeTo(tail, head) since the edge returned will, by this contract, need to be
// Head() == head and Tail() == tail when the reverse must be true to fulfill the
// functionality guaranteed of EdgeTo.
type MutableGraph interface {
	CostGraph
	Mutable

	// Like EdgeBetween in Graph, AddEdgeBetween adds an edge between two nodes.
	// If one or both nodes do not exist, the Graph is expected to add them.
	AddEdgeBetween(e Edge, cost float64)

	// RemoveEdge clears the stored edge between two nodes. Calling this will never
	// remove a node. If the edge does not exist this is a no-op, not an error.
	RemoveEdgeBetween(e Edge)
}

// MutableDirectedGraph is an interface that ensures one can construct an arbitrary directed
// graph. Naturally, a MutableDirectedGraph works for both undirected and directed cases,
// but simply using a MutableGraph may be cleaner. As the documentation for MutableGraph
// notes, however, a graph cannot safely implement MutableGraph and MutableDirectedGraph
// at the same time, because of the functionality of a EdgeTo in DirectedGraph.
type MutableDirectedGraph interface {
	CostDirectedGraph
	Mutable

	// Adds an edge FROM e.Head TO e.Tail. Newer calls overwrite older ones.
	// If the nodes Head or Tail do not exist in the graph, this must add them.
	AddEdgeTo(e Edge, cost float64)

	// Removes an edge FROM e.Head TO e.Tail. If no such edge exists, this is a no-op,
	// not an error.
	RemoveEdgeTo(e Edge)
}

// A function that returns the cost of following an edge
type CostFunc func(Edge) float64

// Estimates the cost of travelling between two nodes
type HeuristicCostFunc func(Node, Node) float64

// Convenience constants for AddEdge and RemoveEdge
const (
	Directed   bool = true
	Undirected      = false
)

// Determines if a MutableGraph implements DirectedGraph and panics if it does.
// This is a utility function to detect unsafe implementations. It's mostly for internal use,
// but is exported since it may be useful to people who use the package for their own tests.
func VetMutableGraph(g MutableGraph) {
	if _, ok := g.(DirectedGraph); ok {
		panic("A MutableGraph implements DirectedGraph; this is unsafe!")
	}
}
<commit_msg>Removed useless convenience constants<commit_after>// Copyright ©2014 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package graph

// All a node needs to do is identify itself. This allows the user to pass in nodes more
// interesting than an int, but also allow us to reap the benefits of having a map-storable,
// ==able type.
type Node interface {
	ID() int
}

// Allows edges to do something more interesting that just be a group of nodes. While the methods
// are called Head and Tail, they are not considered directed unless the given interface specifies
// otherwise.
type Edge interface {
	Head() Node
	Tail() Node
}

// A Graph implements the behavior of an undirected graph.
//
// All methods in Graph are implicitly undirected. Graph algorithms that care about directionality
// will intelligently choose the DirectedGraph behavior if that interface is also implemented,
// even if the function itself only takes in a Graph (or a super-interface of graph).
type Graph interface {
	// NodeExists returns true when node is currently in the graph.
	NodeExists(Node) bool

	// NodeList returns a list of all nodes in no particular order, useful for
	// determining things like if a graph is fully connected. The caller is
	// free to modify this list. Implementations should construct a new list
	// and not return internal representation.
	NodeList() []Node

	// Neighbors returns all nodes connected by any edge to this node.
	Neighbors(Node) []Node

	// EdgeBetween returns an edge between node and neighbor such that
	// Head is one argument and Tail is the other. If no
	// such edge exists, this function returns nil.
	EdgeBetween(node, neighbor Node) Edge
}

// Directed graphs are characterized by having seperable Heads and Tails in their edges.
// That is, if node1 goes to node2, that does not necessarily imply that node2 goes to node1.
//
// While it's possible for a directed graph to have fully reciprocal edges (i.e. the graph is
// symmetric) -- it is not required to be. The graph is also required to implement Graph
// because in many cases it can be useful to know all neighbors regardless of direction.
type DirectedGraph interface {
	Graph
	// Successors gives the nodes connected by OUTBOUND edges.
	// If the graph is an undirected graph, this set is equal to Predecessors.
	Successors(Node) []Node

	// EdgeTo returns an edge between node and successor such that
	// Head returns node and Tail returns successor, if no
	// such edge exists, this function returns nil.
	EdgeTo(node, successor Node) Edge

	// Predecessors gives the nodes connected by INBOUND edges.
	// If the graph is an undirected graph, this set is equal to Successors.
	Predecessors(Node) []Node
}

// Returns all undirected edges in the graph
type EdgeLister interface {
	EdgeList() []Edge
}

type EdgeListGraph interface {
	Graph
	EdgeLister
}

// Returns all directed edges in the graph.
type DirectedEdgeLister interface {
	DirectedEdgeList() []Edge
}

type DirectedEdgeListGraph interface {
	Graph
	DirectedEdgeLister
}

// A crunch graph forces a sparse graph to become a dense graph. That is, if the node IDs are
// [1,4,9,7] it would "crunch" the ids into the contiguous block [0,1,2,3]. Order is not
// required to be preserved between the non-cruched and crunched instances (that means in
// the example above 0 may correspond to 4 or 7 or 9, not necessarily 1).
//
// All dense graphs must have the first ID as 0.
type CrunchGraph interface {
	Graph
	Crunch()
}

// A Graph that implements Coster has an actual cost between adjacent nodes, also known as a
// weighted graph. If a graph implements coster and a function needs to read cost (e.g. A*),
// this function will take precedence over the Uniform Cost function (all weights are 1) if "nil"
// is passed in for the function argument.
//
// If the argument is nil, or the edge is invalid for some reason, this should return math.Inf(1)
type Coster interface {
	Cost(edge Edge) float64
}

type CostGraph interface {
	Coster
	Graph
}

type CostDirectedGraph interface {
	Coster
	DirectedGraph
}

// A graph that implements HeuristicCoster implements a heuristic between any two given nodes.
// Like Coster, if a graph implements this and a function needs a heuristic cost (e.g. A*), this
// function will take precedence over the Null Heuristic (always returns 0) if "nil" is passed in
// for the function argument. If HeuristicCost is not intended to be used, it can be implemented as
// the null heuristic (always returns 0.)
type HeuristicCoster interface {
	// HeuristicCost returns a heuristic cost between any two nodes.
	HeuristicCost(node1, node2 Node) float64
}

// A Mutable is a graph that can have arbitrary nodes and edges added or removed.
//
// Anything implementing Mutable is required to store the actual argument. So if AddNode(myNode) is
// called and later a user calls on the graph graph.NodeList(), the node added by AddNode must be
// an the exact node, not a new node with the same ID.
//
// In any case where conflict is possible (e.g. adding two nodes with the same ID), the later
// call always supercedes the earlier one.
//
// Functions will generally expect one of MutableGraph or MutableDirectedGraph and not Mutable
// itself. That said, any function that takes Mutable[x], the destination mutable should
// always be a different graph than the source.
type Mutable interface {
	// NewNode adds a node with an arbitrary ID and returns the new, unique ID
	// used.
	NewNode() Node

	// Adds a node to the graph. If this is called multiple times for the same ID, the newer node
	// overwrites the old one.
	AddNode(Node)

	// EmptyGraph clears the graph of all nodes and edges.
	EmptyGraph()

	// RemoveNode removes a node from the graph, as well as any edges
	// attached to it. If no such node exists, this is a no-op, not an error.
	RemoveNode(Node)
}

// MutableGraph is an interface ensuring the implementation of the ability to construct
// an arbitrary undirected graph. It is very important to note that any implementation
// of MutableGraph absolutely cannot safely implement the DirectedGraph interface.
//
// A MutableGraph is required to store any Edge argument in the same way Mutable must
// store a Node argument -- any retrieval call is required to return the exact supplied edge.
// This is what makes it incompatible with DirectedGraph.
//
// A call to AddEdgeBetween(Edge{head,tail}) make is so there is simply no way to safely
// return EdgeTo(tail, head) since the edge returned will, by this contract, need to be
// Head() == head and Tail() == tail when the reverse must be true to fulfill the
// functionality guaranteed of EdgeTo.
type MutableGraph interface {
	CostGraph
	Mutable

	// Like EdgeBetween in Graph, AddEdgeBetween adds an edge between two nodes.
	// If one or both nodes do not exist, the Graph is expected to add them.
	AddEdgeBetween(e Edge, cost float64)

	// RemoveEdge clears the stored edge between two nodes. Calling this will never
	// remove a node. If the edge does not exist this is a no-op, not an error.
	RemoveEdgeBetween(e Edge)
}

// MutableDirectedGraph is an interface that ensures one can construct an arbitrary directed
// graph. Naturally, a MutableDirectedGraph works for both undirected and directed cases,
// but simply using a MutableGraph may be cleaner. As the documentation for MutableGraph
// notes, however, a graph cannot safely implement MutableGraph and MutableDirectedGraph
// at the same time, because of the functionality of a EdgeTo in DirectedGraph.
type MutableDirectedGraph interface {
	CostDirectedGraph
	Mutable

	// Adds an edge FROM e.Head TO e.Tail. Newer calls overwrite older ones.
	// If the nodes Head or Tail do not exist in the graph, this must add them.
	AddEdgeTo(e Edge, cost float64)

	// Removes an edge FROM e.Head TO e.Tail. If no such edge exists, this is a no-op,
	// not an error.
	RemoveEdgeTo(e Edge)
}

// A function that returns the cost of following an edge
type CostFunc func(Edge) float64

// Estimates the cost of travelling between two nodes
type HeuristicCostFunc func(Node, Node) float64

// Determines if a MutableGraph implements DirectedGraph and panics if it does.
// This is a utility function to detect unsafe implementations. It's mostly for internal use,
// but is exported since it may be useful to people who use the package for their own tests.
func VetMutableGraph(g MutableGraph) {
	if _, ok := g.(DirectedGraph); ok {
		panic("A MutableGraph implements DirectedGraph; this is unsafe!")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"strconv"
	"text/tabwriter"

	"github.com/coreos-inc/updatectl/client/update/v1"
)

var (
	groupFlags struct {
		label          StringFlag
		channel        StringFlag
		appId          StringFlag
		groupId        StringFlag
		start          int64
		end            int64
		resolution     int64
		updateCount    int64
		updateInterval int64
	}

	cmdGroup = &Command{
		Name:    "group",
		Summary: "Operations that manage groups in an applciation.",
		Subcommands: []*Command{
			cmdGroupList,
			cmdGroupCreate,
			cmdGroupDelete,
			cmdGroupUpdate,
			cmdGroupPause,
			cmdGroupUnpause,
			cmdGroupRollupEvents,
			cmdGroupRollupVersions,
		},
	}

	cmdGroupList = &Command{
		Name:    "group list",
		Usage:   "[OPTION]...",
		Summary: `List all of the groups that exist including their label, token and update state.`,
		Run:     groupList,
	}
	cmdGroupCreate = &Command{
		Name:    "group create",
		Usage:   "[OPTION]...",
		Summary: `Create a new group.`,
		Run:     groupNew,
	}
	cmdGroupDelete = &Command{
		Name:    "group delete",
		Usage:   "[OPTION]...",
		Summary: `Delete a group.`,
		Run:     groupDelete,
	}
	cmdGroupUpdate = &Command{
		Name:        "group update",
		Usage:       "[OPTION]...",
		Description: `Update an existing group.`,
		Run:         groupUpdate,
	}
	cmdGroupPause = &Command{
		Name:    "group pause",
		Usage:   "[OPTION]...",
		Summary: `Pause a group's updates.`,
		Run:     groupPause,
	}
	cmdGroupUnpause = &Command{
		Name:    "group unpause",
		Usage:   "[OPTION]...",
		Summary: `Unpause a group's updates.`,
		Run:     groupUnpause,
	}
	cmdGroupRollupVersions = &Command{
		Name:    "group rollup-versions",
		Usage:   "[OPTION]...",
		Summary: "Rollup versions from events by time.",
		Run:     groupRollupVersions,
	}
	cmdGroupRollupEvents = &Command{
		Name:    "group rollup events",
		Usage:   "[OPTION]...",
		Summary: "Rollup events from events by time.",
		Run:     groupRollupEvents,
	}
)

func init() {
	cmdGroupList.Flags.Var(&groupFlags.appId, "app-id",
		"Application containing the groups to list.")

	cmdGroupDelete.Flags.Var(&groupFlags.appId, "app-id",
		"Application with group to delete.")
	cmdGroupDelete.Flags.Var(&groupFlags.groupId, "group-id",
		"ID of group to delete.")

	cmdGroupCreate.Flags.Var(&groupFlags.appId, "app-id",
		"Application to add group to.")
	cmdGroupCreate.Flags.Var(&groupFlags.groupId, "group-id",
		"ID for the new group.")
	cmdGroupCreate.Flags.Var(&groupFlags.channel, "channel",
		"Channel to associate with the group.")
	cmdGroupCreate.Flags.Var(&groupFlags.label, "label",
		"Label describing the new group.")

	cmdGroupUpdate.Flags.Var(&groupFlags.appId, "app-id",
		"Application containing the group to update.")
	cmdGroupUpdate.Flags.Var(&groupFlags.groupId, "group-id",
		"ID for the group.")
	cmdGroupUpdate.Flags.Var(&groupFlags.label, "label",
		"Label describing the group")
	cmdGroupUpdate.Flags.Var(&groupFlags.channel, "channel",
		"Channel to associate with the group.")
	cmdGroupUpdate.Flags.Int64Var(&groupFlags.updateCount, "update-count",
		-1, "Number of instances per interval")
	cmdGroupUpdate.Flags.Int64Var(&groupFlags.updateInterval,
		"update-interval", -1, "Interval between updates")

	cmdGroupPause.Flags.Var(&groupFlags.appId, "app-id",
		"Application containing the group to pause.")
	cmdGroupPause.Flags.Var(&groupFlags.groupId, "group-id",
		"ID for the group.")

	cmdGroupUnpause.Flags.Var(&groupFlags.appId, "app-id",
		"Application containing the group to unpause.")
	cmdGroupUnpause.Flags.Var(&groupFlags.groupId, "group-id",
		"ID for the group.")

	cmdGroupRollupVersions.Flags.Var(&groupFlags.appId, "app-id",
		"Application containing the group.")
	cmdGroupRollupVersions.Flags.Var(&groupFlags.groupId, "group-id",
		"ID for the group.")
	cmdGroupRollupVersions.Flags.Int64Var(&groupFlags.resolution,
		"resolution", 60, "60, 3600 or 86400 seconds")
	cmdGroupRollupVersions.Flags.Int64Var(&groupFlags.start, "start", 0,
		"Start date filter")
	cmdGroupRollupVersions.Flags.Int64Var(&groupFlags.end, "end", 0,
		"End date filter")

	cmdRollupGroupEvents.Flags.Var(&groupFlags.appId, "app-id",
		"Application containing the group.")
	cmdRollupGroupEvents.Flags.Var(&groupFlags.groupId, "group-id",
		"ID for the group.")
	cmdRollupGroupEvents.Flags.Int64Var(&groupFlags.resolution,
=======
	cmdGroupRollupEvents.Flags.Var(&groupFlags.appId, "app-id",
		"Application containing the group.")
	cmdGroupRollupEvents.Flags.Var(&groupFlags.groupId, "group-id",
		"ID for the group.")
	cmdGroupRollupEvents.Flags.Int64Var(&groupFlags.resolution,
>>>>>>> f4fb7b8... refactor(group): move group commands to subcommands
		"resolution", 60, "60, 3600 or 86400 seconds")
	cmdGroupRollupEvents.Flags.Int64Var(&groupFlags.start, "start", 0,
		"Start date filter")
	cmdGroupRollupEvents.Flags.Int64Var(&groupFlags.end, "end", 0,
		"End date filter")
}

func formatGroup(group *update.Group) string {
	return fmt.Sprintf("%s\t%s\t%s\t%s\t%s\t%v\t%v\n", group.Label, group.AppId, group.ChannelId,
		group.Id, strconv.FormatBool(group.UpdatesPaused), group.UpdateCount, group.UpdateInterval)
}

func groupList(args []string, service *update.Service, out *tabwriter.Writer) int {
	if groupFlags.appId.Get() == nil {
		return ERROR_USAGE
	}

	listCall := service.Group.List(groupFlags.appId.String())
	list, err := listCall.Do()

	if err != nil {
		log.Fatal(err)
	}

	fmt.Fprintln(out, "Label\tApp\tChannel\tId\tUpdatesPaused")
	for _, group := range list.Items {
		fmt.Fprintf(out, "%s", formatGroup(group))
	}

	out.Flush()
	return OK
}

func groupRollupEvents(args []string, service *update.Service, out *tabwriter.Writer) int {
	if groupFlags.appId.Get() == nil || groupFlags.groupId.Get() == nil {
		return ERROR_USAGE
	}

	call := service.Group.Requests.Events.Rollup(
		groupFlags.appId.String(),
		groupFlags.groupId.String(),
		groupFlags.start,
		groupFlags.end,
	)
	call.Resolution(groupFlags.resolution)
	list, err := call.Do()

	if err != nil {
		log.Fatal(err)
	}

	fmt.Fprintln(out, "Version\tType\tResult\tTimestamp\tCount")
	for _, i := range list.Items {
		for _, j := range i.Values {
			fmt.Fprintf(out, "%s\t%s\t%s\t%d\t%d\n",
				i.Version, i.Type, i.Result, j.Timestamp, j.Count)
		}
	}
	out.Flush()
	return OK
}

func groupRollupVersions(args []string, service *update.Service, out *tabwriter.Writer) int {
	if groupFlags.appId.Get() == nil || groupFlags.groupId.Get() == nil {
		return ERROR_USAGE
	}

	call := service.Group.Requests.Versions.Rollup(
		groupFlags.appId.String(),
		groupFlags.groupId.String(),
		groupFlags.start,
		groupFlags.end,
	)
	call.Resolution(groupFlags.resolution)
	list, err := call.Do()

	if err != nil {
		log.Fatal(err)
	}

	fmt.Fprintln(out, "Version\tTimestamp\tCount")
	for _, i := range list.Items {
		for _, j := range i.Values {
			fmt.Fprintf(out, "%s\t%d\t%d\n",
				i.Version, j.Timestamp, j.Count)
		}
	}
	out.Flush()
	return OK
}

func groupNew(args []string, service *update.Service, out *tabwriter.Writer) int {
	if groupFlags.appId.Get() == nil ||
		groupFlags.groupId.Get() == nil ||
		groupFlags.channel.Get() == nil {
		return ERROR_USAGE
	}

	group := &update.Group{
		ChannelId: groupFlags.channel.String(),
		Id:        groupFlags.groupId.String(),
		Label:     groupFlags.label.String(),
	}
	call := service.Group.Insert(groupFlags.appId.String(), group)
	group, err := call.Do()

	if err != nil {
		log.Fatal(err)
	}

	fmt.Fprintf(out, "%s", formatGroup(group))

	out.Flush()
	return OK
}

func groupDelete(args []string, service *update.Service, out *tabwriter.Writer) int {
	if groupFlags.appId.Get() == nil ||
		groupFlags.groupId.Get() == nil {
		return ERROR_USAGE
	}

	call := service.Group.Delete(groupFlags.appId.String(), groupFlags.groupId.String())
	group, err := call.Do()

	if err != nil {
		log.Fatal(err)
	}

	fmt.Fprintf(out, "%s", formatGroup(group))

	out.Flush()
	return OK
}

func groupPause(args []string, service *update.Service, out *tabwriter.Writer) int {
	return setUpdatesPaused(service, out, true)
}

func groupUnpause(args []string, service *update.Service, out *tabwriter.Writer) int {
	return setUpdatesPaused(service, out, false)
}

// Helper function for pause/unpause-group commands
func setUpdatesPaused(service *update.Service, out *tabwriter.Writer, paused bool) int {
	if groupFlags.appId.Get() == nil ||
		groupFlags.groupId.Get() == nil {
		return ERROR_USAGE
	}

	call := service.Group.Get(groupFlags.appId.String(), groupFlags.groupId.String())
	group, err := call.Do()

	if err != nil {
		log.Fatal(err)
	}

	group.UpdatesPaused = paused

	updateCall := service.Group.Patch(groupFlags.appId.String(), groupFlags.groupId.String(), group)
	group, err = updateCall.Do()

	if err != nil {
		log.Fatal(err)
	}

	fmt.Fprintf(out, "%s", formatGroup(group))

	out.Flush()
	return OK
}

func groupUpdate(args []string, service *update.Service, out *tabwriter.Writer) int {
	if groupFlags.appId.Get() == nil ||
		groupFlags.groupId.Get() == nil {
		return ERROR_USAGE
	}

	call := service.Group.Get(groupFlags.appId.String(), groupFlags.groupId.String())
	group, err := call.Do()

	if err != nil {
		log.Fatal(err)
	}

	checkUpdatePooling := false
	if groupFlags.updateCount != -1 {
		group.UpdateCount = groupFlags.updateCount
		checkUpdatePooling = true
	}
	if groupFlags.updateInterval != -1 {
		group.UpdateInterval = groupFlags.updateInterval
		checkUpdatePooling = true
	}
	if groupFlags.label.Get() != nil {
		group.Label = groupFlags.label.String()
	}
	if groupFlags.channel.Get() != nil {
		group.ChannelId = groupFlags.channel.String()
	}

	// set update pooling based on other flags
	// this only changes if the user changed a value
	if checkUpdatePooling {
		if group.UpdateCount == 0 && group.UpdateInterval == 0 {
			group.UpdatePooling = false
		} else {
			group.UpdatePooling = true
		}
	}

	updateCall := service.Group.Patch(groupFlags.appId.String(), groupFlags.groupId.String(), group)
	group, err = updateCall.Do()

	if err != nil {
		log.Fatal(err)
	}

	fmt.Fprintf(out, "%s", formatGroup(group))

	out.Flush()
	return OK
}
<commit_msg>refactor(group): remove rollup from command names<commit_after>package main

import (
	"fmt"
	"log"
	"strconv"
	"text/tabwriter"

	"github.com/coreos-inc/updatectl/client/update/v1"
)

var (
	groupFlags struct {
		label          StringFlag
		channel        StringFlag
		appId          StringFlag
		groupId        StringFlag
		start          int64
		end            int64
		resolution     int64
		updateCount    int64
		updateInterval int64
	}

	cmdGroup = &Command{
		Name:    "group",
		Summary: "Operations that manage groups in an applciation.",
		Subcommands: []*Command{
			cmdGroupList,
			cmdGroupCreate,
			cmdGroupDelete,
			cmdGroupUpdate,
			cmdGroupPause,
			cmdGroupUnpause,
			cmdGroupEvents,
			cmdGroupVersions,
		},
	}

	cmdGroupList = &Command{
		Name:    "group list",
		Usage:   "[OPTION]...",
		Summary: `List all of the groups that exist including their label, token and update state.`,
		Run:     groupList,
	}
	cmdGroupCreate = &Command{
		Name:    "group create",
		Usage:   "[OPTION]...",
		Summary: `Create a new group.`,
		Run:     groupNew,
	}
	cmdGroupDelete = &Command{
		Name:    "group delete",
		Usage:   "[OPTION]...",
		Summary: `Delete a group.`,
		Run:     groupDelete,
	}
	cmdGroupUpdate = &Command{
		Name:        "group update",
		Usage:       "[OPTION]...",
		Description: `Update an existing group.`,
		Run:         groupUpdate,
	}
	cmdGroupPause = &Command{
		Name:    "group pause",
		Usage:   "[OPTION]...",
		Summary: `Pause a group's updates.`,
		Run:     groupPause,
	}
	cmdGroupUnpause = &Command{
		Name:    "group unpause",
		Usage:   "[OPTION]...",
		Summary: `Unpause a group's updates.`,
		Run:     groupUnpause,
	}
	cmdGroupVersions = &Command{
		Name:    "group versions",
		Usage:   "[OPTION]...",
		Summary: "List versions from clients by time.",
		Run:     groupVersions,
	}
	cmdGroupEvents = &Command{
		Name:    "group events",
		Usage:   "[OPTION]...",
		Summary: "List events from clients by time.",
		Run:     groupEvents,
	}
)

func init() {
	cmdGroupList.Flags.Var(&groupFlags.appId, "app-id",
		"Application containing the groups to list.")

	cmdGroupDelete.Flags.Var(&groupFlags.appId, "app-id",
		"Application with group to delete.")
	cmdGroupDelete.Flags.Var(&groupFlags.groupId, "group-id",
		"ID of group to delete.")

	cmdGroupCreate.Flags.Var(&groupFlags.appId, "app-id",
		"Application to add group to.")
	cmdGroupCreate.Flags.Var(&groupFlags.groupId, "group-id",
		"ID for the new group.")
	cmdGroupCreate.Flags.Var(&groupFlags.channel, "channel",
		"Channel to associate with the group.")
	cmdGroupCreate.Flags.Var(&groupFlags.label, "label",
		"Label describing the new group.")

	cmdGroupUpdate.Flags.Var(&groupFlags.appId, "app-id",
		"Application containing the group to update.")
	cmdGroupUpdate.Flags.Var(&groupFlags.groupId, "group-id",
		"ID for the group.")
	cmdGroupUpdate.Flags.Var(&groupFlags.label, "label",
		"Label describing the group")
	cmdGroupUpdate.Flags.Var(&groupFlags.channel, "channel",
		"Channel to associate with the group.")
	cmdGroupUpdate.Flags.Int64Var(&groupFlags.updateCount, "update-count",
		-1, "Number of instances per interval")
	cmdGroupUpdate.Flags.Int64Var(&groupFlags.updateInterval,
		"update-interval", -1, "Interval between updates")

	cmdGroupPause.Flags.Var(&groupFlags.appId, "app-id",
		"Application containing the group to pause.")
	cmdGroupPause.Flags.Var(&groupFlags.groupId, "group-id",
		"ID for the group.")

	cmdGroupUnpause.Flags.Var(&groupFlags.appId, "app-id",
		"Application containing the group to unpause.")
	cmdGroupUnpause.Flags.Var(&groupFlags.groupId, "group-id",
		"ID for the group.")

	cmdGroupVersions.Flags.Var(&groupFlags.appId, "app-id",
		"Application containing the group.")
	cmdGroupVersions.Flags.Var(&groupFlags.groupId, "group-id",
		"ID for the group.")
	cmdGroupVersions.Flags.Int64Var(&groupFlags.resolution,
		"resolution", 60, "60, 3600 or 86400 seconds")
	cmdGroupVersions.Flags.Int64Var(&groupFlags.start, "start", 0,
		"Start date filter")
	cmdGroupVersions.Flags.Int64Var(&groupFlags.end, "end", 0,
		"End date filter")

	cmdGroupEvents.Flags.Var(&groupFlags.appId, "app-id",
		"Application containing the group.")
	cmdGroupEvents.Flags.Var(&groupFlags.groupId, "group-id",
		"ID for the group.")
	cmdGroupEvents.Flags.Int64Var(&groupFlags.resolution,
		"resolution", 60, "60, 3600 or 86400 seconds")
	cmdGroupEvents.Flags.Int64Var(&groupFlags.start, "start", 0,
		"Start date filter")
	cmdGroupEvents.Flags.Int64Var(&groupFlags.end, "end", 0,
		"End date filter")
}

func formatGroup(group *update.Group) string {
	return fmt.Sprintf("%s\t%s\t%s\t%s\t%s\t%v\t%v\n", group.Label, group.AppId, group.ChannelId,
		group.Id, strconv.FormatBool(group.UpdatesPaused), group.UpdateCount, group.UpdateInterval)
}

func groupList(args []string, service *update.Service, out *tabwriter.Writer) int {
	if groupFlags.appId.Get() == nil {
		return ERROR_USAGE
	}

	listCall := service.Group.List(groupFlags.appId.String())
	list, err := listCall.Do()

	if err != nil {
		log.Fatal(err)
	}

	fmt.Fprintln(out, "Label\tApp\tChannel\tId\tUpdatesPaused")
	for _, group := range list.Items {
		fmt.Fprintf(out, "%s", formatGroup(group))
	}

	out.Flush()
	return OK
}

func groupEvents(args []string, service *update.Service, out *tabwriter.Writer) int {
	if groupFlags.appId.Get() == nil || groupFlags.groupId.Get() == nil {
		return ERROR_USAGE
	}

	call := service.Group.Requests.Events.Rollup(
		groupFlags.appId.String(),
		groupFlags.groupId.String(),
		groupFlags.start,
		groupFlags.end,
	)
	call.Resolution(groupFlags.resolution)
	list, err := call.Do()

	if err != nil {
		log.Fatal(err)
	}

	fmt.Fprintln(out, "Version\tType\tResult\tTimestamp\tCount")
	for _, i := range list.Items {
		for _, j := range i.Values {
			fmt.Fprintf(out, "%s\t%s\t%s\t%d\t%d\n",
				i.Version, i.Type, i.Result, j.Timestamp, j.Count)
		}
	}
	out.Flush()
	return OK
}

func groupVersions(args []string, service *update.Service, out *tabwriter.Writer) int {
	if groupFlags.appId.Get() == nil || groupFlags.groupId.Get() == nil {
		return ERROR_USAGE
	}

	call := service.Group.Requests.Versions.Rollup(
		groupFlags.appId.String(),
		groupFlags.groupId.String(),
		groupFlags.start,
		groupFlags.end,
	)
	call.Resolution(groupFlags.resolution)
	list, err := call.Do()

	if err != nil {
		log.Fatal(err)
	}

	fmt.Fprintln(out, "Version\tTimestamp\tCount")
	for _, i := range list.Items {
		for _, j := range i.Values {
			fmt.Fprintf(out, "%s\t%d\t%d\n",
				i.Version, j.Timestamp, j.Count)
		}
	}
	out.Flush()
	return OK
}

func groupNew(args []string, service *update.Service, out *tabwriter.Writer) int {
	if groupFlags.appId.Get() == nil ||
		groupFlags.groupId.Get() == nil ||
		groupFlags.channel.Get() == nil {
		return ERROR_USAGE
	}

	group := &update.Group{
		ChannelId: groupFlags.channel.String(),
		Id:        groupFlags.groupId.String(),
		Label:     groupFlags.label.String(),
	}
	call := service.Group.Insert(groupFlags.appId.String(), group)
	group, err := call.Do()

	if err != nil {
		log.Fatal(err)
	}

	fmt.Fprintf(out, "%s", formatGroup(group))

	out.Flush()
	return OK
}

func groupDelete(args []string, service *update.Service, out *tabwriter.Writer) int {
	if groupFlags.appId.Get() == nil ||
		groupFlags.groupId.Get() == nil {
		return ERROR_USAGE
	}

	call := service.Group.Delete(groupFlags.appId.String(), groupFlags.groupId.String())
	group, err := call.Do()

	if err != nil {
		log.Fatal(err)
	}

	fmt.Fprintf(out, "%s", formatGroup(group))

	out.Flush()
	return OK
}

func groupPause(args []string, service *update.Service, out *tabwriter.Writer) int {
	return setUpdatesPaused(service, out, true)
}

func groupUnpause(args []string, service *update.Service, out *tabwriter.Writer) int {
	return setUpdatesPaused(service, out, false)
}

// Helper function for pause/unpause-group commands
func setUpdatesPaused(service *update.Service, out *tabwriter.Writer, paused bool) int {
	if groupFlags.appId.Get() == nil ||
		groupFlags.groupId.Get() == nil {
		return ERROR_USAGE
	}

	call := service.Group.Get(groupFlags.appId.String(), groupFlags.groupId.String())
	group, err := call.Do()

	if err != nil {
		log.Fatal(err)
	}

	group.UpdatesPaused = paused

	updateCall := service.Group.Patch(groupFlags.appId.String(), groupFlags.groupId.String(), group)
	group, err = updateCall.Do()

	if err != nil {
		log.Fatal(err)
	}

	fmt.Fprintf(out, "%s", formatGroup(group))

	out.Flush()
	return OK
}

func groupUpdate(args []string, service *update.Service, out *tabwriter.Writer) int {
	if groupFlags.appId.Get() == nil ||
		groupFlags.groupId.Get() == nil {
		return ERROR_USAGE
	}

	call := service.Group.Get(groupFlags.appId.String(), groupFlags.groupId.String())
	group, err := call.Do()

	if err != nil {
		log.Fatal(err)
	}

	checkUpdatePooling := false
	if groupFlags.updateCount != -1 {
		group.UpdateCount = groupFlags.updateCount
		checkUpdatePooling = true
	}
	if groupFlags.updateInterval != -1 {
		group.UpdateInterval = groupFlags.updateInterval
		checkUpdatePooling = true
	}
	if groupFlags.label.Get() != nil {
		group.Label = groupFlags.label.String()
	}
	if groupFlags.channel.Get() != nil {
		group.ChannelId = groupFlags.channel.String()
	}

	// set update pooling based on other flags
	// this only changes if the user changed a value
	if checkUpdatePooling {
		if group.UpdateCount == 0 && group.UpdateInterval == 0 {
			group.UpdatePooling = false
		} else {
			group.UpdatePooling = true
		}
	}

	updateCall := service.Group.Patch(groupFlags.appId.String(), groupFlags.groupId.String(), group)
	group, err = updateCall.Do()

	if err != nil {
		log.Fatal(err)
	}

	fmt.Fprintf(out, "%s", formatGroup(group))

	out.Flush()
	return OK
}
<|endoftext|>


<commit_before>package main

import (
    "github.com/PuerkitoBio/goquery"
    "github.com/hu17889/go_spider/core/common/page"
    "github.com/hu17889/go_spider/core/common/request"
    "github.com/hu17889/go_spider/core/pipeline"
    "github.com/hu17889/go_spider/core/spider"
    "net/http"
    "net/url"
    "strings"
    // "fmt"
    "errors"
)

type MyPageProcesser struct {
    cookies []*http.Cookie
}

func NewMyPageProcesser() *MyPageProcesser {
    return &MyPageProcesser{}
}

// Parse html dom here and record the parse result that we want to Page.
// Package goquery (http://godoc.org/github.com/PuerkitoBio/goquery) is used to parse html.
func (this *MyPageProcesser) Process(p *page.Page) {

    if p.GetUrlTag() == "site_login" {
        //fmt.Printf("%v\n", p.GetCookies())
        this.cookies = p.GetCookies()
        // AddTargetRequestWithParams Params:
        //  1. Url.
        //  2. Responce type is "html" or "json" or "jsonp" or "text".
        //  3. The urltag is name for marking url and distinguish different urls in PageProcesser and Pipeline.
        //  4. The method is POST or GET.
        //  5. The postdata is body string sent to sever.
        //  6. The header is header for http request.
        //  7. Cookies
        //  8. Http redirect function
        if len(this.cookies) != 0 {
            p.AddField("info", "get cookies success")
            req := request.NewRequest("http://backadmin.hucong.net/site/index", "html", "site_index", "GET", "", nil, this.cookies, nil, nil)
            p.AddTargetRequestWithParams(req)
        } else {
            p.AddField("info", "get cookies failed")
        }
    } else {
        //fmt.Printf("%v\n", p.GetBodyStr())
        query := p.GetHtmlParser()
        pageTitle := query.Find(".page-content .page-title").Text()

        if len(pageTitle) != 0 {
            p.AddField("page_title", pageTitle)
            p.AddField("info", "login success")
        } else {
            p.AddField("info", "login failed")
        }

    }

    return
    if !p.IsSucc() {
        println(p.Errormsg())
        return
    }

    query := p.GetHtmlParser()
    var urls []string
    query.Find("h3[class='repo-list-name'] a").Each(func(i int, s *goquery.Selection) {
        href, _ := s.Attr("href")
        urls = append(urls, "http://github.com/"+href)
    })
    // these urls will be saved and crawed by other coroutines.
    p.AddTargetRequests(urls, "html")

    name := query.Find(".entry-title .author").Text()
    name = strings.Trim(name, " \t\n")
    repository := query.Find(".entry-title .js-current-repository").Text()
    repository = strings.Trim(repository, " \t\n")
    //readme, _ := query.Find("#readme").Html()
    if name == "" {
        p.SetSkip(true)
    }
    // the entity we want to save by Pipeline
    p.AddField("author", name)
    p.AddField("project", repository)
    //p.AddField("readme", readme)
}

func (this *MyPageProcesser) Finish() {
    fmt.Printf("TODO:before end spider \r\n")
}

// function that prevent redirect for getting cookies
// If CheckRedirect function returns error.New("normal"), the error process after client.Do will ignore the error.
func myRedirect(req *http.Request, via []*http.Request) error {
    return errors.New("normal")
}

func main() {

    // POST data
    post_arg := url.Values{
        "name": {"admin"},
        "pwd":  {"admin"},
    }

    // http header
    header := make(http.Header)
    header.Set("Content-Type", "application/x-www-form-urlencoded")

    // Spider input:
    //  PageProcesser ;
    //  Task name used in Pipeline for record;
    // AddRequest Params:
    //  1. Url.
    //  2. Responce type is "html" or "json" or "jsonp" or "text".
    //  3. The urltag is name for marking url and distinguish different urls in PageProcesser and Pipeline.
    //  4. The method is POST or GET.
    //  5. The postdata is body string sent to sever.
    //  6. The header is header for http request.
    //  7. Cookies
    //  8. Http redirect function
    req := request.NewRequest("http://backadmin.hucong.net/main/user/login", "html", "site_login", "POST", post_arg.Encode(), header, nil, myRedirect, nil)

    spider.NewSpider(NewMyPageProcesser(), "TaskName").
        AddRequest(req).
        AddPipeline(pipeline.NewPipelineConsole()). // Print result on screen
        SetThreadnum(3).                            // Crawl request by three Coroutines
        Run()
}
<commit_msg>Update main.go<commit_after>package main

import (
    "github.com/PuerkitoBio/goquery"
    "github.com/hu17889/go_spider/core/common/page"
    "github.com/hu17889/go_spider/core/common/request"
    "github.com/hu17889/go_spider/core/pipeline"
    "github.com/hu17889/go_spider/core/spider"
    "net/http"
    "net/url"
    "strings"
    "fmt"
    "errors"
)

type MyPageProcesser struct {
    cookies []*http.Cookie
}

func NewMyPageProcesser() *MyPageProcesser {
    return &MyPageProcesser{}
}

// Parse html dom here and record the parse result that we want to Page.
// Package goquery (http://godoc.org/github.com/PuerkitoBio/goquery) is used to parse html.
func (this *MyPageProcesser) Process(p *page.Page) {

    if p.GetUrlTag() == "site_login" {
        //fmt.Printf("%v\n", p.GetCookies())
        this.cookies = p.GetCookies()
        // AddTargetRequestWithParams Params:
        //  1. Url.
        //  2. Responce type is "html" or "json" or "jsonp" or "text".
        //  3. The urltag is name for marking url and distinguish different urls in PageProcesser and Pipeline.
        //  4. The method is POST or GET.
        //  5. The postdata is body string sent to sever.
        //  6. The header is header for http request.
        //  7. Cookies
        //  8. Http redirect function
        if len(this.cookies) != 0 {
            p.AddField("info", "get cookies success")
            req := request.NewRequest("http://backadmin.hucong.net/site/index", "html", "site_index", "GET", "", nil, this.cookies, nil, nil)
            p.AddTargetRequestWithParams(req)
        } else {
            p.AddField("info", "get cookies failed")
        }
    } else {
        //fmt.Printf("%v\n", p.GetBodyStr())
        query := p.GetHtmlParser()
        pageTitle := query.Find(".page-content .page-title").Text()

        if len(pageTitle) != 0 {
            p.AddField("page_title", pageTitle)
            p.AddField("info", "login success")
        } else {
            p.AddField("info", "login failed")
        }

    }

    return
    if !p.IsSucc() {
        println(p.Errormsg())
        return
    }

    query := p.GetHtmlParser()
    var urls []string
    query.Find("h3[class='repo-list-name'] a").Each(func(i int, s *goquery.Selection) {
        href, _ := s.Attr("href")
        urls = append(urls, "http://github.com/"+href)
    })
    // these urls will be saved and crawed by other coroutines.
    p.AddTargetRequests(urls, "html")

    name := query.Find(".entry-title .author").Text()
    name = strings.Trim(name, " \t\n")
    repository := query.Find(".entry-title .js-current-repository").Text()
    repository = strings.Trim(repository, " \t\n")
    //readme, _ := query.Find("#readme").Html()
    if name == "" {
        p.SetSkip(true)
    }
    // the entity we want to save by Pipeline
    p.AddField("author", name)
    p.AddField("project", repository)
    //p.AddField("readme", readme)
}

func (this *MyPageProcesser) Finish() {
    fmt.Printf("TODO:before end spider \r\n")
}

// function that prevent redirect for getting cookies
// If CheckRedirect function returns error.New("normal"), the error process after client.Do will ignore the error.
func myRedirect(req *http.Request, via []*http.Request) error {
    return errors.New("normal")
}

func main() {

    // POST data
    post_arg := url.Values{
        "name": {"admin"},
        "pwd":  {"admin"},
    }

    // http header
    header := make(http.Header)
    header.Set("Content-Type", "application/x-www-form-urlencoded")

    // Spider input:
    //  PageProcesser ;
    //  Task name used in Pipeline for record;
    // AddRequest Params:
    //  1. Url.
    //  2. Responce type is "html" or "json" or "jsonp" or "text".
    //  3. The urltag is name for marking url and distinguish different urls in PageProcesser and Pipeline.
    //  4. The method is POST or GET.
    //  5. The postdata is body string sent to sever.
    //  6. The header is header for http request.
    //  7. Cookies
    //  8. Http redirect function
    req := request.NewRequest("http://backadmin.hucong.net/main/user/login", "html", "site_login", "POST", post_arg.Encode(), header, nil, myRedirect, nil)

    spider.NewSpider(NewMyPageProcesser(), "TaskName").
        AddRequest(req).
        AddPipeline(pipeline.NewPipelineConsole()). // Print result on screen
        SetThreadnum(3).                            // Crawl request by three Coroutines
        Run()
}
<|endoftext|>


<commit_before>package notification

import (
	"github.com/doozr/qbot/queue"
	"fmt"
)

func item(i queue.Item) string {
	if i.Reason == "" {
		return fmt.Sprintf("<@%s>", i.Name)
	}
	return fmt.Sprintf("<@%s> (%s)", i.Name, i.Reason)
}

func finishedWithToken(i queue.Item) string {
	return fmt.Sprintf("%s has finished with the token", item(i))
}

func nowHasToken(i queue.Item) string {
	return fmt.Sprintf("%s now has the token", item(i))
}

func upForGrabs() string {
	return fmt.Sprint("The token is up for grabs")
}

func yielded(i queue.Item) string {
	return fmt.Sprintf("%s has yielded the token", item(i))
}

func ousted(ouster string, i queue.Item) string {
	return fmt.Sprintf("@%s ousted %s", ouster, item(i))
}

func Join(i queue.Item) string {
	return fmt.Sprintf("%s has joined the queue", item(i))
}

func JoinNoReason(i queue.Item) string {
	return fmt.Sprintf("@%s You must provide a reason for joining", i.Name)
}

func JoinActive(i queue.Item) string {
	return fmt.Sprintf("%s", nowHasToken(i))
}

func Leave(i queue.Item) string {
	return fmt.Sprintf("%s has left the queue", item(i))
}

func LeaveActive(i queue.Item, q queue.Queue) string {
	a := q.Active()
	return fmt.Sprintf("%s\n\n%s", Leave(i), nowHasToken(a))
}

func LeaveNoActive(i queue.Item) string {
	return fmt.Sprintf("%s\n\n%s", Leave(i), upForGrabs())
}

func Done(i queue.Item, q queue.Queue) string {
	a := q.Active()
	return fmt.Sprintf("%s\n\n%s",
		finishedWithToken(i), nowHasToken(a))
}

func DoneNoOthers(i queue.Item) string {
	return fmt.Sprintf("%s\n\n%s",
		finishedWithToken(i), upForGrabs())
}

func DoneNotActive(i queue.Item) string {
	return fmt.Sprintf("@%s You cannot be done if you don't have the token", i.Name)
}

func Yield(i queue.Item, q queue.Queue) string {
	a := q.Active()
	return fmt.Sprintf("%s\n\n%s", yielded(i), nowHasToken(a))
}

func YieldNoOthers(i queue.Item) string {
	return fmt.Sprintf("@%s You cannot yield if there is nobody waiting", i.Name)
}

func YieldNotActive(i queue.Item) string {
	return fmt.Sprintf("@%s You cannot yield if you do not have the token", i.Name)
}

func Barge(i queue.Item) string {
	return fmt.Sprintf("%s barged to the front", item(i))
}

func Boot(booter string, i queue.Item) string {
	return fmt.Sprintf("@%s booted %s from the list", booter, item(i))
}

func OustNotBoot(booter string) string {
	return fmt.Sprintf("@%s You must oust the token holder", booter)
}

func Oust(ouster string, i queue.Item, q queue.Queue) string {
	a := q.Active()
	return fmt.Sprintf("%s\n\n%s", ousted(ouster, i), nowHasToken(a))
}

func OustNotActive(ouster string) string {
	return fmt.Sprintf("@%s You can only oust the token holder", ouster)
}

func OustNoOthers(ouster string, i queue.Item) string {
	return ousted(ouster, i)
}
<commit_msg>Tidy up notifications and make token grabbing bold<commit_after>package notification

import (
	"github.com/doozr/qbot/queue"
	"fmt"
)

func item(i queue.Item) string {
	if i.Reason == "" {
		return fmt.Sprintf("<@%s>", i.Name)
	}
	return fmt.Sprintf("<@%s> (%s)", i.Name, i.Reason)
}

func finishedWithToken(i queue.Item) string {
	return fmt.Sprintf("%s has finished with the token", item(i))
}

func nowHasToken(i queue.Item) string {
	return fmt.Sprintf("*%s now has the token*", item(i))
}

func upForGrabs() string {
	return fmt.Sprint("The token is up for grabs")
}

func yielded(i queue.Item) string {
	return fmt.Sprintf("%s has yielded the token", item(i))
}

func ousted(ouster string, i queue.Item) string {
	return fmt.Sprintf("@%s ousted %s", ouster, item(i))
}

func Join(i queue.Item) string {
	return fmt.Sprintf("%s has joined the queue", item(i))
}

func JoinNoReason(i queue.Item) string {
	return fmt.Sprintf("@%s You must provide a reason for joining", i.Name)
}

func JoinActive(i queue.Item) string {
	return fmt.Sprintf("%s", nowHasToken(i))
}

func Leave(i queue.Item) string {
	return fmt.Sprintf("%s has left the queue", item(i))
}

func LeaveActive(i queue.Item, q queue.Queue) string {
	a := q.Active()
	return fmt.Sprintf("%s\n%s", Leave(i), nowHasToken(a))
}

func LeaveNoActive(i queue.Item) string {
	return fmt.Sprintf("%s\n%s", Leave(i), upForGrabs())
}

func Done(i queue.Item, q queue.Queue) string {
	a := q.Active()
	return fmt.Sprintf("%s\n%s",
		finishedWithToken(i), nowHasToken(a))
}

func DoneNoOthers(i queue.Item) string {
	return fmt.Sprintf("%s\n%s",
		finishedWithToken(i), upForGrabs())
}

func DoneNotActive(i queue.Item) string {
	return fmt.Sprintf("@%s You cannot be done if you don't have the token", i.Name)
}

func Yield(i queue.Item, q queue.Queue) string {
	a := q.Active()
	return fmt.Sprintf("%s\n%s", yielded(i), nowHasToken(a))
}

func YieldNoOthers(i queue.Item) string {
	return fmt.Sprintf("@%s You cannot yield if there is nobody waiting", i.Name)
}

func YieldNotActive(i queue.Item) string {
	return fmt.Sprintf("@%s You cannot yield if you do not have the token", i.Name)
}

func Barge(i queue.Item) string {
	return fmt.Sprintf("%s barged to the front", item(i))
}

func Boot(booter string, i queue.Item) string {
	return fmt.Sprintf("@%s booted %s from the list", booter, item(i))
}

func OustNotBoot(booter string) string {
	return fmt.Sprintf("@%s You must oust the token holder", booter)
}

func Oust(ouster string, i queue.Item, q queue.Queue) string {
	a := q.Active()
	return fmt.Sprintf("%s\n%s", ousted(ouster, i), nowHasToken(a))
}

func OustNotActive(ouster string) string {
	return fmt.Sprintf("@%s You can only oust the token holder", ouster)
}

func OustNoOthers(ouster string, i queue.Item) string {
	return ousted(ouster, i)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"log"
	"net/http"
	_ "net/http/pprof"
	"strings"
	"time"

	// "github.com/ncabatoff/gopsutil/process" // use my fork until shirou/gopsutil issue#235 fixed, but needs branch fix-internal-pkgref
	"github.com/prometheus/client_golang/prometheus"
	"github.com/shirou/gopsutil/process"
)

var (
	numprocsDesc = prometheus.NewDesc(
		"namedprocess_namegroup_num_procs",
		"number of processes in this group",
		[]string{"groupname"},
		nil)

	cpusecsDesc = prometheus.NewDesc(
		"namedprocess_namegroup_cpu_seconds_total",
		"cpu usage in seconds",
		[]string{"groupname"},
		nil)

	readbytesDesc = prometheus.NewDesc(
		"namedprocess_namegroup_read_bytes_total",
		"number of bytes read by this group",
		[]string{"groupname"},
		nil)

	writebytesDesc = prometheus.NewDesc(
		"namedprocess_namegroup_write_bytes_total",
		"number of bytes written by this group",
		[]string{"groupname"},
		nil)
)

type (
	dummyResponseWriter struct {
		bytes.Buffer
		header http.Header
	}
)

func (d *dummyResponseWriter) Header() http.Header {
	return d.header
}

func (d *dummyResponseWriter) WriteHeader(code int) {
}

func main() {
	var (
		listenAddress = flag.String("web.listen-address", ":9256",
			"Address on which to expose metrics and web interface.")
		metricsPath = flag.String("web.telemetry-path", "/metrics",
			"Path under which to expose metrics.")
		onceToStdout = flag.Bool("once-to-stdout", false,
			"Don't bind, instead just print the metrics once to stdout and exit")
		procNames = flag.String("procnames", "",
			"comma-seperated list of process names to monitor")
		minIoPercent = flag.Float64("min-io-pct", 10.0,
			"percent of total I/O seen needed to promote out of 'other'")
	)
	flag.Parse()

	var names []string
	for _, s := range strings.Split(*procNames, ",") {
		if s != "" {
			names = append(names, s)
		}
	}

	pc := NewProcessCollector(*minIoPercent, names)

	if err := pc.Init(); err != nil {
		log.Fatalf("Error initializing: %v", err)
	}

	prometheus.MustRegister(pc)

	if *onceToStdout {
		drw := dummyResponseWriter{header: make(http.Header)}
		httpreq, err := http.NewRequest("GET", "/metrics", nil)
		if err != nil {
			log.Fatalf("Error building request: %v", err)
		}

		prometheus.Handler().ServeHTTP(&drw, httpreq)
		drw.Buffer.Truncate(0)

		// We throw away the first result because that first collection primes the pump, and
		// otherwise we won't see our counter metrics.  This is specific to the implementation
		// of NamedProcessCollector.Collect().
		prometheus.Handler().ServeHTTP(&drw, httpreq)
		fmt.Print(drw.String())
		return
	}

	http.Handle(*metricsPath, prometheus.Handler())

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`
			<title>Sensor Exporter</title>

Named Process Exporter

<a href="` + *metricsPath + `">Metrics

			`))
	})
	if err := http.ListenAndServe(*listenAddress, nil); err != nil {
		log.Fatalf("Unable to setup HTTP server: %v", err)
	}
}

type (
	NamedProcessCollector struct {
		minIoPercent  float64
		wantProcNames map[string]struct{}
		// track how much was seen last time so we can report the delta
		groupStats map[string]counts
		tracker    *procTracker
	}

	counts struct {
		cpu        float64
		readbytes  uint64
		writebytes uint64
	}

	groupcounts struct {
		counts
		procs int
	}

	// procSum contains data read from /proc/pid/*
	procSum struct {
		name       string
		cmd        string
		cpu        float64
		readbytes  uint64
		writebytes uint64
		startTime  time.Time
	}

	trackedProc struct {
		lastUpdate time.Time
		lastvals   procSum
		accum      counts
	}

	processId struct {
		pid       int32
		startTime time.Time
	}

	procTracker struct {
		procs map[processId]trackedProc
	}
)

func (ps procSum) String() string {
	cmd := ps.cmd
	if len(cmd) > 20 {
		cmd = cmd[:20]
	}
	return fmt.Sprintf("%20s %20s %7.0f %12d %12d", ps.name, cmd, ps.cpu, ps.readbytes, ps.writebytes)
}

func NewProcessCollector(minIoPercent float64, procnames []string) *NamedProcessCollector {
	pc := NamedProcessCollector{
		minIoPercent:  minIoPercent,
		wantProcNames: make(map[string]struct{}),
		groupStats:    make(map[string]counts),
		tracker:       NewProcTracker(),
	}
	for _, name := range procnames {
		pc.wantProcNames[name] = struct{}{}
	}
	return &pc
}

func (p *NamedProcessCollector) Init() error {
	return p.tracker.update()
}

// Describe implements prometheus.Collector.
func (p *NamedProcessCollector) Describe(ch chan<- *prometheus.Desc) {
	ch <- cpusecsDesc
	ch <- numprocsDesc
	ch <- readbytesDesc
	ch <- writebytesDesc
}

// Collect implements prometheus.Collector.
func (p *NamedProcessCollector) Collect(ch chan<- prometheus.Metric) {
	for gname, gcounts := range p.getGroups() {
		ch <- prometheus.MustNewConstMetric(numprocsDesc,
			prometheus.GaugeValue, float64(gcounts.procs), gname)

		if grpstat, ok := p.groupStats[gname]; ok {
			// It's convenient to treat cpu, readbytes, etc as counters so we can use rate().
			// In practice it doesn't quite work because processes can exit while their group
			// persists, and with that pid's absence our summed value across the group will
			// become smaller.  We'll cheat by simply pretending there was no change to the
			// counter when that happens.

			dcpu := gcounts.cpu - grpstat.cpu
			if dcpu < 0 {
				dcpu = 0
			}
			ch <- prometheus.MustNewConstMetric(cpusecsDesc,
				prometheus.CounterValue,
				dcpu,
				gname)

			drbytes := gcounts.readbytes - grpstat.readbytes
			if drbytes < 0 {
				drbytes = 0
			}
			ch <- prometheus.MustNewConstMetric(readbytesDesc,
				prometheus.CounterValue,
				float64(drbytes),
				gname)

			dwbytes := gcounts.writebytes - grpstat.writebytes
			if dwbytes < 0 {
				dwbytes = 0
			}
			ch <- prometheus.MustNewConstMetric(writebytesDesc,
				prometheus.CounterValue,
				float64(dwbytes),
				gname)
		}

		p.groupStats[gname] = gcounts.counts
	}
}

func (p *NamedProcessCollector) getGroups() map[string]groupcounts {
	if err := p.tracker.update(); err != nil {
		log.Fatalf("Error reading procs: %v", err)
	}

	gcounts := make(map[string]groupcounts)

	for _, pinfo := range p.tracker.procs {
		gname := pinfo.lastvals.name
		if _, ok := p.wantProcNames[gname]; !ok {
			gname = "other"
		}
		cur := gcounts[gname]
		cur.procs++
		cur.counts.cpu += pinfo.accum.cpu
		cur.counts.readbytes += pinfo.accum.readbytes
		cur.counts.writebytes += pinfo.accum.writebytes
		gcounts[gname] = cur
	}

	return gcounts
}

func NewProcTracker() *procTracker {
	return &procTracker{make(map[processId]trackedProc)}
}

// Scan /proc and update oneself.  Rather than allocating a new map each time to detect procs
// that have disappeared, we bump the last update time on those that are still present.  Then
// as a second pass we traverse the map looking for stale procs and removing them.

func (t procTracker) update() error {
	pids, err := process.Pids()
	if err != nil {
		return fmt.Errorf("Error reading procs: %v", err)
	}

	now := time.Now()
	for _, pid := range pids {
		psum, err := getProcSummary(pid)
		if err != nil {
			continue
		}
		procid := processId{pid, psum.startTime}
		var newaccum counts
		if cur, ok := t.procs[procid]; ok {
			newaccum = cur.accum
			newaccum.cpu += psum.cpu - cur.lastvals.cpu
			newaccum.readbytes += psum.readbytes - cur.lastvals.readbytes
			newaccum.writebytes += psum.writebytes - cur.lastvals.writebytes
			// log.Printf("%9d %20s %.1f %6d %6d", procid.pid, psum.name, newaccum.cpu, newaccum.readbytes, newaccum.writebytes)
		}
		t.procs[procid] = trackedProc{now, psum, newaccum}
	}

	for procid, pinfo := range t.procs {
		if pinfo.lastUpdate != now {
			delete(t.procs, procid)
		}
	}

	return nil
}

var (
	ErrUnnamed   = errors.New("unnamed proc")
	ErrNoCommand = errors.New("proc has empty cmdline")
)

func getProcSummary(pid int32) (procSum, error) {
	proc, err := process.NewProcess(pid)
	var psum procSum
	if err != nil {
		// errors happens so routinely (e.g. when we race) that it's not worth reporting IMO
		return psum, err
	}

	times, err := proc.Times()
	if err != nil {
		return psum, err
	}

	name, err := proc.Name()
	if err != nil {
		return psum, err
	}

	if name == "" {
		// these all appear to be kernel processes, which people generally don't care about
		// monitoring directly (i.e. the system OS metrics suffice)
		return psum, ErrUnnamed
	}

	cmdline, err := proc.Cmdline()
	if err != nil {
		return psum, err
	}

	if cmdline == "" {
		// these all appear to be kernel processes, which people generally don't care about
		// monitoring directly (i.e. the system OS metrics suffice)
		return psum, ErrNoCommand
	}

	ios, err := proc.IOCounters()
	if err != nil {
		return psum, err
	}

	ctime, err := proc.CreateTime()
	if err != nil {
		return psum, err
	}

	psum.name = name
	psum.cmd = cmdline
	psum.cpu = times.User + times.System
	psum.writebytes = ios.WriteBytes
	psum.readbytes = ios.ReadBytes
	psum.startTime = time.Unix(ctime, 0)

	return psum, nil
}
<commit_msg>Implement -min-io-pct option.  This provides an alternative to specifying explicitly the names of processes<commit_after>package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"log"
	"net/http"
	_ "net/http/pprof"
	"strings"
	"time"

	// "github.com/ncabatoff/gopsutil/process" // use my fork until shirou/gopsutil issue#235 fixed, but needs branch fix-internal-pkgref
	"github.com/prometheus/client_golang/prometheus"
	"github.com/shirou/gopsutil/process"
)

var (
	numprocsDesc = prometheus.NewDesc(
		"namedprocess_namegroup_num_procs",
		"number of processes in this group",
		[]string{"groupname"},
		nil)

	cpusecsDesc = prometheus.NewDesc(
		"namedprocess_namegroup_cpu_seconds_total",
		"cpu usage in seconds",
		[]string{"groupname"},
		nil)

	readbytesDesc = prometheus.NewDesc(
		"namedprocess_namegroup_read_bytes_total",
		"number of bytes read by this group",
		[]string{"groupname"},
		nil)

	writebytesDesc = prometheus.NewDesc(
		"namedprocess_namegroup_write_bytes_total",
		"number of bytes written by this group",
		[]string{"groupname"},
		nil)
)

type (
	dummyResponseWriter struct {
		bytes.Buffer
		header http.Header
	}
)

func (d *dummyResponseWriter) Header() http.Header {
	return d.header
}

func (d *dummyResponseWriter) WriteHeader(code int) {
}

func main() {
	var (
		listenAddress = flag.String("web.listen-address", ":9256",
			"Address on which to expose metrics and web interface.")
		metricsPath = flag.String("web.telemetry-path", "/metrics",
			"Path under which to expose metrics.")
		onceToStdout = flag.Bool("once-to-stdout", false,
			"Don't bind, instead just print the metrics once to stdout and exit")
		procNames = flag.String("procnames", "",
			"comma-seperated list of process names to monitor")
		minIoPercent = flag.Float64("min-io-pct", 10.0,
			"percent of total I/O seen needed to promote out of 'other'")
	)
	flag.Parse()

	var names []string
	for _, s := range strings.Split(*procNames, ",") {
		if s != "" {
			names = append(names, s)
		}
	}

	pc := NewProcessCollector(*minIoPercent, names)

	if err := pc.Init(); err != nil {
		log.Fatalf("Error initializing: %v", err)
	}

	prometheus.MustRegister(pc)

	if *onceToStdout {
		drw := dummyResponseWriter{header: make(http.Header)}
		httpreq, err := http.NewRequest("GET", "/metrics", nil)
		if err != nil {
			log.Fatalf("Error building request: %v", err)
		}

		prometheus.Handler().ServeHTTP(&drw, httpreq)
		drw.Buffer.Truncate(0)

		// We throw away the first result because that first collection primes the pump, and
		// otherwise we won't see our counter metrics.  This is specific to the implementation
		// of NamedProcessCollector.Collect().
		prometheus.Handler().ServeHTTP(&drw, httpreq)
		fmt.Print(drw.String())
		return
	}

	http.Handle(*metricsPath, prometheus.Handler())

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`
			<title>Sensor Exporter</title>

Named Process Exporter

<a href="` + *metricsPath + `">Metrics

			`))
	})
	if err := http.ListenAndServe(*listenAddress, nil); err != nil {
		log.Fatalf("Unable to setup HTTP server: %v", err)
	}
}

type (
	NamedProcessCollector struct {
		minIoPercent  float64
		wantProcNames map[string]struct{}
		// track how much was seen last time so we can report the delta
		groupStats map[string]counts
		tracker    *procTracker
	}

	counts struct {
		cpu        float64
		readbytes  uint64
		writebytes uint64
	}

	groupcounts struct {
		counts
		procs int
	}

	// procSum contains data read from /proc/pid/*
	procSum struct {
		name       string
		cmd        string
		cpu        float64
		readbytes  uint64
		writebytes uint64
		startTime  time.Time
	}

	trackedProc struct {
		lastUpdate time.Time
		lastvals   procSum
		accum      counts
	}

	processId struct {
		pid       int32
		startTime time.Time
	}

	procTracker struct {
		procs map[processId]trackedProc
		accum counts
	}
)

func (ps procSum) String() string {
	cmd := ps.cmd
	if len(cmd) > 20 {
		cmd = cmd[:20]
	}
	return fmt.Sprintf("%20s %20s %7.0f %12d %12d", ps.name, cmd, ps.cpu, ps.readbytes, ps.writebytes)
}

func NewProcessCollector(minIoPercent float64, procnames []string) *NamedProcessCollector {
	pc := NamedProcessCollector{
		minIoPercent:  minIoPercent,
		wantProcNames: make(map[string]struct{}),
		groupStats:    make(map[string]counts),
		tracker:       NewProcTracker(),
	}
	for _, name := range procnames {
		pc.wantProcNames[name] = struct{}{}
	}
	return &pc
}

func (p *NamedProcessCollector) Init() error {
	_, err := p.tracker.update()
	return err
}

// Describe implements prometheus.Collector.
func (p *NamedProcessCollector) Describe(ch chan<- *prometheus.Desc) {
	ch <- cpusecsDesc
	ch <- numprocsDesc
	ch <- readbytesDesc
	ch <- writebytesDesc
}

// Collect implements prometheus.Collector.
func (p *NamedProcessCollector) Collect(ch chan<- prometheus.Metric) {
	for gname, gcounts := range p.getGroups() {
		ch <- prometheus.MustNewConstMetric(numprocsDesc,
			prometheus.GaugeValue, float64(gcounts.procs), gname)

		if grpstat, ok := p.groupStats[gname]; ok {
			// It's convenient to treat cpu, readbytes, etc as counters so we can use rate().
			// In practice it doesn't quite work because processes can exit while their group
			// persists, and with that pid's absence our summed value across the group will
			// become smaller.  We'll cheat by simply pretending there was no change to the
			// counter when that happens.

			dcpu := gcounts.cpu - grpstat.cpu
			if dcpu < 0 {
				dcpu = 0
			}
			ch <- prometheus.MustNewConstMetric(cpusecsDesc,
				prometheus.CounterValue,
				dcpu,
				gname)

			drbytes := gcounts.readbytes - grpstat.readbytes
			if drbytes < 0 {
				drbytes = 0
			}
			ch <- prometheus.MustNewConstMetric(readbytesDesc,
				prometheus.CounterValue,
				float64(drbytes),
				gname)

			dwbytes := gcounts.writebytes - grpstat.writebytes
			if dwbytes < 0 {
				dwbytes = 0
			}
			ch <- prometheus.MustNewConstMetric(writebytesDesc,
				prometheus.CounterValue,
				float64(dwbytes),
				gname)
		}

		p.groupStats[gname] = gcounts.counts
	}
}

func (p *NamedProcessCollector) getGroups() map[string]groupcounts {
	delta, err := p.tracker.update()
	if err != nil {
		log.Fatalf("Error reading procs: %v", err)
	}

	totdeltaio := float64(delta.readbytes + delta.writebytes)
	gcounts := make(map[string]groupcounts)

	for _, pinfo := range p.tracker.procs {
		gname := pinfo.lastvals.name
		if _, ok := p.wantProcNames[gname]; !ok {
			deltaio := float64(pinfo.accum.readbytes + pinfo.accum.writebytes)
			pct := 100 * deltaio / totdeltaio
			if pct >= p.minIoPercent {
				p.wantProcNames[gname] = struct{}{}
			}

			gname = "other"
		}

		cur := gcounts[gname]
		cur.procs++
		cur.counts.cpu += pinfo.accum.cpu
		cur.counts.readbytes += pinfo.accum.readbytes
		cur.counts.writebytes += pinfo.accum.writebytes
		gcounts[gname] = cur
	}

	return gcounts
}

func NewProcTracker() *procTracker {
	return &procTracker{make(map[processId]trackedProc), counts{}}
}

// Scan /proc and update oneself.  Rather than allocating a new map each time to detect procs
// that have disappeared, we bump the last update time on those that are still present.  Then
// as a second pass we traverse the map looking for stale procs and removing them.

func (t procTracker) update() (delta counts, err error) {
	pids, err := process.Pids()
	if err != nil {
		return delta, fmt.Errorf("Error reading procs: %v", err)
	}

	now := time.Now()
	for _, pid := range pids {
		psum, err := getProcSummary(pid)
		if err != nil {
			continue
		}
		procid := processId{pid, psum.startTime}

		var newaccum counts
		if cur, ok := t.procs[procid]; ok {
			newaccum = cur.accum

			dcpu := psum.cpu - cur.lastvals.cpu
			newaccum.cpu += dcpu
			delta.cpu += dcpu

			drbytes := psum.readbytes - cur.lastvals.readbytes
			newaccum.readbytes += drbytes
			delta.readbytes += drbytes

			dwbytes := psum.writebytes - cur.lastvals.writebytes
			newaccum.writebytes += dwbytes
			delta.writebytes += dwbytes
			// log.Printf("%9d %20s %.1f %6d %6d", procid.pid, psum.name, dcpu, drbytes, dwbytes)
		}
		t.procs[procid] = trackedProc{lastUpdate: now, lastvals: psum, accum: newaccum}
	}

	for procid, pinfo := range t.procs {
		if pinfo.lastUpdate != now {
			delete(t.procs, procid)
		}
	}

	return delta, nil
}

var (
	ErrUnnamed   = errors.New("unnamed proc")
	ErrNoCommand = errors.New("proc has empty cmdline")
)

func getProcSummary(pid int32) (procSum, error) {
	proc, err := process.NewProcess(pid)
	var psum procSum
	if err != nil {
		// errors happens so routinely (e.g. when we race) that it's not worth reporting IMO
		return psum, err
	}

	times, err := proc.Times()
	if err != nil {
		return psum, err
	}

	name, err := proc.Name()
	if err != nil {
		return psum, err
	}

	if name == "" {
		// these all appear to be kernel processes, which people generally don't care about
		// monitoring directly (i.e. the system OS metrics suffice)
		return psum, ErrUnnamed
	}

	cmdline, err := proc.Cmdline()
	if err != nil {
		return psum, err
	}

	if cmdline == "" {
		// these all appear to be kernel processes, which people generally don't care about
		// monitoring directly (i.e. the system OS metrics suffice)
		return psum, ErrNoCommand
	}

	ios, err := proc.IOCounters()
	if err != nil {
		return psum, err
	}

	ctime, err := proc.CreateTime()
	if err != nil {
		return psum, err
	}

	psum.name = name
	psum.cmd = cmdline
	psum.cpu = times.User + times.System
	psum.writebytes = ios.WriteBytes
	psum.readbytes = ios.ReadBytes
	psum.startTime = time.Unix(ctime, 0)

	return psum, nil
}
<|endoftext|>


<commit_before>package examples

import (
	"fmt"
	c "jecolasurdo/go-chaining"
	"jecolasurdo/go-chaining/behavior"
	"strconv"
	"testing"

	"github.com/stretchr/testify/assert"
)

func addOne(value *interface{}) (*interface{}, error) {
	var result interface{} = (*value).(int) + 1
	return &result, nil
}

func multiplyBySix(value *interface{}) (*interface{}, error) {
	var result interface{} = (*value).(int) * 6
	return &result, nil
}

func convertToString(value *interface{}) (*interface{}, error) {
	var result interface{} = strconv.Itoa((*value).(int))
	return &result, nil
}

func sendToThePrinter(value *interface{}) error {
	_, err := fmt.Print(value)
	return err
}

func Test_ChainItAllTogether(t *testing.T) {
	chain := c.New()
	var initialValue interface{} = 1
	initialBehavior := c.ActionArg{
		Value:    &initialValue,
		Behavior: behavior.InjectSuppliedValue,
	}
	chain.ApplyUnaryIface(addOne, initialBehavior)
	// chain.ApplyUnaryIface(multiplyBySix, c.ActionArg{})
	// chain.ApplyUnaryIface(convertToString, c.ActionArg{})
	// chain.ApplyUnary(sendToThePrinter, c.ActionArg{})

	result, err := chain.Flush()
	assert.Equal(t, 2, (*result).(int))
	assert.Nil(t, err)
}
<commit_msg>Removed part of example that requires feature that's not built.<commit_after>package examples

import (
	c "jecolasurdo/go-chaining"
	"jecolasurdo/go-chaining/behavior"
	"strconv"
	"testing"

	"github.com/stretchr/testify/assert"
)

func addOne(value *interface{}) (*interface{}, error) {
	var result interface{} = (*value).(int) + 1
	return &result, nil
}

func multiplyBySix(value *interface{}) (*interface{}, error) {
	var result interface{} = (*value).(int) * 6
	return &result, nil
}

func convertToString(value *interface{}) (*interface{}, error) {
	var result interface{} = strconv.Itoa((*value).(int))
	return &result, nil
}

func Test_ChainItAllTogether(t *testing.T) {
	chain := c.New()
	var initialValue interface{} = 1
	initialBehavior := c.ActionArg{
		Value:    &initialValue,
		Behavior: behavior.InjectSuppliedValue,
	}
	chain.ApplyUnaryIface(addOne, initialBehavior)
	chain.ApplyUnaryIface(multiplyBySix, c.ActionArg{})
	chain.ApplyUnaryIface(convertToString, c.ActionArg{})

	result, err := chain.Flush()
	assert.Equal(t, "12", (*result).(string))
	assert.Nil(t, err)
}
<|endoftext|>