<commit_before>package weibo

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"reflect"
	"strings"

	"github.com/google/go-querystring/query"
)

const (
	libraryVersion  = "0.1"
	weiboApiVersion = "2"
	defaultBaseURL  = "https://api.weibo.com/"
	userAgent       = "go-weibo/" + libraryVersion
)

// A Client manages communication with the Weibo API.
type Client struct {
	// HTTP client used to communcate with the API.
	client *http.Client

	// Access Token
	accessToken string

	// Base URL for API requests.
	BaseURL *url.URL

	// User agent used when communicating with the Weibo API.
	UserAgent string

	// Services used for talking to different parts of the Weibo API.
	Statuses *StatusesService
}

// ListOptions specifies the optional parameters to various List methods that
// support pagination.
type ListOptions struct {
	// For paginated result sets, page of results to retrieve.
	Page int `url:"page,omitempty"`

	// For paginated result sets, the number of results to include per page.
	PerPage int `url:"count,omitempty"`
}

// addOptions adds the parameters in opt as URL query parameters to string.
// opt must be a struct whose fields may contain "url" tags.
func addOptions(s string, opt interface{}) (string, error) {
	v := reflect.ValueOf(opt)
	if v.Kind() == reflect.Ptr && v.IsNil() {
		return s, nil
	}

	u, err := url.Parse(s)
	if err != nil {
		return s, err
	}

	qs, err := query.Values(opt)
	if err != nil {
		return s, err
	}

	u.RawQuery = qs.Encode()
	return u.String(), nil
}

// NewClient returns a new Weibo API client.
func NewClient(accessToken string) *Client {
	baseURL, _ := url.Parse(defaultBaseURL)

	c := &Client{client: http.DefaultClient, accessToken: accessToken, BaseURL: baseURL, UserAgent: userAgent}
	c.Statuses = &StatusesService{client: c}

	return c
}

// NewRequest creates an API request.  A relative URL can be provided in urlStr,
// in which case it is resolved relative to the BaseURL of the Client.
// Relative URLs should always be specified without a preceding slash. If
// specified, the value pointed to by body is JSON encoded and included as the
// request body.
func (c *Client) NewRequest(method, urlString string, body interface{}) (*http.Request, error) {
	if strings.HasPrefix(urlString, "/") {
		urlString = weiboApiVersion + urlString
	} else {
		urlString = weiboApiVersion + "/" + urlString
	}

	rel, err := url.Parse(urlString)
	if err != nil {
		return nil, err
	}

	u := c.BaseURL.ResolveReference(rel)

	buf := new(bytes.Buffer)
	if body != nil {
		if err := json.NewEncoder(buf).Encode(body); err != nil {
			return nil, err
		}
	}

	req, err := http.NewRequest(method, u.String(), buf)
	if err != nil {
		return nil, err
	}

	req.Header.Add("User-Agent", c.UserAgent)
	return req, nil
}

// Response is a Weibo API response.
// This wraps the standrad http.Response returned from Weibo.
// For future use, maybe.
type Response struct {
	*http.Response
}

// newResponse creates a new Response for the provided http.Response.
func newResponse(r *http.Response) *Response {
	response := &Response{Response: r}

	return response
}

// Do sends an API request and returns the API response.  The API response is
// JSON decoded and stored in the value pointed to by v, or returned as an
// error if an API error has occured.  If v implements the io.Writer
// interface, the raw response body will be written to v, without attempting
// to first decode it.
func (c *Client) Do(req *http.Request, v interface{}) (*Response, error) {
	resp, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()

	response := newResponse(resp)

	if err := CheckResponse(resp); err != nil {
		// even though there was an error, we still return the response
		// in case the caller wants to inspect it further
		return response, err
	}

	if v != nil {
		if w, ok := v.(io.Writer); ok {
			io.Copy(w, resp.Body)
		} else {
			err = json.NewDecoder(resp.Body).Decode(v)
		}
	}

	return response, err
}

// An Error Response reports one or more errors caused by an API request.
//
// Weibo API docs: http://open.weibo.com/wiki/Error_code
type ErrorResponse struct {
	Response   *http.Response // HTTP response that caused this error
	RequestURL string         `json:"request"`    // request on which the error occured
	ErrorCode  int            `json:"error_code"` // error_code
	Message    string         `json:"error"`      // error message
}

func (r *ErrorResponse) Error() string {
	return fmt.Sprintf("%v %v: %d %v %v",
		r.Response.Request.Method, r.RequestURL,
		r.Response.StatusCode, r.ErrorCode, r.Message)
}

// CheckResponse checks the API response for errors, and returns them if
// present.  A response is considered an error if it has a status code outside
// the 200 range. API error responses are expected to have either no response
// body, or a JSON response body that maps to ErrorResponse.  Any other
// response body will be silently ignored.
func CheckResponse(r *http.Response) error {
	if c := r.StatusCode; 200 <= c && c <= 299 {
		return nil
	}
	errorResponse := &ErrorResponse{Response: r}
	data, err := ioutil.ReadAll(r.Body)
	if err == nil && data != nil {
		json.Unmarshal(data, errorResponse)
	}
	return errorResponse
}

// Bool is a helper routine that allocates a new bool value
// to store v and returns a pointer to it.
func Bool(v bool) *bool {
	p := new(bool)
	*p = v
	return p
}

// Int is a helper routine that allocates a new int32 value
// to store v and returns a pointer to it, but unlike Int32
// its argument value is an int.
func Int(v int) *int {
	p := new(int)
	*p = v
	return p
}

// String is a helper routine that allocates a new string value
// to store v and returns a pointer to it.
func String(v string) *string {
	p := new(string)
	*p = v
	return p
}
<commit_msg>Add access_token in request header.<commit_after>package weibo

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"reflect"
	"strings"

	"github.com/google/go-querystring/query"
)

const (
	libraryVersion  = "0.1"
	weiboApiVersion = "2"
	defaultBaseURL  = "https://api.weibo.com/"
	userAgent       = "go-weibo/" + libraryVersion
)

// A Client manages communication with the Weibo API.
type Client struct {
	// HTTP client used to communcate with the API.
	client *http.Client

	// Access Token
	accessToken string

	// Base URL for API requests.
	BaseURL *url.URL

	// User agent used when communicating with the Weibo API.
	UserAgent string

	// Services used for talking to different parts of the Weibo API.
	Statuses *StatusesService
}

// ListOptions specifies the optional parameters to various List methods that
// support pagination.
type ListOptions struct {
	// For paginated result sets, page of results to retrieve.
	Page int `url:"page,omitempty"`

	// For paginated result sets, the number of results to include per page.
	PerPage int `url:"count,omitempty"`
}

// addOptions adds the parameters in opt as URL query parameters to string.
// opt must be a struct whose fields may contain "url" tags.
func addOptions(s string, opt interface{}) (string, error) {
	v := reflect.ValueOf(opt)
	if v.Kind() == reflect.Ptr && v.IsNil() {
		return s, nil
	}

	u, err := url.Parse(s)
	if err != nil {
		return s, err
	}

	qs, err := query.Values(opt)
	if err != nil {
		return s, err
	}

	u.RawQuery = qs.Encode()
	return u.String(), nil
}

// NewClient returns a new Weibo API client.
func NewClient(accessToken string) *Client {
	baseURL, _ := url.Parse(defaultBaseURL)

	c := &Client{client: http.DefaultClient, accessToken: accessToken, BaseURL: baseURL, UserAgent: userAgent}
	c.Statuses = &StatusesService{client: c}

	return c
}

// NewRequest creates an API request.  A relative URL can be provided in urlStr,
// in which case it is resolved relative to the BaseURL of the Client.
// Relative URLs should always be specified without a preceding slash. If
// specified, the value pointed to by body is JSON encoded and included as the
// request body.
func (c *Client) NewRequest(method, urlString string, body interface{}) (*http.Request, error) {
	if strings.HasPrefix(urlString, "/") {
		urlString = weiboApiVersion + urlString
	} else {
		urlString = weiboApiVersion + "/" + urlString
	}

	rel, err := url.Parse(urlString)
	if err != nil {
		return nil, err
	}

	u := c.BaseURL.ResolveReference(rel)

	buf := new(bytes.Buffer)
	if body != nil {
		if err := json.NewEncoder(buf).Encode(body); err != nil {
			return nil, err
		}
	}

	req, err := http.NewRequest(method, u.String(), buf)
	if err != nil {
		return nil, err
	}

	req.Header.Add("User-Agent", c.UserAgent)
	req.Header.Add("Authorization", "OAuth2 "+c.accessToken)
	return req, nil
}

// Response is a Weibo API response.
// This wraps the standrad http.Response returned from Weibo.
// For future use, maybe.
type Response struct {
	*http.Response
}

// newResponse creates a new Response for the provided http.Response.
func newResponse(r *http.Response) *Response {
	response := &Response{Response: r}

	return response
}

// Do sends an API request and returns the API response.  The API response is
// JSON decoded and stored in the value pointed to by v, or returned as an
// error if an API error has occured.  If v implements the io.Writer
// interface, the raw response body will be written to v, without attempting
// to first decode it.
func (c *Client) Do(req *http.Request, v interface{}) (*Response, error) {
	resp, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()

	response := newResponse(resp)

	if err := CheckResponse(resp); err != nil {
		// even though there was an error, we still return the response
		// in case the caller wants to inspect it further
		return response, err
	}

	if v != nil {
		if w, ok := v.(io.Writer); ok {
			io.Copy(w, resp.Body)
		} else {
			err = json.NewDecoder(resp.Body).Decode(v)
		}
	}

	return response, err
}

// An Error Response reports one or more errors caused by an API request.
//
// Weibo API docs: http://open.weibo.com/wiki/Error_code
type ErrorResponse struct {
	Response   *http.Response // HTTP response that caused this error
	RequestURL string         `json:"request"`    // request on which the error occured
	ErrorCode  int            `json:"error_code"` // error_code
	Message    string         `json:"error"`      // error message
}

func (r *ErrorResponse) Error() string {
	return fmt.Sprintf("%v %v: %d %v %v",
		r.Response.Request.Method, r.RequestURL,
		r.Response.StatusCode, r.ErrorCode, r.Message)
}

// CheckResponse checks the API response for errors, and returns them if
// present.  A response is considered an error if it has a status code outside
// the 200 range. API error responses are expected to have either no response
// body, or a JSON response body that maps to ErrorResponse.  Any other
// response body will be silently ignored.
func CheckResponse(r *http.Response) error {
	if c := r.StatusCode; 200 <= c && c <= 299 {
		return nil
	}
	errorResponse := &ErrorResponse{Response: r}
	data, err := ioutil.ReadAll(r.Body)
	if err == nil && data != nil {
		json.Unmarshal(data, errorResponse)
	}
	return errorResponse
}

// Bool is a helper routine that allocates a new bool value
// to store v and returns a pointer to it.
func Bool(v bool) *bool {
	p := new(bool)
	*p = v
	return p
}

// Int is a helper routine that allocates a new int32 value
// to store v and returns a pointer to it, but unlike Int32
// its argument value is an int.
func Int(v int) *int {
	p := new(int)
	*p = v
	return p
}

// String is a helper routine that allocates a new string value
// to store v and returns a pointer to it.
func String(v string) *string {
	p := new(string)
	*p = v
	return p
}
<|endoftext|>


<commit_before>package client

const (
	Network = "tcp"
	// queue size
	queueSize = 8
	// read buffer size
	bufferSize = 8192
	// min packet length
	minPacketLength = 12

	// \x00REQ
	req    = 5391697
	reqStr = "\x00REQ"
	// \x00RES
	res    = 5391699
	resStr = "\x00RES"

	// package data type
	dtCanDo          = 1
	dtCantDo         = 2
	dtResetAbilities = 3
	dtPreSleep       = 4
	dtNoop           = 6
	dtJobCreated     = 8
	dtGrabJob        = 9
	dtNoJob          = 10
	dtJobAssign      = 11
	dtWorkStatus     = 12
	dtWorkComplete   = 13
	dtWorkFail       = 14
	dtGetStatus      = 15
	dtEchoReq        = 16
	dtEchoRes        = 17
	dtError          = 19
	dtStatusRes      = 20
	dtSetClientId    = 22
	dtCanDoTimeout   = 23
	dtAllYours       = 24
	dtWorkException  = 25
	dtWorkData       = 28
	dtWorkWarning    = 29
	dtGrabJobUniq    = 30
	dtJobAssignUniq  = 31

	dtSubmitJob       = 7
	dtSubmitJobBg     = 18
	dtSubmitJobHigh   = 21
	dtSubmitJobHighBg = 32
	dtSubmitJobLow    = 33
	dtSubmitJobLowBg  = 34

	WorkComplate  = dtWorkComplete
	WorkData      = dtWorkData
	WorkStatus    = dtWorkStatus
	WorkWarning   = dtWorkWarning
	WorkFail      = dtWorkFail
	WorkException = dtWorkException
)

const (
	// Job type
	JobNormal = iota
	// low level
	JobLow
	// high level
	JobHigh
)

func getBuffer(l int) (buf []byte) {
	// TODO add byte buffer pool
	buf = make([]byte, l)
	return
}
<commit_msg>fixed typo and keep WorkComplate for downward compatibility<commit_after>package client

const (
	Network = "tcp"
	// queue size
	queueSize = 8
	// read buffer size
	bufferSize = 8192
	// min packet length
	minPacketLength = 12

	// \x00REQ
	req    = 5391697
	reqStr = "\x00REQ"
	// \x00RES
	res    = 5391699
	resStr = "\x00RES"

	// package data type
	dtCanDo          = 1
	dtCantDo         = 2
	dtResetAbilities = 3
	dtPreSleep       = 4
	dtNoop           = 6
	dtJobCreated     = 8
	dtGrabJob        = 9
	dtNoJob          = 10
	dtJobAssign      = 11
	dtWorkStatus     = 12
	dtWorkComplete   = 13
	dtWorkFail       = 14
	dtGetStatus      = 15
	dtEchoReq        = 16
	dtEchoRes        = 17
	dtError          = 19
	dtStatusRes      = 20
	dtSetClientId    = 22
	dtCanDoTimeout   = 23
	dtAllYours       = 24
	dtWorkException  = 25
	dtWorkData       = 28
	dtWorkWarning    = 29
	dtGrabJobUniq    = 30
	dtJobAssignUniq  = 31

	dtSubmitJob       = 7
	dtSubmitJobBg     = 18
	dtSubmitJobHigh   = 21
	dtSubmitJobHighBg = 32
	dtSubmitJobLow    = 33
	dtSubmitJobLowBg  = 34

	WorkComplate  = dtWorkComplete
	WorkComplete  = dtWorkComplete
	WorkData      = dtWorkData
	WorkStatus    = dtWorkStatus
	WorkWarning   = dtWorkWarning
	WorkFail      = dtWorkFail
	WorkException = dtWorkException
)

const (
	// Job type
	JobNormal = iota
	// low level
	JobLow
	// high level
	JobHigh
)

func getBuffer(l int) (buf []byte) {
	// TODO add byte buffer pool
	buf = make([]byte, l)
	return
}
<|endoftext|>


<commit_before>package client

import (
	"fmt"
	consul "github.com/hashicorp/consul/api"
	"github.com/hashicorp/go-multierror"
	"github.com/hashicorp/nomad/nomad/structs"
	"log"
	"time"
)

const (
	syncInterval = 5 * time.Second
)

type trackedService struct {
	allocId string
	task    *structs.Task
	service *structs.Service
}

type ConsulClient struct {
	client     *consul.Client
	logger     *log.Logger
	shutdownCh chan struct{}

	trackedServices map[string]*trackedService
}

func NewConsulClient(logger *log.Logger, consulAddr string) (*ConsulClient, error) {
	var err error
	var c *consul.Client
	cfg := consul.DefaultConfig()
	cfg.Address = consulAddr
	if c, err = consul.NewClient(cfg); err != nil {
		return nil, err
	}

	consulClient := ConsulClient{
		client:          c,
		logger:          logger,
		trackedServices: make(map[string]*trackedService),
		shutdownCh:      make(chan struct{}),
	}

	return &consulClient, nil
}

func (c *ConsulClient) Register(task *structs.Task, allocID string) error {
	var mErr multierror.Error
	for _, service := range task.Services {
		if err := c.registerService(service, task, allocID); err != nil {
			mErr.Errors = append(mErr.Errors, err)
		}
		ts := &trackedService{
			allocId: allocID,
			task:    task,
			service: service,
		}
		c.trackedServices[service.Id] = ts

	}

	return mErr.ErrorOrNil()
}

func (c *ConsulClient) Deregister(task *structs.Task) error {
	var mErr multierror.Error
	for _, service := range task.Services {
		c.logger.Printf("[INFO] De-Registering service %v with Consul", service.Name)
		if err := c.deregisterService(service.Id); err != nil {
			c.logger.Printf("[ERROR] Error in de-registering service %v from Consul", service.Name)
			mErr.Errors = append(mErr.Errors, err)
		}
		delete(c.trackedServices, service.Id)
	}
	return mErr.ErrorOrNil()
}

func (c *ConsulClient) ShutDown() {
	close(c.shutdownCh)
}

func (c *ConsulClient) findPortAndHostForLabel(portLabel string, task *structs.Task) (string, int) {
	for _, network := range task.Resources.Networks {
		if p, ok := network.MapLabelToValues()[portLabel]; ok {
			return network.IP, p
		}
	}
	return "", 0
}

func (c *ConsulClient) SyncWithConsul() {
	sync := time.After(syncInterval)
	agent := c.client.Agent()

	for {
		select {
		case <-sync:
			sync = time.After(syncInterval)
			var consulServices map[string]*consul.AgentService
			var err error

			// Get the list of the services that Consul knows about
			if consulServices, err = agent.Services(); err != nil {
				c.logger.Printf("[DEBUG] Error while syncing services with Consul: %v", err)
				continue
			}

			// See if we have services that Consul doesn't know about yet.
			// Register with Consul the services which are not registered
			for serviceId := range c.trackedServices {
				if _, ok := consulServices[serviceId]; !ok {
					ts := c.trackedServices[serviceId]
					c.registerService(ts.service, ts.task, ts.allocId)
				}
			}

			// See if consul thinks we have some services which are not running
			// anymore on the node. We de-register those services
			for serviceId := range consulServices {
				if serviceId == "consul" {
					continue
				}
				if _, ok := c.trackedServices[serviceId]; !ok {
					if err := c.deregisterService(serviceId); err != nil {
						c.logger.Printf("[DEBUG] Error while de-registering service with ID: %s", serviceId)
					}
				}
			}
		case <-c.shutdownCh:
			c.logger.Printf("[INFO] Shutting down Consul Client")
			return
		}
	}
}

func (c *ConsulClient) registerService(service *structs.Service, task *structs.Task, allocID string) error {
	var mErr multierror.Error
	service.Id = fmt.Sprintf("%s-%s", allocID, task.Name)
	host, port := c.findPortAndHostForLabel(service.PortLabel, task)
	if host == "" || port == 0 {
		return fmt.Errorf("The port:%s marked for registration of service: %s couldn't be found", service.PortLabel, service.Name)
	}
	checks := c.makeChecks(service, host, port)
	asr := &consul.AgentServiceRegistration{
		ID:      service.Id,
		Name:    service.Name,
		Tags:    service.Tags,
		Port:    port,
		Address: host,
		Checks:  checks,
	}
	if err := c.client.Agent().ServiceRegister(asr); err != nil {
		c.logger.Printf("[ERROR] Error while registering service %v with Consul: %v", service.Name, err)
		mErr.Errors = append(mErr.Errors, err)
	}
	return mErr.ErrorOrNil()
}

func (c *ConsulClient) deregisterService(serviceId string) error {
	if err := c.client.Agent().ServiceDeregister(serviceId); err != nil {
		return err
	}
	return nil
}

func (c *ConsulClient) makeChecks(service *structs.Service, ip string, port int) []*consul.AgentServiceCheck {
	var checks []*consul.AgentServiceCheck
	for _, check := range service.Checks {
		c := &consul.AgentServiceCheck{
			Interval: check.Interval.String(),
			Timeout:  check.Timeout.String(),
		}
		switch check.Type {
		case structs.ServiceCheckHTTP:
			c.HTTP = fmt.Sprintf("%s://%s:%d/%s", check.Protocol, ip, port, check.Http)
		case structs.ServiceCheckTCP:
			c.TCP = fmt.Sprintf("%s:%d", ip, port)
		case structs.ServiceCheckScript:
			c.Script = check.Script // TODO This needs to include the path of the alloc dir and based on driver types
		}
		checks = append(checks, c)
	}
	return checks
}
<commit_msg>Added a lock around modification of tracked services map<commit_after>package client

import (
	"fmt"
	consul "github.com/hashicorp/consul/api"
	"github.com/hashicorp/go-multierror"
	"github.com/hashicorp/nomad/nomad/structs"
	"log"
	"sync"
	"time"
)

const (
	syncInterval = 5 * time.Second
)

type trackedService struct {
	allocId string
	task    *structs.Task
	service *structs.Service
}

type ConsulClient struct {
	client     *consul.Client
	logger     *log.Logger
	shutdownCh chan struct{}

	trackedServices map[string]*trackedService
	trackedSrvLock  sync.Mutex
}

func NewConsulClient(logger *log.Logger, consulAddr string) (*ConsulClient, error) {
	var err error
	var c *consul.Client
	cfg := consul.DefaultConfig()
	cfg.Address = consulAddr
	if c, err = consul.NewClient(cfg); err != nil {
		return nil, err
	}

	consulClient := ConsulClient{
		client:          c,
		logger:          logger,
		trackedServices: make(map[string]*trackedService),
		shutdownCh:      make(chan struct{}),
	}

	return &consulClient, nil
}

func (c *ConsulClient) Register(task *structs.Task, allocID string) error {
	var mErr multierror.Error
	for _, service := range task.Services {
		if err := c.registerService(service, task, allocID); err != nil {
			mErr.Errors = append(mErr.Errors, err)
		}
		ts := &trackedService{
			allocId: allocID,
			task:    task,
			service: service,
		}
		c.trackedSrvLock.Lock()
		c.trackedServices[service.Id] = ts
		c.trackedSrvLock.Unlock()
	}

	return mErr.ErrorOrNil()
}

func (c *ConsulClient) Deregister(task *structs.Task) error {
	var mErr multierror.Error
	for _, service := range task.Services {
		c.logger.Printf("[INFO] De-Registering service %v with Consul", service.Name)
		if err := c.deregisterService(service.Id); err != nil {
			c.logger.Printf("[ERROR] Error in de-registering service %v from Consul", service.Name)
			mErr.Errors = append(mErr.Errors, err)
		}
		c.trackedSrvLock.Lock()
		delete(c.trackedServices, service.Id)
		c.trackedSrvLock.Unlock()
	}
	return mErr.ErrorOrNil()
}

func (c *ConsulClient) ShutDown() {
	close(c.shutdownCh)
}

func (c *ConsulClient) findPortAndHostForLabel(portLabel string, task *structs.Task) (string, int) {
	for _, network := range task.Resources.Networks {
		if p, ok := network.MapLabelToValues()[portLabel]; ok {
			return network.IP, p
		}
	}
	return "", 0
}

func (c *ConsulClient) SyncWithConsul() {
	sync := time.After(syncInterval)
	agent := c.client.Agent()

	for {
		select {
		case <-sync:
			sync = time.After(syncInterval)
			var consulServices map[string]*consul.AgentService
			var err error

			// Get the list of the services that Consul knows about
			if consulServices, err = agent.Services(); err != nil {
				c.logger.Printf("[DEBUG] Error while syncing services with Consul: %v", err)
				continue
			}

			// See if we have services that Consul doesn't know about yet.
			// Register with Consul the services which are not registered
			for serviceId := range c.trackedServices {
				if _, ok := consulServices[serviceId]; !ok {
					ts := c.trackedServices[serviceId]
					c.registerService(ts.service, ts.task, ts.allocId)
				}
			}

			// See if consul thinks we have some services which are not running
			// anymore on the node. We de-register those services
			for serviceId := range consulServices {
				if serviceId == "consul" {
					continue
				}
				if _, ok := c.trackedServices[serviceId]; !ok {
					if err := c.deregisterService(serviceId); err != nil {
						c.logger.Printf("[DEBUG] Error while de-registering service with ID: %s", serviceId)
					}
				}
			}
		case <-c.shutdownCh:
			c.logger.Printf("[INFO] Shutting down Consul Client")
			return
		}
	}
}

func (c *ConsulClient) registerService(service *structs.Service, task *structs.Task, allocID string) error {
	var mErr multierror.Error
	service.Id = fmt.Sprintf("%s-%s", allocID, task.Name)
	host, port := c.findPortAndHostForLabel(service.PortLabel, task)
	if host == "" || port == 0 {
		return fmt.Errorf("The port:%s marked for registration of service: %s couldn't be found", service.PortLabel, service.Name)
	}
	checks := c.makeChecks(service, host, port)
	asr := &consul.AgentServiceRegistration{
		ID:      service.Id,
		Name:    service.Name,
		Tags:    service.Tags,
		Port:    port,
		Address: host,
		Checks:  checks,
	}
	if err := c.client.Agent().ServiceRegister(asr); err != nil {
		c.logger.Printf("[ERROR] Error while registering service %v with Consul: %v", service.Name, err)
		mErr.Errors = append(mErr.Errors, err)
	}
	return mErr.ErrorOrNil()
}

func (c *ConsulClient) deregisterService(serviceId string) error {
	if err := c.client.Agent().ServiceDeregister(serviceId); err != nil {
		return err
	}
	return nil
}

func (c *ConsulClient) makeChecks(service *structs.Service, ip string, port int) []*consul.AgentServiceCheck {
	var checks []*consul.AgentServiceCheck
	for _, check := range service.Checks {
		c := &consul.AgentServiceCheck{
			Interval: check.Interval.String(),
			Timeout:  check.Timeout.String(),
		}
		switch check.Type {
		case structs.ServiceCheckHTTP:
			c.HTTP = fmt.Sprintf("%s://%s:%d/%s", check.Protocol, ip, port, check.Http)
		case structs.ServiceCheckTCP:
			c.TCP = fmt.Sprintf("%s:%d", ip, port)
		case structs.ServiceCheckScript:
			c.Script = check.Script // TODO This needs to include the path of the alloc dir and based on driver types
		}
		checks = append(checks, c)
	}
	return checks
}
<|endoftext|>


<commit_before>package blocks

import (
	"github.com/bitly/go-simplejson"
	"log"
	"math/rand"
	"strings"
	"time"
)

var (
	msg        *simplejson.Json
	raw_string = "∰ ∈ ∉ ∌ ∂ According to ISO 10646-1:2000, sections D.7 and 2.3c, a device receiving UTF-8 shall interpret a \"malformed sequence in the same way that it interprets a character &that is outside тіршілік-тынысының қолайлы дамыту, қауіпсіздігін, the adopted subset\" and “characters that are not within the adopted 남한산성(南漢山城)은 광주시, κόσμε 성남시, 산성으로, 784-16에 속해있다. subset shall be indicated to the user” by a receiving device. It can cost just $125 to buy a package of bees, and there’ is no real maintenance involved. Bees are typically bred in the south and shipped north in April, sent to beekeepers in a cage the size of a lunch box that can be mailed through the United States Postal Service. To buy a mature hive that is already producing honey, like the ones the Durst Organization has, can cost $300 to as high as $1,000. “People look at pogosto ne da razlikovati od nestrupenih vrst ter da žrtev na začetku nemalokrat zgleda neprizadeto, je treba v primeru ugriza takoj poiskati zdravniško pomoč moms and say, ‘oh they’re just low income and that’s it,'” she said. “I think parents have changed. विकिपीडिया  यह यथासम्भव निष्पक्ष दृष्टिकोण वाली सूOur income may still be low, but we’re more educated”"
	options    = []string{"a", "b", "c", "d", "e", "f"}
)

func Random(b *Block) {

	type randomRule struct {
		Period int
	}

	rule := &randomRule{
		Period: 1,
	}

	msg := simplejson.New()
	c := time.Tick(time.Duration(rule.Period) * time.Second)
	r := rand.New(rand.NewSource(99))

	for {
		select {
		case r := <-b.Routes["set_rule"]:
			log.Println(r.Msg)
			unmarshal(r, &rule)
			log.Println("recieved new Period", rule.Period)
			c = time.Tick(time.Duration(rule.Period) * time.Second)
		case now := <-c:
			a := int64(r.Float64() * 10000000000)
			strTime := now.UnixNano() - a
			msg.Set("t", int64(strTime/1000000))
			msg.Set("a", 10)

			msg.Set("random_int", rand.Intn(2)+1)

			randints := make([]int, rand.Intn(10))
			for i, _ := range randints {
				randints[i] = rand.Intn(100)
			}
			msg.Set("random_integers", randints)

			idx0 := rand.Intn(len(options))
			idx1 := rand.Intn(len(options))
			msg.Set("option", options[idx0])

			nestJson, _ := simplejson.NewJson([]byte("{}"))
			l := rand.Intn(20) + 10
			d := make([]string, l)
			string_bank := strings.Fields(raw_string)
			for i, _ := range d {
				d[i] = string_bank[rand.Intn(len(string_bank))]
			}
			nestJson.Set("d", strings.Join(d, " ")+".")
			nestJson.Set("e", rand.Float32()*8888)
			nestJson.Set("f", rand.Float32()-rand.Float32()*32)
			nestJson.Set("nestedOption", options[idx1])
			msg.Set("c", nestJson)
			msg.Set("e", rand.Float32()*8888)

			broadcast(b.OutChans, msg)
		case msg := <-b.AddChan:
			updateOutChans(msg, b)
		}
	}
}
<commit_msg>put the random block's random_int back to it's full range<commit_after>package blocks

import (
	"github.com/bitly/go-simplejson"
	"log"
	"math/rand"
	"strings"
	"time"
)

var (
	msg        *simplejson.Json
	raw_string = "∰ ∈ ∉ ∌ ∂ According to ISO 10646-1:2000, sections D.7 and 2.3c, a device receiving UTF-8 shall interpret a \"malformed sequence in the same way that it interprets a character &that is outside тіршілік-тынысының қолайлы дамыту, қауіпсіздігін, the adopted subset\" and “characters that are not within the adopted 남한산성(南漢山城)은 광주시, κόσμε 성남시, 산성으로, 784-16에 속해있다. subset shall be indicated to the user” by a receiving device. It can cost just $125 to buy a package of bees, and there’ is no real maintenance involved. Bees are typically bred in the south and shipped north in April, sent to beekeepers in a cage the size of a lunch box that can be mailed through the United States Postal Service. To buy a mature hive that is already producing honey, like the ones the Durst Organization has, can cost $300 to as high as $1,000. “People look at pogosto ne da razlikovati od nestrupenih vrst ter da žrtev na začetku nemalokrat zgleda neprizadeto, je treba v primeru ugriza takoj poiskati zdravniško pomoč moms and say, ‘oh they’re just low income and that’s it,'” she said. “I think parents have changed. विकिपीडिया  यह यथासम्भव निष्पक्ष दृष्टिकोण वाली सूOur income may still be low, but we’re more educated”"
	options    = []string{"a", "b", "c", "d", "e", "f"}
)

func Random(b *Block) {

	type randomRule struct {
		Period int
	}

	rule := &randomRule{
		Period: 1,
	}

	msg := simplejson.New()
	c := time.Tick(time.Duration(rule.Period) * time.Second)
	r := rand.New(rand.NewSource(99))

	for {
		select {
		case r := <-b.Routes["set_rule"]:
			log.Println(r.Msg)
			unmarshal(r, &rule)
			log.Println("recieved new Period", rule.Period)
			c = time.Tick(time.Duration(rule.Period) * time.Second)
		case now := <-c:
			a := int64(r.Float64() * 10000000000)
			strTime := now.UnixNano() - a
			msg.Set("t", int64(strTime/1000000))
			msg.Set("a", 10)

			msg.Set("random_int", rand.Intn(10)+1)

			randints := make([]int, rand.Intn(10))
			for i, _ := range randints {
				randints[i] = rand.Intn(100)
			}
			msg.Set("random_integers", randints)

			idx0 := rand.Intn(len(options))
			idx1 := rand.Intn(len(options))
			msg.Set("option", options[idx0])

			nestJson, _ := simplejson.NewJson([]byte("{}"))
			l := rand.Intn(20) + 10
			d := make([]string, l)
			string_bank := strings.Fields(raw_string)
			for i, _ := range d {
				d[i] = string_bank[rand.Intn(len(string_bank))]
			}
			nestJson.Set("d", strings.Join(d, " ")+".")
			nestJson.Set("e", rand.Float32()*8888)
			nestJson.Set("f", rand.Float32()-rand.Float32()*32)
			nestJson.Set("nestedOption", options[idx1])
			msg.Set("c", nestJson)
			msg.Set("e", rand.Float32()*8888)

			broadcast(b.OutChans, msg)
		case msg := <-b.AddChan:
			updateOutChans(msg, b)
		}
	}
}
<|endoftext|>


<commit_before>package blinker

import (
	"fmt"
	"github.com/golang/glog"
	"io"
	"os"
	"os/exec"
	"path/filepath"
)

type serviceImpl struct {
	settings Settings
}

func getPath(root, country, region, id string) string {
	return filepath.Join(root, fmt.Sprintf("%s-%s-%s", country, region, id))
}

func NewService(settings Settings) (Service, error) {

	impl := &serviceImpl{
		settings: settings,
	}
	return impl, nil
}

func (this *serviceImpl) GetImage(country, region, id string) (bytes io.ReadCloser, size int64, err error) {
	path := getPath(this.settings.FsSettings.RootDir, country, region, id)
	glog.Infoln("Reading from file", path)

	f, err := os.Open(path)
	if err != nil {
		return
	}

	stat, err := f.Stat()
	if err != nil {
		return
	}

	bytes = f
	size = stat.Size()
	return
}

func (this *serviceImpl) ExecAlpr(country, region, id string, image io.ReadCloser) (stdout []byte, err error) {
	path := getPath(this.settings.FsSettings.RootDir, country, region, id)

	glog.Infoln("Saving to file", path)

	dst, err := os.Create(path)
	defer dst.Close()

	if err != nil {
		return
	}

	_, err = io.Copy(dst, image)
	if err != nil {
		return
	}

	alpr := &AlprCommand{
		Country: country,
		Region:  region,
		Path:    path,
	}

	stdout, err = alpr.Execute()
	if err != nil {
		return
	}

	// copy the results
	json, err := os.Create(path + ".json")
	defer json.Close()

	json.Write(stdout)

	return
}

func (this *serviceImpl) Close() {
	glog.Infoln("Service closed")
}

func (this *AlprCommand) Execute() (stdout []byte, err error) {
	path, err := exec.LookPath("alpr")
	if err != nil {
		return nil, err
	}

	cmd := exec.Command(path, "-c", this.Country, "-t", this.Region, "-j", this.Path)
	return cmd.Output()
}
<commit_msg>Additional logging<commit_after>package blinker

import (
	"fmt"
	"github.com/golang/glog"
	"io"
	"os"
	"os/exec"
	"path/filepath"
)

type serviceImpl struct {
	settings Settings
}

func getPath(root, country, region, id string) string {
	return filepath.Join(root, fmt.Sprintf("%s-%s-%s", country, region, id))
}

func NewService(settings Settings) (Service, error) {

	impl := &serviceImpl{
		settings: settings,
	}
	return impl, nil
}

func (this *serviceImpl) GetImage(country, region, id string) (bytes io.ReadCloser, size int64, err error) {
	path := getPath(this.settings.FsSettings.RootDir, country, region, id)
	glog.Infoln("Reading from file", path)

	f, err := os.Open(path)
	if err != nil {
		return
	}

	stat, err := f.Stat()
	if err != nil {
		return
	}

	bytes = f
	size = stat.Size()
	return
}

func (this *serviceImpl) ExecAlpr(country, region, id string, image io.ReadCloser) (stdout []byte, err error) {
	path := getPath(this.settings.FsSettings.RootDir, country, region, id)

	glog.Infoln("ExecAlpr: saving to file", path)

	dst, err := os.Create(path)
	defer dst.Close()

	if err != nil {
		return
	}

	_, err = io.Copy(dst, image)
	if err != nil {
		return
	}

	alpr := &AlprCommand{
		Country: country,
		Region:  region,
		Path:    path,
	}

	stdout, err = alpr.Execute()
	if err != nil {
		return
	}

	// copy the results
	json, err := os.Create(path + ".json")

	defer json.Close()

	glog.Infoln("ExecAlpr: saving results to", json.Name())
	json.Write(stdout)

	return
}

func (this *serviceImpl) Close() {
	glog.Infoln("Service closed")
}

func (this *AlprCommand) Execute() (stdout []byte, err error) {
	cmd := exec.Command("alpr", "-c", this.Country, "-t", this.Region, "-j", this.Path)
	glog.Infoln("exec command:", cmd)
	return cmd.Output()
}
<|endoftext|>


<commit_before>package main

import (
	"code.cloudfoundry.org/lager"
	"github.com/concourse/atc"
	"github.com/jessevdk/go-flags"
	"github.com/tedsuo/ifrit"
)

type GardenBackend struct{}

func (cmd WorkerCommand) lessenRequirements(command *flags.Command) {
	command.FindOptionByLongName("baggageclaim-volumes").Required = false
}

func (cmd *WorkerCommand) gardenRunner(logger lager.Logger, args []string) (atc.Worker, ifrit.Runner, error) {
	err := cmd.checkRoot()
	if err != nil {
		return atc.Worker{}, nil, err
	}

	return cmd.houdiniRunner(logger, "darwin")
}

func (cmd *WorkerCommand) baggageclaimRunner(logger lager.Logger) (ifrit.Runner, error) {
	return cmd.naiveBaggageclaimRunner(logger)
}
<commit_msg>remove root user check from darwin worker<commit_after>package main

import (
	"code.cloudfoundry.org/lager"
	"github.com/concourse/atc"
	"github.com/jessevdk/go-flags"
	"github.com/tedsuo/ifrit"
)

type GardenBackend struct{}

func (cmd WorkerCommand) lessenRequirements(command *flags.Command) {
	command.FindOptionByLongName("baggageclaim-volumes").Required = false
}

func (cmd *WorkerCommand) gardenRunner(logger lager.Logger, args []string) (atc.Worker, ifrit.Runner, error) {
	return cmd.houdiniRunner(logger, "darwin")
}

func (cmd *WorkerCommand) baggageclaimRunner(logger lager.Logger) (ifrit.Runner, error) {
	return cmd.naiveBaggageclaimRunner(logger)
}
<|endoftext|>


<commit_before>package mysql

import (
	"fmt"
	"hash/adler32"
	"strconv"
	"strings"
	"time"

	"github.com/funkygao/fae/config"
	"github.com/funkygao/fae/servant/mysql"
	"github.com/funkygao/gafka/cmd/kateway/job"
	"github.com/funkygao/golib/idgen"
)

const (
	lookupPool     = "ShardLookup"
	appLookupTable = "AppLookup"
	AppPool        = "AppShard"

	sqlInsertAppLookup = "INSERT IGNORE INTO AppLookup(entityId, shardId, name, ctime) VALUES(?,?,?,?)"
)

type mysqlStore struct {
	idgen *idgen.IdGenerator
	mc    *mysql.MysqlCluster
}

func New(id string, cf *config.ConfigMysql) (job.JobStore, error) {
	if cf == nil {
		return nil, fmt.Errorf("job store: empty mysql config")
	}

	wid, err := strconv.Atoi(id)
	if err != nil {
		return nil, err
	}

	ig, err := idgen.NewIdGenerator(wid)
	if err != nil {
		return nil, err
	}

	cf.DefaultLookupTable = appLookupTable
	return &mysqlStore{
		idgen: ig,
		mc:    mysql.New(cf),
	}, nil
}

func (this *mysqlStore) CreateJob(shardId int, appid, topic string) (err error) {
	// first, insert into app if not present
	aid, table := App_id(appid), this.table(topic)
	_, _, err = this.mc.Exec(lookupPool, appLookupTable, 0, sqlInsertAppLookup,
		aid, shardId, appid, time.Now())
	if err != nil {
		return
	}

	// create the job table and job histrory table
	sql := fmt.Sprintf(`
CREATE TABLE %s (
    app_id bigint unsigned NOT NULL DEFAULT 0,
    job_id bigint unsigned NOT NULL DEFAULT 0,
    payload blob,
    ctime timestamp NOT NULL DEFAULT 0,
    mtime timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
    due_time timestamp NULL DEFAULT NULL COMMENT "end time point of the event",
    PRIMARY KEY (app_id, job_id),
    KEY(due_time)
) ENGINE = INNODB DEFAULT CHARSET utf8
		`, table)
	_, _, err = this.mc.Exec(AppPool, table, aid, sql)
	if err != nil {
		return
	}

	historyTable := table + "_history"
	sql = fmt.Sprintf(`
CREATE TABLE %s (
    app_id bigint unsigned NOT NULL DEFAULT 0,
    job_id bigint unsigned NOT NULL DEFAULT 0,
    payload blob,
    ctime timestamp NOT NULL DEFAULT 0,
    due_time timestamp NULL DEFAULT NULL,
    PRIMARY KEY (app_id, job_id),
    KEY(due_time),
    KEY(ctime),
) ENGINE = INNODB DEFAULT CHARSET utf8
		`, historyTable)
	_, _, err = this.mc.Exec(AppPool, historyTable, aid, sql)
	return
}

func (this *mysqlStore) Add(appid, topic string, payload []byte, delay time.Duration) (jobId string, err error) {
	jid := this.nextId()
	table, aid := this.table(topic), App_id(appid)
	t0 := time.Now()
	t1 := t0.Add(delay)
	sql := fmt.Sprintf("INSERT INTO %s(app_id, job_id, payload, ctime, due_time) VALUES(?,?,?,?,?)", table)
	_, _, err = this.mc.Exec(AppPool, table, aid, sql,
		aid, jid, payload, t0, t1)
	jobId = strconv.FormatInt(jid, 10)
	return
}

func (this *mysqlStore) Delete(appid, topic, jobId string) (err error) {
	// TODO race condition with actor worker
	var jid int64
	jid, err = strconv.ParseInt(jobId, 10, 64)
	if err != nil {
		return
	}

	var affectedRows int64
	table, aid := this.table(topic), App_id(appid)
	sql := fmt.Sprintf("DELETE FROM %s WHERE app_id=? AND job_id=?", table)
	affectedRows, _, err = this.mc.Exec(AppPool, table, aid, sql,
		aid, jid)
	if affectedRows == 0 {
		err = job.ErrNothingDeleted
	}

	return
}

func (this *mysqlStore) Name() string {
	return "mysql"
}

func (this *mysqlStore) Start() error {
	this.mc.Warmup()
	return nil
}

func (this *mysqlStore) Stop() {
	this.mc.Close()
}

func (this *mysqlStore) table(topic string) string {
	return strings.Replace(topic, ".", "_", -1)
}

// App_id convert a string appid to hash int which is used to locate shard.
func App_id(appid string) int {
	return int(adler32.Checksum([]byte(appid)))
}
<commit_msg>memo: mysql blob has limit size of 64KB<commit_after>package mysql

import (
	"fmt"
	"hash/adler32"
	"strconv"
	"strings"
	"time"

	"github.com/funkygao/fae/config"
	"github.com/funkygao/fae/servant/mysql"
	"github.com/funkygao/gafka/cmd/kateway/job"
	"github.com/funkygao/golib/idgen"
)

const (
	lookupPool     = "ShardLookup"
	appLookupTable = "AppLookup"
	AppPool        = "AppShard"

	sqlInsertAppLookup = "INSERT IGNORE INTO AppLookup(entityId, shardId, name, ctime) VALUES(?,?,?,?)"
)

type mysqlStore struct {
	idgen *idgen.IdGenerator
	mc    *mysql.MysqlCluster
}

func New(id string, cf *config.ConfigMysql) (job.JobStore, error) {
	if cf == nil {
		return nil, fmt.Errorf("job store: empty mysql config")
	}

	wid, err := strconv.Atoi(id)
	if err != nil {
		return nil, err
	}

	ig, err := idgen.NewIdGenerator(wid)
	if err != nil {
		return nil, err
	}

	cf.DefaultLookupTable = appLookupTable
	return &mysqlStore{
		idgen: ig,
		mc:    mysql.New(cf),
	}, nil
}

func (this *mysqlStore) CreateJob(shardId int, appid, topic string) (err error) {
	// first, insert into app if not present
	aid, table := App_id(appid), this.table(topic)
	_, _, err = this.mc.Exec(lookupPool, appLookupTable, 0, sqlInsertAppLookup,
		aid, shardId, appid, time.Now())
	if err != nil {
		return
	}

	// create the job table and job histrory table
	// in mysql InnoDB, blob is []byte while text is string, both length limit 1<<16(64KB)
	sql := fmt.Sprintf(`
CREATE TABLE %s (
    app_id bigint unsigned NOT NULL DEFAULT 0,
    job_id bigint unsigned NOT NULL DEFAULT 0,
    payload blob,
    ctime timestamp NOT NULL DEFAULT 0,
    mtime timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
    due_time timestamp NULL DEFAULT NULL COMMENT "end time point of the event",
    PRIMARY KEY (app_id, job_id),
    KEY(due_time)
) ENGINE = INNODB DEFAULT CHARSET utf8
		`, table)
	_, _, err = this.mc.Exec(AppPool, table, aid, sql)
	if err != nil {
		return
	}

	historyTable := table + "_history"
	sql = fmt.Sprintf(`
CREATE TABLE %s (
    app_id bigint unsigned NOT NULL DEFAULT 0,
    job_id bigint unsigned NOT NULL DEFAULT 0,
    payload blob,
    ctime timestamp NOT NULL DEFAULT 0,
    due_time timestamp NULL DEFAULT NULL,
    PRIMARY KEY (app_id, job_id),
    KEY(due_time),
    KEY(ctime),
) ENGINE = INNODB DEFAULT CHARSET utf8
		`, historyTable)
	_, _, err = this.mc.Exec(AppPool, historyTable, aid, sql)
	return
}

func (this *mysqlStore) Add(appid, topic string, payload []byte, delay time.Duration) (jobId string, err error) {
	jid := this.nextId()
	table, aid := this.table(topic), App_id(appid)
	t0 := time.Now()
	t1 := t0.Add(delay)
	sql := fmt.Sprintf("INSERT INTO %s(app_id, job_id, payload, ctime, due_time) VALUES(?,?,?,?,?)", table)
	_, _, err = this.mc.Exec(AppPool, table, aid, sql,
		aid, jid, payload, t0, t1)
	jobId = strconv.FormatInt(jid, 10)
	return
}

func (this *mysqlStore) Delete(appid, topic, jobId string) (err error) {
	// TODO race condition with actor worker
	var jid int64
	jid, err = strconv.ParseInt(jobId, 10, 64)
	if err != nil {
		return
	}

	var affectedRows int64
	table, aid := this.table(topic), App_id(appid)
	sql := fmt.Sprintf("DELETE FROM %s WHERE app_id=? AND job_id=?", table)
	affectedRows, _, err = this.mc.Exec(AppPool, table, aid, sql,
		aid, jid)
	if affectedRows == 0 {
		err = job.ErrNothingDeleted
	}

	return
}

func (this *mysqlStore) Name() string {
	return "mysql"
}

func (this *mysqlStore) Start() error {
	this.mc.Warmup()
	return nil
}

func (this *mysqlStore) Stop() {
	this.mc.Close()
}

func (this *mysqlStore) table(topic string) string {
	return strings.Replace(topic, ".", "_", -1)
}

// App_id convert a string appid to hash int which is used to locate shard.
func App_id(appid string) int {
	return int(adler32.Checksum([]byte(appid)))
}
<|endoftext|>


<commit_before>package mc

import (
	"math/rand"
	"time"

	"crypto/md5"
	"io"
	"math"
	"os"

	"path/filepath"

	"fmt"

	"github.com/materials-commons/gohandy/file"
	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/app/flow"
	"github.com/materials-commons/mcstore/pkg/files"
	"github.com/materials-commons/mcstore/server/mcstore"
)

var _ = fmt.Println

// projectUploader holds the starting state for a project upload. It controls
// how many threads process requests.
type projectUploader struct {
	db         ProjectDB
	numThreads int
}

// upload will upload the project. It starts a parallel walker to process entries.
func (p *projectUploader) upload() error {
	db := p.db
	project := db.Project()

	// create a func to process entries. Since we have some state to close over we create
	// a closure here to call.
	fn := func(done <-chan struct{}, entries <-chan files.TreeEntry, result chan<- string) {
		uploader := newUploader(p.db, project)
		uploader.uploadEntries(done, entries, result)
	}

	walker := files.PWalker{
		NumParallel: p.numThreads,
		ProcessFn:   fn,
		ProcessDirs: true,
	}

	_, errc := walker.PWalk(project.Path)
	err := <-errc
	return err
}

// uploader holds the state for one upload thread.
type uploader struct {
	db         ProjectDB
	serverAPI  *mcstore.ServerAPI
	project    *Project
	minWait    int
	maxWait    int
	retryCount int
}

// defaultMinWaitBeforeRetry is the default minimum wait time before
// a request is retried.
const defaultMinWaitBeforeRetry = 100

// defaultMaxWaitBeforeRetry is the default max wait time before
// a request is retried.
const defaultMaxWaitBeforeRetry = 5000

// retryForever means we should retry requests forever. If requests shouldn't
// be retried forever then the upload.retryCount should be set to a positive
// number.
const retryForever = -1

// newUploader creates a new uploader. It creates a clone of the database,
// sets minWait to defaultMinWaitBeforeRetry, maxWait to defaultMaxWaitBeforeRetry
// and retryCount to retryForever.
func newUploader(db ProjectDB, project *Project) *uploader {
	return &uploader{
		db:         db.Clone(),
		project:    project,
		serverAPI:  mcstore.NewServerAPI(),
		minWait:    defaultMinWaitBeforeRetry,
		maxWait:    defaultMaxWaitBeforeRetry,
		retryCount: retryForever,
	}
}

// uploadEntries waits on the entries and done channels. When  entry comes in on
// the entries channel it will process it. If an something comes in on the done
// channel then it will stop processing and return from the go routine.
func (u *uploader) uploadEntries(done <-chan struct{}, entries <-chan files.TreeEntry, result chan<- string) {
	for entry := range entries {
		select {
		case result <- u.uploadEntry(entry):
		case <-done:
			return
		}
	}
}

// uploadEntry identifies the type of entry it is processing (file or directory) and
// processes it appropriately.
func (u *uploader) uploadEntry(entry files.TreeEntry) string {
	switch {
	case entry.Finfo.IsDir():
		u.handleDirEntry(entry)
	default:
		u.handleFileEntry(entry)
	}
	return ""
}

// handleDirEntry handles processing of directories. It checks if the directory already exists in the
// local database. If it doesn't it will create the directory on the server and insert the directory
// into its local database.
func (u *uploader) handleDirEntry(entry files.TreeEntry) {
	path := filepath.ToSlash(entry.Finfo.Name())
	_, err := u.db.FindDirectory(path)
	switch {
	case err == app.ErrNotFound:
		u.createDirectory(entry)
	case err != nil:
		app.Log.Panicf("Local database returned err, panic!: %s", err)
	default:
		// directory already known nothing to do
		return
	}
}

// createDirectory creates a new directory entry on the server.
func (u *uploader) createDirectory(entry files.TreeEntry) {
	dirPath := filepath.ToSlash(entry.Path)
	req := mcstore.DirectoryRequest{
		ProjectName: u.project.Name,
		ProjectID:   u.project.ProjectID,
		Path:        dirPath,
	}
	dirID := u.getDirectoryWithRetry(req)
	dir := &Directory{
		DirectoryID: dirID,
		Path:        dirPath,
	}
	if _, err := u.db.InsertDirectory(dir); err != nil {
		app.Log.Panicf("Local database returned err, panic!: %s", err)
	}
}

// getDirectoryWithRetry will make the server API GetDirectory call. If it fails
// it will retry (dependent on retry settings).
func (u *uploader) getDirectoryWithRetry(req mcstore.DirectoryRequest) string {
	var dirID string
	fn := func() bool {
		var err error
		if dirID, err = u.serverAPI.GetDirectory(req); err != nil {
			return false
		}
		return true
	}
	u.withRetry(fn)
	return dirID
}

// handleFileEntry will handle processing file entries. It will check if the file
// has already been uploaded. If it hasn't it will upload the file to the server.
func (u *uploader) handleFileEntry(entry files.TreeEntry) {
	if dir := u.getDirByPath(filepath.Dir(entry.Path)); dir == nil {
		app.Log.Panicf("Should have found dir")
	} else {
		file := u.getFileByName(entry.Finfo.Name(), dir.ID)
		switch {
		case file == nil:
			u.uploadFile(entry, file, dir)
		case entry.Finfo.ModTime().Unix() > file.MTime.Unix():
			u.uploadFile(entry, file, dir)
		default:
			// nothing to do
		}
	}
}

// getDirByPath looks up a path in the local database. It normalizes
// the file separator.
func (u *uploader) getDirByPath(path string) *Directory {
	path = filepath.ToSlash(path)
	dir, err := u.db.FindDirectory(path)
	switch {
	case err == app.ErrNotFound:
		return nil
	case err != nil:
		app.Log.Panicf("Local database returned err, panic!: %s", err)
		return nil
	default:
		// directory already known nothing to do
		return dir
	}
}

// getFileByName looks up a file by name and directory id in the local database.
func (u *uploader) getFileByName(name string, dirID int64) *File {
	f, err := u.db.FindFile(name, dirID)
	switch {
	case err == app.ErrNotFound:
		return nil
	case err != nil:
		app.Log.Panicf("Local database returned err, panic!: %s", err)
		return nil
	default:
		return f
	}
}

// uploadFile will upload the file blocks to the server. It will create an upload request
// and then start uploading the blocks. When it completes it will update the local database
// state for the file.
func (u *uploader) uploadFile(entry files.TreeEntry, file *File, dir *Directory) {
	uploadResponse, checksum := u.getUploadResponse(dir.DirectoryID, entry)
	requestID := uploadResponse.RequestID
	// TODO: do something with the starting block (its ignored for now)
	var n int
	var err error
	chunkNumber := 1

	f, _ := os.Open(entry.Path)
	defer f.Close()
	buf := make([]byte, 1024*1024)
	totalChunks := numChunks(entry.Finfo.Size())
	var uploadResp *mcstore.UploadChunkResponse
	for {
		n, err = f.Read(buf)
		if n != 0 {
			// send bytes
			req := &flow.Request{
				FlowChunkNumber:  int32(chunkNumber),
				FlowTotalChunks:  totalChunks,
				FlowChunkSize:    int32(n),
				FlowTotalSize:    entry.Finfo.Size(),
				FlowIdentifier:   requestID,
				FlowFileName:     entry.Finfo.Name(),
				FlowRelativePath: "",
				ProjectID:        u.project.ProjectID,
				DirectoryID:      dir.DirectoryID,
				Chunk:            buf[:n],
			}
			uploadResp = u.sendFlowDataWithRetry(req)
			fmt.Printf("uploadResp = %#v\n", uploadResp)
			if uploadResp.Done {
				break
			}
			chunkNumber++
		}
		if err != nil {
			break
		}
	}

	if err != nil && err != io.EOF {
		app.Log.Errorf("Unable to complete read on file for upload: %s", entry.Path)
	} else {
		// done, so update the database with the entry.
		if file == nil || file.Checksum != checksum {
			// create new entry
			newFile := File{
				FileID:     uploadResp.FileID,
				Name:       entry.Finfo.Name(),
				Checksum:   checksum,
				Size:       entry.Finfo.Size(),
				MTime:      entry.Finfo.ModTime(),
				LastUpload: time.Now(),
				Directory:  dir.ID,
			}
			u.db.InsertFile(&newFile)
		} else {
			// update existing entry
			file.MTime = entry.Finfo.ModTime()
			file.LastUpload = time.Now()
			u.db.UpdateFile(file)
		}
	}
}

// getUploadResponse sends an upload request to the server and gets the response.
func (u *uploader) getUploadResponse(directoryID string, entry files.TreeEntry) (*mcstore.CreateUploadResponse, string) {
	// retry forever
	checksum, _ := file.HashStr(md5.New(), entry.Path)
	chunkSize := int32(1000 * 1000)
	uploadReq := mcstore.CreateUploadRequest{
		ProjectID:   u.project.ProjectID,
		DirectoryID: directoryID,
		FileName:    entry.Finfo.Name(),
		FileSize:    entry.Finfo.Size(),
		ChunkSize:   chunkSize,
		FileMTime:   entry.Finfo.ModTime().Format(time.RFC1123),
		Checksum:    checksum,
	}
	resp := u.createUploadRequestWithRetry(uploadReq)
	return resp, checksum
}

// createUplaodRequestWithRetry will make the server API CreateUploadRequest call. If it fails
// it will retry (dependent on retry settings).
func (u *uploader) createUploadRequestWithRetry(uploadReq mcstore.CreateUploadRequest) *mcstore.CreateUploadResponse {
	var resp *mcstore.CreateUploadResponse
	fn := func() bool {
		var err error
		if resp, err = u.serverAPI.CreateUploadRequest(uploadReq); err != nil {
			fmt.Println("CreateUploadRequest returned err", err)
			return false
		}
		return true
	}
	u.withRetry(fn)
	return resp
}

// numChunks determines the number of chunks that will be sent to the server.
func numChunks(size int64) int32 {
	d := float64(size) / float64(1024*1024)
	n := int(math.Ceil(d))
	return int32(n)
}

// createUplaodRequestWithRetry will make the server API SendFlowData call. If it fails
// it will retry (dependent on retry settings).
func (u *uploader) sendFlowDataWithRetry(req *flow.Request) *mcstore.UploadChunkResponse {
	var resp *mcstore.UploadChunkResponse
	fn := func() bool {
		var err error
		if resp, err = u.serverAPI.SendFlowData(req); err != nil {
			fmt.Println("SendFlowData returned err", err)
			return false
		}
		return true
	}
	u.withRetry(fn)
	return resp
}

// withRetry will retry the given func. The func should return true
// when it is successful. It will retry retryCount times (forever
// if retryCount is set to retryForever). When a call fails it will
// sleep a random amount of time bounded by minWait and maxWait before
// it retries the request.
func (u *uploader) withRetry(fn func() bool) {
	retryCounter := 0
	for {
		if fn() {
			break
		}

		if u.retryCount != retryForever {
			retryCounter++
			if retryCounter > u.retryCount {
				app.Log.Panicf("Retries exceeded aborting")
			}
		}
		u.sleepRandom()
	}
}

func (u *uploader) sleepRandom() {
	// sleep a random amount between minWait and maxWait
	rand.Seed(time.Now().Unix())
	randomSleepTime := rand.Intn(u.maxWait) + u.minWait
	time.Sleep(time.Duration(randomSleepTime))
}
<commit_msg>Correctly set the total number of blocks. Remove debug statements. Only retry once.<commit_after>package mc

import (
	"math/rand"
	"time"

	"crypto/md5"
	"io"
	"math"
	"os"

	"path/filepath"

	"fmt"

	"github.com/materials-commons/gohandy/file"
	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/app/flow"
	"github.com/materials-commons/mcstore/pkg/files"
	"github.com/materials-commons/mcstore/server/mcstore"
)

var _ = fmt.Println

// projectUploader holds the starting state for a project upload. It controls
// how many threads process requests.
type projectUploader struct {
	db         ProjectDB
	numThreads int
}

// upload will upload the project. It starts a parallel walker to process entries.
func (p *projectUploader) upload() error {
	db := p.db
	project := db.Project()

	// create a func to process entries. Since we have some state to close over we create
	// a closure here to call.
	fn := func(done <-chan struct{}, entries <-chan files.TreeEntry, result chan<- string) {
		uploader := newUploader(p.db, project)
		uploader.retryCount = 1
		uploader.uploadEntries(done, entries, result)
	}

	walker := files.PWalker{
		NumParallel: p.numThreads,
		ProcessFn:   fn,
		ProcessDirs: true,
	}

	_, errc := walker.PWalk(project.Path)
	err := <-errc
	return err
}

// uploader holds the state for one upload thread.
type uploader struct {
	db         ProjectDB
	serverAPI  *mcstore.ServerAPI
	project    *Project
	minWait    int
	maxWait    int
	retryCount int
}

// defaultMinWaitBeforeRetry is the default minimum wait time before
// a request is retried.
const defaultMinWaitBeforeRetry = 100

// defaultMaxWaitBeforeRetry is the default max wait time before
// a request is retried.
const defaultMaxWaitBeforeRetry = 5000

// retryForever means we should retry requests forever. If requests shouldn't
// be retried forever then the upload.retryCount should be set to a positive
// number.
const retryForever = -1

// newUploader creates a new uploader. It creates a clone of the database,
// sets minWait to defaultMinWaitBeforeRetry, maxWait to defaultMaxWaitBeforeRetry
// and retryCount to retryForever.
func newUploader(db ProjectDB, project *Project) *uploader {
	return &uploader{
		db:         db.Clone(),
		project:    project,
		serverAPI:  mcstore.NewServerAPI(),
		minWait:    defaultMinWaitBeforeRetry,
		maxWait:    defaultMaxWaitBeforeRetry,
		retryCount: retryForever,
	}
}

// uploadEntries waits on the entries and done channels. When  entry comes in on
// the entries channel it will process it. If an something comes in on the done
// channel then it will stop processing and return from the go routine.
func (u *uploader) uploadEntries(done <-chan struct{}, entries <-chan files.TreeEntry, result chan<- string) {
	for entry := range entries {
		select {
		case result <- u.uploadEntry(entry):
		case <-done:
			return
		}
	}
}

// uploadEntry identifies the type of entry it is processing (file or directory) and
// processes it appropriately.
func (u *uploader) uploadEntry(entry files.TreeEntry) string {
	switch {
	case entry.Finfo.IsDir():
		u.handleDirEntry(entry)
	default:
		u.handleFileEntry(entry)
	}
	return ""
}

// handleDirEntry handles processing of directories. It checks if the directory already exists in the
// local database. If it doesn't it will create the directory on the server and insert the directory
// into its local database.
func (u *uploader) handleDirEntry(entry files.TreeEntry) {
	path := filepath.ToSlash(entry.Finfo.Name())
	_, err := u.db.FindDirectory(path)
	switch {
	case err == app.ErrNotFound:
		u.createDirectory(entry)
	case err != nil:
		app.Log.Panicf("Local database returned err, panic!: %s", err)
	default:
		// directory already known nothing to do
		return
	}
}

// createDirectory creates a new directory entry on the server.
func (u *uploader) createDirectory(entry files.TreeEntry) {
	dirPath := filepath.ToSlash(entry.Path)
	req := mcstore.DirectoryRequest{
		ProjectName: u.project.Name,
		ProjectID:   u.project.ProjectID,
		Path:        dirPath,
	}
	dirID := u.getDirectoryWithRetry(req)
	dir := &Directory{
		DirectoryID: dirID,
		Path:        dirPath,
	}
	if _, err := u.db.InsertDirectory(dir); err != nil {
		app.Log.Panicf("Local database returned err, panic!: %s", err)
	}
}

// getDirectoryWithRetry will make the server API GetDirectory call. If it fails
// it will retry (dependent on retry settings).
func (u *uploader) getDirectoryWithRetry(req mcstore.DirectoryRequest) string {
	var dirID string
	fn := func() bool {
		var err error
		if dirID, err = u.serverAPI.GetDirectory(req); err != nil {
			return false
		}
		return true
	}
	u.withRetry(fn)
	return dirID
}

// handleFileEntry will handle processing file entries. It will check if the file
// has already been uploaded. If it hasn't it will upload the file to the server.
func (u *uploader) handleFileEntry(entry files.TreeEntry) {
	if dir := u.getDirByPath(filepath.Dir(entry.Path)); dir == nil {
		app.Log.Panicf("Should have found dir")
	} else {
		file := u.getFileByName(entry.Finfo.Name(), dir.ID)
		switch {
		case file == nil:
			u.uploadFile(entry, file, dir)
		case entry.Finfo.ModTime().Unix() > file.MTime.Unix():
			u.uploadFile(entry, file, dir)
		default:
			// nothing to do
		}
	}
}

// getDirByPath looks up a path in the local database. It normalizes
// the file separator.
func (u *uploader) getDirByPath(path string) *Directory {
	path = filepath.ToSlash(path)
	dir, err := u.db.FindDirectory(path)
	switch {
	case err == app.ErrNotFound:
		return nil
	case err != nil:
		app.Log.Panicf("Local database returned err, panic!: %s", err)
		return nil
	default:
		// directory already known nothing to do
		return dir
	}
}

// getFileByName looks up a file by name and directory id in the local database.
func (u *uploader) getFileByName(name string, dirID int64) *File {
	f, err := u.db.FindFile(name, dirID)
	switch {
	case err == app.ErrNotFound:
		return nil
	case err != nil:
		app.Log.Panicf("Local database returned err, panic!: %s", err)
		return nil
	default:
		return f
	}
}

// uploadFile will upload the file blocks to the server. It will create an upload request
// and then start uploading the blocks. When it completes it will update the local database
// state for the file.
func (u *uploader) uploadFile(entry files.TreeEntry, file *File, dir *Directory) {
	uploadResponse, checksum := u.getUploadResponse(dir.DirectoryID, entry)
	requestID := uploadResponse.RequestID
	// TODO: do something with the starting block (its ignored for now)
	var n int
	var err error
	chunkNumber := 1

	f, _ := os.Open(entry.Path)
	defer f.Close()
	buf := make([]byte, 1024*1024)
	totalChunks := numChunks(entry.Finfo.Size())
	var uploadResp *mcstore.UploadChunkResponse
	for {
		n, err = f.Read(buf)
		if n != 0 {
			// send bytes
			req := &flow.Request{
				FlowChunkNumber:  int32(chunkNumber),
				FlowTotalChunks:  totalChunks,
				FlowChunkSize:    int32(n),
				FlowTotalSize:    entry.Finfo.Size(),
				FlowIdentifier:   requestID,
				FlowFileName:     entry.Finfo.Name(),
				FlowRelativePath: "",
				ProjectID:        u.project.ProjectID,
				DirectoryID:      dir.DirectoryID,
				Chunk:            buf[:n],
			}
			uploadResp = u.sendFlowDataWithRetry(req)
			if uploadResp.Done {
				break
			}
			chunkNumber++
		}
		if err != nil {
			break
		}
	}

	if !uploadResp.Done {
		app.Log.Errorf("uploadResp not done %#v\n", uploadResp)
		return
	}

	if err != nil && err != io.EOF {
		app.Log.Errorf("Unable to complete read on file for upload: %s", entry.Path)
	} else {
		// done, so update the database with the entry.
		if file == nil || file.Checksum != checksum {
			// create new entry
			newFile := File{
				FileID:     uploadResp.FileID,
				Name:       entry.Finfo.Name(),
				Checksum:   checksum,
				Size:       entry.Finfo.Size(),
				MTime:      entry.Finfo.ModTime(),
				LastUpload: time.Now(),
				Directory:  dir.ID,
			}
			u.db.InsertFile(&newFile)
		} else {
			// update existing entry
			file.MTime = entry.Finfo.ModTime()
			file.LastUpload = time.Now()
			u.db.UpdateFile(file)
		}
	}
}

// getUploadResponse sends an upload request to the server and gets the response.
func (u *uploader) getUploadResponse(directoryID string, entry files.TreeEntry) (*mcstore.CreateUploadResponse, string) {
	// retry forever
	checksum, _ := file.HashStr(md5.New(), entry.Path)
	chunkSize := int32(1024 * 1024)
	uploadReq := mcstore.CreateUploadRequest{
		ProjectID:   u.project.ProjectID,
		DirectoryID: directoryID,
		FileName:    entry.Finfo.Name(),
		FileSize:    entry.Finfo.Size(),
		ChunkSize:   chunkSize,
		FileMTime:   entry.Finfo.ModTime().Format(time.RFC1123),
		Checksum:    checksum,
	}
	resp := u.createUploadRequestWithRetry(uploadReq)
	return resp, checksum
}

// createUplaodRequestWithRetry will make the server API CreateUploadRequest call. If it fails
// it will retry (dependent on retry settings).
func (u *uploader) createUploadRequestWithRetry(uploadReq mcstore.CreateUploadRequest) *mcstore.CreateUploadResponse {
	var resp *mcstore.CreateUploadResponse
	fn := func() bool {
		var err error
		if resp, err = u.serverAPI.CreateUploadRequest(uploadReq); err != nil {
			return false
		}
		return true
	}
	u.withRetry(fn)
	return resp
}

// numChunks determines the number of chunks that will be sent to the server.
func numChunks(size int64) int32 {
	d := float64(size) / float64(1024*1024)
	n := int(math.Ceil(d))
	return int32(n)
}

// createUplaodRequestWithRetry will make the server API SendFlowData call. If it fails
// it will retry (dependent on retry settings).
func (u *uploader) sendFlowDataWithRetry(req *flow.Request) *mcstore.UploadChunkResponse {
	var resp *mcstore.UploadChunkResponse
	fn := func() bool {
		var err error
		if resp, err = u.serverAPI.SendFlowData(req); err != nil {
			return false
		}
		return true
	}
	u.withRetry(fn)
	return resp
}

// withRetry will retry the given func. The func should return true
// when it is successful. It will retry retryCount times (forever
// if retryCount is set to retryForever). When a call fails it will
// sleep a random amount of time bounded by minWait and maxWait before
// it retries the request.
func (u *uploader) withRetry(fn func() bool) {
	retryCounter := 0
	for {
		if fn() {
			break
		}

		if u.retryCount != retryForever {
			retryCounter++
			if retryCounter > u.retryCount {
				app.Log.Panicf("Retries exceeded aborting")
			}
		}
		u.sleepRandom()
	}
}

func (u *uploader) sleepRandom() {
	// sleep a random amount between minWait and maxWait
	rand.Seed(time.Now().Unix())
	randomSleepTime := rand.Intn(u.maxWait) + u.minWait
	time.Sleep(time.Duration(randomSleepTime))
}
<|endoftext|>


<commit_before>package collector

import (
	"errors"
	"fmt"
	"net/url"
	"os"

	"github.com/zimmski/tirion/shm"
)

type CollectorShm struct {
	shm *shm.Shm
}

func (c *CollectorShm) InitAgent(pid int32, metricCount int32) (*url.URL, error) {
	var u = &url.URL{
		Scheme: "shm",
		Path:   fmt.Sprintf("/proc/%d", pid),
	}

	err := c.initShm(u.Path, true, metricCount)

	if err != nil {
		return nil, err
	}

	return u, nil
}

func (c *CollectorShm) InitClient(u *url.URL, metricCount int32) error {
	if _, err := os.Stat(u.Path); os.IsNotExist(err) {
		return errors.New(fmt.Sprintf("Cannot open shm path: %v", err))
	}

	return c.initShm(u.Path, false, metricCount)
}

func (c *CollectorShm) initShm(filename string, create bool, count int32) error {
	var err error

	c.shm, err = shm.NewShm(filename, create, count)

	if err != nil {
		return errors.New(fmt.Sprintf("Open shared memory: %v", err))
	}

	err = c.shm.Read()

	if err != nil {
		return errors.New(fmt.Sprintf("Read shared memory: %v", err))
	}

	return nil
}

func (c *CollectorShm) Data() []float32 {
	return c.shm.Data()
}

func (c *CollectorShm) Close() error {
	return c.shm.Close()
}

func (c *CollectorShm) Add(i int32, v float32) float32 {
	return c.shm.Add(i, v)
}

func (c *CollectorShm) Dec(i int32) float32 {
	return c.shm.Dec(i)
}

func (c *CollectorShm) Inc(i int32) float32 {
	return c.shm.Inc(i)
}

func (c *CollectorShm) Sub(i int32, v float32) float32 {
	return c.shm.Sub(i, v)
}
<commit_msg>forgot to delete the old shm file<commit_after><|endoftext|>


<commit_before>package collector

// +build linux freebsd
// +build !nozfs

import (
	"errors"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/log"
)

type zfsMetricValue int

const zfsErrorValue = zfsMetricValue(-1)

var zfsNotAvailableError = errors.New("ZFS / ZFS statistics are not available")

type zfsSysctl string
type zfsSubsystemName string

const (
	arc = zfsSubsystemName("zfs_arc")
)

// Metrics

type zfsMetric struct {
	subsystem zfsSubsystemName // The Prometheus subsystem name.
	name      string           // The Prometheus name of the metric.
	sysctl    zfsSysctl        // The sysctl of the ZFS metric.
}

func (m *zfsMetric) ConstMetric(value zfsMetricValue) prometheus.Metric {
	return prometheus.MustNewConstMetric(
		prometheus.NewDesc(
			prometheus.BuildFQName(Namespace, string(m.subsystem), m.name),
			m.name,
			nil,
			nil,
		),
		prometheus.UntypedValue,
		float64(value),
	)
}

type datasetMetric struct {
	subsystem zfsSubsystemName
	name      string
}

// Collector

func init() {
	Factories["zfs"] = NewZFSCollector
}

type zfsCollector struct {
	zfsMetrics []zfsMetric
}

func NewZFSCollector() (Collector, error) {
	return &zfsCollector{
		zfsMetrics: []zfsMetric{
			{arc, "mru_size", zfsSysctl("kstat.zfs.misc.arcstats.p")},
			{arc, "size", zfsSysctl("kstat.zfs.misc.arcstats.size")},
			{arc, "target_min_size", zfsSysctl("kstat.zfs.misc.arcstats.c_min")},
			{arc, "target_size", zfsSysctl("kstat.zfs.misc.arcstats.c")},
			{arc, "target_max_size", zfsSysctl("kstat.zfs.misc.arcstats.c_max")},
			{arc, "hits", zfsSysctl("kstat.zfs.misc.arcstats.hits")},
			{arc, "misses", zfsSysctl("kstat.zfs.misc.arcstats.misses")},
		},
	}, nil
}

func (c *zfsCollector) Update(ch chan<- prometheus.Metric) (err error) {

	err = c.PrepareUpdate()
	switch {
	case err == zfsNotAvailableError:
		log.Debug(err)
		return nil
	case err != nil:
		return err
	}

	// Arcstats

	err = c.updateArcstats(ch)
	if err != nil {
		return err
	}

	return err
}
<commit_msg>Rename zfs_arc subsystem.<commit_after>package collector

// +build linux freebsd
// +build !nozfs

import (
	"errors"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/log"
)

type zfsMetricValue int

const zfsErrorValue = zfsMetricValue(-1)

var zfsNotAvailableError = errors.New("ZFS / ZFS statistics are not available")

type zfsSysctl string
type zfsSubsystemName string

const (
	arc = zfsSubsystemName("zfsArc")
)

// Metrics

type zfsMetric struct {
	subsystem zfsSubsystemName // The Prometheus subsystem name.
	name      string           // The Prometheus name of the metric.
	sysctl    zfsSysctl        // The sysctl of the ZFS metric.
}

func (m *zfsMetric) ConstMetric(value zfsMetricValue) prometheus.Metric {
	return prometheus.MustNewConstMetric(
		prometheus.NewDesc(
			prometheus.BuildFQName(Namespace, string(m.subsystem), m.name),
			m.name,
			nil,
			nil,
		),
		prometheus.UntypedValue,
		float64(value),
	)
}

type datasetMetric struct {
	subsystem zfsSubsystemName
	name      string
}

// Collector

func init() {
	Factories["zfs"] = NewZFSCollector
}

type zfsCollector struct {
	zfsMetrics []zfsMetric
}

func NewZFSCollector() (Collector, error) {
	return &zfsCollector{
		zfsMetrics: []zfsMetric{
			{arc, "mru_size", zfsSysctl("kstat.zfs.misc.arcstats.p")},
			{arc, "size", zfsSysctl("kstat.zfs.misc.arcstats.size")},
			{arc, "target_min_size", zfsSysctl("kstat.zfs.misc.arcstats.c_min")},
			{arc, "target_size", zfsSysctl("kstat.zfs.misc.arcstats.c")},
			{arc, "target_max_size", zfsSysctl("kstat.zfs.misc.arcstats.c_max")},
			{arc, "hits", zfsSysctl("kstat.zfs.misc.arcstats.hits")},
			{arc, "misses", zfsSysctl("kstat.zfs.misc.arcstats.misses")},
		},
	}, nil
}

func (c *zfsCollector) Update(ch chan<- prometheus.Metric) (err error) {

	err = c.PrepareUpdate()
	switch {
	case err == zfsNotAvailableError:
		log.Debug(err)
		return nil
	case err != nil:
		return err
	}

	// Arcstats

	err = c.updateArcstats(ch)
	if err != nil {
		return err
	}

	return err
}
<|endoftext|>


<commit_before>// returns data points from Win32_OperatingSystem

package collectors

import (
	"log"

	"github.com/StackExchange/wmi"
	"github.com/prometheus/client_golang/prometheus"
)

// A OSCollector is a Prometheus collector for WMI OperatingSystem metrics
type OSCollector struct {
	FreePhysicalMemory     *prometheus.Desc
	FreeSpaceInPagingFiles *prometheus.Desc
	FreeVirtualMemory      *prometheus.Desc
	NumberOfProcesses      *prometheus.Desc
	NumberOfUsers          *prometheus.Desc
}

// NewOSCollector ...
func NewOSCollector() *OSCollector {

	return &OSCollector{
		FreePhysicalMemory: prometheus.NewDesc(
			prometheus.BuildFQName(wmiNamespace, "os", "free_physical_memory"),
			"Free physical memory.",
			nil,
			nil,
		),

		FreeSpaceInPagingFiles: prometheus.NewDesc(
			prometheus.BuildFQName(wmiNamespace, "os", "free_space_in_paging_files"),
			"Free space in paging files.",
			nil,
			nil,
		),

		FreeVirtualMemory: prometheus.NewDesc(
			prometheus.BuildFQName(wmiNamespace, "os", "free_virtual_memory"),
			"Free virtual memory.",
			nil,
			nil,
		),

		NumberOfProcesses: prometheus.NewDesc(
			prometheus.BuildFQName(wmiNamespace, "os", "number_of_processes"),
			"No. of processes running on the system.",
			nil,
			nil,
		),

		NumberOfUsers: prometheus.NewDesc(
			prometheus.BuildFQName(wmiNamespace, "os", "number_of_users"),
			"No. of users logged in.",
			nil,
			nil,
		),
	}
}

// Collect sends the metric values for each metric
// to the provided prometheus Metric channel.
func (c *OSCollector) Collect(ch chan<- prometheus.Metric) {
	if desc, err := c.collect(ch); err != nil {
		log.Println("[ERROR] failed collecting process metrics:", desc, err)
		return
	}
}

// Describe sends the descriptors of each metric over to the provided channel.
// The corresponding metric values are sent separately.
func (c *OSCollector) Describe(ch chan<- *prometheus.Desc) {

	ch <- c.FreePhysicalMemory
	ch <- c.FreeSpaceInPagingFiles
	ch <- c.FreeVirtualMemory
	ch <- c.NumberOfProcesses
	ch <- c.NumberOfUsers
}

type Win32_OperatingSystem struct {
	FreePhysicalMemory     uint64
	FreeSpaceInPagingFiles uint64
	FreeVirtualMemory      uint64
	NumberOfProcesses      uint32
	NumberOfUsers          uint32
}

func (c *OSCollector) collect(ch chan<- prometheus.Metric) (*prometheus.Desc, error) {
	var dst []Win32_OperatingSystem
	q := wmi.CreateQuery(&dst, "")
	if err := wmi.Query(q, &dst); err != nil {
		return nil, err
	}

	ch <- prometheus.MustNewConstMetric(
		c.FreePhysicalMemory,
		prometheus.GaugeValue,
		float64(dst[0].FreePhysicalMemory),
	)

	ch <- prometheus.MustNewConstMetric(
		c.FreeSpaceInPagingFiles,
		prometheus.GaugeValue,
		float64(dst[0].FreeSpaceInPagingFiles),
	)

	ch <- prometheus.MustNewConstMetric(
		c.FreeVirtualMemory,
		prometheus.GaugeValue,
		float64(dst[0].FreeVirtualMemory),
	)

	ch <- prometheus.MustNewConstMetric(
		c.NumberOfProcesses,
		prometheus.GaugeValue,
		float64(dst[0].NumberOfProcesses),
	)

	ch <- prometheus.MustNewConstMetric(
		c.NumberOfUsers,
		prometheus.GaugeValue,
		float64(dst[0].NumberOfUsers),
	)

	return nil, nil
}
<commit_msg>os: expose SizeStoredInPagingFiles, TotalVirtualMemorySize, TotalVisibleMemorySize<commit_after>// returns data points from Win32_OperatingSystem
// https://msdn.microsoft.com/en-us/library/aa394239(v=vs.85).aspx

package collectors

import (
	"log"

	"github.com/StackExchange/wmi"
	"github.com/prometheus/client_golang/prometheus"
)

// A OSCollector is a Prometheus collector for WMI OperatingSystem metrics
type OSCollector struct {
	FreePhysicalMemory      *prometheus.Desc
	FreeSpaceInPagingFiles  *prometheus.Desc
	FreeVirtualMemory       *prometheus.Desc
	NumberOfProcesses       *prometheus.Desc
	NumberOfUsers           *prometheus.Desc
	SizeStoredInPagingFiles *prometheus.Desc
	TotalVirtualMemorySize  *prometheus.Desc
	TotalVisibleMemorySize  *prometheus.Desc
}

// NewOSCollector ...
func NewOSCollector() *OSCollector {

	return &OSCollector{
		FreePhysicalMemory: prometheus.NewDesc(
			prometheus.BuildFQName(wmiNamespace, "os", "free_physical_memory"),
			"Number, in kilobytes, of physical memory currently unused and available.",
			nil,
			nil,
		),

		FreeSpaceInPagingFiles: prometheus.NewDesc(
			prometheus.BuildFQName(wmiNamespace, "os", "free_space_in_paging_files"),
			"Number, in kilobytes, that can be mapped into the operating system paging files without causing any other pages to be swapped out.",
			nil,
			nil,
		),

		FreeVirtualMemory: prometheus.NewDesc(
			prometheus.BuildFQName(wmiNamespace, "os", "free_virtual_memory"),
			"Number, in kilobytes, of virtual memory currently unused and available.",
			nil,
			nil,
		),

		NumberOfProcesses: prometheus.NewDesc(
			prometheus.BuildFQName(wmiNamespace, "os", "number_of_processes"),
			"Number of process contexts currently loaded or running on the operating system.",
			nil,
			nil,
		),

		NumberOfUsers: prometheus.NewDesc(
			prometheus.BuildFQName(wmiNamespace, "os", "number_of_users"),
			"Number of user sessions for which the operating system is storing state information currently.",
			nil,
			nil,
		),

		SizeStoredInPagingFiles: prometheus.NewDesc(
			prometheus.BuildFQName(wmiNamespace, "os", "size_stored_in_paging_files"),
			"Total number of kilobytes that can be stored in the operating system paging files.",
			nil,
			nil,
		),

		TotalVirtualMemorySize: prometheus.NewDesc(
			prometheus.BuildFQName(wmiNamespace, "os", "total_virtual_memory_size"),
			"Number, in kilobytes, of virtual memory.",
			nil,
			nil,
		),

		TotalVisibleMemorySize: prometheus.NewDesc(
			prometheus.BuildFQName(wmiNamespace, "os", "total_visible_memory_size"),
			"Total amount, in kilobytes, of physical memory available to the operating system.",
			nil,
			nil,
		),
	}
}

// Collect sends the metric values for each metric
// to the provided prometheus Metric channel.
func (c *OSCollector) Collect(ch chan<- prometheus.Metric) {
	if desc, err := c.collect(ch); err != nil {
		log.Println("[ERROR] failed collecting process metrics:", desc, err)
		return
	}
}

// Describe sends the descriptors of each metric over to the provided channel.
// The corresponding metric values are sent separately.
func (c *OSCollector) Describe(ch chan<- *prometheus.Desc) {

	ch <- c.FreePhysicalMemory
	ch <- c.FreeSpaceInPagingFiles
	ch <- c.FreeVirtualMemory
	ch <- c.NumberOfProcesses
	ch <- c.NumberOfUsers
	ch <- c.SizeStoredInPagingFiles
	ch <- c.TotalVirtualMemorySize
	ch <- c.TotalVisibleMemorySize
}

type Win32_OperatingSystem struct {
	FreePhysicalMemory      uint64
	FreeSpaceInPagingFiles  uint64
	FreeVirtualMemory       uint64
	NumberOfProcesses       uint32
	NumberOfUsers           uint32
	SizeStoredInPagingFiles uint64
	TotalVirtualMemorySize  uint64
	TotalVisibleMemorySize  uint64
}

func (c *OSCollector) collect(ch chan<- prometheus.Metric) (*prometheus.Desc, error) {
	var dst []Win32_OperatingSystem
	q := wmi.CreateQuery(&dst, "")
	if err := wmi.Query(q, &dst); err != nil {
		return nil, err
	}

	ch <- prometheus.MustNewConstMetric(
		c.FreePhysicalMemory,
		prometheus.GaugeValue,
		float64(dst[0].FreePhysicalMemory),
	)

	ch <- prometheus.MustNewConstMetric(
		c.FreeSpaceInPagingFiles,
		prometheus.GaugeValue,
		float64(dst[0].FreeSpaceInPagingFiles),
	)

	ch <- prometheus.MustNewConstMetric(
		c.FreeVirtualMemory,
		prometheus.GaugeValue,
		float64(dst[0].FreeVirtualMemory),
	)

	ch <- prometheus.MustNewConstMetric(
		c.NumberOfProcesses,
		prometheus.GaugeValue,
		float64(dst[0].NumberOfProcesses),
	)

	ch <- prometheus.MustNewConstMetric(
		c.NumberOfUsers,
		prometheus.GaugeValue,
		float64(dst[0].NumberOfUsers),
	)

	ch <- prometheus.MustNewConstMetric(
		c.SizeStoredInPagingFiles,
		prometheus.GaugeValue,
		float64(dst[0].SizeStoredInPagingFiles),
	)

	ch <- prometheus.MustNewConstMetric(
		c.TotalVirtualMemorySize,
		prometheus.GaugeValue,
		float64(dst[0].TotalVirtualMemorySize),
	)

	ch <- prometheus.MustNewConstMetric(
		c.TotalVisibleMemorySize,
		prometheus.GaugeValue,
		float64(dst[0].TotalVisibleMemorySize),
	)

	return nil, nil
}
<|endoftext|>


<commit_before>package command

import (
	"flag"
	"fmt"
	"github.com/hashicorp/serf/cli"
	"strings"
)

// EventCommand is a Command implementation that queries a running
// Serf agent what members are part of the cluster currently.
type EventCommand struct{}

func (c *EventCommand) Help() string {
	helpText := `
Usage: serf event [options] name payload

  Dispatches a custom event across the Serf cluster.

Options:

  -coalesce=true/false      Whether this event can be coalesced. This means
                            that repeated events of the same name within a
                            short period of time are ignored, except the last
                            one received. Default is true.
  -rpc-addr=127.0.0.1:7373  RPC address of the Serf agent.
`
	return strings.TrimSpace(helpText)
}

func (c *EventCommand) Run(args []string, ui cli.Ui) int {
	var coalesce bool

	cmdFlags := flag.NewFlagSet("event", flag.ContinueOnError)
	cmdFlags.Usage = func() { ui.Output(c.Help()) }
	cmdFlags.BoolVar(&coalesce, "coalesce", true, "coalesce")
	rpcAddr := RPCAddrFlag(cmdFlags)
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	args = cmdFlags.Args()
	if len(args) < 1 {
		ui.Error("An event name must be specified.")
		ui.Error("")
		ui.Error(c.Help())
		return 1
	} else if len(args) > 2 {
		ui.Error("Too many command line arguments. Only a name and payload must be specified.")
		ui.Error("")
		ui.Error(c.Help())
		return 1
	}

	event := args[0]
	var payload []byte
	if len(args) == 2 {
		payload = []byte(args[1])
	}

	client, err := RPCClient(*rpcAddr)
	if err != nil {
		ui.Error(fmt.Sprintf("Error connecting to Serf agent: %s", err))
		return 1
	}
	defer client.Close()

	if err := client.UserEvent(event, payload, coalesce); err != nil {
		ui.Error(fmt.Sprintf("Error sending event: %s", err))
		return 1
	}

	return 0
}

func (c *EventCommand) Synopsis() string {
	return "Send a custom event through the Serf cluster"
}
<commit_msg>command: event outputs feedback<commit_after>package command

import (
	"flag"
	"fmt"
	"github.com/hashicorp/serf/cli"
	"strings"
)

// EventCommand is a Command implementation that queries a running
// Serf agent what members are part of the cluster currently.
type EventCommand struct{}

func (c *EventCommand) Help() string {
	helpText := `
Usage: serf event [options] name payload

  Dispatches a custom event across the Serf cluster.

Options:

  -coalesce=true/false      Whether this event can be coalesced. This means
                            that repeated events of the same name within a
                            short period of time are ignored, except the last
                            one received. Default is true.
  -rpc-addr=127.0.0.1:7373  RPC address of the Serf agent.
`
	return strings.TrimSpace(helpText)
}

func (c *EventCommand) Run(args []string, ui cli.Ui) int {
	var coalesce bool

	cmdFlags := flag.NewFlagSet("event", flag.ContinueOnError)
	cmdFlags.Usage = func() { ui.Output(c.Help()) }
	cmdFlags.BoolVar(&coalesce, "coalesce", true, "coalesce")
	rpcAddr := RPCAddrFlag(cmdFlags)
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	args = cmdFlags.Args()
	if len(args) < 1 {
		ui.Error("An event name must be specified.")
		ui.Error("")
		ui.Error(c.Help())
		return 1
	} else if len(args) > 2 {
		ui.Error("Too many command line arguments. Only a name and payload must be specified.")
		ui.Error("")
		ui.Error(c.Help())
		return 1
	}

	event := args[0]
	var payload []byte
	if len(args) == 2 {
		payload = []byte(args[1])
	}

	client, err := RPCClient(*rpcAddr)
	if err != nil {
		ui.Error(fmt.Sprintf("Error connecting to Serf agent: %s", err))
		return 1
	}
	defer client.Close()

	if err := client.UserEvent(event, payload, coalesce); err != nil {
		ui.Error(fmt.Sprintf("Error sending event: %s", err))
		return 1
	}

	ui.Output(fmt.Sprintf("Event '%s' dispatched! Coalescing enabled: %#v",
		event, coalesce))
	return 0
}

func (c *EventCommand) Synopsis() string {
	return "Send a custom event through the Serf cluster"
}
<|endoftext|>


<commit_before>/**********************************************************\
|                                                          |
|                          hprose                          |
|                                                          |
| Official WebSite: http://www.hprose.com/                 |
|                   http://www.hprose.org/                 |
|                                                          |
\**********************************************************/
/**********************************************************\
 *                                                        *
 * hprose/jsonrpc_service_filter.go                       *
 *                                                        *
 * jsonrpc service filter for Go.                         *
 *                                                        *
 * LastModified: May 22, 2015                             *
 * Author: Ma Bingyao <andot@hprose.com>                  *
 *                                                        *
\**********************************************************/

package hprose

import (
	"bytes"
	"encoding/json"
)

// JSONRPCServiceFilter is a JSONRPC Service Filter
type JSONRPCServiceFilter struct{}

// InputFilter for JSONRPC Service
func (filter JSONRPCServiceFilter) InputFilter(data []byte, context Context) []byte {
	if (len(data) > 0) && (data[0] == '[' || data[0] == '{') {
		context.SetString("format", "jsonrpc")
		var request map[string]interface{}
		if err := json.Unmarshal(data, &request); err != nil {
			return data
		}
		if id, ok := request["id"]; ok {
			context.SetInterface("id", id)
		} else {
			context.SetInterface("id", nil)
		}
		if version, ok := request["version"].(string); ok {
			context.SetString("version", version)
		} else if jsonrpc, ok := request["jsonrpc"].(string); ok {
			context.SetString("version", jsonrpc)
		} else {
			context.SetString("version", "1.0")
		}
		buf := new(bytes.Buffer)
		writer := NewWriter(buf, true)
		if method, ok := request["method"].(string); ok && method != "" {
			if err := buf.WriteByte(TagCall); err != nil {
				return data
			}
			if err := writer.WriteString(method); err != nil {
				return data
			}
			if params, ok := request["params"].([]interface{}); ok && params != nil && len(params) > 0 {
				if err := writer.Serialize(params); err != nil {
					return data
				}
			}
		}
		buf.WriteByte(TagEnd)
		data = buf.Bytes()
	}
	return data
}

// OutputFilter for JSONRPC Service
func (filter JSONRPCServiceFilter) OutputFilter(data []byte, context Context) []byte {
	if format, ok := context.GetString("format"); ok && format == "jsonrpc" {
		response := make(map[string]interface{})
		if version, ok := context.GetString("version"); ok && version != "2.0" {
			if version == "1.1" {
				response["version"] = "1.1"
			}
			response["result"] = nil
			response["error"] = nil
		} else {
			response["jsonrpc"] = "2.0"
		}
		response["id"], _ = context.GetInterface("id")
		if len(data) == 0 {
			data, _ = json.Marshal(response)
			return data
		}
		istream := NewBytesReader(data)
		reader := NewReader(istream, false)
		reader.JSONCompatible = true
		for tag, err := istream.ReadByte(); err == nil && tag != TagEnd; tag, err = istream.ReadByte() {
			switch tag {
			case TagResult:
				reader.Reset()
				var result interface{}
				reader.Unserialize(&result)
				if err != nil {
					e := make(map[string]interface{})
					e["code"] = -1
					e["message"] = err.Error()
					response["error"] = e
				} else {
					response["result"] = result
				}
			case TagError:
				reader.Reset()
				e := make(map[string]interface{})
				e["code"] = -1
				if message, err := reader.ReadString(); err == nil {
					e["message"] = message
				} else {
					e["message"] = err.Error()
				}
			default:
				data, _ = json.Marshal(response)
				return data
			}
		}
		data, _ = json.Marshal(response)
		return data
	}
	return data
}
<commit_msg>Added batch support for jsonrpc service.<commit_after>/**********************************************************\
|                                                          |
|                          hprose                          |
|                                                          |
| Official WebSite: http://www.hprose.com/                 |
|                   http://www.hprose.org/                 |
|                                                          |
\**********************************************************/
/**********************************************************\
 *                                                        *
 * hprose/jsonrpc_service_filter.go                       *
 *                                                        *
 * jsonrpc service filter for Go.                         *
 *                                                        *
 * LastModified: May 25, 2015                             *
 * Author: Ma Bingyao <andot@hprose.com>                  *
 *                                                        *
\**********************************************************/

package hprose

import (
	"bytes"
	"encoding/json"
)

// JSONRPCServiceFilter is a JSONRPC Service Filter
type JSONRPCServiceFilter struct{}

// InputFilter for JSONRPC Service
func (filter JSONRPCServiceFilter) InputFilter(data []byte, context Context) []byte {
	if (len(data) > 0) && (data[0] == '[' || data[0] == '{') {
		var requests []map[string]interface{}
		if data[0] == '[' {
			if err := json.Unmarshal(data, &requests); err != nil {
				return data
			}
		} else {
			requests = make([]map[string]interface{}, 1)
			if err := json.Unmarshal(data, &requests[0]); err != nil {
				return data
			}
		}
		buf := new(bytes.Buffer)
		writer := NewWriter(buf, true)
		n := len(requests)
		jsonrpc := make([]map[string]interface{}, n)
		for i, request := range requests {
			j := make(map[string]interface{})
			if id, ok := request["id"]; ok {
				j["id"] = id
			} else {
				j["id"] = nil
			}
			if version, ok := request["version"]; ok {
				j["version"] = version
			} else if jsonrpc, ok := request["jsonrpc"]; ok {
				j["version"] = jsonrpc
			} else {
				j["version"] = "1.0"
			}
			jsonrpc[i] = j
			if method, ok := request["method"].(string); ok && method != "" {
				buf.WriteByte(TagCall)
				writer.WriteString(method)
				if params, ok := request["params"].([]interface{}); ok && params != nil && len(params) > 0 {
					writer.Serialize(params)
				}
			}
		}
		buf.WriteByte(TagEnd)
		data = buf.Bytes()
		context.SetInterface("jsonrpc", jsonrpc)
	}
	return data
}

func initResponse(j map[string]interface{}, err interface{}) map[string]interface{} {
	response := make(map[string]interface{})
	if version, ok := j["version"].(string); ok && version != "2.0" {
		if version == "1.1" {
			response["version"] = "1.1"
		}
		response["result"] = nil
		response["error"] = err
	} else {
		response["jsonrpc"] = "2.0"
	}
	response["id"] = j["id"]
	return response
}

func jsonrpcError(jsonrpc []map[string]interface{}, err error) []byte {
	n := len(jsonrpc)
	responses := make([]map[string]interface{}, n)
	for i, j := range jsonrpc {
		responses[i] = initResponse(j, err.Error())
	}
	if n == 1 {
		data, _ := json.Marshal(responses[0])
		return data
	}
	data, _ := json.Marshal(responses)
	return data
}

// OutputFilter for JSONRPC Service
func (filter JSONRPCServiceFilter) OutputFilter(data []byte, context Context) []byte {
	if jsonrpc, ok := context.GetInterface("jsonrpc"); ok {
		jsonrpc := jsonrpc.([]map[string]interface{})
		n := len(jsonrpc)
		responses := make([]map[string]interface{}, n)
		istream := NewBytesReader(data)
		reader := NewReader(istream, false)
		reader.JSONCompatible = true
		tag, err := istream.ReadByte()
		if err != nil {
			return jsonrpcError(jsonrpc, err)
		}
		i := 0
		for {
			response := initResponse(jsonrpc[i], nil)
			if tag == TagResult {
				reader.Reset()
				var result interface{}
				reader.Unserialize(&result)
				response["result"] = result
				tag, _ = istream.ReadByte()
			} else if tag == TagError {
				reader.Reset()
				e := make(map[string]interface{})
				e["code"] = -1
				message, _ := reader.ReadString()
				e["message"] = message
				tag, _ = istream.ReadByte()
			}
			responses[i] = response
			i++
			if tag == TagEnd {
				break
			}
		}
		if n == 1 {
			data, _ := json.Marshal(responses[0])
			return data
		}
		data, _ := json.Marshal(responses)
		return data
	}
	return data
}
<|endoftext|>