<commit_before>package lytics

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"
)

// Client defines the supported subset of the Lytics API. The Lytics API may contain other features
// that have been added or deprecated since the last update of this SDK.
//
// Lytics reserves the right to deprecate or modify endpoints at any time and does not guarantee
// backwards compatibility or SemVer versioning for early version of API.
//
// For more information see the Lytics API documentation at
// http://getlytics.com/developers/rest-api
//
//  Original Author: Mark Hayden
//  Contributions:   Mark Hayden
//  Version:         0.0.1
//
// Examples
// Coming Soon

const (
	apiBase        = "https://api.lytics.io/api"
	updated        = "2015-09-15"
	apiVersion     = "Beta 1.0.0"
	libraryVersion = "0.0.1"
)

// Client bundles the data necessary to interact with the vast majority of Lytics REST endpoints.
type Client struct {
	baseURL    string
	apiKey     string
	dataApiKey string
	client     *http.Client
	Scan       *SegmentScanner
}

// ApiResp is the core api response for all Lytics endpoints. In some instances the "Status" is returned
// as a string rather than an int. This is a known but and will be addressed / updated.
type ApiResp struct {
	Status  interface{} `json:"status"`
	Message string      `json:"message"`
	Meta    Meta        `json:"meta"`
	Next    string      `json:"_next"`
	Total   int         `json:"total"`
	Data    interface{} `json:"data"`
}

type Meta struct {
	Format   string   `json:"name"`
	Name     []string `json:"by_fields"`
	ByFields []string `json:"by_fields"`
}

// NewLytics creates a new client instance. This contains the segment pager, segment details
// and maintains all core data used throughout this SDK
func NewLytics(apiKey, dataApiKey interface{}) *Client {
	l := Client{
		baseURL: apiBase,
		client:  http.DefaultClient,
	}

	// set the apikey if not null
	if apiKey != nil {
		l.apiKey = apiKey.(string)
	}

	// set the dataapikey if not null
	if dataApiKey != nil {
		l.dataApiKey = dataApiKey.(string)
	}

	return &l
}

// BaseUrl returns the base url used in all calls for this client.
func (l *Client) BaseUrl() string {
	return l.baseURL
}

// ApiKey returns the API key configured for this client.
func (l *Client) ApiKey() string {
	return l.apiKey
}

// DataApiKey returns the public API key configured for this client.
func (l *Client) DataApiKey() string {
	return l.dataApiKey
}

// Client returns the HTTP client configured for this client.
func (l *Client) Client() *http.Client {
	return l.client
}

// SetClient updates the HTTP client for this client. Used to alter main client
// handling for instances such as AppEngine
func (l *Client) SetClient(c *http.Client) {
	l.client = c
}

// PrepUrl handles the parsing and setup for all api calls. The encoded url string is passed
// along with a set of param. Params are looped and injected into the master url
func (l *Client) PrepUrl(endpoint string, params interface{}, dataKey bool) (string, error) {
	// parse the url into native http.URL
	url, err := url.Parse(fmt.Sprintf("%s/%s", l.BaseUrl(), endpoint))
	if err != nil {
		return "", err
	}

	// add the api key
	values := url.Query()

	// if there is a data key use that by default, if not use the main api key
	// assumption here is that if its a data key there are specific reasons
	if dataKey {
		values.Add("key", l.dataApiKey)
	} else {
		values.Add("key", l.apiKey)
	}

	// add additional params
	if params != nil {
		for key, value := range params.(map[string]string) {
			values.Add(key, value)
		}
	}

	// encode the final url so we can return string and make call
	url.RawQuery = values.Encode()

	return url.String(), nil
}

// Do handles executing all http requests for the SDK. Takes a httpRequest and parses
// the response into the master api struct as well as a specific data type.
func (l *Client) Do(r *http.Request, response, data interface{}) error {
	// make the request
	res, err := l.client.Do(r)
	if err != nil {
		return err
	}
	defer res.Body.Close()

	// get the response
	b, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return err
	}

	// if we have an invalid response code error out
	if res.StatusCode >= 301 {
		return errors.New(fmt.Sprintf("Received non-successful response: %d", res.StatusCode))
	}

	// if we have some struct to unmarshal body into, do that and return
	if response != nil {
		return buildRespJSON(b, response, data)
	}

	return nil
}

// Get prepares a get request and then executes using the Do method
func (l *Client) Get(endpoint string, params interface{}, body io.Reader, response, data interface{}) error {
	method := "GET"

	// get the formatted endpoint url
	path, err := l.PrepUrl(endpoint, params, false)
	if err != nil {
		return err
	}

	// build the request
	r, _ := http.NewRequest(method, path, body)

	// execute the request
	err = l.Do(r, response, data)
	if err != nil {
		return err
	}

	return nil
}

// buildRespJSON handles the first round of unmarshaling into the master Api Response struct
func buildRespJSON(b []byte, response, data interface{}) error {
	var err error

	err = json.Unmarshal(b, response)
	if err != nil {
		return err
	}

	err = buildDataJSON(response.(*ApiResp).Data, &data)
	if err != nil {
		return err
	}

	return nil
}

// buildDataJSON is a helper function that translates the data object into a speciic
// data type based upon value passed in. This simplifies working with the various
// responses throughout the SDK
func buildDataJSON(data, parse interface{}) error {
	// first we have to marshal this since its already been unmarshaled
	marsh, err := json.Marshal(data)
	if err != nil {
		return err
	}

	// then unmarshal to desired format
	err = json.Unmarshal(marsh, parse)
	return err
}

// parseLyticsTime translates a timestamp as returned by Lytics into a Go standard timestamp.
func parseLyticsTime(ts string) (time.Time, error) {
	var err error

	i, err := strconv.ParseInt(ts, 10, 64)
	if err != nil {
		return time.Now(), err
	}

	tm := time.Unix((i / 1000), 0)

	return tm, err
}

// formatLyticsTime translates a timestamp into a human-readable form.
func formatLyticsTime(t *time.Time) string {
	return t.Format("Mon, 2 Jan 2006 15:04:05 -0700")
}

// parseLyticsURL joins params with a string using : notation
func parseLyticsURL(url string, params map[string]string) string {
	out := url

	for key, value := range params {
		out = strings.Replace(out, fmt.Sprintf(":%s", key), value, -1)
	}

	return out
}
<commit_msg>prevent nil pointer if no client<commit_after>package lytics

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"
)

// Client defines the supported subset of the Lytics API. The Lytics API may contain other features
// that have been added or deprecated since the last update of this SDK.
//
// Lytics reserves the right to deprecate or modify endpoints at any time and does not guarantee
// backwards compatibility or SemVer versioning for early version of API.
//
// For more information see the Lytics API documentation at
// http://getlytics.com/developers/rest-api
//
//  Original Author: Mark Hayden
//  Contributions:   Mark Hayden
//  Version:         0.0.1
//
// Examples
// Coming Soon

const (
	apiBase        = "https://api.lytics.io/api"
	updated        = "2015-09-15"
	apiVersion     = "Beta 1.0.0"
	libraryVersion = "0.0.1"
)

// Client bundles the data necessary to interact with the vast majority of Lytics REST endpoints.
type Client struct {
	baseURL    string
	apiKey     string
	dataApiKey string
	client     *http.Client
	Scan       *SegmentScanner
}

// ApiResp is the core api response for all Lytics endpoints. In some instances the "Status" is returned
// as a string rather than an int. This is a known but and will be addressed / updated.
type ApiResp struct {
	Status  interface{} `json:"status"`
	Message string      `json:"message"`
	Meta    Meta        `json:"meta"`
	Next    string      `json:"_next"`
	Total   int         `json:"total"`
	Data    interface{} `json:"data"`
}

type Meta struct {
	Format   string   `json:"name"`
	Name     []string `json:"by_fields"`
	ByFields []string `json:"by_fields"`
}

// NewLytics creates a new client instance. This contains the segment pager, segment details
// and maintains all core data used throughout this SDK
func NewLytics(apiKey, dataApiKey interface{}, httpclient *http.Client) *Client {
	l := Client{
		baseURL: apiBase,
	}

	if httpclient != nil {
		l.client = httpclient
	} else {
		l.client = http.DefaultClient
	}

	// set the apikey if not null
	if apiKey != nil {
		l.apiKey = apiKey.(string)
	}

	// set the dataapikey if not null
	if dataApiKey != nil {
		l.dataApiKey = dataApiKey.(string)
	}

	return &l
}

// BaseUrl returns the base url used in all calls for this client.
func (l *Client) BaseUrl() string {
	return l.baseURL
}

// ApiKey returns the API key configured for this client.
func (l *Client) ApiKey() string {
	return l.apiKey
}

// DataApiKey returns the public API key configured for this client.
func (l *Client) DataApiKey() string {
	return l.dataApiKey
}

// Client returns the HTTP client configured for this client.
func (l *Client) Client() *http.Client {
	return l.client
}

// SetClient updates the HTTP client for this client. Used to alter main client
// handling for instances such as AppEngine
func (l *Client) SetClient(c *http.Client) {
	l.client = c
}

// PrepUrl handles the parsing and setup for all api calls. The encoded url string is passed
// along with a set of param. Params are looped and injected into the master url
func (l *Client) PrepUrl(endpoint string, params interface{}, dataKey bool) (string, error) {
	// parse the url into native http.URL
	url, err := url.Parse(fmt.Sprintf("%s/%s", l.BaseUrl(), endpoint))
	if err != nil {
		return "", err
	}

	// add the api key
	values := url.Query()

	// if there is a data key use that by default, if not use the main api key
	// assumption here is that if its a data key there are specific reasons
	if dataKey {
		values.Add("key", l.dataApiKey)
	} else {
		values.Add("key", l.apiKey)
	}

	// add additional params
	if params != nil {
		for key, value := range params.(map[string]string) {
			values.Add(key, value)
		}
	}

	// encode the final url so we can return string and make call
	url.RawQuery = values.Encode()

	return url.String(), nil
}

// Do handles executing all http requests for the SDK. Takes a httpRequest and parses
// the response into the master api struct as well as a specific data type.
func (l *Client) Do(r *http.Request, response, data interface{}) error {
	// make the request
	res, err := l.client.Do(r)
	if err != nil {
		return err
	}
	defer res.Body.Close()

	// get the response
	b, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return err
	}

	// if we have an invalid response code error out
	if res.StatusCode >= 301 {
		return errors.New(fmt.Sprintf("Received non-successful response: %d", res.StatusCode))
	}

	// if we have some struct to unmarshal body into, do that and return
	if response != nil {
		return buildRespJSON(b, response, data)
	}

	return nil
}

// Get prepares a get request and then executes using the Do method
func (l *Client) Get(endpoint string, params interface{}, body io.Reader, response, data interface{}) error {
	method := "GET"

	// get the formatted endpoint url
	path, err := l.PrepUrl(endpoint, params, false)
	if err != nil {
		return err
	}

	// build the request
	r, _ := http.NewRequest(method, path, body)

	// execute the request
	err = l.Do(r, response, data)
	if err != nil {
		return err
	}

	return nil
}

// buildRespJSON handles the first round of unmarshaling into the master Api Response struct
func buildRespJSON(b []byte, response, data interface{}) error {
	var err error

	err = json.Unmarshal(b, response)
	if err != nil {
		return err
	}

	err = buildDataJSON(response.(*ApiResp).Data, &data)
	if err != nil {
		return err
	}

	return nil
}

// buildDataJSON is a helper function that translates the data object into a speciic
// data type based upon value passed in. This simplifies working with the various
// responses throughout the SDK
func buildDataJSON(data, parse interface{}) error {
	// first we have to marshal this since its already been unmarshaled
	marsh, err := json.Marshal(data)
	if err != nil {
		return err
	}

	// then unmarshal to desired format
	err = json.Unmarshal(marsh, parse)
	return err
}

// parseLyticsTime translates a timestamp as returned by Lytics into a Go standard timestamp.
func parseLyticsTime(ts string) (time.Time, error) {
	var err error

	i, err := strconv.ParseInt(ts, 10, 64)
	if err != nil {
		return time.Now(), err
	}

	tm := time.Unix((i / 1000), 0)

	return tm, err
}

// formatLyticsTime translates a timestamp into a human-readable form.
func formatLyticsTime(t *time.Time) string {
	return t.Format("Mon, 2 Jan 2006 15:04:05 -0700")
}

// parseLyticsURL joins params with a string using : notation
func parseLyticsURL(url string, params map[string]string) string {
	out := url

	for key, value := range params {
		out = strings.Replace(out, fmt.Sprintf(":%s", key), value, -1)
	}

	return out
}
<|endoftext|>


<commit_before>package generator

import (
	"encoding/xml"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"socialapi/config"
	"socialapi/workers/helper"
	"socialapi/workers/sitemap/models"
	"time"

	"github.com/koding/logging"
	"github.com/robfig/cron"
)

type Controller struct {
	log          logging.Logger
	fileSelector FileSelector
	fileName     string
}

const (
	// TODO change this later
	// SCHEDULE = "0 0-59/15 * * * *"
	SCHEDULE    = "* * * * * *"
)

var (
	cronJob *cron.Cron
)

func New(log logging.Logger) (*Controller, error) {
	c := &Controller{
		log:          log,
		fileSelector: SimpleFileSelector{},
	}
	c.generate()

	return c, nil
}

func (c *Controller) initCron() error {
	cronJob := cron.New()
	if err := cronJob.AddFunc(SCHEDULE, c.generate); err != nil {
		return err
	}
	cronJob.Start()

	return nil
}

func (c *Controller) Shutdown() {
	cronJob.Stop()
}

func (c *Controller) generate() {
	c.fileName = c.fileSelector.Select()

	els, err := c.fetchElements()
	if err != nil {
		c.log.Error("Could not fetch updated elements: %s", err)
		return
	}

	container := c.buildContainer(els)

	// TODO fetch current file and unmarshall data
	s, err := c.getCurrentSet()
	if err != nil {
		c.log.Error("Could not get current set: %s", err)
		return
	}

	if err := c.updateFile(container, s); err != nil {
		c.log.Error("Could not update file: %s", err)
	}
}

func (c *Controller) fetchElements() ([]*models.SitemapItem, error) {
	key := c.prepareCacheKey()

	redisConn := helper.MustGetRedisConn()
	members, err := redisConn.GetSetMembers(key)
	if err != nil {
		return nil, err
	}

	if len(members) == 0 {
		return nil, errors.New("members not found")
	}

	els := make([]*models.SitemapItem, len(members))
	for index, v := range members {
		i := &models.SitemapItem{}
		value, err := redisConn.String(v)
		if err != nil {
			c.log.Error("Could not get item data: %s", err)
			continue
		}

		if err := i.Populate(value); err != nil {
			c.log.Error("Could not update item %s: %s", value, err)
			continue
		}
		els[index] = i
	}

	return els, nil
}

func (c *Controller) getCurrentSet() (*models.ItemSet, error) {
	// check if this is a new sitemap file or not
	n := fmt.Sprintf("%s.xml", c.fileName)
	if _, err := os.Stat(n); os.IsNotExist(err) {
		return models.NewItemSet(), nil
	}
	input, err := ioutil.ReadFile(n)
	if err != nil {
		return nil, err
	}

	s := models.NewItemSet()
	if err := xml.Unmarshal(input, s); err != nil {
		return nil, err
	}

	return s, nil
}

func (c *Controller) buildContainer(items []*models.SitemapItem) *models.ItemContainer {
	container := models.NewItemContainer()
	for _, v := range items {
		item := v.Definition(config.Get().Uri)
		switch v.Status {
		case models.STATUS_ADD:
			container.Add = append(container.Add, item)
		case models.STATUS_DELETE:
			container.Delete = append(container.Delete, item)
		case models.STATUS_UPDATE:
			item.LastModified = time.Now().UTC().Format(time.RFC3339)
			container.Update = append(container.Update, item)
		}
	}

	return container
}

func (c *Controller) updateFile(container *models.ItemContainer, set *models.ItemSet) error {
	set.Populate(container)

	// TODO no need for indent in prod
	header := []byte(xml.Header)
	res, err := xml.MarshalIndent(set, " ", "  ")
	if err != nil {
		fmt.Printf("hatali %s", err)
	}
	res = append(header, res...)

	c.write(res)

	return nil
}

func (c *Controller) write(input []byte) {
	output, err := os.Create(c.fileName)
	if err != nil {
		panic(err)
	}
	defer func() {
		if err := output.Close(); err != nil {
			panic(err)
		}
	}()
	if _, err := output.Write(input); err != nil {
		panic(err)
	}

}
<commit_msg>SItemap: Change generator write method name to MustWrite<commit_after>package generator

import (
	"encoding/xml"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"socialapi/config"
	"socialapi/workers/helper"
	"socialapi/workers/sitemap/models"
	"time"

	"github.com/koding/logging"
	"github.com/robfig/cron"
)

type Controller struct {
	log          logging.Logger
	fileSelector FileSelector
	fileName     string
}

const (
	// TODO change this later
	// SCHEDULE = "0 0-59/15 * * * *"
	SCHEDULE    = "* * * * * *"
)

var (
	cronJob *cron.Cron
)

func New(log logging.Logger) (*Controller, error) {
	c := &Controller{
		log:          log,
		fileSelector: SimpleFileSelector{},
	}
	c.generate()

	return c, nil
}

func (c *Controller) initCron() error {
	cronJob := cron.New()
	if err := cronJob.AddFunc(SCHEDULE, c.generate); err != nil {
		return err
	}
	cronJob.Start()

	return nil
}

func (c *Controller) Shutdown() {
	cronJob.Stop()
}

func (c *Controller) generate() {
	c.fileName = c.fileSelector.Select()

	els, err := c.fetchElements()
	if err != nil {
		c.log.Error("Could not fetch updated elements: %s", err)
		return
	}

	container := c.buildContainer(els)

	// TODO fetch current file and unmarshall data
	s, err := c.getCurrentSet()
	if err != nil {
		c.log.Error("Could not get current set: %s", err)
		return
	}

	if err := c.updateFile(container, s); err != nil {
		c.log.Error("Could not update file: %s", err)
	}
}

func (c *Controller) fetchElements() ([]*models.SitemapItem, error) {
	key := c.prepareCacheKey()

	redisConn := helper.MustGetRedisConn()
	members, err := redisConn.GetSetMembers(key)
	if err != nil {
		return nil, err
	}

	if len(members) == 0 {
		return nil, errors.New("members not found")
	}

	els := make([]*models.SitemapItem, len(members))
	for index, v := range members {
		i := &models.SitemapItem{}
		value, err := redisConn.String(v)
		if err != nil {
			c.log.Error("Could not get item data: %s", err)
			continue
		}

		if err := i.Populate(value); err != nil {
			c.log.Error("Could not update item %s: %s", value, err)
			continue
		}
		els[index] = i
	}

	return els, nil
}

func (c *Controller) getCurrentSet() (*models.ItemSet, error) {
	// check if this is a new sitemap file or not
	n := fmt.Sprintf("%s.xml", c.fileName)
	if _, err := os.Stat(n); os.IsNotExist(err) {
		return models.NewItemSet(), nil
	}
	input, err := ioutil.ReadFile(n)
	if err != nil {
		return nil, err
	}

	s := models.NewItemSet()
	if err := xml.Unmarshal(input, s); err != nil {
		return nil, err
	}

	return s, nil
}

func (c *Controller) buildContainer(items []*models.SitemapItem) *models.ItemContainer {
	container := models.NewItemContainer()
	for _, v := range items {
		item := v.Definition(config.Get().Uri)
		switch v.Status {
		case models.STATUS_ADD:
			container.Add = append(container.Add, item)
		case models.STATUS_DELETE:
			container.Delete = append(container.Delete, item)
		case models.STATUS_UPDATE:
			item.LastModified = time.Now().UTC().Format(time.RFC3339)
			container.Update = append(container.Update, item)
		}
	}

	return container
}

func (c *Controller) updateFile(container *models.ItemContainer, set *models.ItemSet) error {
	set.Populate(container)

	// TODO no need for indent in prod
	header := []byte(xml.Header)
	res, err := xml.MarshalIndent(set, " ", "  ")
	if err != nil {
		fmt.Printf("hatali %s", err)
	}
	res = append(header, res...)

	c.MustWrite(res)

	return nil
}

func (c *Controller) MustWrite(input []byte) {
	n := fmt.Sprintf("%s.xml", c.fileName)

	output, err := os.Create(n)
	if err != nil {
		panic(err)
	}
	defer func() {
		if err := output.Close(); err != nil {
			panic(err)
		}
	}()
	if _, err := output.Write(input); err != nil {
		panic(err)
	}

}
<|endoftext|>


<commit_before>package filegenerator

import (
	"fmt"
	"io"
	"strings"

	"github.com/golang/protobuf/protoc-gen-go/descriptor"
	"github.com/golang/protobuf/protoc-gen-go/generator"
)

type FileGenerator struct {
	w      io.Writer
	indent uint
}

func New(w io.Writer) *FileGenerator {
	return &FileGenerator{
		w: w,
	}
}

func (fg *FileGenerator) In() {
	fg.indent++
}

func (fg *FileGenerator) Out() {
	fg.indent--
}

func (fg *FileGenerator) P(format string, a ...interface{}) error {
	var err error

	// If format is empty, avoid printing just whitespaces.
	if format != "" {
		_, err = fmt.Fprintf(fg.w, strings.Repeat("    ", int(fg.indent)))
		if err != nil {
			return err
		}

		_, err = fmt.Fprintf(fg.w, format, a...)
		if err != nil {
			return err
		}
	}

	_, err = fmt.Fprintf(fg.w, "\n")
	if err != nil {
		return err
	}

	return nil
}

func (fg *FileGenerator) Generate(file *descriptor.FileDescriptorProto) {
	fg.P(`package %s`, file.GetPackage())
	fg.P("")

	fg.P("/*")
	fg.In()
	fg.P("This file is generated by protoc-gen-gopherjs, DO NOT EDIT.")
	fg.Out()
	fg.P("*/")
	fg.P("")

	fg.P(`import "github.com/gopherjs/gopherjs/js"`)
	fg.P("")

	for _, msg := range file.GetMessageType() {
		fg.generateProtoMessage(file, msg)
	}
}

func (fg *FileGenerator) generateProtoMessage(file *descriptor.FileDescriptorProto, message *descriptor.DescriptorProto) {
	ccTypeName := generator.CamelCase(message.GetName())

	fg.P(`type %s struct {`, ccTypeName)
	fg.In()
	fg.P(`*js.Object`)
	for _, field := range message.GetField() {
		fg.P(`%s %s `+"`js:"+`"%s"`+"`", generator.CamelCase(field.GetName()), GoType(message, field), field.GetJsonName())
	}
	fg.Out()
	fg.P(`}`)
}

// GoType returns a string representing the type name
func GoType(message *descriptor.DescriptorProto, field *descriptor.FieldDescriptorProto) (typ string) {
	switch *field.Type {
	case descriptor.FieldDescriptorProto_TYPE_DOUBLE:
		typ = "float64"
	case descriptor.FieldDescriptorProto_TYPE_FLOAT:
		typ = "float32"
	case descriptor.FieldDescriptorProto_TYPE_INT64, descriptor.FieldDescriptorProto_TYPE_SFIXED64, descriptor.FieldDescriptorProto_TYPE_SINT64:
		typ = "int64"
	case descriptor.FieldDescriptorProto_TYPE_UINT64, descriptor.FieldDescriptorProto_TYPE_FIXED64:
		typ = "uint64"
	case descriptor.FieldDescriptorProto_TYPE_INT32, descriptor.FieldDescriptorProto_TYPE_SFIXED32, descriptor.FieldDescriptorProto_TYPE_SINT32:
		typ = "int32"
	case descriptor.FieldDescriptorProto_TYPE_UINT32, descriptor.FieldDescriptorProto_TYPE_FIXED32:
		typ = "uint32"
	case descriptor.FieldDescriptorProto_TYPE_BOOL:
		typ = "bool"
	case descriptor.FieldDescriptorProto_TYPE_STRING:
		typ = "string"
	case descriptor.FieldDescriptorProto_TYPE_BYTES:
		typ = "[]byte"
	case descriptor.FieldDescriptorProto_TYPE_ENUM, descriptor.FieldDescriptorProto_TYPE_GROUP, descriptor.FieldDescriptorProto_TYPE_MESSAGE:
		typ = field.GetTypeName()
	default:
		panic("unknown type for " + field.GetName())
	}
	if needsStar(field, field.Extendee == nil, message != nil) {
		typ = "*" + typ
	}
	if isRepeated(field) {
		typ = "[]" + typ
	}
	return
}

func needsStar(field *descriptor.FieldDescriptorProto, proto3 bool, allowOneOf bool) bool {
	if isRepeated(field) &&
		(*field.Type != descriptor.FieldDescriptorProto_TYPE_MESSAGE) &&
		(*field.Type != descriptor.FieldDescriptorProto_TYPE_GROUP) {
		return false
	}
	if *field.Type == descriptor.FieldDescriptorProto_TYPE_BYTES {
		return false
	}
	if field.OneofIndex != nil && allowOneOf &&
		(*field.Type != descriptor.FieldDescriptorProto_TYPE_MESSAGE) &&
		(*field.Type != descriptor.FieldDescriptorProto_TYPE_GROUP) {
		return false
	}
	if proto3 &&
		(*field.Type != descriptor.FieldDescriptorProto_TYPE_MESSAGE) &&
		(*field.Type != descriptor.FieldDescriptorProto_TYPE_GROUP) {
		return false
	}
	return true
}

// Is this field repeated?
func isRepeated(field *descriptor.FieldDescriptorProto) bool {
	return field.Label != nil && *field.Label == descriptor.FieldDescriptorProto_LABEL_REPEATED
}
<commit_msg>Use tab in filegenerator<commit_after>package filegenerator

import (
	"fmt"
	"io"
	"strings"

	"github.com/golang/protobuf/protoc-gen-go/descriptor"
	"github.com/golang/protobuf/protoc-gen-go/generator"
)

type FileGenerator struct {
	w      io.Writer
	indent uint
}

func New(w io.Writer) *FileGenerator {
	return &FileGenerator{
		w: w,
	}
}

func (fg *FileGenerator) In() {
	fg.indent++
}

func (fg *FileGenerator) Out() {
	fg.indent--
}

func (fg *FileGenerator) P(format string, a ...interface{}) error {
	var err error

	// If format is empty, avoid printing just whitespaces.
	if format != "" {
		_, err = fmt.Fprintf(fg.w, strings.Repeat("\t", int(fg.indent)))
		if err != nil {
			return err
		}

		_, err = fmt.Fprintf(fg.w, format, a...)
		if err != nil {
			return err
		}
	}

	_, err = fmt.Fprintf(fg.w, "\n")
	if err != nil {
		return err
	}

	return nil
}

func (fg *FileGenerator) Generate(file *descriptor.FileDescriptorProto) {
	fg.P(`package %s`, file.GetPackage())
	fg.P("")

	fg.P("/*")
	fg.P("This file is generated by protoc-gen-gopherjs, DO NOT EDIT.")
	fg.P("*/")
	fg.P("")

	fg.P(`import "github.com/gopherjs/gopherjs/js"`)
	fg.P("")

	for _, msg := range file.GetMessageType() {
		fg.generateProtoMessage(file, msg)
	}
}

func (fg *FileGenerator) generateProtoMessage(file *descriptor.FileDescriptorProto, message *descriptor.DescriptorProto) {
	ccTypeName := generator.CamelCase(message.GetName())

	fg.P(`type %s struct {`, ccTypeName)
	fg.In()
	fg.P(`*js.Object`)
	for _, field := range message.GetField() {
		fg.P(`%s %s `+"`js:"+`"%s"`+"`", generator.CamelCase(field.GetName()), GoType(message, field), field.GetJsonName())
	}
	fg.Out()
	fg.P(`}`)
}

// GoType returns a string representing the type name
func GoType(message *descriptor.DescriptorProto, field *descriptor.FieldDescriptorProto) (typ string) {
	switch *field.Type {
	case descriptor.FieldDescriptorProto_TYPE_DOUBLE:
		typ = "float64"
	case descriptor.FieldDescriptorProto_TYPE_FLOAT:
		typ = "float32"
	case descriptor.FieldDescriptorProto_TYPE_INT64, descriptor.FieldDescriptorProto_TYPE_SFIXED64, descriptor.FieldDescriptorProto_TYPE_SINT64:
		typ = "int64"
	case descriptor.FieldDescriptorProto_TYPE_UINT64, descriptor.FieldDescriptorProto_TYPE_FIXED64:
		typ = "uint64"
	case descriptor.FieldDescriptorProto_TYPE_INT32, descriptor.FieldDescriptorProto_TYPE_SFIXED32, descriptor.FieldDescriptorProto_TYPE_SINT32:
		typ = "int32"
	case descriptor.FieldDescriptorProto_TYPE_UINT32, descriptor.FieldDescriptorProto_TYPE_FIXED32:
		typ = "uint32"
	case descriptor.FieldDescriptorProto_TYPE_BOOL:
		typ = "bool"
	case descriptor.FieldDescriptorProto_TYPE_STRING:
		typ = "string"
	case descriptor.FieldDescriptorProto_TYPE_BYTES:
		typ = "[]byte"
	case descriptor.FieldDescriptorProto_TYPE_ENUM, descriptor.FieldDescriptorProto_TYPE_GROUP, descriptor.FieldDescriptorProto_TYPE_MESSAGE:
		typ = field.GetTypeName()
	default:
		panic("unknown type for " + field.GetName())
	}
	if needsStar(field, field.Extendee == nil, message != nil) {
		typ = "*" + typ
	}
	if isRepeated(field) {
		typ = "[]" + typ
	}
	return
}

func needsStar(field *descriptor.FieldDescriptorProto, proto3 bool, allowOneOf bool) bool {
	if isRepeated(field) &&
		(*field.Type != descriptor.FieldDescriptorProto_TYPE_MESSAGE) &&
		(*field.Type != descriptor.FieldDescriptorProto_TYPE_GROUP) {
		return false
	}
	if *field.Type == descriptor.FieldDescriptorProto_TYPE_BYTES {
		return false
	}
	if field.OneofIndex != nil && allowOneOf &&
		(*field.Type != descriptor.FieldDescriptorProto_TYPE_MESSAGE) &&
		(*field.Type != descriptor.FieldDescriptorProto_TYPE_GROUP) {
		return false
	}
	if proto3 &&
		(*field.Type != descriptor.FieldDescriptorProto_TYPE_MESSAGE) &&
		(*field.Type != descriptor.FieldDescriptorProto_TYPE_GROUP) {
		return false
	}
	return true
}

// Is this field repeated?
func isRepeated(field *descriptor.FieldDescriptorProto) bool {
	return field.Label != nil && *field.Label == descriptor.FieldDescriptorProto_LABEL_REPEATED
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"fmt"
	"html/template"
	"log"
	"net/smtp"
)

var mail_template = template.Must(template.ParseFiles("templates/mail.html"))

func mail(payment Payment) error {
	// Open a connection to the server.
	conn, err := smtp.Dial(fmt.Sprintf("%s:%s",
		getenv("SMTP_HOST"),
		getenv("SMTP_PORT")))
	if err != nil {
		return err
	}
	defer conn.Close()

	// (Possibly) start encryption.
	if ok, params := conn.Extension("STARTTLS"); ok {
		log.Println("STARTTLS supported with params", params)
		conn.StartTLS(&tls.Config{
			ServerName: getenv("SMTP_HOST"),
		})
	}

	// Authenticate to the server.
	err = conn.Auth(smtp.PlainAuth("",
		getenv("SMTP_USER"),
		getenv("SMTP_PASS"),
		getenv("SMTP_HOST")))
	if err != nil {
		return err
	}

	// Set up the mail headers.
	conn.Mail(getenv("SMTP_FROM"))
	conn.Rcpt(payment.Email)
	conn.Rcpt("acm@umn.edu")

	// Create the request body.
	w, err := conn.Data()
	if err != nil {
		return err
	}
	defer w.Close()

	// Write the mail headers.
	ioutil.WriteString(w, fmt.Sprintf("To: %s\n", payment.Email))
	ioutil.WriteString(w, "Receipt from payacmumn")

	// Render the request body and return.
	return mail_template.Execute(w, payment)
}
<commit_msg>I can totally read, I swear.<commit_after>package main

import (
	"crypto/tls"
	"fmt"
	"html/template"
	"io"
	"log"
	"net/smtp"
)

var mail_template = template.Must(template.ParseFiles("templates/mail.html"))

func mail(payment Payment) error {
	// Open a connection to the server.
	conn, err := smtp.Dial(fmt.Sprintf("%s:%s",
		getenv("SMTP_HOST"),
		getenv("SMTP_PORT")))
	if err != nil {
		return err
	}
	defer conn.Close()

	// (Possibly) start encryption.
	if ok, params := conn.Extension("STARTTLS"); ok {
		log.Println("STARTTLS supported with params", params)
		conn.StartTLS(&tls.Config{
			ServerName: getenv("SMTP_HOST"),
		})
	}

	// Authenticate to the server.
	err = conn.Auth(smtp.PlainAuth("",
		getenv("SMTP_USER"),
		getenv("SMTP_PASS"),
		getenv("SMTP_HOST")))
	if err != nil {
		return err
	}

	// Set up the mail headers.
	conn.Mail(getenv("SMTP_FROM"))
	conn.Rcpt(payment.Email)
	conn.Rcpt("acm@umn.edu")

	// Create the request body.
	w, err := conn.Data()
	if err != nil {
		return err
	}
	defer w.Close()

	// Write the mail headers.
	io.WriteString(w, fmt.Sprintf("To: %s\n", payment.Email))
	io.WriteString(w, "Receipt from payacmumn")

	// Render the request body and return.
	return mail_template.Execute(w, payment)
}
<|endoftext|>


<commit_before>package restpc

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func Test_NewError_Twice(t *testing.T) {
	err := NewError(InvalidArgument, "something is missing", nil)
	assert.Equal(t, "something is missing", err.Message())
	assert.Equal(t, "something is missing", err.Error())
	assert.Equal(t, InvalidArgument, err.Code())

	err = NewError(Unavailable, "not sure what", err)
	assert.Equal(t, "something is missing", err.Message())
	assert.Equal(t, "something is missing", err.Error())
	assert.Equal(t, InvalidArgument, err.Code())
}

func unimplementedHandler(req Request) (*Response, error) {
	err := fmt.Errorf("just an unexposed message")
	return nil, NewError(Unimplemented, "we didn't implement this", err).Add("name", "June The Girl")
}

func TestErrorFull(t *testing.T) {
	handlerName := "unimplementedHandler"

	origErrorDispatcher := errorDispatcher
	defer func() {
		errorDispatcher = origErrorDispatcher
	}()
	var rpcErr RPCError
	var request Request
	errorDispatcher = func(requestArg Request, rpcErrArg RPCError) {
		rpcErr = rpcErrArg
		request = requestArg
	}
	handlerFunc := TranslateHandler(unimplementedHandler)
	w := httptest.NewRecorder()
	r, _ := http.NewRequest("GET", "", nil)
	handlerFunc(w, r)
	assert.Equal(t, http.StatusNotImplemented, w.Code)
	time.Sleep(500 * time.Millisecond)
	if rpcErr == nil {
		panic("rpcErr == nil")
	}
	if !strings.HasSuffix(request.HandlerName(), "."+handlerName) {
		t.Errorf("request.HandlerName()=%v", request.HandlerName())
		return
	}
	handlerNameFull := request.HandlerName()
	assert.Equal(t, Unimplemented, rpcErr.Code())
	assert.Equal(t, "we didn't implement this", rpcErr.Error())
	assert.Equal(t, "we didn't implement this", rpcErr.Message())
	assert.Equal(t, "just an unexposed message", rpcErr.Private().Error())
	assert.Equal(t, "June The Girl", rpcErr.Details()["name"])
	{
		tb := rpcErr.Traceback("")
		assert.Equal(t, 6, len(tb.Callers()))
		assert.Equal(t, 6, len(tb.Records()))
	}
	{
		tb := rpcErr.Traceback(handlerNameFull)
		assert.Equal(t, 6, len(tb.Callers()))
		records := tb.Records()
		if len(records) != 1 {
			for _, record := range records {
				t.Log(record)
			}
			t.Errorf("len(records) = %v", len(records))
			return
		}
		record := records[0]
		if !strings.HasSuffix(record.File(), "/error_test.go") {
			t.Errorf("record.File()=%v", record.File())
		}
		assert.Equal(t, handlerNameFull, record.Function())
		assert.Equal(t, handlerName, record.FunctionLocal())
		assert.Equal(t, 28, record.Line())
		mapRecords := tb.MapRecords()
		if len(mapRecords) != 1 {
			t.Errorf("len(mapRecords) = %v", len(mapRecords))
			return
		}
		mapRecord := mapRecords[0]
		assert.Equal(t, record.File(), mapRecord["file"])
		assert.Equal(t, record.Function(), mapRecord["function"])
		assert.Equal(t, record.FunctionLocal(), mapRecord["functionLocal"])
		assert.Equal(t, record.Line(), mapRecord["line"])
	}
}
<commit_msg>error_test.go: last tests for traceback<commit_after>package restpc

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func Test_NewError_Twice(t *testing.T) {
	err := NewError(InvalidArgument, "something is missing", nil)
	assert.Equal(t, "something is missing", err.Message())
	assert.Equal(t, "something is missing", err.Error())
	assert.Equal(t, InvalidArgument, err.Code())

	err = NewError(Unavailable, "not sure what", err)
	assert.Equal(t, "something is missing", err.Message())
	assert.Equal(t, "something is missing", err.Error())
	assert.Equal(t, InvalidArgument, err.Code())
}

func unimplementedHandler(req Request) (*Response, error) {
	err := fmt.Errorf("just an unexposed message")
	return nil, NewError(Unimplemented, "we didn't implement this", err).Add("name", "June The Girl")
}

func TestErrorFull(t *testing.T) {
	handlerName := "unimplementedHandler"

	origErrorDispatcher := errorDispatcher
	defer func() {
		errorDispatcher = origErrorDispatcher
	}()
	var rpcErr RPCError
	var request Request
	errorDispatcher = func(requestArg Request, rpcErrArg RPCError) {
		rpcErr = rpcErrArg
		request = requestArg
	}
	handlerFunc := TranslateHandler(unimplementedHandler)
	w := httptest.NewRecorder()
	r, _ := http.NewRequest("GET", "", nil)
	handlerFunc(w, r)
	assert.Equal(t, http.StatusNotImplemented, w.Code)
	time.Sleep(500 * time.Millisecond)
	if rpcErr == nil {
		panic("rpcErr == nil")
	}
	if !strings.HasSuffix(request.HandlerName(), "."+handlerName) {
		t.Errorf("request.HandlerName()=%v", request.HandlerName())
		return
	}
	handlerNameFull := request.HandlerName()
	assert.Equal(t, Unimplemented, rpcErr.Code())
	assert.Equal(t, "we didn't implement this", rpcErr.Error())
	assert.Equal(t, "we didn't implement this", rpcErr.Message())
	assert.Equal(t, "just an unexposed message", rpcErr.Private().Error())
	assert.Equal(t, "June The Girl", rpcErr.Details()["name"])
	{
		tb := rpcErr.Traceback("")
		assert.Equal(t, 6, len(tb.Callers()))
		assert.Equal(t, 6, len(tb.Records()))
	}
	{
		tb := rpcErr.Traceback(handlerNameFull)
		assert.Equal(t, 6, len(tb.Callers()))
		records := tb.Records()
		if len(records) != 1 {
			for _, record := range records {
				t.Log(record)
			}
			t.Errorf("len(records) = %v", len(records))
			return
		}
		record := records[0]
		if !strings.HasSuffix(record.File(), "/error_test.go") {
			t.Errorf("record.File()=%v", record.File())
		}
		assert.Equal(t, handlerNameFull, record.Function())
		assert.Equal(t, handlerName, record.FunctionLocal())
		assert.Equal(t, 28, record.Line())
		mapRecords := tb.MapRecords()
		if len(mapRecords) != 1 {
			t.Errorf("len(mapRecords) = %v", len(mapRecords))
			return
		}
		mapRecord := mapRecords[0]
		assert.Equal(t, record.File(), mapRecord["file"])
		assert.Equal(t, record.Function(), mapRecord["function"])
		assert.Equal(t, record.FunctionLocal(), mapRecord["functionLocal"])
		assert.Equal(t, record.Line(), mapRecord["line"])
	}
	{
		tb := rpcErr.Traceback("")
		assert.Equal(t, 6, len(tb.Callers()))
		assert.Equal(t, 6, len(tb.Records()))
	}
}

func TestFunctionLocalNoPanic(t *testing.T) {
	record := &tracebackRecordImp{}
	assert.Equal(t, "", record.FunctionLocal())
}

func Test_tracebackImp_empty(t *testing.T) {
	tb := &tracebackImp{}
	assert.Equal(t, 0, len(tb.Callers()))
	assert.Equal(t, 0, len(tb.Records()))
}
<|endoftext|>


<commit_before>/*
Copyright IBM Corp 2016 All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
		 http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/json"
	"errors"
	"fmt"

	"github.com/hyperledger/fabric/core/chaincode/shim"
)

type isval []byte

type MarketerStruct struct {
	EId                   string `json:"EId"`
	TaxId                 string `json:"TaxId"`
	BeginDate             string `json:"BeginDate"`
	MarketerTypeFlag      string `json:"MarketerTypeFlag"`
	MarketerType          string `json:"MarketerType"`
	MarketerRole          string `json:"MarketerRole"`
	MarketerStatus        string `json:"MarketerStatus"`
	LegalName             string `json:"LegalName"`
	Gender                string `json:"Gender"`
	DoB                   string `json:"DoB"`
	RegStateName          string `json:"RegStateName"`
	MarketerEffectiveDate string `json:"MarketerEffectiveDate"`
	MarketerEndDate       string `json:"MarketerEndDate"`
	FirstName             string `json:"FirstName"`
	LastName              string `json:"LastName"`
	BusinessAddress       string `json:"BusinessAddress"`
	City                  string `json:"City"`
	State                 string `json:"State"`
	PostalCode            string `json:"PostalCode"`
	PhoneNumber           string `json:"PhoneNumber"`
	EMail                 string `json:"EMail"`
	MarketerEaRole        string `json:"“MarketerEaRole”"`
	OwnerRole             string `json:"OwnerRole"`
	OrgName               string `json:"OrgName"`
}

// SimpleChaincode example simple Chaincode implementation
type SimpleChaincode struct {
}

func main() {
	err := shim.Start(new(SimpleChaincode))

	fmt.Println("****** Starting to send information to my ledger")

	if err != nil {
		fmt.Printf("Error starting Simple chaincode: %s", err)
	}
}

// Init resets all the things
func (t *SimpleChaincode) Init(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	if len(args) != 1 {
		return nil, errors.New("Incorrect number of arguments. Expecting 1")
	}

	err := stub.PutState("hello_world", []byte(args[0]))

	if err != nil {
		return nil, err
	}

	return nil, nil
}

// Invoke isur entry point to invoke a chaincode function
func (t *SimpleChaincode) Invoke(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	fmt.Println("invoke is running " + function)

	// Handle different functions
	if function == "init" {
		return t.Init(stub, "init", args)
	} else if function == "write" {
		return t.write(stub, args)
	}
	fmt.Println("invoke did not find func: " + function)

	return nil, errors.New("Received unknown function invocation: " + function)
}

// Query is our entry point for queries
func (t *SimpleChaincode) Query(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	fmt.Println("query is running " + function)

	// Handle different functions
	if function == "read" { //read a variable
		return t.read(stub, args)
	}

	fmt.Println("query did not find func: " + function)

	return nil, errors.New("Received unknown function query: " + function)
}

// write - invoke function to write key/value pair
func (t *SimpleChaincode) write(stub shim.ChaincodeStubInterface, args []string) ([]byte, error) {

	var key string
	var err error
	var dupMktr string = "Marketer exists!"

	mktrStruct := MarketerStruct{
		EId:                   args[0],
		TaxId:                 args[1],
		BeginDate:             args[2],
		MarketerTypeFlag:      args[3],
		MarketerType:          args[4],
		MarketerRole:          args[5],
		MarketerStatus:        args[6],
		LegalName:             args[7],
		Gender:                args[8],
		DoB:                   args[9],
		RegStateName:          args[10],
		MarketerEffectiveDate: args[11],
		MarketerEndDate:       args[12],
		FirstName:             args[13],
		LastName:              args[14],
		BusinessAddress:       args[15],
		City:                  args[16],
		State:                 args[17],
		PostalCode:            args[18],
		PhoneNumber:           args[19],
		EMail:                 args[20],
		MarketerEaRole:        args[21],
		OwnerRole:             args[22],
		OrgName:               args[23],
	}

	mktrStructBytes, err := json.Marshal(mktrStruct)
	_ = err //ignore errors
	key = args[0]
	//t.read(stub, args)
	isval, err := t.read(stub, args)
	if isval == nil {
		stub.PutState(key, mktrStructBytes)
		fmt.Println("*** successfully wrote marketer to state")
	} else {
		fmt.Println("****duplicate entry")
		dupErrMsg, _ := json.Marshal(dupMktr)
		return dupErrMsg, errors.New("duplicate entry")
	}
	if err != nil {
		dupErrMsg, _ := json.Marshal(dupMktr)
		return dupErrMsg, errors.New("duplicate entry")
	}

	return nil, nil
}

// read - query function to read key/value pair
func (t *SimpleChaincode) read(stub shim.ChaincodeStubInterface, args []string) ([]byte, error) {
	var key, jsonResp string
	var err error

	var retrievedStruct MarketerStruct
	key = args[0]
	retrievedBytes, err := stub.GetState(key)
	json.Unmarshal(retrievedBytes, retrievedStruct)

	fmt.Println("Retrieved struct: ", retrievedStruct)

	if err != nil {
		jsonResp = "{\"Error\":\"Failed to get state for " + key + "\"}"
		return nil, errors.New(jsonResp)
	}

	return retrievedBytes, nil
}
<commit_msg>Added account and assignment code<commit_after>/*
Copyright IBM Corp 2016 All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
		 http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/json"
	"errors"
	"fmt"

	"github.com/hyperledger/fabric/core/chaincode/shim"
)

type isval []byte

type MarketerStruct struct {
	EId                   string `json:"EId"`
	TaxId                 string `json:"TaxId"`
	BeginDate             string `json:"BeginDate"`
	MarketerTypeFlag      string `json:"MarketerTypeFlag"`
	MarketerType          string `json:"MarketerType"`
	MarketerRole          string `json:"MarketerRole"`
	MarketerStatus        string `json:"MarketerStatus"`
	LegalName             string `json:"LegalName"`
	Gender                string `json:"Gender"`
	DoB                   string `json:"DoB"`
	RegStateName          string `json:"RegStateName"`
	MarketerEffectiveDate string `json:"MarketerEffectiveDate"`
	MarketerEndDate       string `json:"MarketerEndDate"`
	FirstName             string `json:"FirstName"`
	LastName              string `json:"LastName"`
	BusinessAddress       string `json:"BusinessAddress"`
	City                  string `json:"City"`
	State                 string `json:"State"`
	PostalCode            string `json:"PostalCode"`
	PhoneNumber           string `json:"PhoneNumber"`
	EMail                 string `json:"EMail"`
	MarketerEaRole        string `json:"“MarketerEaRole”"`
	OwnerRole             string `json:"OwnerRole"`
	OrgName               string `json:"OrgName"`
}

type AccountStruct struct {
	policyPrefix               string `json:"policyPrefix"`
	accountNumber              string `json:"accountNumber"`
	internalAccountName        string `json:"internalAccountName"`
	accountStatus              string `json:"accountStatus"`
	accountStatusEffectiveDate string `json:"accountStatusEffectiveDate"`
	validationStatus           string `json:"validationStatus"`
	accountEffectiveDate       string `json:"accountEffectiveDate"`
	marketerProduct            string `json:"marketerProduct"`
	disclosureStatus           string `json:"disclosureStatus"`
	disclosureEffectiveDate    string `json:"disclosureEffectiveDate"`
}

type AssignmentStruct struct {
	AssignmentId            string `json:"AssignmentRoleType"`
	AssignmentRoleType      string `json:"AssignmentRoleType"`
	SplitPercentage         string `json:"SplitPercentage"`
	AssignmentEffectiveDate string `json:"AssignmentEffectiveDate"`
	AssignmentStatus        string `json:"AssignmentStatus"`
	AssignmentEndDate       string `json:"AssignmentEndDate"`
	SplitEffectiveDate      string `json:"SplitEffectiveDate"`
	OwnerEId                string `json:"OwnerEId"`
	OwnerRole               string `json:"OwnerRole"`
	OrgName                 string `json:"OrgName"`
	policyPrefix            string `json:"policyPrefix"`
	accountNumber           string `json:"accountNumber"`
	EId                     string `json:"EId"`
	TaxId                   string `json:"TaxId"`
	BeginDate               string `json:"BeginDate"`
	MarketerTypeFlag        string `json:"MarketerTypeFlag"`
	MarketerType            string `json:"MarketerType"`
	MarketerRole            string `json:"MarketerRole"`
	MarketerStatus          string `json:"MarketerStatus"`
	LegalName               string `json:"LegalName"`
	Gender                  string `json:"Gender"`
	DoB                     string `json:"DoB"`
	RegStateName            string `json:"RegStateName"`
	MarketerEffectiveDate   string `json:"MarketerEffectiveDate"`
	MarketerEndDate         string `json:"MarketerEndDate"`
	FirstName               string `json:"FirstName"`
	LastName                string `json:"LastName"`
	EMail                   string `json:"EMail"`
	MarketerEaRole          string `json:"“MarketerEaRole”"`
}

// SimpleChaincode example simple Chaincode implementation
type SimpleChaincode struct {
}

func main() {
	err := shim.Start(new(SimpleChaincode))

	fmt.Println("****** Starting to send information to my ledger")

	if err != nil {
		fmt.Printf("Error starting Simple chaincode: %s", err)
	}
}

// Init resets all the things
func (t *SimpleChaincode) Init(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	if len(args) != 1 {
		return nil, errors.New("Incorrect number of arguments. Expecting 1")
	}

	err := stub.PutState("hello_world", []byte(args[0]))

	if err != nil {
		return nil, err
	}

	return nil, nil
}

// Invoke isur entry point to invoke a chaincode function
func (t *SimpleChaincode) Invoke(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	fmt.Println("invoke is running " + function)

	// Handle different functions
	if function == "init" {
		return t.Init(stub, "init", args)
	} else if function == "write" {
		return t.write(stub, args)
	} else if function == "writeAcc" {
		return t.writeAcc(stub, args)
	} else if function == "assign" {
		return t.assign(stub, args)
	}
	fmt.Println("invoke did not find func: " + function)

	return nil, errors.New("Received unknown function invocation: " + function)
}

// Query is our entry point for queries
func (t *SimpleChaincode) Query(stub shim.ChaincodeStubInterface, function string, args []string) ([]byte, error) {
	fmt.Println("query is running " + function)

	// Handle different functions
	if function == "read" { //read a variable
		return t.read(stub, args)
	}

	fmt.Println("query did not find func: " + function)

	return nil, errors.New("Received unknown function query: " + function)
}

// write - invoke function to write key/value pair
func (t *SimpleChaincode) write(stub shim.ChaincodeStubInterface, args []string) ([]byte, error) {

	var key string
	var err error

	mktrStruct := MarketerStruct{
		EId:                   args[0],
		TaxId:                 args[1],
		BeginDate:             args[2],
		MarketerTypeFlag:      args[3],
		MarketerType:          args[4],
		MarketerRole:          args[5],
		MarketerStatus:        args[6],
		LegalName:             args[7],
		Gender:                args[8],
		DoB:                   args[9],
		RegStateName:          args[10],
		MarketerEffectiveDate: args[11],
		MarketerEndDate:       args[12],
		FirstName:             args[13],
		LastName:              args[14],
		BusinessAddress:       args[15],
		City:                  args[16],
		State:                 args[17],
		PostalCode:            args[18],
		PhoneNumber:           args[19],
		EMail:                 args[20],
		MarketerEaRole:        args[21],
		OwnerRole:             args[22],
		OrgName:               args[23],
	}

	mktrStructBytes, err := json.Marshal(mktrStruct)
	_ = err //ignore errors
	key = args[0]
	//t.read(stub, args)
	isval, err := t.read(stub, args)
	if isval == nil {
		stub.PutState(key, mktrStructBytes)
		fmt.Println("*** successfully wrote marketer to state")
	} else {
		fmt.Println("****duplicate entry")
		dupMktrArr := []byte("Marketer exists!")
		return dupMktrArr, errors.New("duplicate entry")
	}

	successMsgArr := []byte("Marketer added succesfully!")

	return successMsgArr, nil
}

func (t *SimpleChaincode) writeAcc(stub shim.ChaincodeStubInterface, args []string) ([]byte, error) {

	var key string
	var err error

	accStruct := AccountStruct{
		accountNumber:              args[0],
		policyPrefix:               args[1],
		internalAccountName:        args[2],
		accountStatus:              args[3],
		accountStatusEffectiveDate: args[4],
		validationStatus:           args[5],
		accountEffectiveDate:       args[6],
		marketerProduct:            args[7],
		disclosureStatus:           args[8],
		disclosureEffectiveDate:    args[9],
	}

	accStructBytes, err := json.Marshal(accStruct)
	_ = err //ignore errors
	key = args[0]
	stub.PutState(key, accStructBytes)
	fmt.Println("*** successfully wrote account to state")

	successMsgArr := []byte("Account added succesfully!")

	return successMsgArr, nil
}

func (t *SimpleChaincode) assign(stub shim.ChaincodeStubInterface, args []string) ([]byte, error) {

	var key string
	var err error

	assignStruct := AssignmentStruct{
		AssignmentId:            args[0],
		AssignmentRoleType:      args[1],
		SplitPercentage:         args[2],
		AssignmentEffectiveDate: args[3],
		AssignmentStatus:        args[4],
		AssignmentEndDate:       args[5],
		SplitEffectiveDate:      args[6],
		OwnerEId:                args[7],
		OwnerRole:               args[8],
		OrgName:                 args[9],
		policyPrefix:            args[10],
		accountNumber:           args[11],
		EId:                     args[12],
		TaxId:                   args[13],
		BeginDate:               args[14],
		MarketerTypeFlag:        args[15],
		MarketerType:            args[16],
		MarketerRole:            args[17],
		MarketerStatus:          args[18],
		LegalName:               args[19],
		Gender:                  args[20],
		DoB:                     args[21],
		RegStateName:            args[22],
		MarketerEffectiveDate:   args[23],
		MarketerEndDate:         args[24],
		FirstName:               args[25],
		LastName:                args[26],
		EMail:                   args[27],
		MarketerEaRole:          args[28],
	}

	assignStructBytes, err := json.Marshal(assignStruct)
	_ = err //ignore errors
	key = args[0]

	stub.PutState(key, assignStructBytes)
	fmt.Println("*** successfully wrote assignemt to state")

	successMsgArr := []byte("Assignment added succesfully!")

	return successMsgArr, nil
}

// read - query function to read key/value pair
func (t *SimpleChaincode) read(stub shim.ChaincodeStubInterface, args []string) ([]byte, error) {
	var key, jsonResp string
	var err error

	var retrievedStruct MarketerStruct
	key = args[0]
	retrievedBytes, err := stub.GetState(key)
	json.Unmarshal(retrievedBytes, retrievedStruct)

	fmt.Println("Retrieved struct: ", retrievedStruct)

	if err != nil {
		jsonResp = "{\"Error\":\"Failed to get state for " + key + "\"}"
		return nil, errors.New(jsonResp)
	}

	return retrievedBytes, nil
}
<|endoftext|>


<commit_before>package persister

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync/atomic"
	"time"

	"devroom.ru/lomik/carbon/points"

	"github.com/Sirupsen/logrus"
	"github.com/grobian/go-whisper"
)

// Whisper write data to *.wsp files
type Whisper struct {
	in          chan *points.Points
	exit        chan bool
	schemas     *WhisperSchemas
	rootPath    string
	graphPrefix string
	commited    uint64 // (updateOperations << 32) + commitedPoints
	created     uint32 // counter
}

// NewWhisper create instance of Whisper
func NewWhisper(rootPath string, schemas *WhisperSchemas, in chan *points.Points) *Whisper {
	return &Whisper{
		in:       in,
		exit:     make(chan bool),
		schemas:  schemas,
		rootPath: rootPath,
	}
}

// SetGraphPrefix for internal cache metrics
func (p *Whisper) SetGraphPrefix(prefix string) {
	p.graphPrefix = prefix
}

// Stat sends internal statistics to cache
func (p *Whisper) Stat(metric string, value float64) {
	p.in <- points.OnePoint(
		fmt.Sprintf("%s%s", p.graphPrefix, metric),
		value,
		time.Now().Unix(),
	)
}

func (p *Whisper) store(values *points.Points) {
	// @TODO: lock or shard by hash(metricName), no thread safe
	path := filepath.Join(p.rootPath, strings.Replace(values.Metric, ".", "/", -1)+".wsp")

	w, err := whisper.Open(path)
	if err != nil {
		schema := p.schemas.match(values.Metric)
		if schema == nil {
			logrus.Errorf("No storage schema defined for %s", values.Metric)
			return
		}

		logrus.Infof("Creating %s: %s, retention %#v (section %#v)",
			values.Metric, path, schema.retentionStr, schema.name)

		if err = os.MkdirAll(filepath.Dir(path), os.ModeDir|os.ModePerm); err != nil {
			logrus.Error(err)
			return
		}

		w, err = whisper.Create(path, schema.retentions, whisper.Last, 0.5)
		if err != nil {
			logrus.Warningf("Failed to create new whisper file %s: %s", path, err.Error())
			return
		}

		atomic.AddUint32(&p.created, 1)
	}

	points := make([]*whisper.TimeSeriesPoint, len(values.Data))
	for i, r := range values.Data {
		points[i] = &whisper.TimeSeriesPoint{Time: int(r.Timestamp), Value: r.Value}
	}

	w.UpdateMany(points)

	atomic.AddUint64(&p.commited, (uint64(1)<<32)+uint64(len(values.Data)))

	w.Close()
}

func (p *Whisper) worker(in chan *points.Points) {
	for {
		select {
		case <-p.exit:
			break
		case values := <-in:
			p.store(values)
		}
	}
}

func (p *Whisper) statWorker() {
	ticker := time.NewTicker(time.Minute)
	defer ticker.Stop()

	for {
		select {
		case <-p.exit:
			break
		case <-ticker.C:
			// @send stat
			cnt := atomic.LoadUint64(&p.commited)
			atomic.AddUint64(&p.commited, -cnt)
			p.Stat("updateOperations", float64(cnt>>32))
			p.Stat("commitedPoints", float64(cnt%(1<<32)))
			if float64(cnt>>32) > 0 {
				p.Stat("pointsPerUpdate", float64(cnt%(1<<32))/float64(cnt>>32))
			} else {
				p.Stat("pointsPerUpdate", 0.0)
			}

			created := atomic.LoadUint32(&p.created)
			atomic.AddUint32(&p.created, -created)

			p.Stat("created", float64(created))
		}
	}
}

// Start worker
func (p *Whisper) Start() {
	go p.statWorker()
	go p.worker(p.in)
}

// Stop worker
func (p *Whisper) Stop() {
	close(p.exit)
}
<commit_msg>more atomic whisper stats<commit_after>package persister

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync/atomic"
	"time"

	"devroom.ru/lomik/carbon/points"

	"github.com/Sirupsen/logrus"
	"github.com/grobian/go-whisper"
)

// Whisper write data to *.wsp files
type Whisper struct {
	in          chan *points.Points
	exit        chan bool
	schemas     *WhisperSchemas
	rootPath    string
	graphPrefix string
	commited    uint64 // (updateOperations << 32) + commitedPoints
	created     uint32 // counter
}

// NewWhisper create instance of Whisper
func NewWhisper(rootPath string, schemas *WhisperSchemas, in chan *points.Points) *Whisper {
	return &Whisper{
		in:       in,
		exit:     make(chan bool),
		schemas:  schemas,
		rootPath: rootPath,
	}
}

// SetGraphPrefix for internal cache metrics
func (p *Whisper) SetGraphPrefix(prefix string) {
	p.graphPrefix = prefix
}

// Stat sends internal statistics to cache
func (p *Whisper) Stat(metric string, value float64) {
	p.in <- points.OnePoint(
		fmt.Sprintf("%s%s", p.graphPrefix, metric),
		value,
		time.Now().Unix(),
	)
}

func (p *Whisper) store(values *points.Points) {
	// @TODO: lock or shard by hash(metricName), no thread safe
	path := filepath.Join(p.rootPath, strings.Replace(values.Metric, ".", "/", -1)+".wsp")

	w, err := whisper.Open(path)
	if err != nil {
		schema := p.schemas.match(values.Metric)
		if schema == nil {
			logrus.Errorf("No storage schema defined for %s", values.Metric)
			return
		}

		logrus.Infof("Creating %s: %s, retention %#v (section %#v)",
			values.Metric, path, schema.retentionStr, schema.name)

		if err = os.MkdirAll(filepath.Dir(path), os.ModeDir|os.ModePerm); err != nil {
			logrus.Error(err)
			return
		}

		w, err = whisper.Create(path, schema.retentions, whisper.Last, 0.5)
		if err != nil {
			logrus.Warningf("Failed to create new whisper file %s: %s", path, err.Error())
			return
		}

		atomic.AddUint32(&p.created, 1)
	}

	points := make([]*whisper.TimeSeriesPoint, len(values.Data))
	for i, r := range values.Data {
		points[i] = &whisper.TimeSeriesPoint{Time: int(r.Timestamp), Value: r.Value}
	}

	w.UpdateMany(points)

	atomic.AddUint64(&p.commited, (uint64(1)<<32)+uint64(len(values.Data)))

	w.Close()
}

func (p *Whisper) worker(in chan *points.Points) {
	for {
		select {
		case <-p.exit:
			break
		case values := <-in:
			p.store(values)
		}
	}
}

func (p *Whisper) statWorker() {
	ticker := time.NewTicker(time.Minute)
	defer ticker.Stop()

	for {
		select {
		case <-p.exit:
			break
		case <-ticker.C:
			// @send stat
			cnt := atomic.LoadUint64(&p.commited)
			atomic.AddUint64(&p.commited, -cnt)

			created := atomic.LoadUint32(&p.created)
			atomic.AddUint32(&p.created, -created)

			p.Stat("updateOperations", float64(cnt>>32))
			p.Stat("commitedPoints", float64(cnt%(1<<32)))
			if float64(cnt>>32) > 0 {
				p.Stat("pointsPerUpdate", float64(cnt%(1<<32))/float64(cnt>>32))
			} else {
				p.Stat("pointsPerUpdate", 0.0)
			}

			p.Stat("created", float64(created))
		}
	}
}

// Start worker
func (p *Whisper) Start() {
	go p.statWorker()
	go p.worker(p.in)
}

// Stop worker
func (p *Whisper) Stop() {
	close(p.exit)
}
<|endoftext|>


<commit_before>package persister

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"

	whisper "github.com/go-graphite/go-whisper"
	"go.uber.org/zap"

	"github.com/lomik/go-carbon/helper"
	"github.com/lomik/go-carbon/points"
	"github.com/lomik/go-carbon/tags"
	"github.com/lomik/zapwriter"
)

const storeMutexCount = 32768

type StoreFunc func(metric string)

// Whisper write data to *.wsp files
type Whisper struct {
	helper.Stoppable
	recv                    func(chan bool) string
	pop                     func(string) (*points.Points, bool)
	confirm                 func(*points.Points)
	onCreateTagged          func(string)
	tagsEnabled             bool
	schemas                 WhisperSchemas
	aggregation             *WhisperAggregation
	workersCount            int
	rootPath                string
	created                 uint32 // counter
	throttledCreates        uint32 // counter
	updateOperations        uint32 // counter
	committedPoints         uint32 // counter
	sparse                  bool
	flock                   bool
	hashFilenames           bool
	maxUpdatesPerSecond     int
	maxCreatesPerSecond     int
	hardMaxCreatesPerSecond bool
	throttleTicker          *ThrottleTicker
	maxCreatesTicker        *ThrottleTicker
	storeMutex              [storeMutexCount]sync.Mutex
	mockStore               func() (StoreFunc, func())
	logger                  *zap.Logger
	createLogger            *zap.Logger
	// blockThrottleNs        uint64 // sum ns counter
	// blockQueueGetNs        uint64 // sum ns counter
	// blockAvoidConcurrentNs uint64 // sum ns counter
	// blockUpdateManyNs      uint64 // sum ns counter
}

// NewWhisper create instance of Whisper
func NewWhisper(
	rootPath string,
	schemas WhisperSchemas,
	aggregation *WhisperAggregation,
	recv func(chan bool) string,
	pop func(string) (*points.Points, bool),
	confirm func(*points.Points)) *Whisper {

	return &Whisper{
		recv:                recv,
		pop:                 pop,
		confirm:             confirm,
		schemas:             schemas,
		aggregation:         aggregation,
		workersCount:        1,
		rootPath:            rootPath,
		maxUpdatesPerSecond: 0,
		logger:              zapwriter.Logger("persister"),
		createLogger:        zapwriter.Logger("whisper:new"),
	}
}

// SetMaxUpdatesPerSecond enable throttling
func (p *Whisper) SetMaxUpdatesPerSecond(maxUpdatesPerSecond int) {
	p.maxUpdatesPerSecond = maxUpdatesPerSecond
}

// SetMaxCreatesPerSecond enable throttling
func (p *Whisper) SetMaxCreatesPerSecond(maxCreatesPerSecond int) {
	p.maxCreatesPerSecond = maxCreatesPerSecond
}

// SetHardMaxCreatesPerSecond enable throttling
func (p *Whisper) SetHardMaxCreatesPerSecond(hardMaxCreatesPerSecond bool) {
	p.hardMaxCreatesPerSecond = hardMaxCreatesPerSecond
}

// GetMaxUpdatesPerSecond returns current throttling speed
func (p *Whisper) GetMaxUpdatesPerSecond() int {
	return p.maxUpdatesPerSecond
}

// SetWorkers count
func (p *Whisper) SetWorkers(count int) {
	if count >= 1 {
		p.workersCount = count
	} else {
		p.workersCount = 1
	}
}

// SetSparse creation
func (p *Whisper) SetSparse(sparse bool) {
	p.sparse = sparse
}

// SetFLock on create and open
func (p *Whisper) SetFLock(flock bool) {
	p.flock = flock
}

func (p *Whisper) SetHashFilenames(v bool) {
	p.hashFilenames = v
}

func (p *Whisper) SetMockStore(fn func() (StoreFunc, func())) {
	p.mockStore = fn
}

func (p *Whisper) SetTagsEnabled(v bool) {
	p.tagsEnabled = v
}

func (p *Whisper) SetOnCreateTagged(fn func(string)) {
	p.onCreateTagged = fn
}

func fnv32(key string) uint32 {
	hash := uint32(2166136261)
	const prime32 = uint32(16777619)
	for i := 0; i < len(key); i++ {
		hash *= prime32
		hash ^= uint32(key[i])
	}
	return hash
}

func (p *Whisper) updateMany(w *whisper.Whisper, path string, points []*whisper.TimeSeriesPoint) {
	defer func() {
		if r := recover(); r != nil {
			p.logger.Error("UpdateMany panic recovered",
				zap.String("path", path),
				zap.String("traceback", fmt.Sprint(r)),
			)
		}
	}()

	// start = time.Now()
	w.UpdateMany(points)
}

func (p *Whisper) store(metric string) {
	// avoid concurrent store same metric
	// @TODO: may be flock?
	// start := time.Now()
	mutexIndex := fnv32(metric) % storeMutexCount
	p.storeMutex[mutexIndex].Lock()
	// atomic.AddUint64(&p.blockAvoidConcurrentNs, uint64(time.Since(start).Nanoseconds()))
	defer p.storeMutex[mutexIndex].Unlock()

	var path string
	if p.tagsEnabled && strings.IndexByte(metric, ';') >= 0 {
		path = tags.FilePath(p.rootPath, metric, p.hashFilenames) + ".wsp"
	} else {
		path = filepath.Join(p.rootPath, strings.Replace(metric, ".", "/", -1)+".wsp")
	}

	w, err := whisper.OpenWithOptions(path, &whisper.Options{
		FLock: p.flock,
	})
	if err != nil {
		// create new whisper if file not exists
		if !os.IsNotExist(err) {
			p.logger.Error("failed to open whisper file", zap.String("path", path), zap.Error(err))
			return
		}

		// max creates throttling
		select {
		case keep := <-p.maxCreatesTicker.C:
			if !keep {
				p.pop(metric)
				atomic.AddUint32(&p.throttledCreates, 1)
				p.logger.Error("metric creation throttled",
					zap.String("name", metric),
					zap.String("operation", "create"),
					zap.Bool("dropped", true),
				)
				return
			}

			// pass
		default:
			atomic.AddUint32(&p.throttledCreates, 1)
			p.logger.Error("metric creation throttled",
				zap.String("name", metric),
				zap.String("operation", "create"),
				zap.Bool("dropped", false),
			)
			return
		}

		schema, ok := p.schemas.Match(metric)
		if !ok {
			p.logger.Error("no storage schema defined for metric", zap.String("metric", metric))
			return
		}

		aggr := p.aggregation.match(metric)
		if aggr == nil {
			p.logger.Error("no storage aggregation defined for metric", zap.String("metric", metric))
			return
		}

		if err = os.MkdirAll(filepath.Dir(path), os.ModeDir|os.ModePerm); err != nil {
			p.logger.Error("mkdir failed",
				zap.String("dir", filepath.Dir(path)),
				zap.Error(err),
				zap.String("path", path),
			)
			return
		}

		w, err = whisper.CreateWithOptions(path, schema.Retentions, aggr.aggregationMethod, float32(aggr.xFilesFactor), &whisper.Options{
			Sparse: p.sparse,
			FLock:  p.flock,
		})
		if err != nil {
			p.logger.Error("create new whisper file failed",
				zap.String("path", path),
				zap.Error(err),
				zap.String("retention", schema.RetentionStr),
				zap.String("schema", schema.Name),
				zap.String("aggregation", aggr.name),
				zap.Float64("xFilesFactor", aggr.xFilesFactor),
				zap.String("method", aggr.aggregationMethodStr),
			)
			return
		}

		if p.tagsEnabled && p.onCreateTagged != nil && strings.IndexByte(metric, ';') >= 0 {
			p.onCreateTagged(metric)
		}

		p.createLogger.Debug("created",
			zap.String("path", path),
			zap.String("retention", schema.RetentionStr),
			zap.String("schema", schema.Name),
			zap.String("aggregation", aggr.name),
			zap.Float64("xFilesFactor", aggr.xFilesFactor),
			zap.String("method", aggr.aggregationMethodStr),
		)

		atomic.AddUint32(&p.created, 1)
	}

	values, exists := p.pop(metric)
	if !exists {
		return
	}
	if p.confirm != nil {
		defer p.confirm(values)
	}

	points := make([]*whisper.TimeSeriesPoint, len(values.Data))
	for i, r := range values.Data {
		points[i] = &whisper.TimeSeriesPoint{Time: int(r.Timestamp), Value: r.Value}
	}

	atomic.AddUint32(&p.committedPoints, uint32(len(values.Data)))
	atomic.AddUint32(&p.updateOperations, 1)

	// start = time.Now()
	p.updateMany(w, path, points)

	w.Close()
	// atomic.AddUint64(&p.blockUpdateManyNs, uint64(time.Since(start).Nanoseconds()))
}

func (p *Whisper) worker(exit chan bool) {
	storeFunc := p.store
	var doneCb func()
	if p.mockStore != nil {
		storeFunc, doneCb = p.mockStore()
	}

LOOP:
	for {
		// start := time.Now()
		select {
		case <-p.throttleTicker.C:
			// atomic.AddUint64(&p.blockThrottleNs, uint64(time.Since(start).Nanoseconds()))
			// pass
		case <-exit:
			return
		}

		// start = time.Now()
		metric := p.recv(exit)
		// atomic.AddUint64(&p.blockQueueGetNs, uint64(time.Since(start).Nanoseconds()))
		if metric == "" {
			// exit closed
			break LOOP
		}
		storeFunc(metric)
		if doneCb != nil {
			doneCb()
		}
	}
}

// Stat callback
func (p *Whisper) Stat(send helper.StatCallback) {
	updateOperations := atomic.LoadUint32(&p.updateOperations)
	committedPoints := atomic.LoadUint32(&p.committedPoints)
	atomic.AddUint32(&p.updateOperations, -updateOperations)
	atomic.AddUint32(&p.committedPoints, -committedPoints)

	created := atomic.LoadUint32(&p.created)
	atomic.AddUint32(&p.created, -created)

	throttledCreates := atomic.LoadUint32(&p.throttledCreates)
	atomic.AddUint32(&p.throttledCreates, -throttledCreates)

	send("updateOperations", float64(updateOperations))
	send("committedPoints", float64(committedPoints))
	if updateOperations > 0 {
		send("pointsPerUpdate", float64(committedPoints)/float64(updateOperations))
	} else {
		send("pointsPerUpdate", 0.0)
	}

	send("created", float64(created))
	send("throttledCreates", float64(throttledCreates))

	send("maxUpdatesPerSecond", float64(p.maxUpdatesPerSecond))
	send("workers", float64(p.workersCount))

	// helper.SendAndSubstractUint64("blockThrottleNs", &p.blockThrottleNs, send)
	// helper.SendAndSubstractUint64("blockQueueGetNs", &p.blockQueueGetNs, send)
	// helper.SendAndSubstractUint64("blockAvoidConcurrentNs", &p.blockAvoidConcurrentNs, send)
	// helper.SendAndSubstractUint64("blockUpdateManyNs", &p.blockUpdateManyNs, send)

}

// Start worker
func (p *Whisper) Start() error {
	return p.StartFunc(func() error {

		p.throttleTicker = NewThrottleTicker(p.maxUpdatesPerSecond)
		if p.hardMaxCreatesPerSecond {
			p.maxCreatesTicker = NewHardThrottleTicker(p.maxCreatesPerSecond)
		} else {
			p.maxCreatesTicker = NewThrottleTicker(p.maxCreatesPerSecond)
		}

		for i := 0; i < p.workersCount; i++ {
			p.Go(p.worker)
		}

		return nil
	})
}

func (p *Whisper) Stop() {
	p.StopFunc(func() {
		p.throttleTicker.Stop()
		p.maxCreatesTicker.Stop()
	})
}
<commit_msg>Export number of max creates per second<commit_after>package persister

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"

	whisper "github.com/go-graphite/go-whisper"
	"go.uber.org/zap"

	"github.com/lomik/go-carbon/helper"
	"github.com/lomik/go-carbon/points"
	"github.com/lomik/go-carbon/tags"
	"github.com/lomik/zapwriter"
)

const storeMutexCount = 32768

type StoreFunc func(metric string)

// Whisper write data to *.wsp files
type Whisper struct {
	helper.Stoppable
	recv                    func(chan bool) string
	pop                     func(string) (*points.Points, bool)
	confirm                 func(*points.Points)
	onCreateTagged          func(string)
	tagsEnabled             bool
	schemas                 WhisperSchemas
	aggregation             *WhisperAggregation
	workersCount            int
	rootPath                string
	created                 uint32 // counter
	throttledCreates        uint32 // counter
	updateOperations        uint32 // counter
	committedPoints         uint32 // counter
	sparse                  bool
	flock                   bool
	hashFilenames           bool
	maxUpdatesPerSecond     int
	maxCreatesPerSecond     int
	hardMaxCreatesPerSecond bool
	throttleTicker          *ThrottleTicker
	maxCreatesTicker        *ThrottleTicker
	storeMutex              [storeMutexCount]sync.Mutex
	mockStore               func() (StoreFunc, func())
	logger                  *zap.Logger
	createLogger            *zap.Logger
	// blockThrottleNs        uint64 // sum ns counter
	// blockQueueGetNs        uint64 // sum ns counter
	// blockAvoidConcurrentNs uint64 // sum ns counter
	// blockUpdateManyNs      uint64 // sum ns counter
}

// NewWhisper create instance of Whisper
func NewWhisper(
	rootPath string,
	schemas WhisperSchemas,
	aggregation *WhisperAggregation,
	recv func(chan bool) string,
	pop func(string) (*points.Points, bool),
	confirm func(*points.Points)) *Whisper {

	return &Whisper{
		recv:                recv,
		pop:                 pop,
		confirm:             confirm,
		schemas:             schemas,
		aggregation:         aggregation,
		workersCount:        1,
		rootPath:            rootPath,
		maxUpdatesPerSecond: 0,
		logger:              zapwriter.Logger("persister"),
		createLogger:        zapwriter.Logger("whisper:new"),
	}
}

// SetMaxUpdatesPerSecond enable throttling
func (p *Whisper) SetMaxUpdatesPerSecond(maxUpdatesPerSecond int) {
	p.maxUpdatesPerSecond = maxUpdatesPerSecond
}

// SetMaxCreatesPerSecond enable throttling
func (p *Whisper) SetMaxCreatesPerSecond(maxCreatesPerSecond int) {
	p.maxCreatesPerSecond = maxCreatesPerSecond
}

// SetHardMaxCreatesPerSecond enable throttling
func (p *Whisper) SetHardMaxCreatesPerSecond(hardMaxCreatesPerSecond bool) {
	p.hardMaxCreatesPerSecond = hardMaxCreatesPerSecond
}

// GetMaxUpdatesPerSecond returns current throttling speed
func (p *Whisper) GetMaxUpdatesPerSecond() int {
	return p.maxUpdatesPerSecond
}

// SetWorkers count
func (p *Whisper) SetWorkers(count int) {
	if count >= 1 {
		p.workersCount = count
	} else {
		p.workersCount = 1
	}
}

// SetSparse creation
func (p *Whisper) SetSparse(sparse bool) {
	p.sparse = sparse
}

// SetFLock on create and open
func (p *Whisper) SetFLock(flock bool) {
	p.flock = flock
}

func (p *Whisper) SetHashFilenames(v bool) {
	p.hashFilenames = v
}

func (p *Whisper) SetMockStore(fn func() (StoreFunc, func())) {
	p.mockStore = fn
}

func (p *Whisper) SetTagsEnabled(v bool) {
	p.tagsEnabled = v
}

func (p *Whisper) SetOnCreateTagged(fn func(string)) {
	p.onCreateTagged = fn
}

func fnv32(key string) uint32 {
	hash := uint32(2166136261)
	const prime32 = uint32(16777619)
	for i := 0; i < len(key); i++ {
		hash *= prime32
		hash ^= uint32(key[i])
	}
	return hash
}

func (p *Whisper) updateMany(w *whisper.Whisper, path string, points []*whisper.TimeSeriesPoint) {
	defer func() {
		if r := recover(); r != nil {
			p.logger.Error("UpdateMany panic recovered",
				zap.String("path", path),
				zap.String("traceback", fmt.Sprint(r)),
			)
		}
	}()

	// start = time.Now()
	w.UpdateMany(points)
}

func (p *Whisper) store(metric string) {
	// avoid concurrent store same metric
	// @TODO: may be flock?
	// start := time.Now()
	mutexIndex := fnv32(metric) % storeMutexCount
	p.storeMutex[mutexIndex].Lock()
	// atomic.AddUint64(&p.blockAvoidConcurrentNs, uint64(time.Since(start).Nanoseconds()))
	defer p.storeMutex[mutexIndex].Unlock()

	var path string
	if p.tagsEnabled && strings.IndexByte(metric, ';') >= 0 {
		path = tags.FilePath(p.rootPath, metric, p.hashFilenames) + ".wsp"
	} else {
		path = filepath.Join(p.rootPath, strings.Replace(metric, ".", "/", -1)+".wsp")
	}

	w, err := whisper.OpenWithOptions(path, &whisper.Options{
		FLock: p.flock,
	})
	if err != nil {
		// create new whisper if file not exists
		if !os.IsNotExist(err) {
			p.logger.Error("failed to open whisper file", zap.String("path", path), zap.Error(err))
			return
		}

		// max creates throttling
		select {
		case keep := <-p.maxCreatesTicker.C:
			if !keep {
				p.pop(metric)
				atomic.AddUint32(&p.throttledCreates, 1)
				p.logger.Error("metric creation throttled",
					zap.String("name", metric),
					zap.String("operation", "create"),
					zap.Bool("dropped", true),
				)
				return
			}

			// pass
		default:
			atomic.AddUint32(&p.throttledCreates, 1)
			p.logger.Error("metric creation throttled",
				zap.String("name", metric),
				zap.String("operation", "create"),
				zap.Bool("dropped", false),
			)
			return
		}

		schema, ok := p.schemas.Match(metric)
		if !ok {
			p.logger.Error("no storage schema defined for metric", zap.String("metric", metric))
			return
		}

		aggr := p.aggregation.match(metric)
		if aggr == nil {
			p.logger.Error("no storage aggregation defined for metric", zap.String("metric", metric))
			return
		}

		if err = os.MkdirAll(filepath.Dir(path), os.ModeDir|os.ModePerm); err != nil {
			p.logger.Error("mkdir failed",
				zap.String("dir", filepath.Dir(path)),
				zap.Error(err),
				zap.String("path", path),
			)
			return
		}

		w, err = whisper.CreateWithOptions(path, schema.Retentions, aggr.aggregationMethod, float32(aggr.xFilesFactor), &whisper.Options{
			Sparse: p.sparse,
			FLock:  p.flock,
		})
		if err != nil {
			p.logger.Error("create new whisper file failed",
				zap.String("path", path),
				zap.Error(err),
				zap.String("retention", schema.RetentionStr),
				zap.String("schema", schema.Name),
				zap.String("aggregation", aggr.name),
				zap.Float64("xFilesFactor", aggr.xFilesFactor),
				zap.String("method", aggr.aggregationMethodStr),
			)
			return
		}

		if p.tagsEnabled && p.onCreateTagged != nil && strings.IndexByte(metric, ';') >= 0 {
			p.onCreateTagged(metric)
		}

		p.createLogger.Debug("created",
			zap.String("path", path),
			zap.String("retention", schema.RetentionStr),
			zap.String("schema", schema.Name),
			zap.String("aggregation", aggr.name),
			zap.Float64("xFilesFactor", aggr.xFilesFactor),
			zap.String("method", aggr.aggregationMethodStr),
		)

		atomic.AddUint32(&p.created, 1)
	}

	values, exists := p.pop(metric)
	if !exists {
		return
	}
	if p.confirm != nil {
		defer p.confirm(values)
	}

	points := make([]*whisper.TimeSeriesPoint, len(values.Data))
	for i, r := range values.Data {
		points[i] = &whisper.TimeSeriesPoint{Time: int(r.Timestamp), Value: r.Value}
	}

	atomic.AddUint32(&p.committedPoints, uint32(len(values.Data)))
	atomic.AddUint32(&p.updateOperations, 1)

	// start = time.Now()
	p.updateMany(w, path, points)

	w.Close()
	// atomic.AddUint64(&p.blockUpdateManyNs, uint64(time.Since(start).Nanoseconds()))
}

func (p *Whisper) worker(exit chan bool) {
	storeFunc := p.store
	var doneCb func()
	if p.mockStore != nil {
		storeFunc, doneCb = p.mockStore()
	}

LOOP:
	for {
		// start := time.Now()
		select {
		case <-p.throttleTicker.C:
			// atomic.AddUint64(&p.blockThrottleNs, uint64(time.Since(start).Nanoseconds()))
			// pass
		case <-exit:
			return
		}

		// start = time.Now()
		metric := p.recv(exit)
		// atomic.AddUint64(&p.blockQueueGetNs, uint64(time.Since(start).Nanoseconds()))
		if metric == "" {
			// exit closed
			break LOOP
		}
		storeFunc(metric)
		if doneCb != nil {
			doneCb()
		}
	}
}

// Stat callback
func (p *Whisper) Stat(send helper.StatCallback) {
	updateOperations := atomic.LoadUint32(&p.updateOperations)
	committedPoints := atomic.LoadUint32(&p.committedPoints)
	atomic.AddUint32(&p.updateOperations, -updateOperations)
	atomic.AddUint32(&p.committedPoints, -committedPoints)

	created := atomic.LoadUint32(&p.created)
	atomic.AddUint32(&p.created, -created)

	throttledCreates := atomic.LoadUint32(&p.throttledCreates)
	atomic.AddUint32(&p.throttledCreates, -throttledCreates)

	send("updateOperations", float64(updateOperations))
	send("committedPoints", float64(committedPoints))
	if updateOperations > 0 {
		send("pointsPerUpdate", float64(committedPoints)/float64(updateOperations))
	} else {
		send("pointsPerUpdate", 0.0)
	}

	send("created", float64(created))
	send("throttledCreates", float64(throttledCreates))
	send("maxCreatesPerSecond", float64(p.maxCreatesPerSecond))

	send("maxUpdatesPerSecond", float64(p.maxUpdatesPerSecond))
	send("workers", float64(p.workersCount))

	// helper.SendAndSubstractUint64("blockThrottleNs", &p.blockThrottleNs, send)
	// helper.SendAndSubstractUint64("blockQueueGetNs", &p.blockQueueGetNs, send)
	// helper.SendAndSubstractUint64("blockAvoidConcurrentNs", &p.blockAvoidConcurrentNs, send)
	// helper.SendAndSubstractUint64("blockUpdateManyNs", &p.blockUpdateManyNs, send)

}

// Start worker
func (p *Whisper) Start() error {
	return p.StartFunc(func() error {

		p.throttleTicker = NewThrottleTicker(p.maxUpdatesPerSecond)
		if p.hardMaxCreatesPerSecond {
			p.maxCreatesTicker = NewHardThrottleTicker(p.maxCreatesPerSecond)
		} else {
			p.maxCreatesTicker = NewThrottleTicker(p.maxCreatesPerSecond)
		}

		for i := 0; i < p.workersCount; i++ {
			p.Go(p.worker)
		}

		return nil
	})
}

func (p *Whisper) Stop() {
	p.StopFunc(func() {
		p.throttleTicker.Stop()
		p.maxCreatesTicker.Stop()
	})
}
<|endoftext|>


<commit_before>// Copyright 2012 Junqing Tan <ivan@mysqlab.net> and The Go Authors
// Use of this source code is governed by a BSD-style
// Part of source code is from Go fcgi package

package fcgiclient

import (
	"bufio"
	"bytes"
	"encoding/binary"
	"sync"
	"errors"
	"io"
	"net"
	"strconv"
)

const FCGI_LISTENSOCK_FILENO uint8 = 0
const FCGI_HEADER_LEN uint8 = 8
const VERSION_1 uint8 = 1
const FCGI_NULL_REQUEST_ID uint8 = 0
const FCGI_KEEP_CONN uint8 = 1

const (
	FCGI_BEGIN_REQUEST uint8 = iota + 1
	FCGI_ABORT_REQUEST
	FCGI_END_REQUEST
	FCGI_PARAMS
	FCGI_STDIN
	FCGI_STDOUT
	FCGI_STDERR
	FCGI_DATA
	FCGI_GET_VALUES
	FCGI_GET_VALUES_RESULT
	FCGI_UNKNOWN_TYPE
	FCGI_MAXTYPE = FCGI_UNKNOWN_TYPE
)

const (
	FCGI_RESPONDER uint8 = iota + 1
	FCGI_AUTHORIZER
	FCGI_FILTER
)

const (
	FCGI_REQUEST_COMPLETE uint8 = iota
	FCGI_CANT_MPX_CONN
	FCGI_OVERLOADED
	FCGI_UNKNOWN_ROLE
)

const (
	FCGI_MAX_CONNS  string = "MAX_CONNS"
	FCGI_MAX_REQS   string = "MAX_REQS"
	FCGI_MPXS_CONNS string = "MPXS_CONNS"
)

const (
	maxWrite = 6553500 // maximum record body
	maxPad   = 255
)

type header struct {
	Version       uint8
	Type          uint8
	Id            uint16
	ContentLength uint16
	PaddingLength uint8
	Reserved      uint8
}

// for padding so we don't have to allocate all the time
// not synchronized because we don't care what the contents are
var pad [maxPad]byte

func (h *header) init(recType uint8, reqId uint16, contentLength int) {
	h.Version = 1
	h.Type = recType
	h.Id = reqId
	h.ContentLength = uint16(contentLength)
	h.PaddingLength = uint8(-contentLength & 7)
}

type record struct {
	h   header
	buf [maxWrite + maxPad]byte
}

func (rec *record) read(r io.Reader) (err error) {
	if err = binary.Read(r, binary.BigEndian, &rec.h); err != nil {
		return err
	}
	if rec.h.Version != 1 {
		return errors.New("fcgi: invalid header version")
	}
	n := int(rec.h.ContentLength) + int(rec.h.PaddingLength)
	if _, err = io.ReadFull(r, rec.buf[:n]); err != nil {
		return err
	}
	return nil
}

func (r *record) content() []byte {
	return r.buf[:r.h.ContentLength]
}

type FCGIClient struct {
	mutex     sync.Mutex
	rwc       io.ReadWriteCloser
	h         header
	buf 	  bytes.Buffer
	keepAlive bool
}

func New(h string, args ...interface{}) (fcgi *FCGIClient, err error) {
	var conn net.Conn
	if len(args) != 1 {
		err = errors.New("fcgi: not enough params")
		return
	}
	switch args[0].(type) {
	case int:
		addr := h + ":" + strconv.FormatInt(int64(args[0].(int)), 10)
		conn, err = net.Dial("tcp", addr)
	case string:
		addr := h + ":" + args[0].(string)
		conn, err = net.Dial("unix", addr)
	default:
		err = errors.New("fcgi: we only accept int (port) or string (socket) params.")
	}
	fcgi = &FCGIClient{
		rwc:       conn,
		keepAlive: false,
	}
	return
}

func (this *FCGIClient) writeRecord(recType uint8, reqId uint16, content []byte) ( err error) {
	this.mutex.Lock()
	defer this.mutex.Unlock()
	this.buf.Reset()
	this.h.init(recType, reqId, len(content))
	if err := binary.Write(&this.buf, binary.BigEndian, this.h); err != nil {
		return err
	}
	if _, err := this.buf.Write(content); err != nil {
		return err
	}
	if _, err := this.buf.Write(pad[:this.h.PaddingLength]); err != nil {
		return err
	}
	_, err = this.rwc.Write(this.buf.Bytes())
	return err
}

func (this *FCGIClient) writeBeginRequest(reqId uint16, role uint16, flags uint8) error {
	b := [8]byte{byte(role >> 8), byte(role), flags}
	return this.writeRecord(FCGI_BEGIN_REQUEST, reqId, b[:])
}

func (this *FCGIClient) writeEndRequest(reqId uint16, appStatus int, protocolStatus uint8) error {
	b := make([]byte, 8)
	binary.BigEndian.PutUint32(b, uint32(appStatus))
	b[4] = protocolStatus
	return this.writeRecord(FCGI_END_REQUEST, reqId, b)
}

func (this *FCGIClient) writePairs(recType uint8, reqId uint16, pairs map[string]string) error {
	w := newWriter(this, recType, reqId)
	b := make([]byte, 8)
	for k, v := range pairs {
		n := encodeSize(b, uint32(len(k)))
		n += encodeSize(b[n:], uint32(len(v)))
		if _, err := w.Write(b[:n]); err != nil {
			return err
		}
		if _, err := w.WriteString(k); err != nil {
			return err
		}
		if _, err := w.WriteString(v); err != nil {
			return err
		}
	}
	w.Close()
	return nil
}

func readSize(s []byte) (uint32, int) {
	if len(s) == 0 {
		return 0, 0
	}
	size, n := uint32(s[0]), 1
	if size&(1<<7) != 0 {
		if len(s) < 4 {
			return 0, 0
		}
		n = 4
		size = binary.BigEndian.Uint32(s)
		size &^= 1 << 31
	}
	return size, n
}

func readString(s []byte, size uint32) string {
	if size > uint32(len(s)) {
		return ""
	}
	return string(s[:size])
}

func encodeSize(b []byte, size uint32) int {
	if size > 127 {
		size |= 1 << 31
		binary.BigEndian.PutUint32(b, size)
		return 4
	}
	b[0] = byte(size)
	return 1
}

// bufWriter encapsulates bufio.Writer but also closes the underlying stream when
// Closed.
type bufWriter struct {
	closer io.Closer
	*bufio.Writer
}

func (w *bufWriter) Close() error {
	if err := w.Writer.Flush(); err != nil {
		w.closer.Close()
		return err
	}
	return w.closer.Close()
}

func newWriter(c *FCGIClient, recType uint8, reqId uint16) *bufWriter {
	s := &streamWriter{c: c, recType: recType, reqId: reqId}
	w := bufio.NewWriterSize(s, maxWrite)
	return &bufWriter{s, w}
}

// streamWriter abstracts out the separation of a stream into discrete records.
// It only writes maxWrite bytes at a time.
type streamWriter struct {
	c       *FCGIClient
	recType uint8
	reqId   uint16
}

func (w *streamWriter) Write(p []byte) (int, error) {
	nn := 0
	for len(p) > 0 {
		n := len(p)
		if n > maxWrite {
			n = maxWrite
		}
		if err := w.c.writeRecord(w.recType, w.reqId, p[:n]); err != nil {
			return nn, err
		}
		nn += n
		p = p[n:]
	}
	return nn, nil
}

func (w *streamWriter) Close() error {
	// send empty record to close the stream
	return w.c.writeRecord(w.recType, w.reqId, nil)
}

func (this *FCGIClient) Request(env map[string]string, reqStr string) (ret []byte, err error) {

	var reqId uint16 = 1

	err = this.writeBeginRequest(reqId, uint16(FCGI_RESPONDER), 0)
	if err != nil {
		return
	}
	err = this.writePairs(FCGI_PARAMS, reqId, env)
	if err != nil {
		return
	}
	if len(reqStr) > 0 {
		err = this.writeRecord(FCGI_STDIN, reqId, []byte(reqStr))
		if err != nil {
			return
		}
	}

	rec := &record{}
	err = rec.read(this.rwc)
	if err != nil {
		return
	}

	ret = rec.content()

	return
}
<commit_msg>able to accept chunked fcgi response; fix: listen to unix socket<commit_after>// Copyright 2012 Junqing Tan <ivan@mysqlab.net> and The Go Authors
// Use of this source code is governed by a BSD-style
// Part of source code is from Go fcgi package

package fcgiclient

import (
	"bufio"
	"bytes"
	"encoding/binary"
	"sync"
	"errors"
	"io"
	"net"
	"strconv"
)

const FCGI_LISTENSOCK_FILENO uint8 = 0
const FCGI_HEADER_LEN uint8 = 8
const VERSION_1 uint8 = 1
const FCGI_NULL_REQUEST_ID uint8 = 0
const FCGI_KEEP_CONN uint8 = 1

const (
	FCGI_BEGIN_REQUEST uint8 = iota + 1
	FCGI_ABORT_REQUEST
	FCGI_END_REQUEST
	FCGI_PARAMS
	FCGI_STDIN
	FCGI_STDOUT
	FCGI_STDERR
	FCGI_DATA
	FCGI_GET_VALUES
	FCGI_GET_VALUES_RESULT
	FCGI_UNKNOWN_TYPE
	FCGI_MAXTYPE = FCGI_UNKNOWN_TYPE
)

const (
	FCGI_RESPONDER uint8 = iota + 1
	FCGI_AUTHORIZER
	FCGI_FILTER
)

const (
	FCGI_REQUEST_COMPLETE uint8 = iota
	FCGI_CANT_MPX_CONN
	FCGI_OVERLOADED
	FCGI_UNKNOWN_ROLE
)

const (
	FCGI_MAX_CONNS  string = "MAX_CONNS"
	FCGI_MAX_REQS   string = "MAX_REQS"
	FCGI_MPXS_CONNS string = "MPXS_CONNS"
)

const (
	maxWrite = 6553500 // maximum record body
	maxPad   = 255
)

type header struct {
	Version       uint8
	Type          uint8
	Id            uint16
	ContentLength uint16
	PaddingLength uint8
	Reserved      uint8
}

// for padding so we don't have to allocate all the time
// not synchronized because we don't care what the contents are
var pad [maxPad]byte

func (h *header) init(recType uint8, reqId uint16, contentLength int) {
	h.Version = 1
	h.Type = recType
	h.Id = reqId
	h.ContentLength = uint16(contentLength)
	h.PaddingLength = uint8(-contentLength & 7)
}

type record struct {
	h   header
	buf [maxWrite + maxPad]byte
}

func (rec *record) read(r io.Reader) (buf []byte, err error) {
	if err = binary.Read(r, binary.BigEndian, &rec.h); err != nil {
		return
	}
	if rec.h.Version != 1 {
    err = errors.New("fcgi: invalid header version")
		return
	}
  if rec.h.Type == FCGI_END_REQUEST {
    err = io.EOF
    return
  }
  n := int(rec.h.ContentLength) + int(rec.h.PaddingLength)
	if n, err = io.ReadFull(r, rec.buf[:n]); err != nil {
		return
	}
  buf = rec.buf[:int(rec.h.ContentLength)]

	return
}

type FCGIClient struct {
	mutex     sync.Mutex
	rwc       io.ReadWriteCloser
	h         header
	buf 	  bytes.Buffer
	keepAlive bool
}

func New(h string, args ...interface{}) (fcgi *FCGIClient, err error) {
	var conn net.Conn
	if len(args) != 1 {
		err = errors.New("fcgi: not enough params")
		return
	}
	switch args[0].(type) {
	case int:
		addr := h + ":" + strconv.FormatInt(int64(args[0].(int)), 10)
		conn, err = net.Dial("tcp", addr)
	case string:
		addr := args[0].(string)
		conn, err = net.Dial("unix", addr)
	default:
		err = errors.New("fcgi: we only accept int (port) or string (socket) params.")
	}
	fcgi = &FCGIClient{
		rwc:       conn,
		keepAlive: false,
	}
	return
}

func (this *FCGIClient) writeRecord(recType uint8, reqId uint16, content []byte) ( err error) {
	this.mutex.Lock()
	defer this.mutex.Unlock()
	this.buf.Reset()
	this.h.init(recType, reqId, len(content))
	if err := binary.Write(&this.buf, binary.BigEndian, this.h); err != nil {
		return err
	}
	if _, err := this.buf.Write(content); err != nil {
		return err
	}
	if _, err := this.buf.Write(pad[:this.h.PaddingLength]); err != nil {
		return err
	}
	_, err = this.rwc.Write(this.buf.Bytes())
	return err
}

func (this *FCGIClient) writeBeginRequest(reqId uint16, role uint16, flags uint8) error {
	b := [8]byte{byte(role >> 8), byte(role), flags}
	return this.writeRecord(FCGI_BEGIN_REQUEST, reqId, b[:])
}

func (this *FCGIClient) writeEndRequest(reqId uint16, appStatus int, protocolStatus uint8) error {
	b := make([]byte, 8)
	binary.BigEndian.PutUint32(b, uint32(appStatus))
	b[4] = protocolStatus
	return this.writeRecord(FCGI_END_REQUEST, reqId, b)
}

func (this *FCGIClient) writePairs(recType uint8, reqId uint16, pairs map[string]string) error {
	w := newWriter(this, recType, reqId)
	b := make([]byte, 8)
	for k, v := range pairs {
		n := encodeSize(b, uint32(len(k)))
		n += encodeSize(b[n:], uint32(len(v)))
		if _, err := w.Write(b[:n]); err != nil {
			return err
		}
		if _, err := w.WriteString(k); err != nil {
			return err
		}
		if _, err := w.WriteString(v); err != nil {
			return err
		}
	}
	w.Close()
	return nil
}

func readSize(s []byte) (uint32, int) {
	if len(s) == 0 {
		return 0, 0
	}
	size, n := uint32(s[0]), 1
	if size&(1<<7) != 0 {
		if len(s) < 4 {
			return 0, 0
		}
		n = 4
		size = binary.BigEndian.Uint32(s)
		size &^= 1 << 31
	}
	return size, n
}

func readString(s []byte, size uint32) string {
	if size > uint32(len(s)) {
		return ""
	}
	return string(s[:size])
}

func encodeSize(b []byte, size uint32) int {
	if size > 127 {
		size |= 1 << 31
		binary.BigEndian.PutUint32(b, size)
		return 4
	}
	b[0] = byte(size)
	return 1
}

// bufWriter encapsulates bufio.Writer but also closes the underlying stream when
// Closed.
type bufWriter struct {
	closer io.Closer
	*bufio.Writer
}

func (w *bufWriter) Close() error {
	if err := w.Writer.Flush(); err != nil {
		w.closer.Close()
		return err
	}
	return w.closer.Close()
}

func newWriter(c *FCGIClient, recType uint8, reqId uint16) *bufWriter {
	s := &streamWriter{c: c, recType: recType, reqId: reqId}
	w := bufio.NewWriterSize(s, maxWrite)
	return &bufWriter{s, w}
}

// streamWriter abstracts out the separation of a stream into discrete records.
// It only writes maxWrite bytes at a time.
type streamWriter struct {
	c       *FCGIClient
	recType uint8
	reqId   uint16
}

func (w *streamWriter) Write(p []byte) (int, error) {
	nn := 0
	for len(p) > 0 {
		n := len(p)
		if n > maxWrite {
			n = maxWrite
		}
		if err := w.c.writeRecord(w.recType, w.reqId, p[:n]); err != nil {
			return nn, err
		}
		nn += n
		p = p[n:]
	}
	return nn, nil
}

func (w *streamWriter) Close() error {
	// send empty record to close the stream
	return w.c.writeRecord(w.recType, w.reqId, nil)
}

func (this *FCGIClient) Request(env map[string]string, reqStr string) (ret []byte, err error) {

	var reqId uint16 = 1

	err = this.writeBeginRequest(reqId, uint16(FCGI_RESPONDER), 0)
	if err != nil {
		return
	}
	err = this.writePairs(FCGI_PARAMS, reqId, env)
	if err != nil {
		return
	}
	if len(reqStr) > 0 {
		err = this.writeRecord(FCGI_STDIN, reqId, []byte(reqStr))
		if err != nil {
			return
		}
	}

	rec := &record{}
	for {
    buf, err := rec.read(this.rwc)
  	if err != nil {
  		break
  	}
    ret = append(ret, buf...)
  }

	return
}
<|endoftext|>