<commit_before>package models

import (
	// Go Builtin Packages
	"bytes"
	"compress/gzip"
	"compress/zlib"
	"crypto/sha256"
	"encoding/base64"
	"fmt"
	"io"
	"log"
	"net/http"
	"regexp"
	"strings"
	"time"

	"github.com/lib/pq"

	// Local Packages
	"github.com/adayoung/gae-pastebin/pastebin/utils"
	"github.com/adayoung/gae-pastebin/pastebin/utils/storage"
)

type Paste struct {
	PasteID  string    `db:"paste_id"`
	UserID   string    `db:"user_id"`
	Title    string    `db:"title"`
	Content  []byte    `db:"content"`
	Tags     []string  `db:"tags"`
	Format   string    `db:"format"`
	Date     time.Time `db:"date"`
	Gzip     bool      `db:"gzip"`
	Zlib     bool      `db:"zlib"`
	uContent string    `db:"-"` // Private content, for validation and processing
	GDriveID string    `db:"gdriveid"`
	GDriveDL string    `db:"gdrivedl"`
}

func genpasteKey(p *Paste) string {
	timestamp := p.Date.Format(time.StampNano)

	maxContentHashLen := 2 * 1024 // let's read only upto the first 2kbs
	hasher := sha256.New()
	hasher.Write([]byte(timestamp))
	if len(p.Content) > maxContentHashLen {
		hasher.Write(p.Content[:maxContentHashLen])
	} else {
		hasher.Write(p.Content)
	}
	// This, because Gwawr was here, and Gwawr is awesome!
	digest := base64.URLEncoding.WithPadding(base64.NoPadding).EncodeToString(hasher.Sum(nil))

	paste_id := digest[:8] // This is probably a silly way to go about it xD
	// We're such trolls, we don't even check for collisions ^_^
	// FIXME: check for collisions D:
	return paste_id
}

type ValidationError struct {
	What string // What is invalid
	Why  string // Why is it invalid
}

func (e *ValidationError) Error() string {
	return fmt.Sprintf("%s - %s", e.What, e.Why)
}

func (p *Paste) Validate() error {
	// ... this looks more like a cleaner than a validator O_o

	// A paste must have content!@
	if !(len(p.uContent) > 0) {
		return &ValidationError{"Content", "Oops, we need 'content' for this."}
	}

	// Title - truncate title to 50
	if len(p.Title) > 50 {
		p.Title = p.Title[:50]
	}

	// Force format to 'plain' if nothing valid is specified
	if !(p.Format == "plain" || p.Format == "html") {
		p.Format = "plain"
	}

	// Tags - accept a maximum of 15 tags only, each of max length 15
	// Tags - must consist of alphanumeric characters only
	filter_exp := regexp.MustCompile("[^A-Za-z0-9]+")
	for index := 0; index < len(p.Tags); index++ {
		p.Tags[index] = filter_exp.ReplaceAllString(p.Tags[index], "")
		p.Tags[index] = strings.ToLower(strings.TrimSpace(p.Tags[index]))
		if len(p.Tags[index]) > 15 {
			p.Tags[index] = p.Tags[index][:15]
		}
	}
	if len(p.Tags) > 15 {
		p.Tags = p.Tags[:15]
	}

	var u_tags []string
	u_tag_map := make(map[string]string)
	for _, tag := range p.Tags {
		if u_tag_map[tag] == "" {
			if len(tag) > 0 {
				u_tags = append(u_tags, tag)
			}
		}
		u_tag_map[tag] = tag
	}

	p.Tags = u_tags // Yay no more duplicates!

	// return &ValidationError{"Huh?", "Why?"}
	return nil
}

func (p *Paste) save(r *http.Request, score float64) (string, error) {
	if err := p.Validate(); err == nil {
		// Compress content here, AFTER validation

		paste_id := genpasteKey(p)
		log.Printf("INFO: Creating new paste with paste_id [%s] [%.1f]", paste_id, score)
		p.PasteID = paste_id

		havetoken := (r.Form.Get("destination") == "gdrive")

		p.Gzip = false
		p.Zlib = false
		if havetoken == true {
			// TODO: This should should probably happen in a goroutine
			err = p.saveToDrive(r, paste_id)
			if err != nil {
				return "", err
			}
		} else {
			var content bytes.Buffer
			w := gzip.NewWriter(&content)
			w.Write([]byte(p.uContent))
			w.Close()
			p.Content = content.Bytes()
			p.Gzip = true
		}

		if err := p.saveToDB(score); err != nil {
			return "", err
		}

		return paste_id, nil
	} else {
		return "", err
	}
}

func (p *Paste) saveToDB(score float64) error {
	if len(p.Content) > (2 * 1024 * 1024) {
		return fmt.Errorf("Paste content is still over 2MB after compression.")
	}
	pasteSQL := `INSERT INTO pastebin (
			paste_id, user_id, title, content, tags,
			format, date, gzip, zlib, gdriveid, gdrivedl, rcscore
		) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
	`
	pasteSQL = storage.DB.Rebind(pasteSQL)
	_, err := storage.DB.Exec(pasteSQL,
		p.PasteID, p.UserID, p.Title, p.Content, pq.Array(p.Tags),
		p.Format, p.Date, p.Gzip, p.Zlib, p.GDriveID, p.GDriveDL, score,
	)
	return err
}

type pasteContent interface {
	Write([]byte) (int, error)
}

func (p *Paste) ZContent(r *http.Request, pc pasteContent) error {
	if len(p.GDriveID) > 0 {
		if err := p.loadFromDrive(r); err != nil {
			return err
		}
	}

	if !(len(p.Content) > 0) {
		if err := p.loadContent(); err != nil {
			return err
		}
	}

	if p.Zlib {
		// Decompress content and write out the response
		var zbuffer io.Reader
		zbuffer = bytes.NewReader(p.Content)
		ureader, _ := zlib.NewReader(zbuffer)
		io.Copy(pc, ureader)
	} else if p.Format == "plain" && p.Gzip {
		var zbuffer io.Reader
		zbuffer = bytes.NewReader(p.Content)
		ureader, _ := gzip.NewReader(zbuffer)
		io.Copy(pc, ureader)
	} else {
		buffer := bytes.NewReader(p.Content)
		io.Copy(pc, buffer)
	}

	return nil
}

func NewPaste(r *http.Request, score float64) (string, error) {
	var paste Paste

	if err := utils.ProcessForm(r); err != nil {
		return "", err
	}

	// TODO: we do some magic with the received score here :D

	// if usr := user.Current(c); usr != nil {
	// 	paste.UserID = usr.ID
	// }

	paste.Title = r.Form.Get("title")
	paste.uContent = r.Form.Get("content")

	paste.Tags = strings.Split(r.Form.Get("tags"), " ")
	paste.Format = r.Form.Get("format")
	paste.Date = time.Now()

	paste_id, err := paste.save(r, score)
	if err != nil {
		return "", err
	}

	return paste_id, nil
}

func GetPaste(paste_id string, withContent, withTags bool) (*Paste, error) {
	var paste Paste

	query := "SELECT"
	query = query + " paste_id, user_id, title,"
	if withContent {
		query = query + " content,"
	}
	query = query + " format, date, gzip, zlib, gdriveid, gdrivedl"
	query = query + " FROM pastebin WHERE paste_id=?"

	query = storage.DB.Rebind(query)
	err := storage.DB.QueryRowx(query, paste_id).StructScan(&paste)

	if withTags {
		if err == nil { // FIXME: There has to be a better way of doing this
			query = "SELECT tags FROM pastebin WHERE paste_id=?"
			query = storage.DB.Rebind(query)
			err = storage.DB.QueryRow(query, paste_id).Scan(pq.Array(&paste.Tags))
		}
	}

	return &paste, err
}

func (p *Paste) loadContent() error {
	query := "SELECT content FROM pastebin WHERE paste_id=?"
	query = storage.DB.Rebind(query)
	err := storage.DB.QueryRow(query, p.PasteID).Scan(&p.Content)
	return err
}

func (p *Paste) Delete() error {
	paste_id := p.PasteID

	query := "DELETE FROM pastebin WHERE paste_id=?"
	query = storage.DB.Rebind(query)
	if _, err := storage.DB.Exec(query, paste_id); err == nil {
		log.Printf("INFO: Delete paste with paste_id [%s]", paste_id)
	} else {
		return err
	}
	/*
		// FIXME: This is duplicated from pastebin.clean
		// Clear counter shards here
		var shardc_dkeys []*datastore.Key
		c_key := datastore.NewKey(c, "GeneralCounterShardConfig", paste_id, 0, nil)
		shardc_dkeys = append(shardc_dkeys, c_key)

		shard_keys := datastore.NewQuery("GeneralCounterShard").Filter("Name =", paste_id).KeysOnly()
		if shard_dkeys, cerr := shard_keys.GetAll(c, nil); cerr == nil {
			derr := datastore.DeleteMulti(c, shard_dkeys)
			if derr != nil {
				return derr
			}
		} else {
			return cerr
		}

		err = datastore.DeleteMulti(c, shardc_dkeys)
		if err != nil {
			return err
		}
	*/
	return nil
}
<commit_msg>Right, let's not be fancy with scanning for tags..<commit_after>package models

import (
	// Go Builtin Packages
	"bytes"
	"compress/gzip"
	"compress/zlib"
	"crypto/sha256"
	"encoding/base64"
	"fmt"
	"io"
	"log"
	"net/http"
	"regexp"
	"strings"
	"time"

	"github.com/lib/pq"

	// Local Packages
	"github.com/adayoung/gae-pastebin/pastebin/utils"
	"github.com/adayoung/gae-pastebin/pastebin/utils/storage"
)

type Paste struct {
	PasteID  string    `db:"paste_id"`
	UserID   string    `db:"user_id"`
	Title    string    `db:"title"`
	Content  []byte    `db:"content"`
	Tags     []string  `db:"tags"`
	Format   string    `db:"format"`
	Date     time.Time `db:"date"`
	Gzip     bool      `db:"gzip"`
	Zlib     bool      `db:"zlib"`
	uContent string    `db:"-"` // Private content, for validation and processing
	GDriveID string    `db:"gdriveid"`
	GDriveDL string    `db:"gdrivedl"`
}

func genpasteKey(p *Paste) string {
	timestamp := p.Date.Format(time.StampNano)

	maxContentHashLen := 2 * 1024 // let's read only upto the first 2kbs
	hasher := sha256.New()
	hasher.Write([]byte(timestamp))
	if len(p.Content) > maxContentHashLen {
		hasher.Write(p.Content[:maxContentHashLen])
	} else {
		hasher.Write(p.Content)
	}
	// This, because Gwawr was here, and Gwawr is awesome!
	digest := base64.URLEncoding.WithPadding(base64.NoPadding).EncodeToString(hasher.Sum(nil))

	paste_id := digest[:8] // This is probably a silly way to go about it xD
	// We're such trolls, we don't even check for collisions ^_^
	// FIXME: check for collisions D:
	return paste_id
}

type ValidationError struct {
	What string // What is invalid
	Why  string // Why is it invalid
}

func (e *ValidationError) Error() string {
	return fmt.Sprintf("%s - %s", e.What, e.Why)
}

func (p *Paste) Validate() error {
	// ... this looks more like a cleaner than a validator O_o

	// A paste must have content!@
	if !(len(p.uContent) > 0) {
		return &ValidationError{"Content", "Oops, we need 'content' for this."}
	}

	// Title - truncate title to 50
	if len(p.Title) > 50 {
		p.Title = p.Title[:50]
	}

	// Force format to 'plain' if nothing valid is specified
	if !(p.Format == "plain" || p.Format == "html") {
		p.Format = "plain"
	}

	// Tags - accept a maximum of 15 tags only, each of max length 15
	// Tags - must consist of alphanumeric characters only
	filter_exp := regexp.MustCompile("[^A-Za-z0-9]+")
	for index := 0; index < len(p.Tags); index++ {
		p.Tags[index] = filter_exp.ReplaceAllString(p.Tags[index], "")
		p.Tags[index] = strings.ToLower(strings.TrimSpace(p.Tags[index]))
		if len(p.Tags[index]) > 15 {
			p.Tags[index] = p.Tags[index][:15]
		}
	}
	if len(p.Tags) > 15 {
		p.Tags = p.Tags[:15]
	}

	var u_tags []string
	u_tag_map := make(map[string]string)
	for _, tag := range p.Tags {
		if u_tag_map[tag] == "" {
			if len(tag) > 0 {
				u_tags = append(u_tags, tag)
			}
		}
		u_tag_map[tag] = tag
	}

	p.Tags = u_tags // Yay no more duplicates!

	// return &ValidationError{"Huh?", "Why?"}
	return nil
}

func (p *Paste) save(r *http.Request, score float64) (string, error) {
	if err := p.Validate(); err == nil {
		// Compress content here, AFTER validation

		paste_id := genpasteKey(p)
		log.Printf("INFO: Creating new paste with paste_id [%s] [%.1f]", paste_id, score)
		p.PasteID = paste_id

		havetoken := (r.Form.Get("destination") == "gdrive")

		p.Gzip = false
		p.Zlib = false
		if havetoken == true {
			// TODO: This should should probably happen in a goroutine
			err = p.saveToDrive(r, paste_id)
			if err != nil {
				return "", err
			}
		} else {
			var content bytes.Buffer
			w := gzip.NewWriter(&content)
			w.Write([]byte(p.uContent))
			w.Close()
			p.Content = content.Bytes()
			p.Gzip = true
		}

		if err := p.saveToDB(score); err != nil {
			return "", err
		}

		return paste_id, nil
	} else {
		return "", err
	}
}

func (p *Paste) saveToDB(score float64) error {
	if len(p.Content) > (2 * 1024 * 1024) {
		return fmt.Errorf("Paste content is still over 2MB after compression.")
	}
	pasteSQL := `INSERT INTO pastebin (
			paste_id, user_id, title, content, tags,
			format, date, gzip, zlib, gdriveid, gdrivedl, rcscore
		) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
	`
	pasteSQL = storage.DB.Rebind(pasteSQL)
	_, err := storage.DB.Exec(pasteSQL,
		p.PasteID, p.UserID, p.Title, p.Content, pq.Array(p.Tags),
		p.Format, p.Date, p.Gzip, p.Zlib, p.GDriveID, p.GDriveDL, score,
	)
	return err
}

type pasteContent interface {
	Write([]byte) (int, error)
}

func (p *Paste) ZContent(r *http.Request, pc pasteContent) error {
	if len(p.GDriveID) > 0 {
		if err := p.loadFromDrive(r); err != nil {
			return err
		}
	}

	if !(len(p.Content) > 0) {
		if err := p.loadContent(); err != nil {
			return err
		}
	}

	if p.Zlib {
		// Decompress content and write out the response
		var zbuffer io.Reader
		zbuffer = bytes.NewReader(p.Content)
		ureader, _ := zlib.NewReader(zbuffer)
		io.Copy(pc, ureader)
	} else if p.Format == "plain" && p.Gzip {
		var zbuffer io.Reader
		zbuffer = bytes.NewReader(p.Content)
		ureader, _ := gzip.NewReader(zbuffer)
		io.Copy(pc, ureader)
	} else {
		buffer := bytes.NewReader(p.Content)
		io.Copy(pc, buffer)
	}

	return nil
}

func NewPaste(r *http.Request, score float64) (string, error) {
	var paste Paste

	if err := utils.ProcessForm(r); err != nil {
		return "", err
	}

	// TODO: we do some magic with the received score here :D

	// if usr := user.Current(c); usr != nil {
	// 	paste.UserID = usr.ID
	// }

	paste.Title = r.Form.Get("title")
	paste.uContent = r.Form.Get("content")

	paste.Tags = strings.Split(r.Form.Get("tags"), " ")
	paste.Format = r.Form.Get("format")
	paste.Date = time.Now()

	paste_id, err := paste.save(r, score)
	if err != nil {
		return "", err
	}

	return paste_id, nil
}

func GetPaste(paste_id string, withContent, withTags bool) (*Paste, error) {
	var paste Paste

	query := "SELECT"
	query = query + " paste_id, user_id, title, tags,"
	if withContent {
		query = query + " content,"
	}
	query = query + " format, date, gzip, zlib, gdriveid, gdrivedl"
	query = query + " FROM pastebin WHERE paste_id=?"

	query = storage.DB.Rebind(query)
	row := storage.DB.QueryRow(query, paste_id)

	var err error
	if withContent {
		err = row.Scan(
			&paste.PasteID, &paste.UserID, &paste.Title, pq.Array(&paste.Tags),
			&paste.Content, &paste.Format, &paste.Date, &paste.Gzip,
			&paste.Zlib, &paste.GDriveID, &paste.GDriveDL,
		)
	} else {
		err = row.Scan(
			&paste.PasteID, &paste.UserID, &paste.Title, pq.Array(&paste.Tags),
			&paste.Format, &paste.Date, &paste.Gzip, &paste.Zlib,
			&paste.GDriveID, &paste.GDriveDL,
		)
	}

	return &paste, err
}

func (p *Paste) loadContent() error {
	query := "SELECT content FROM pastebin WHERE paste_id=?"
	query = storage.DB.Rebind(query)
	err := storage.DB.QueryRow(query, p.PasteID).Scan(&p.Content)
	return err
}

func (p *Paste) Delete() error {
	paste_id := p.PasteID

	query := "DELETE FROM pastebin WHERE paste_id=?"
	query = storage.DB.Rebind(query)
	if _, err := storage.DB.Exec(query, paste_id); err == nil {
		log.Printf("INFO: Delete paste with paste_id [%s]", paste_id)
	} else {
		return err
	}
	/*
		// FIXME: This is duplicated from pastebin.clean
		// Clear counter shards here
		var shardc_dkeys []*datastore.Key
		c_key := datastore.NewKey(c, "GeneralCounterShardConfig", paste_id, 0, nil)
		shardc_dkeys = append(shardc_dkeys, c_key)

		shard_keys := datastore.NewQuery("GeneralCounterShard").Filter("Name =", paste_id).KeysOnly()
		if shard_dkeys, cerr := shard_keys.GetAll(c, nil); cerr == nil {
			derr := datastore.DeleteMulti(c, shard_dkeys)
			if derr != nil {
				return derr
			}
		} else {
			return cerr
		}

		err = datastore.DeleteMulti(c, shardc_dkeys)
		if err != nil {
			return err
		}
	*/
	return nil
}
<|endoftext|>


<commit_before>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package otlptext // import "go.opentelemetry.io/collector/exporter/loggingexporter/internal/otlptext"

import (
	"go.opentelemetry.io/collector/pdata/plog"
)

// NewTextLogsMarshaler returns a plog.Marshaler to encode to OTLP text bytes.
func NewTextLogsMarshaler() plog.Marshaler {
	return textLogsMarshaler{}
}

type textLogsMarshaler struct{}

// MarshalLogs plog.Logs to OTLP text.
func (textLogsMarshaler) MarshalLogs(ld plog.Logs) ([]byte, error) {
	buf := dataBuffer{}
	rls := ld.ResourceLogs()
	for i := 0; i < rls.Len(); i++ {
		buf.logEntry("ResourceLog #%d", i)
		rl := rls.At(i)
		buf.logEntry("Resource SchemaURL: %s", rl.SchemaUrl())
		buf.logAttributes("Resource attributes", rl.Resource().Attributes())
		ills := rl.ScopeLogs()
		for j := 0; j < ills.Len(); j++ {
			buf.logEntry("ScopeLogs #%d", j)
			ils := ills.At(j)
			buf.logEntry("ScopeLogs SchemaURL: %s", ils.SchemaUrl())
			buf.logInstrumentationScope(ils.Scope())

			logs := ils.LogRecords()
			for k := 0; k < logs.Len(); k++ {
				buf.logEntry("LogRecord #%d", k)
				lr := logs.At(k)
				buf.logEntry("ObservedTimestamp: %s", lr.ObservedTimestamp())
				buf.logEntry("Timestamp: %s", lr.Timestamp())
				buf.logEntry("Severity: %s", lr.SeverityText())
				buf.logEntry("Body: %s", attributeValueToString(lr.Body()))
				buf.logAttributes("Attributes", lr.Attributes())
				buf.logEntry("Trace ID: %s", lr.TraceID().HexString())
				buf.logEntry("Span ID: %s", lr.SpanID().HexString())
				buf.logEntry("Flags: %d", lr.Flags())
			}
		}
	}

	return buf.buf.Bytes(), nil
}
<commit_msg>Add log severity number to logging exporter (#6101)<commit_after>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package otlptext // import "go.opentelemetry.io/collector/exporter/loggingexporter/internal/otlptext"

import (
	"go.opentelemetry.io/collector/pdata/plog"
)

// NewTextLogsMarshaler returns a plog.Marshaler to encode to OTLP text bytes.
func NewTextLogsMarshaler() plog.Marshaler {
	return textLogsMarshaler{}
}

type textLogsMarshaler struct{}

// MarshalLogs plog.Logs to OTLP text.
func (textLogsMarshaler) MarshalLogs(ld plog.Logs) ([]byte, error) {
	buf := dataBuffer{}
	rls := ld.ResourceLogs()
	for i := 0; i < rls.Len(); i++ {
		buf.logEntry("ResourceLog #%d", i)
		rl := rls.At(i)
		buf.logEntry("Resource SchemaURL: %s", rl.SchemaUrl())
		buf.logAttributes("Resource attributes", rl.Resource().Attributes())
		ills := rl.ScopeLogs()
		for j := 0; j < ills.Len(); j++ {
			buf.logEntry("ScopeLogs #%d", j)
			ils := ills.At(j)
			buf.logEntry("ScopeLogs SchemaURL: %s", ils.SchemaUrl())
			buf.logInstrumentationScope(ils.Scope())

			logs := ils.LogRecords()
			for k := 0; k < logs.Len(); k++ {
				buf.logEntry("LogRecord #%d", k)
				lr := logs.At(k)
				buf.logEntry("ObservedTimestamp: %s", lr.ObservedTimestamp())
				buf.logEntry("Timestamp: %s", lr.Timestamp())
				buf.logEntry("SeverityText: %s", lr.SeverityText())
				buf.logEntry("SeverityNumber: %s(%d)", lr.SeverityNumber(), lr.SeverityNumber())
				buf.logEntry("Body: %s", attributeValueToString(lr.Body()))
				buf.logAttributes("Attributes", lr.Attributes())
				buf.logEntry("Trace ID: %s", lr.TraceID().HexString())
				buf.logEntry("Span ID: %s", lr.SpanID().HexString())
				buf.logEntry("Flags: %d", lr.Flags())
			}
		}
	}

	return buf.buf.Bytes(), nil
}
<|endoftext|>


<commit_before>package ecc

import (
	"bytes"
	b64 "encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"time"

	"github.com/socketplane/ecc/Godeps/_workspace/src/github.com/mitchellh/cli"
)

// Embedded Consul Client
// Quick and Dirty way to embed Consul with any golang based application without the
// additional step of installing the Consul application in the host system

// Consul Agent related functions

var started bool
var OfflineSupport bool = true

func Start(serverMode bool, bootstrap bool, bindInterface string, dataDir string) error {
	bindAddress := ""
	if bindInterface != "" {
		intf, err := net.InterfaceByName(bindInterface)
		if err != nil {
			log.Printf("Error : %v", err)
			return err
		}
		addrs, err := intf.Addrs()
		if err == nil {
			for i := 0; i < len(addrs); i++ {
				addr := addrs[i].String()
				ip, _, _ := net.ParseCIDR(addr)
				if ip != nil && ip.To4() != nil {
					bindAddress = ip.To4().String()
				}
			}
		}
	}
	errCh := make(chan int)
	go startConsul(serverMode, bootstrap, bindAddress, dataDir, errCh)

	select {
	case <-errCh:
		return errors.New("Error starting Consul Agent")
	case <-time.After(time.Second * 5):
	}
	go populateKVStoreFromCache()
	return nil
}

func startConsul(serverMode bool, bootstrap bool, bindAddress string, dataDir string, eCh chan int) {
	args := []string{"agent", "-data-dir", dataDir}

	if serverMode {
		args = append(args, "-server")
	}

	if bootstrap {
		args = append(args, "-bootstrap")
	}

	if bindAddress != "" {
		args = append(args, "-bind")
		args = append(args, bindAddress)
	}

	ret := Execute(args...)

	if ret != 0 {
		eCh <- ret
	}
}

func HasStarted() bool {
	return started
}

func Join(address string) error {
	ret := Execute("join", address)

	if ret != 0 {
		log.Println("Error (%d) joining %s with Consul peers", ret, address)
		return errors.New("Error adding member")
	}
	return nil
}

func Leave() error {
	ret := Execute("leave")
	if ret != 0 {
		log.Println("Error Leaving Consul membership")
		return errors.New("Error leaving Consul cluster")
	}
	time.Sleep(time.Second * 3)
	return nil
}

// Execute function is borrowed from Consul's main.go
func Execute(args ...string) int {

	for _, arg := range args {
		if arg == "-v" || arg == "--version" {
			newArgs := make([]string, len(args)+1)
			newArgs[0] = "version"
			copy(newArgs[1:], args)
			args = newArgs
			break
		}
	}

	cli := &cli.CLI{
		Args:     args,
		Commands: Commands,
		HelpFunc: cli.BasicHelpFunc("consul"),
	}

	exitCode, err := cli.Run()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error executing CLI: %s\n", err.Error())
		return 1
	}

	return exitCode
}

const CONSUL_BASE_URL = "http://localhost:8500/v1/"

func ConsulGet(url string) (string, bool) {
	resp, err := http.Get(url)
	if err != nil {
		log.Printf("Error (%v) in Get for %s\n", err, url)
		return "", false
	}
	defer resp.Body.Close()
	log.Printf("Status of Get %s %d for %s", resp.Status, resp.StatusCode, url)
	if resp.StatusCode >= 200 && resp.StatusCode < 300 {
		var jsonBody []consulBody
		body, err := ioutil.ReadAll(resp.Body)
		err = json.Unmarshal(body, &jsonBody)
		existingValue, err := b64.StdEncoding.DecodeString(jsonBody[0].Value)
		if err != nil {
			return "", false
		}
		return string(existingValue[:]), true
	} else {
		return "", false
	}
}

// Consul KV Store related

const CONSUL_KV_BASE_URL = "http://localhost:8500/v1/kv/"

type consulBody struct {
	CreateIndex int    `json:"CreateIndex,omitempty"`
	ModifyIndex int    `json:"ModifyIndex,omitempty"`
	Key         string `json:"Key,omitempty"`
	Flags       int    `json:"Flags,omitempty"`
	Value       string `json:"Value,omitempty"`
}

func GetAll(store string) ([][]byte, []int, bool) {
	if OfflineSupport && !started {
		return getAllFromCache(store)
	}
	url := CONSUL_KV_BASE_URL + store + "?recurse"
	resp, err := http.Get(url)
	if err != nil {
		log.Printf("Error (%v) in Get for %s\n", err, url)
		return nil, nil, false
	}
	defer resp.Body.Close()
	log.Printf("Status of Get %s %d for %s", resp.Status, resp.StatusCode, url)
	if resp.StatusCode >= 400 && resp.StatusCode < 500 {
		return nil, nil, false
	} else if resp.StatusCode >= 200 && resp.StatusCode < 300 {
		var jsonBody []consulBody
		valueArr := make([][]byte, 0)
		indexArr := make([]int, 0)
		body, _ := ioutil.ReadAll(resp.Body)
		err = json.Unmarshal(body, &jsonBody)
		for _, body := range jsonBody {
			existingValue, _ := b64.StdEncoding.DecodeString(body.Value)
			valueArr = append(valueArr, existingValue)
			indexArr = append(indexArr, body.ModifyIndex)
		}
		return valueArr, indexArr, true
	} else {
		return nil, nil, false
	}
}

func Get(store string, key string) ([]byte, int, bool) {
	if OfflineSupport && !started {
		return getFromCache(store, key)
	}
	url := CONSUL_KV_BASE_URL + store + "/" + key
	resp, err := http.Get(url)
	if err != nil {
		log.Printf("Error (%v) in Get for %s\n", err, url)
		return nil, 0, false
	}
	defer resp.Body.Close()
	log.Printf("Status of Get %s %d for %s", resp.Status, resp.StatusCode, url)
	if resp.StatusCode >= 400 && resp.StatusCode < 500 {
		return nil, 0, false
	} else if resp.StatusCode >= 200 && resp.StatusCode < 300 {
		var jsonBody []consulBody
		body, err := ioutil.ReadAll(resp.Body)
		err = json.Unmarshal(body, &jsonBody)
		existingValue, err := b64.StdEncoding.DecodeString(jsonBody[0].Value)
		if err != nil {
			return nil, jsonBody[0].ModifyIndex, false
		}
		return existingValue, jsonBody[0].ModifyIndex, true
	} else {
		return nil, 0, false
	}
}

const (
	OK = iota
	OUTDATED
	ERROR
)

type eccerror int

func Put(store string, key string, value []byte, oldValue []byte) eccerror {
	if OfflineSupport && !started {
		return putInCache(store, key, value, oldValue)
	}
	existingValue, casIndex, ok := Get(store, key)
	if ok && !bytes.Equal(oldValue, existingValue) {
		return OUTDATED
	}
	url := fmt.Sprintf("%s%s/%s?cas=%d", CONSUL_KV_BASE_URL, store, key, casIndex)
	log.Printf("Updating KV pair for %s %s %s %d", url, key, value, casIndex)
	req, err := http.NewRequest("PUT", url, bytes.NewBuffer(value))

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		log.Println("Error creating KV pair for ", url, err)
		return ERROR
	}
	defer resp.Body.Close()

	body, _ := ioutil.ReadAll(resp.Body)
	if string(body) == "false" {
		// Let the application retry based on return value
		// return Put(store, key, value, oldValue)
		return OUTDATED
	}
	return OK
}

func Delete(store string, key string) eccerror {
	if OfflineSupport && !started {
		return deleteFromCache(store, key)
	}
	url := fmt.Sprintf("%s%s/%s", CONSUL_KV_BASE_URL, store, key)
	log.Printf("Deleting KV pair for %s", url)
	req, err := http.NewRequest("DELETE", url, nil)

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		log.Println("Error creating KV pair for ", url, err)
		return ERROR
	}
	defer resp.Body.Close()

	body, _ := ioutil.ReadAll(resp.Body)
	log.Println(string(body))
	return OK
}

type Store struct {
	cache map[string][]byte
}

var cache map[string]Store = make(map[string]Store)

func getAllFromCache(storeName string) ([][]byte, []int, bool) {
	store, ok := cache[storeName]
	if !ok {
		return nil, nil, false
	}
	vals := make([][]byte, 0)
	for _, val := range store.cache {
		vals = append(vals, val)
	}
	return vals, nil, true
}

func getFromCache(storeName string, key string) ([]byte, int, bool) {
	store, ok := cache[storeName]
	if !ok {
		return nil, 0, false
	}
	val, ok := store.cache[key]
	return val, 0, ok
}

func putInCache(storeName string, key string, value []byte, oldValue []byte) eccerror {
	store, ok := cache[storeName]
	if !ok {
		store = Store{make(map[string][]byte)}
		cache[storeName] = store
	}
	store.cache[key] = value
	return OK
}

func deleteFromCache(storeName string, key string) eccerror {
	store, ok := cache[storeName]
	if ok {
		delete(store.cache, key)
	}
	return OK
}

func populateKVStoreFromCache() {
	if !OfflineSupport {
		return
	}
	// Populating KV store from Cache -> consul datastore needs some time interval
	// between consul daemon start and first addition to the KV store.
	// Bad things happen otherwise.

	time.Sleep(time.Second * 3)
	started = true
	for storeName, store := range cache {
		for key, val := range store.cache {
			go Put(storeName, key, val, nil)
		}
		delete(cache, storeName)
	}
}
<commit_msg>Removing an unneccessary time.Sleep() which was added during dev<commit_after>package ecc

import (
	"bytes"
	b64 "encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"time"

	"github.com/socketplane/ecc/Godeps/_workspace/src/github.com/mitchellh/cli"
)

// Embedded Consul Client
// Quick and Dirty way to embed Consul with any golang based application without the
// additional step of installing the Consul application in the host system

// Consul Agent related functions

var started bool
var OfflineSupport bool = true

func Start(serverMode bool, bootstrap bool, bindInterface string, dataDir string) error {
	bindAddress := ""
	if bindInterface != "" {
		intf, err := net.InterfaceByName(bindInterface)
		if err != nil {
			log.Printf("Error : %v", err)
			return err
		}
		addrs, err := intf.Addrs()
		if err == nil {
			for i := 0; i < len(addrs); i++ {
				addr := addrs[i].String()
				ip, _, _ := net.ParseCIDR(addr)
				if ip != nil && ip.To4() != nil {
					bindAddress = ip.To4().String()
				}
			}
		}
	}
	errCh := make(chan int)
	go startConsul(serverMode, bootstrap, bindAddress, dataDir, errCh)

	select {
	case <-errCh:
		return errors.New("Error starting Consul Agent")
	case <-time.After(time.Second * 5):
	}
	go populateKVStoreFromCache()
	return nil
}

func startConsul(serverMode bool, bootstrap bool, bindAddress string, dataDir string, eCh chan int) {
	args := []string{"agent", "-data-dir", dataDir}

	if serverMode {
		args = append(args, "-server")
	}

	if bootstrap {
		args = append(args, "-bootstrap")
	}

	if bindAddress != "" {
		args = append(args, "-bind")
		args = append(args, bindAddress)
	}

	ret := Execute(args...)

	if ret != 0 {
		eCh <- ret
	}
}

func HasStarted() bool {
	return started
}

func Join(address string) error {
	ret := Execute("join", address)

	if ret != 0 {
		log.Println("Error (%d) joining %s with Consul peers", ret, address)
		return errors.New("Error adding member")
	}
	return nil
}

func Leave() error {
	ret := Execute("leave")
	if ret != 0 {
		log.Println("Error Leaving Consul membership")
		return errors.New("Error leaving Consul cluster")
	}
	time.Sleep(time.Second * 3)
	return nil
}

// Execute function is borrowed from Consul's main.go
func Execute(args ...string) int {

	for _, arg := range args {
		if arg == "-v" || arg == "--version" {
			newArgs := make([]string, len(args)+1)
			newArgs[0] = "version"
			copy(newArgs[1:], args)
			args = newArgs
			break
		}
	}

	cli := &cli.CLI{
		Args:     args,
		Commands: Commands,
		HelpFunc: cli.BasicHelpFunc("consul"),
	}

	exitCode, err := cli.Run()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error executing CLI: %s\n", err.Error())
		return 1
	}

	return exitCode
}

const CONSUL_BASE_URL = "http://localhost:8500/v1/"

func ConsulGet(url string) (string, bool) {
	resp, err := http.Get(url)
	if err != nil {
		log.Printf("Error (%v) in Get for %s\n", err, url)
		return "", false
	}
	defer resp.Body.Close()
	log.Printf("Status of Get %s %d for %s", resp.Status, resp.StatusCode, url)
	if resp.StatusCode >= 200 && resp.StatusCode < 300 {
		var jsonBody []consulBody
		body, err := ioutil.ReadAll(resp.Body)
		err = json.Unmarshal(body, &jsonBody)
		existingValue, err := b64.StdEncoding.DecodeString(jsonBody[0].Value)
		if err != nil {
			return "", false
		}
		return string(existingValue[:]), true
	} else {
		return "", false
	}
}

// Consul KV Store related

const CONSUL_KV_BASE_URL = "http://localhost:8500/v1/kv/"

type consulBody struct {
	CreateIndex int    `json:"CreateIndex,omitempty"`
	ModifyIndex int    `json:"ModifyIndex,omitempty"`
	Key         string `json:"Key,omitempty"`
	Flags       int    `json:"Flags,omitempty"`
	Value       string `json:"Value,omitempty"`
}

func GetAll(store string) ([][]byte, []int, bool) {
	if OfflineSupport && !started {
		return getAllFromCache(store)
	}
	url := CONSUL_KV_BASE_URL + store + "?recurse"
	resp, err := http.Get(url)
	if err != nil {
		log.Printf("Error (%v) in Get for %s\n", err, url)
		return nil, nil, false
	}
	defer resp.Body.Close()
	log.Printf("Status of Get %s %d for %s", resp.Status, resp.StatusCode, url)
	if resp.StatusCode >= 400 && resp.StatusCode < 500 {
		return nil, nil, false
	} else if resp.StatusCode >= 200 && resp.StatusCode < 300 {
		var jsonBody []consulBody
		valueArr := make([][]byte, 0)
		indexArr := make([]int, 0)
		body, _ := ioutil.ReadAll(resp.Body)
		err = json.Unmarshal(body, &jsonBody)
		for _, body := range jsonBody {
			existingValue, _ := b64.StdEncoding.DecodeString(body.Value)
			valueArr = append(valueArr, existingValue)
			indexArr = append(indexArr, body.ModifyIndex)
		}
		return valueArr, indexArr, true
	} else {
		return nil, nil, false
	}
}

func Get(store string, key string) ([]byte, int, bool) {
	if OfflineSupport && !started {
		return getFromCache(store, key)
	}
	url := CONSUL_KV_BASE_URL + store + "/" + key
	resp, err := http.Get(url)
	if err != nil {
		log.Printf("Error (%v) in Get for %s\n", err, url)
		return nil, 0, false
	}
	defer resp.Body.Close()
	log.Printf("Status of Get %s %d for %s", resp.Status, resp.StatusCode, url)
	if resp.StatusCode >= 400 && resp.StatusCode < 500 {
		return nil, 0, false
	} else if resp.StatusCode >= 200 && resp.StatusCode < 300 {
		var jsonBody []consulBody
		body, err := ioutil.ReadAll(resp.Body)
		err = json.Unmarshal(body, &jsonBody)
		existingValue, err := b64.StdEncoding.DecodeString(jsonBody[0].Value)
		if err != nil {
			return nil, jsonBody[0].ModifyIndex, false
		}
		return existingValue, jsonBody[0].ModifyIndex, true
	} else {
		return nil, 0, false
	}
}

const (
	OK = iota
	OUTDATED
	ERROR
)

type eccerror int

func Put(store string, key string, value []byte, oldValue []byte) eccerror {
	if OfflineSupport && !started {
		return putInCache(store, key, value, oldValue)
	}
	existingValue, casIndex, ok := Get(store, key)
	if ok && !bytes.Equal(oldValue, existingValue) {
		return OUTDATED
	}
	url := fmt.Sprintf("%s%s/%s?cas=%d", CONSUL_KV_BASE_URL, store, key, casIndex)
	log.Printf("Updating KV pair for %s %s %s %d", url, key, value, casIndex)
	req, err := http.NewRequest("PUT", url, bytes.NewBuffer(value))

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		log.Println("Error creating KV pair for ", url, err)
		return ERROR
	}
	defer resp.Body.Close()

	body, _ := ioutil.ReadAll(resp.Body)
	if string(body) == "false" {
		// Let the application retry based on return value
		// return Put(store, key, value, oldValue)
		return OUTDATED
	}
	return OK
}

func Delete(store string, key string) eccerror {
	if OfflineSupport && !started {
		return deleteFromCache(store, key)
	}
	url := fmt.Sprintf("%s%s/%s", CONSUL_KV_BASE_URL, store, key)
	log.Printf("Deleting KV pair for %s", url)
	req, err := http.NewRequest("DELETE", url, nil)

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		log.Println("Error creating KV pair for ", url, err)
		return ERROR
	}
	defer resp.Body.Close()

	body, _ := ioutil.ReadAll(resp.Body)
	log.Println(string(body))
	return OK
}

type Store struct {
	cache map[string][]byte
}

var cache map[string]Store = make(map[string]Store)

func getAllFromCache(storeName string) ([][]byte, []int, bool) {
	store, ok := cache[storeName]
	if !ok {
		return nil, nil, false
	}
	vals := make([][]byte, 0)
	for _, val := range store.cache {
		vals = append(vals, val)
	}
	return vals, nil, true
}

func getFromCache(storeName string, key string) ([]byte, int, bool) {
	store, ok := cache[storeName]
	if !ok {
		return nil, 0, false
	}
	val, ok := store.cache[key]
	return val, 0, ok
}

func putInCache(storeName string, key string, value []byte, oldValue []byte) eccerror {
	store, ok := cache[storeName]
	if !ok {
		store = Store{make(map[string][]byte)}
		cache[storeName] = store
	}
	store.cache[key] = value
	return OK
}

func deleteFromCache(storeName string, key string) eccerror {
	store, ok := cache[storeName]
	if ok {
		delete(store.cache, key)
	}
	return OK
}

func populateKVStoreFromCache() {
	if !OfflineSupport {
		return
	}
	started = true
	for storeName, store := range cache {
		for key, val := range store.cache {
			go Put(storeName, key, val, nil)
		}
		delete(cache, storeName)
	}
}
<|endoftext|>


<commit_before>package smsclub

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"golang.org/x/text/encoding/charmap"
	"golang.org/x/text/transform"
)

type methodAPI int

const (
	mSend methodAPI = iota
	mStatus
	mBalance
)

var (
	epoint = "https://gate.smsclub.mobi/http"
	mapper = map[methodAPI]string{
		mSend:    "httpsendsms",
		mStatus:  "httpgetsmsstate",
		mBalance: "httpgetbalance",
	}
	makeURL = func(m methodAPI) string {
		return fmt.Sprintf("%s/%s.php", epoint, mapper[m])
	}
)

// SMSer is interface for https://smsclub.mobi/en/pages/show/api
type SMSer interface {
	// Balance returns values for balance and credit.
	Balance() (float64, float64, error)

	// LifeTime sets life time of SMS, which is specified in minutes.
	LifeTime(d time.Duration) error

	// Send sends SMS text message from "alphaName" to recipients.
	Send(from, text string, to ...string) ([]string, error)

	// Status gets list of SMS identifiers and returns statuses for ones.
	Status(ids ...string) ([]string, error)
}

// New returns SMSer interface
func New(user, pass string) SMSer {
	return &client{
		user: user,
		pass: pass,
	}
}

type client struct {
	user  string
	pass  string
	ltime int
}

func (c *client) makeForm(f url.Values) (url.Values, error) {
	if c.user == "" || c.pass == "" {
		return nil, fmt.Errorf("smsclub: username or password is empty")
	}
	if f == nil {
		f = url.Values{}
	}
	f.Set("username", c.user)
	f.Set("password", c.pass)
	if c.ltime > 0 {
		f.Set("lifetime", strconv.Itoa(c.ltime))
	}
	return f, nil
}

func (c *client) callAPI(m methodAPI, v url.Values) ([]string, error) {
	frm, err := c.makeForm(v)
	if err != nil {
		return nil, err
	}

	res, err := http.PostForm(makeURL(m), frm)
	defer func() {
		err := res.Body.Close()
		if err != nil {
			panic(err)
		}
	}()
	if err != nil {
		return nil, err
	}

	if res.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("smsclub: server returns %d %s", res.StatusCode, res.Status)
	}

	bts, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}

	var out []string
	for _, s := range strings.Split(string(bts), "
") {
		if strings.Contains(s, "=") || strings.TrimSpace(s) == "" {
			continue
		}
		out = append(out, s)
	}

	if m != mBalance && !strings.Contains(string(bts), "=IDS") {
		return nil, fmt.Errorf("%s", strings.Join(out, "\n"))
	}

	if out == nil {
		return nil, io.EOF
	}

	return out, nil
}

func (c *client) Balance() (float64, float64, error) {
	res, err := c.callAPI(mBalance, nil)
	if err != nil {
		return 0.0, 0.0, err
	}

	var bal, cre float64
	if len(res) == 2 {
		bal, _ = strconv.ParseFloat(res[0], 64)
		cre, _ = strconv.ParseFloat(res[1], 64)
	}

	return bal, cre, nil
}

func (c *client) LifeTime(d time.Duration) error {
	if d < 0 {
		return fmt.Errorf("smsclub: invalid duration value %d", d)
	}
	c.ltime = int(d.Minutes())
	return nil
}

func (c *client) Send(from, text string, to ...string) ([]string, error) {
	toBase64 := func(s string) string {
		return base64.StdEncoding.EncodeToString([]byte(s))
	}
	toWin1251 := func(s string) string {
		b := new(bytes.Buffer)
		w := transform.NewWriter(b, charmap.Windows1251.NewEncoder())
		_, _ = w.Write([]byte(s))
		return b.String()
	}

	form := url.Values{
		"from": []string{from},
		"text": []string{toBase64(toWin1251(text))},
		"to":   []string{strings.Join(to, ";")},
	}

	return c.callAPI(mSend, form)
}

func (c *client) Status(ids ...string) ([]string, error) {
	form := url.Values{
		"smscid": []string{strings.Join(ids, ";")},
	}

	res, err := c.callAPI(mStatus, form)
	if err != nil {
		return nil, err
	}

	for i := range res {
		spl := strings.Split(res[i], ":")
		if len(spl) == 2 {
			res[i] = strings.TrimSpace(spl[1])
		}
	}

	return res, nil
}

func (c *client) String() string {
	return fmt.Sprintf("%s %s %d", c.user, c.pass, c.ltime)
}
<commit_msg>go1.7<commit_after>package smsclub

import (
	"bytes"
	"context"
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"golang.org/x/text/encoding/charmap"
	"golang.org/x/text/transform"
)

type methodAPI int

const (
	mSend methodAPI = iota
	mStatus
	mBalance
)

var (
	epoint = "https://gate.smsclub.mobi/http"
	mapper = map[methodAPI]string{
		mSend:    "httpsendsms",
		mStatus:  "httpgetsmsstate",
		mBalance: "httpgetbalance",
	}
	makeURL = func(m methodAPI) string {
		return fmt.Sprintf("%s/%s.php", epoint, mapper[m])
	}
)

// SMSer is interface for https://smsclub.mobi/en/pages/show/api
type SMSer interface {
	// Balance returns values for balance and credit.
	Balance() (float64, float64, error)

	// LifeTime sets life time of SMS, which is specified in minutes.
	LifeTime(d time.Duration) error

	// Send sends SMS text message from "alphaName" to recipients.
	Send(from, text string, to ...string) ([]string, error)

	// Status gets list of SMS identifiers and returns statuses for ones.
	Status(ids ...string) ([]string, error)

	// Timeout sets timeout for calls Balance, Send and Status
	Timeout(d time.Duration) error
}

// New returns SMSer interface
func New(user, pass string) SMSer {
	return &client{
		user: user,
		pass: pass,
	}
}

type client struct {
	user    string
	pass    string
	ltime   int
	timeout time.Duration
}

func (c *client) makeForm(f url.Values) (url.Values, error) {
	if c.user == "" || c.pass == "" {
		return nil, fmt.Errorf("smsclub: username or password is empty")
	}
	if f == nil {
		f = url.Values{}
	}
	f.Set("username", c.user)
	f.Set("password", c.pass)
	if c.ltime > 0 {
		f.Set("lifetime", strconv.Itoa(c.ltime))
	}
	return f, nil
}

func closeBody(c io.Closer) {
	if c == nil {
		return
	}
	err := c.Close()
	if err != nil {
		panic(err)
	}
}

func (c *client) callAPI(m methodAPI, v url.Values) ([]string, error) {
	frm, err := c.makeForm(v)
	if err != nil {
		return nil, err
	}

	req, err := http.NewRequest("POST", makeURL(m), strings.NewReader(frm.Encode()))
	if err != nil {
		return nil, err
	}

	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	if c.timeout > 0 {
		ctx, cancel := context.WithTimeout(context.Background(), c.timeout)
		defer cancel()
		req = req.WithContext(ctx)
	}

	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer closeBody(res.Body)

	if res.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("smsclub: server returns %d %s", res.StatusCode, res.Status)
	}

	bts, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}

	var out []string
	for _, s := range strings.Split(string(bts), "
") {
		if strings.Contains(s, "=") || strings.TrimSpace(s) == "" {
			continue
		}
		out = append(out, s)
	}

	if m != mBalance && !strings.Contains(string(bts), "=IDS") {
		return nil, fmt.Errorf("%s", strings.Join(out, "\n"))
	}

	if out == nil {
		return nil, io.EOF
	}

	return out, nil
}

func (c *client) Balance() (float64, float64, error) {
	res, err := c.callAPI(mBalance, nil)
	if err != nil {
		return 0.0, 0.0, err
	}

	var bal, cre float64
	if len(res) == 2 {
		bal, _ = strconv.ParseFloat(res[0], 64)
		cre, _ = strconv.ParseFloat(res[1], 64)
	}

	return bal, cre, nil
}

func (c *client) LifeTime(d time.Duration) error {
	if d < 0 {
		return fmt.Errorf("smsclub: invalid duration value %d", d)
	}
	c.ltime = int(d.Minutes())
	return nil
}

func (c *client) Send(from, text string, to ...string) ([]string, error) {
	toBase64 := func(s string) string {
		return base64.StdEncoding.EncodeToString([]byte(s))
	}
	toWin1251 := func(s string) string {
		b := new(bytes.Buffer)
		w := transform.NewWriter(b, charmap.Windows1251.NewEncoder())
		_, _ = w.Write([]byte(s))
		return b.String()
	}

	form := url.Values{
		"from": []string{from},
		"text": []string{toBase64(toWin1251(text))},
		"to":   []string{strings.Join(to, ";")},
	}

	return c.callAPI(mSend, form)
}

func (c *client) Status(ids ...string) ([]string, error) {
	form := url.Values{
		"smscid": []string{strings.Join(ids, ";")},
	}

	res, err := c.callAPI(mStatus, form)
	if err != nil {
		return nil, err
	}

	for i := range res {
		spl := strings.Split(res[i], ":")
		if len(spl) == 2 {
			res[i] = strings.TrimSpace(spl[1])
		}
	}

	return res, nil
}

func (c *client) String() string {
	return fmt.Sprintf("%s %s %d", c.user, c.pass, c.ltime)
}

func (c *client) Timeout(d time.Duration) error {
	if d < 0 {
		return fmt.Errorf("smsclub: invalid duration value %d", d)
	}
	c.timeout = d
	return nil
}
<|endoftext|>


<commit_before>package mgr

import (
	"errors"
	"fmt"
	"io"
	"strings"
	"time"

	"github.com/json-iterator/go"
	"github.com/qiniu/pandora-go-sdk/base/reqerr"

	"github.com/qiniu/logkit/conf"
	"github.com/qiniu/logkit/parser"
	"github.com/qiniu/logkit/parser/grok"
	"github.com/qiniu/logkit/reader"
	"github.com/qiniu/logkit/router"
	"github.com/qiniu/logkit/sender"
	"github.com/qiniu/logkit/transforms"
	. "github.com/qiniu/logkit/utils/models"
)

const DefaultTryTimes = 3

// RawData 从 reader 模块中根据 type 获取字符串形式的样例日志
func RawData(readerConfig conf.MapConf) (string, error) {
	if readerConfig == nil {
		return "", fmt.Errorf("reader config cannot be empty")
	}

	rd, err := reader.NewReader(readerConfig, true)
	if err != nil {
		return "", err
	}
	defer rd.Close()

	var rawData string

	// DataReader 设定超时，普通 Reader 设定重试
	if dr, ok := rd.(reader.DataReader); ok {
		type dataErr struct {
			data string
			err  error
		}
		// Note: 添加一位缓冲保证 goroutine 在 runner 已经超时的情况下能够正常退出，避免资源泄露
		readChan := make(chan dataErr, 1)
		go func() {
			data, _, err := dr.ReadData()
			if err != nil && err != io.EOF {
				readChan <- dataErr{"", err}
				return
			}

			p, err := jsoniter.MarshalIndent(data, "", "  ")
			readChan <- dataErr{string(p), err}
		}()

		timeout := time.NewTimer(time.Minute)
		select {
		case de := <-readChan:
			rawData, err = de.data, de.err
			if err != nil {
				return "", fmt.Errorf("reader %q - error: %v", rd.Name(), err)
			}

		case <-timeout.C:
			return "", fmt.Errorf("reader %q read data timeout, no data received", rd.Name())
		}

	} else {
		tryCount := DefaultTryTimes
		for {
			if tryCount <= 0 {
				return "", fmt.Errorf("reader %q read line timeout, no data received", rd.Name())
			}
			tryCount--

			rawData, err = rd.ReadLine()
			if err != nil && err != io.EOF {
				return "", fmt.Errorf("reader %q - error: %v", rd.Name(), err)
			}
			if err == io.EOF {
				return rawData, nil
			}

			if len(rawData) > 0 {
				break
			}
		}
	}

	if len(rawData) >= DefaultMaxBatchSize {
		err = errors.New("data size large than 2M and will be discard")
		return "", fmt.Errorf("reader %q - error: %v", rd.Name(), err)
	}
	return rawData, nil
}

//parse模块中各种type的日志都能获取解析后的数据
func ParseData(parserConfig conf.MapConf) (parsedData []Data, err error) {
	parserConfig = parser.ConvertWebParserConfig(parserConfig)
	if parserConfig == nil {
		err = fmt.Errorf("parser config was empty after web config convet")
		return
	}

	sampleData, err := getSampleData(parserConfig)
	if err != nil {
		return nil, err
	}
	parserRegistry := parser.NewRegistry()
	logParser, err := parserRegistry.NewLogParser(parserConfig)
	if err != nil {
		return nil, err
	}
	sampleData, err = checkSampleData(sampleData, logParser)
	if err != nil {
		return nil, err
	}

	parsedData, err = logParser.Parse(sampleData)
	err = checkErr(err, logParser.Name())
	if err != nil {
		return nil, err
	}

	return
}

func TransformData(transformerConfig map[string]interface{}) ([]Data, error) {
	if transformerConfig == nil {
		err := fmt.Errorf("transformer config cannot be empty")
		return nil, err
	}

	create, err := getTransformerCreator(transformerConfig)
	if err != nil {
		return nil, err
	}

	data, err := getDataFromTransformConfig(transformerConfig)
	if err != nil {
		return nil, err
	}

	transformer, err := getTransformer(transformerConfig, create)
	if err != nil {
		return nil, err
	}

	// Transform data
	transformedData, transErr := transformer.Transform(data)
	se, ok := transErr.(*StatsError)
	if ok {
		transErr = se.ErrorDetail
	}
	if transErr != nil {
		se, ok := transErr.(*StatsError)
		if ok {
			transErr = se.ErrorDetail
		}
		err := fmt.Errorf("transform processing error %v", transErr)
		return nil, err
	}
	return transformedData, nil
}

func SendData(senderConfig map[string]interface{}) error {
	if senderConfig == nil {
		return fmt.Errorf("sender config cannot be empty")
	}

	sendersConf, err := getSendersConfig(senderConfig)
	if err != nil {
		return err
	}

	senders, err := getSenders(sendersConf)
	if err != nil {
		return err
	}

	senderCnt := len(senders)
	router, err := getRouter(senderConfig, senderCnt)
	if err != nil {
		return err
	}

	datas, err := getDataFromSenderConfig(senderConfig)
	if err != nil {
		return err
	}

	senderDataList := classifySenderData(datas, router, len(senders))
	for index, s := range senders {
		if err := trySend(s, senderDataList[index], DefaultTryTimes); err != nil {
			return err
		}
	}

	return nil
}

func getSendersConfig(senderConfig map[string]interface{}) ([]conf.MapConf, error) {
	config := senderConfig[KeySendConfig]
	byteSendersConfig, err := jsoniter.Marshal(config)
	if err != nil {
		return nil, err
	}
	var sendersConf []conf.MapConf
	if jsonErr := jsoniter.Unmarshal(byteSendersConfig, &sendersConf); jsonErr != nil {
		return nil, jsonErr
	}
	if sendersConf == nil {
		return nil, fmt.Errorf("sender config cannot be empty")
	}
	return sendersConf, nil
}

func getDataFromSenderConfig(senderConfig map[string]interface{}) ([]Data, error) {
	var datas = []Data{}
	rawData := senderConfig[KeySampleLog]
	rawDataStr, ok := rawData.(string)
	if !ok {
		err := fmt.Errorf("missing param %s", KeySampleLog)
		return nil, err
	}
	if rawDataStr == "" {
		err := fmt.Errorf("sender fetched empty sample log")
		return nil, err
	}

	if jsonErr := jsoniter.Unmarshal([]byte(rawDataStr), &datas); jsonErr != nil {
		return nil, jsonErr
	}
	return datas, nil
}

func getSenders(sendersConf []conf.MapConf) ([]sender.Sender, error) {
	senders := make([]sender.Sender, 0)
	sr := sender.NewRegistry()
	for i, senderConfig := range sendersConf {
		senderConfig[sender.KeyFaultTolerant] = "false"
		s, err := sr.NewSender(senderConfig, "")
		if err != nil {
			return nil, err
		}
		senders = append(senders, s)
		delete(sendersConf[i], sender.InnerUserAgent)
	}
	return senders, nil
}

func getRouter(senderConfig map[string]interface{}, senderCnt int) (*router.Router, error) {
	routerConf, err := getRouterConfig(senderConfig)
	if err != nil {
		return nil, err
	}

	router, err := router.NewSenderRouter(routerConf, senderCnt)
	if err != nil {
		return nil, fmt.Errorf("sender router error, %v", err)
	}
	return router, nil
}

func getRouterConfig(senderConfig map[string]interface{}) (router.RouterConfig, error) {
	config := senderConfig[KeyRouterConfig]
	byteRouterConfig, err := jsoniter.Marshal(config)
	if err != nil {
		return router.RouterConfig{}, err
	}

	var routerConfig router.RouterConfig
	if jsonErr := jsoniter.Unmarshal(byteRouterConfig, &routerConfig); jsonErr != nil {
		return router.RouterConfig{}, jsonErr
	}
	return routerConfig, nil
}

// trySend 尝试发送数据，如果此时runner退出返回false，其他情况无论是达到最大重试次数还是发送成功，都返回true
func trySend(s sender.Sender, datas []Data, times int) (err error) {
	if times <= 0 {
		times = DefaultTryTimes
	}
	if len(datas) <= 0 {
		return nil
	}
	cnt := 0
	for {
		if cnt >= times {
			break
		}
		err = s.Send(datas)
		if se, ok := err.(*StatsError); ok {
			err = se.ErrorDetail
		}

		if err != nil {
			cnt++
			se, ok := err.(*reqerr.SendError)
			if ok {
				datas = sender.ConvertDatas(se.GetFailDatas())
				continue
			}
			if cnt < times {
				continue
			}
			err = fmt.Errorf("retry send %v times, but still error %v, discard datas %v ... total %v lines", cnt, err, datas, len(datas))
			return err
		}
		break
	}
	return
}

func getSampleData(parserConfig conf.MapConf) ([]string, error) {
	parserType, _ := parserConfig.GetString(parser.KeyParserType)
	rawData, _ := parserConfig.GetStringOr(KeySampleLog, "")
	var sampleData []string

	switch parserType {
	case parser.TypeCSV, parser.TypeJSON, parser.TypeRaw, parser.TypeNginx, parser.TypeEmpty, parser.TypeKafkaRest, parser.TypeLogv1:
		sampleData = append(sampleData, rawData)
	case parser.TypeSyslog:
		sampleData = strings.Split(rawData, "\n")
		sampleData = append(sampleData, parser.PandoraParseFlushSignal)
	case parser.TypeMySQL:
		sampleData = strings.Split(rawData, "\n")
		sampleData = append(sampleData, parser.PandoraParseFlushSignal)
	case parser.TypeGrok:
		grokMode, _ := parserConfig.GetString(parser.KeyGrokMode)
		if grokMode != grok.ModeMulti {
			sampleData = append(sampleData, rawData)
		} else {
			sampleData = []string{rawData}
		}
	default:
		errMsg := fmt.Sprintf("parser type <%v> is not supported yet", parserType)
		return nil, errors.New(errMsg)
	}

	return sampleData, nil
}

func checkSampleData(sampleData []string, logParser parser.Parser) ([]string, error) {
	if len(sampleData) <= 0 {
		_, ok := logParser.(parser.Flushable)
		if ok {
			sampleData = []string{parser.PandoraParseFlushSignal}
		} else {
			err := fmt.Errorf("parser [%v] fetched 0 lines", logParser.Name())
			return nil, err
		}
	}
	return sampleData, nil
}

func checkErr(err error, parserName string) error {
	se, ok := err.(*StatsError)
	var errorCnt int64
	if ok {
		errorCnt = se.Errors
		err = se.ErrorDetail
	} else if err != nil {
		errorCnt = 1
	}
	if err != nil {
		errMsg := fmt.Sprintf("parser %s, error %v ", parserName, err.Error())
		err = fmt.Errorf("%v parse line errors occured, same as %v", errorCnt, errors.New(errMsg))
	}

	return err
}

func getTransformerCreator(transformerConfig map[string]interface{}) (transforms.Creator, error) {
	transformKeyType, ok := transformerConfig[transforms.KeyType]
	if !ok {
		err := fmt.Errorf("missing param %s", transforms.KeyType)
		return nil, err
	}
	transformKeyTypeStr, ok := transformKeyType.(string)
	if !ok {
		err := fmt.Errorf("param %s must be of type string", transforms.KeyType)
		return nil, err
	}

	create, ok := transforms.Transformers[transformKeyTypeStr]
	if !ok {
		err := fmt.Errorf("transformer of type %v not exist", transformKeyTypeStr)
		return nil, err
	}
	return create, nil
}

func getDataFromTransformConfig(transformerConfig map[string]interface{}) ([]Data, error) {
	rawData, ok := transformerConfig[KeySampleLog]
	if !ok {
		err := fmt.Errorf("missing param %s", KeySampleLog)
		return nil, err
	}
	rawDataStr, ok := rawData.(string)
	if !ok {
		err := fmt.Errorf("missing param %s", KeySampleLog)
		return nil, err
	}
	if rawDataStr == "" {
		err := fmt.Errorf("transformer fetched empty sample log")
		return nil, err
	}

	var data = []Data{}
	var singleData Data
	if jsonErr := jsoniter.Unmarshal([]byte(rawDataStr), &singleData); jsonErr != nil {
		if jsonErr = jsoniter.Unmarshal([]byte(rawDataStr), &data); jsonErr != nil {
			return nil, jsonErr
		}
	} else {
		// is single log, and method transformer.transform(data []sender.Data) accept a param of slice type
		data = append(data, singleData)
	}
	return data, nil
}

func getTransformer(transConfig map[string]interface{}, create transforms.Creator) (transforms.Transformer, error) {
	trans := create()
	transformerConfig := convertWebTransformerConfig(transConfig)
	delete(transformerConfig, KeySampleLog)
	bts, jsonErr := jsoniter.Marshal(transformerConfig)
	if jsonErr != nil {
		return nil, jsonErr
	}
	if jsonErr = jsoniter.Unmarshal(bts, trans); jsonErr != nil {
		return nil, jsonErr
	}

	if trans, ok := trans.(transforms.Initialize); ok {
		if err := trans.Init(); err != nil {
			return nil, err
		}
	}
	return trans, nil
}
<commit_msg>fix parser<commit_after>package mgr

import (
	"errors"
	"fmt"
	"io"
	"strings"
	"time"

	"github.com/json-iterator/go"
	"github.com/qiniu/pandora-go-sdk/base/reqerr"

	"github.com/qiniu/logkit/conf"
	"github.com/qiniu/logkit/parser"
	"github.com/qiniu/logkit/parser/grok"
	"github.com/qiniu/logkit/reader"
	"github.com/qiniu/logkit/router"
	"github.com/qiniu/logkit/sender"
	"github.com/qiniu/logkit/transforms"
	. "github.com/qiniu/logkit/utils/models"
)

const DefaultTryTimes = 3

// RawData 从 reader 模块中根据 type 获取字符串形式的样例日志
func RawData(readerConfig conf.MapConf) (string, error) {
	if readerConfig == nil {
		return "", fmt.Errorf("reader config cannot be empty")
	}

	rd, err := reader.NewReader(readerConfig, true)
	if err != nil {
		return "", err
	}
	defer rd.Close()

	var rawData string

	// DataReader 设定超时，普通 Reader 设定重试
	if dr, ok := rd.(reader.DataReader); ok {
		type dataErr struct {
			data string
			err  error
		}
		// Note: 添加一位缓冲保证 goroutine 在 runner 已经超时的情况下能够正常退出，避免资源泄露
		readChan := make(chan dataErr, 1)
		go func() {
			data, _, err := dr.ReadData()
			if err != nil && err != io.EOF {
				readChan <- dataErr{"", err}
				return
			}

			p, err := jsoniter.MarshalIndent(data, "", "  ")
			readChan <- dataErr{string(p), err}
		}()

		timeout := time.NewTimer(time.Minute)
		select {
		case de := <-readChan:
			rawData, err = de.data, de.err
			if err != nil {
				return "", fmt.Errorf("reader %q - error: %v", rd.Name(), err)
			}

		case <-timeout.C:
			return "", fmt.Errorf("reader %q read data timeout, no data received", rd.Name())
		}

	} else {
		tryCount := DefaultTryTimes
		for {
			if tryCount <= 0 {
				return "", fmt.Errorf("reader %q read line timeout, no data received", rd.Name())
			}
			tryCount--

			rawData, err = rd.ReadLine()
			if err != nil && err != io.EOF {
				return "", fmt.Errorf("reader %q - error: %v", rd.Name(), err)
			}
			if err == io.EOF {
				return rawData, nil
			}

			if len(rawData) > 0 {
				break
			}
		}
	}

	if len(rawData) >= DefaultMaxBatchSize {
		err = errors.New("data size large than 2M and will be discard")
		return "", fmt.Errorf("reader %q - error: %v", rd.Name(), err)
	}
	return rawData, nil
}

//parse模块中各种type的日志都能获取解析后的数据
func ParseData(parserConfig conf.MapConf) (parsedData []Data, err error) {
	parserConfig = parser.ConvertWebParserConfig(parserConfig)
	if parserConfig == nil {
		err = fmt.Errorf("parser config was empty after web config convet")
		return
	}

	sampleData, err := getSampleData(parserConfig)
	if err != nil {
		return nil, err
	}
	parserRegistry := parser.NewRegistry()
	logParser, err := parserRegistry.NewLogParser(parserConfig)
	if err != nil {
		return nil, err
	}
	sampleData, err = checkSampleData(sampleData, logParser)
	if err != nil {
		return nil, err
	}

	parsedData, err = logParser.Parse(sampleData)
	err = checkErr(err, logParser.Name())
	if err != nil {
		return nil, err
	}

	return
}

func TransformData(transformerConfig map[string]interface{}) ([]Data, error) {
	if transformerConfig == nil {
		err := fmt.Errorf("transformer config cannot be empty")
		return nil, err
	}

	create, err := getTransformerCreator(transformerConfig)
	if err != nil {
		return nil, err
	}

	data, err := getDataFromTransformConfig(transformerConfig)
	if err != nil {
		return nil, err
	}

	transformer, err := getTransformer(transformerConfig, create)
	if err != nil {
		return nil, err
	}

	// Transform data
	transformedData, transErr := transformer.Transform(data)
	se, ok := transErr.(*StatsError)
	if ok {
		transErr = se.ErrorDetail
	}
	if transErr != nil {
		se, ok := transErr.(*StatsError)
		if ok {
			transErr = se.ErrorDetail
		}
		err := fmt.Errorf("transform processing error %v", transErr)
		return nil, err
	}
	return transformedData, nil
}

func SendData(senderConfig map[string]interface{}) error {
	if senderConfig == nil {
		return fmt.Errorf("sender config cannot be empty")
	}

	sendersConf, err := getSendersConfig(senderConfig)
	if err != nil {
		return err
	}

	senders, err := getSenders(sendersConf)
	if err != nil {
		return err
	}

	senderCnt := len(senders)
	router, err := getRouter(senderConfig, senderCnt)
	if err != nil {
		return err
	}

	datas, err := getDataFromSenderConfig(senderConfig)
	if err != nil {
		return err
	}

	senderDataList := classifySenderData(datas, router, len(senders))
	for index, s := range senders {
		if err := trySend(s, senderDataList[index], DefaultTryTimes); err != nil {
			return err
		}
	}

	return nil
}

func getSendersConfig(senderConfig map[string]interface{}) ([]conf.MapConf, error) {
	config := senderConfig[KeySendConfig]
	byteSendersConfig, err := jsoniter.Marshal(config)
	if err != nil {
		return nil, err
	}
	var sendersConf []conf.MapConf
	if jsonErr := jsoniter.Unmarshal(byteSendersConfig, &sendersConf); jsonErr != nil {
		return nil, jsonErr
	}
	if sendersConf == nil {
		return nil, fmt.Errorf("sender config cannot be empty")
	}
	return sendersConf, nil
}

func getDataFromSenderConfig(senderConfig map[string]interface{}) ([]Data, error) {
	var datas = []Data{}
	rawData := senderConfig[KeySampleLog]
	rawDataStr, ok := rawData.(string)
	if !ok {
		err := fmt.Errorf("missing param %s", KeySampleLog)
		return nil, err
	}
	if rawDataStr == "" {
		err := fmt.Errorf("sender fetched empty sample log")
		return nil, err
	}

	if jsonErr := jsoniter.Unmarshal([]byte(rawDataStr), &datas); jsonErr != nil {
		return nil, jsonErr
	}
	return datas, nil
}

func getSenders(sendersConf []conf.MapConf) ([]sender.Sender, error) {
	senders := make([]sender.Sender, 0)
	sr := sender.NewRegistry()
	for i, senderConfig := range sendersConf {
		senderConfig[sender.KeyFaultTolerant] = "false"
		s, err := sr.NewSender(senderConfig, "")
		if err != nil {
			return nil, err
		}
		senders = append(senders, s)
		delete(sendersConf[i], sender.InnerUserAgent)
	}
	return senders, nil
}

func getRouter(senderConfig map[string]interface{}, senderCnt int) (*router.Router, error) {
	routerConf, err := getRouterConfig(senderConfig)
	if err != nil {
		return nil, err
	}

	router, err := router.NewSenderRouter(routerConf, senderCnt)
	if err != nil {
		return nil, fmt.Errorf("sender router error, %v", err)
	}
	return router, nil
}

func getRouterConfig(senderConfig map[string]interface{}) (router.RouterConfig, error) {
	config := senderConfig[KeyRouterConfig]
	byteRouterConfig, err := jsoniter.Marshal(config)
	if err != nil {
		return router.RouterConfig{}, err
	}

	var routerConfig router.RouterConfig
	if jsonErr := jsoniter.Unmarshal(byteRouterConfig, &routerConfig); jsonErr != nil {
		return router.RouterConfig{}, jsonErr
	}
	return routerConfig, nil
}

// trySend 尝试发送数据，如果此时runner退出返回false，其他情况无论是达到最大重试次数还是发送成功，都返回true
func trySend(s sender.Sender, datas []Data, times int) (err error) {
	if times <= 0 {
		times = DefaultTryTimes
	}
	if len(datas) <= 0 {
		return nil
	}
	cnt := 0
	for {
		if cnt >= times {
			break
		}
		err = s.Send(datas)
		if se, ok := err.(*StatsError); ok {
			err = se.ErrorDetail
		}

		if err != nil {
			cnt++
			se, ok := err.(*reqerr.SendError)
			if ok {
				datas = sender.ConvertDatas(se.GetFailDatas())
				continue
			}
			if cnt < times {
				continue
			}
			err = fmt.Errorf("retry send %v times, but still error %v, discard datas %v ... total %v lines", cnt, err, datas, len(datas))
			return err
		}
		break
	}
	return
}

func getSampleData(parserConfig conf.MapConf) ([]string, error) {
	parserType, _ := parserConfig.GetString(parser.KeyParserType)
	rawData, _ := parserConfig.GetStringOr(KeySampleLog, "")
	var sampleData []string

	switch parserType {
	case parser.TypeCSV, parser.TypeJSON, parser.TypeRaw, parser.TypeNginx, parser.TypeEmpty, parser.TypeKafkaRest, parser.TypeLogv1:
		sampleData = append(sampleData, rawData)
	case parser.TypeSyslog:
		sampleData = strings.Split(rawData, "\n")
		sampleData = append(sampleData, parser.PandoraParseFlushSignal)
	case parser.TypeMySQL:
		sampleData = strings.Split(rawData, "\n")
		sampleData = append(sampleData, parser.PandoraParseFlushSignal)
	case parser.TypeGrok:
		grokMode, _ := parserConfig.GetString(parser.KeyGrokMode)
		if grokMode != grok.ModeMulti {
			sampleData = append(sampleData, rawData)
		} else {
			sampleData = []string{rawData}
		}
	default:
		sampleData = append(sampleData, rawData)
	}

	return sampleData, nil
}

func checkSampleData(sampleData []string, logParser parser.Parser) ([]string, error) {
	if len(sampleData) <= 0 {
		_, ok := logParser.(parser.Flushable)
		if ok {
			sampleData = []string{parser.PandoraParseFlushSignal}
		} else {
			err := fmt.Errorf("parser [%v] fetched 0 lines", logParser.Name())
			return nil, err
		}
	}
	return sampleData, nil
}

func checkErr(err error, parserName string) error {
	se, ok := err.(*StatsError)
	var errorCnt int64
	if ok {
		errorCnt = se.Errors
		err = se.ErrorDetail
	} else if err != nil {
		errorCnt = 1
	}
	if err != nil {
		errMsg := fmt.Sprintf("parser %s, error %v ", parserName, err.Error())
		err = fmt.Errorf("%v parse line errors occured, same as %v", errorCnt, errors.New(errMsg))
	}

	return err
}

func getTransformerCreator(transformerConfig map[string]interface{}) (transforms.Creator, error) {
	transformKeyType, ok := transformerConfig[transforms.KeyType]
	if !ok {
		err := fmt.Errorf("missing param %s", transforms.KeyType)
		return nil, err
	}
	transformKeyTypeStr, ok := transformKeyType.(string)
	if !ok {
		err := fmt.Errorf("param %s must be of type string", transforms.KeyType)
		return nil, err
	}

	create, ok := transforms.Transformers[transformKeyTypeStr]
	if !ok {
		err := fmt.Errorf("transformer of type %v not exist", transformKeyTypeStr)
		return nil, err
	}
	return create, nil
}

func getDataFromTransformConfig(transformerConfig map[string]interface{}) ([]Data, error) {
	rawData, ok := transformerConfig[KeySampleLog]
	if !ok {
		err := fmt.Errorf("missing param %s", KeySampleLog)
		return nil, err
	}
	rawDataStr, ok := rawData.(string)
	if !ok {
		err := fmt.Errorf("missing param %s", KeySampleLog)
		return nil, err
	}
	if rawDataStr == "" {
		err := fmt.Errorf("transformer fetched empty sample log")
		return nil, err
	}

	var data = []Data{}
	var singleData Data
	if jsonErr := jsoniter.Unmarshal([]byte(rawDataStr), &singleData); jsonErr != nil {
		if jsonErr = jsoniter.Unmarshal([]byte(rawDataStr), &data); jsonErr != nil {
			return nil, jsonErr
		}
	} else {
		// is single log, and method transformer.transform(data []sender.Data) accept a param of slice type
		data = append(data, singleData)
	}
	return data, nil
}

func getTransformer(transConfig map[string]interface{}, create transforms.Creator) (transforms.Transformer, error) {
	trans := create()
	transformerConfig := convertWebTransformerConfig(transConfig)
	delete(transformerConfig, KeySampleLog)
	bts, jsonErr := jsoniter.Marshal(transformerConfig)
	if jsonErr != nil {
		return nil, jsonErr
	}
	if jsonErr = jsoniter.Unmarshal(bts, trans); jsonErr != nil {
		return nil, jsonErr
	}

	if trans, ok := trans.(transforms.Initialize); ok {
		if err := trans.Init(); err != nil {
			return nil, err
		}
	}
	return trans, nil
}
<|endoftext|>


<commit_before>// Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.

package mid

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http/httputil"
	"runtime"

	"github.com/go-ego/ego"
)

var (
	dunno     = []byte("???")
	centerDot = []byte("·")
	dot       = []byte(".")
	slash     = []byte("/")
)

// Recovery returns a middleware that recovers from any panics and writes a 500 if there was one.
func Recovery() ego.HandlerFunc {
	return RecoveryWithWriter(ego.DefaultErrorWriter)
}

func RecoveryWithWriter(out io.Writer) ego.HandlerFunc {
	var logger *log.Logger
	if out != nil {
		logger = log.New(out, "\n\n\x1b[31m", log.LstdFlags)
	}
	return func(c *ego.Context) {
		defer func() {
			if err := recover(); err != nil {
				if logger != nil {
					stack := stack(3)
					httprequest, _ := httputil.DumpRequest(c.Request, false)
					logger.Printf("[Recovery] panic recovered:\n%s\n%s\n%s%s", string(httprequest), err, stack, reset)
				}
				c.AbortWithStatus(500)
			}
		}()
		c.Next()
	}
}

// stack returns a nicely formated stack frame, skipping skip frames
func stack(skip int) []byte {
	buf := new(bytes.Buffer) // the returned data
	// As we loop, we open files and read them. These variables record the currently
	// loaded file.
	var lines [][]byte
	var lastFile string
	for i := skip; ; i++ { // Skip the expected number of frames
		pc, file, line, ok := runtime.Caller(i)
		if !ok {
			break
		}
		// Print this much at least.  If we can't find the source, it won't show.
		fmt.Fprintf(buf, "%s:%d (0x%x)\n", file, line, pc)
		if file != lastFile {
			data, err := ioutil.ReadFile(file)
			if err != nil {
				continue
			}
			lines = bytes.Split(data, []byte{'\n'})
			lastFile = file
		}
		fmt.Fprintf(buf, "\t%s: %s\n", function(pc), source(lines, line))
	}
	return buf.Bytes()
}

// source returns a space-trimmed slice of the n'th line.
func source(lines [][]byte, n int) []byte {
	n-- // in stack trace, lines are 1-indexed but our array is 0-indexed
	if n < 0 || n >= len(lines) {
		return dunno
	}
	return bytes.TrimSpace(lines[n])
}

// function returns, if possible, the name of the function containing the PC.
func function(pc uintptr) []byte {
	fn := runtime.FuncForPC(pc)
	if fn == nil {
		return dunno
	}
	name := []byte(fn.Name())
	// The name includes the path name to the package, which is unnecessary
	// since the file name is already included.  Plus, it has center dots.
	// That is, we see
	//	runtime/debug.*T·ptrmethod
	// and want
	//	*T.ptrmethod
	// Also the package path might contains dot (e.g. code.google.com/...),
	// so first eliminate the path prefix
	if lastslash := bytes.LastIndex(name, slash); lastslash >= 0 {
		name = name[lastslash+1:]
	}
	if period := bytes.Index(name, dot); period >= 0 {
		name = name[period+1:]
	}
	name = bytes.Replace(name, centerDot, dot, -1)
	return name
}
<commit_msg>add panic time log<commit_after>// Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.

package mid

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http/httputil"
	"runtime"
	"time"

	"github.com/go-ego/ego"
)

var (
	dunno     = []byte("???")
	centerDot = []byte("·")
	dot       = []byte(".")
	slash     = []byte("/")
)

// Recovery returns a middleware that recovers from any panics and writes a 500 if there was one.
func Recovery() ego.HandlerFunc {
	return RecoveryWithWriter(ego.DefaultErrorWriter)
}

func RecoveryWithWriter(out io.Writer) ego.HandlerFunc {
	var logger *log.Logger
	if out != nil {
		logger = log.New(out, "\n\n\x1b[31m", log.LstdFlags)
	}
	return func(c *ego.Context) {
		defer func() {
			if err := recover(); err != nil {
				if logger != nil {
					stack := stack(3)
					httprequest, _ := httputil.DumpRequest(c.Request, false)
					logger.Printf("[Recovery] %s panic recovered:\n%s\n%s\n%s%s",
						timeFormat(time.Now()), string(httprequest), err, stack, reset)
				}
				c.AbortWithStatus(500)
			}
		}()
		c.Next()
	}
}

// stack returns a nicely formated stack frame, skipping skip frames
func stack(skip int) []byte {
	buf := new(bytes.Buffer) // the returned data
	// As we loop, we open files and read them. These variables record the currently
	// loaded file.
	var (
		lines    [][]byte
		lastFile string
	)

	for i := skip; ; i++ { // Skip the expected number of frames
		pc, file, line, ok := runtime.Caller(i)
		if !ok {
			break
		}
		// Print this much at least.  If we can't find the source, it won't show.
		fmt.Fprintf(buf, "%s:%d (0x%x)\n", file, line, pc)
		if file != lastFile {
			data, err := ioutil.ReadFile(file)
			if err != nil {
				continue
			}
			lines = bytes.Split(data, []byte{'\n'})
			lastFile = file
		}
		fmt.Fprintf(buf, "\t%s: %s\n", function(pc), source(lines, line))
	}
	return buf.Bytes()
}

// source returns a space-trimmed slice of the n'th line.
func source(lines [][]byte, n int) []byte {
	n-- // in stack trace, lines are 1-indexed but our array is 0-indexed
	if n < 0 || n >= len(lines) {
		return dunno
	}
	return bytes.TrimSpace(lines[n])
}

// function returns, if possible, the name of the function containing the PC.
func function(pc uintptr) []byte {
	fn := runtime.FuncForPC(pc)
	if fn == nil {
		return dunno
	}
	name := []byte(fn.Name())
	// The name includes the path name to the package, which is unnecessary
	// since the file name is already included.  Plus, it has center dots.
	// That is, we see
	//	runtime/debug.*T·ptrmethod
	// and want
	//	*T.ptrmethod
	// Also the package path might contains dot (e.g. code.google.com/...),
	// so first eliminate the path prefix
	if lastslash := bytes.LastIndex(name, slash); lastslash >= 0 {
		name = name[lastslash+1:]
	}
	if period := bytes.Index(name, dot); period >= 0 {
		name = name[period+1:]
	}
	name = bytes.Replace(name, centerDot, dot, -1)
	return name
}

func timeFormat(t time.Time) string {
	var timeString = t.Format("2006/01/02 - 15:04:05")
	return timeString
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/kelseyhightower/packer-builder-googlecompute/builder/googlecompute"
	"github.com/mitchellh/packer/packer/plugin"
)

func main() {
	plugin.ServeBuilder(new(googlecompute.Builder))
}
<commit_msg>plugin/builder-googlecompute: compile<commit_after>package main

import (
	"github.com/mitchellh/packer/builder/googlecompute"
	"github.com/mitchellh/packer/packer/plugin"
)

func main() {
	server, err := plugin.Server()
	if err != nil {
		panic(err)
	}
	server.RegisterBuilder(new(googlecompute.Builder))
	server.Serve()
}
<|endoftext|>


<commit_before>package main

// keys.go contains functions for generating and printing keys.

import (
	"fmt"
	"os"
	"path/filepath"
	"strconv"

	"github.com/spf13/cobra"

	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/types"
)

const (
	// The header for all siakg files. Do not change.
	FileHeader = "siakg"
)

// A KeyPair is the object that gets saved to disk for a signature key. All the
// information necessary to sign a transaction is in the struct, and the struct
// can be directly written to disk.
type KeyPair struct {
	Header           string
	Version          string
	Index            int
	SecretKey        crypto.SecretKey
	PublicKey        crypto.PublicKey
	UnlockConditions types.UnlockConditions
}

// generateKeys will generate a set of keys and save the keyfiles to disk.
func generateKeys(*cobra.Command, []string) {
	// Check that the key requirements make sense.
	if config.Siakg.RequiredKeys == 0 {
		fmt.Println("An address with 0 required keys is not useful.")
		return
	}
	if config.Siakg.TotalKeys < config.Siakg.RequiredKeys {
		fmt.Printf("Total Keys (%v) must be greater than or equal to Required Keys (%v)\n", config.Siakg.TotalKeys, config.Siakg.RequiredKeys)
		return
	}

	fmt.Printf("Creating key '%s' with %v total keys and %v required keys.\n", config.Siakg.KeyName, config.Siakg.TotalKeys, config.Siakg.RequiredKeys)

	// Generate 'TotalKeys' keyparis and fill out the metadata.
	keys := make([]KeyPair, config.Siakg.TotalKeys)
	for i := range keys {
		keys[i].Header = FileHeader
		keys[i].Version = Version
		keys[i].Index = i
	}

	// Add the keys to each keypair.
	for i := 0; i < config.Siakg.TotalKeys; i++ {
		var err error
		keys[i].SecretKey, keys[i].PublicKey, err = crypto.GenerateSignatureKeys()
		if err != nil {
			fmt.Println(err)
			return
		}
	}

	// Generate the unlock conditions and add them to each KeyPair object.
	uc := types.UnlockConditions{
		Timelock:      0,
		NumSignatures: uint64(config.Siakg.RequiredKeys),
	}
	for i := range keys {
		uc.PublicKeys = append(uc.PublicKeys, types.SiaPublicKey{
			Algorithm: types.SignatureEd25519,
			Key:       string(keys[i].PublicKey[:]),
		})
	}
	for i := range keys {
		keys[i].UnlockConditions = uc
	}

	// Save the KeyPairs to disk.
	if config.Siakg.Folder != "" {
		err := os.MkdirAll(config.Siakg.Folder, 0700)
		if err != nil {
			fmt.Println(err)
			return
		}
	}
	for i, key := range keys {
		err := encoding.WriteFile(filepath.Join(config.Siakg.Folder, config.Siakg.KeyName)+"_Key"+strconv.Itoa(i)+FileExtension, key)
		if err != nil {
			fmt.Println(err)
			return
		}
	}

	fmt.Printf("Success, the address for this set of keys is: %x\n", uc.UnlockHash())
}

// printKey opens a keyfile and prints the contents.
func printKey(*cobra.Command, []string) {
	var kp KeyPair
	err := encoding.ReadFile(config.Address.Filename, &kp)
	if err != nil {
		fmt.Println(err)
		return
	}

	fmt.Printf("Found a key for a %v of %v address.\n", kp.UnlockConditions.NumSignatures, len(kp.UnlockConditions.PublicKeys))
	fmt.Printf("The address is: %x\n", kp.UnlockConditions.UnlockHash())
}
<commit_msg>check that keys are valid<commit_after>package main

// keys.go contains functions for generating and printing keys.

import (
	"fmt"
	"os"
	"path/filepath"
	"strconv"

	"github.com/spf13/cobra"

	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/types"
)

const (
	// The header for all siakg files. Do not change.
	FileHeader = "siakg"
)

// A KeyPair is the object that gets saved to disk for a signature key. All the
// information necessary to sign a transaction is in the struct, and the struct
// can be directly written to disk.
type KeyPair struct {
	Header           string
	Version          string
	Index            int
	SecretKey        crypto.SecretKey
	PublicKey        crypto.PublicKey
	UnlockConditions types.UnlockConditions
}

// generateKeys will generate a set of keys and save the keyfiles to disk.
func generateKeys(*cobra.Command, []string) {
	// Check that the key requirements make sense.
	if config.Siakg.RequiredKeys == 0 {
		fmt.Println("An address with 0 required keys is not useful.")
		return
	}
	if config.Siakg.TotalKeys < config.Siakg.RequiredKeys {
		fmt.Printf("Total Keys (%v) must be greater than or equal to Required Keys (%v)\n", config.Siakg.TotalKeys, config.Siakg.RequiredKeys)
		return
	}

	fmt.Printf("Creating key '%s' with %v total keys and %v required keys.\n", config.Siakg.KeyName, config.Siakg.TotalKeys, config.Siakg.RequiredKeys)

	// Generate 'TotalKeys' keyparis and fill out the metadata.
	keys := make([]KeyPair, config.Siakg.TotalKeys)
	for i := range keys {
		keys[i].Header = FileHeader
		keys[i].Version = Version
		keys[i].Index = i
	}

	// Add the keys to each keypair.
	for i := 0; i < config.Siakg.TotalKeys; i++ {
		var err error
		keys[i].SecretKey, keys[i].PublicKey, err = crypto.GenerateSignatureKeys()
		if err != nil {
			fmt.Println(err)
			return
		}
	}

	// Generate the unlock conditions and add them to each KeyPair object.
	uc := types.UnlockConditions{
		Timelock:      0,
		NumSignatures: uint64(config.Siakg.RequiredKeys),
	}
	for i := range keys {
		uc.PublicKeys = append(uc.PublicKeys, types.SiaPublicKey{
			Algorithm: types.SignatureEd25519,
			Key:       string(keys[i].PublicKey[:]),
		})
	}
	for i := range keys {
		keys[i].UnlockConditions = uc
	}

	// Save the KeyPairs to disk.
	if config.Siakg.Folder != "" {
		err := os.MkdirAll(config.Siakg.Folder, 0700)
		if err != nil {
			fmt.Println(err)
			return
		}
	}
	for i, key := range keys {
		err := encoding.WriteFile(filepath.Join(config.Siakg.Folder, config.Siakg.KeyName)+"_Key"+strconv.Itoa(i)+FileExtension, key)
		if err != nil {
			fmt.Println(err)
			return
		}
	}

	// Load the keys from disk back into memory, then verifiy that the keys on
	// disk are able to sign outputs in transactions.
	loadedKeys := make([]KeyPair, config.Siakg.TotalKeys)
	for i := 0; i < len(loadedKeys); i++ {
		err := encoding.ReadFile(filepath.Join(config.Siakg.Folder, config.Siakg.KeyName)+"_Key"+strconv.Itoa(i)+FileExtension, &loadedKeys[i])
		if err != nil {
			fmt.Println(err)
			return
		}
	}

	// Check that the keys can be used to spend transactions. Load them back into memory.
	for _, loadedKey := range loadedKeys {
		if loadedKey.UnlockConditions.UnlockHash() != uc.UnlockHash() {
			fmt.Println("corruption occured while saving the keys to disk")
		}
	}
	// Create a transaction for the keys to sign.
	txn := types.Transaction{
		SiafundInputs: []types.SiafundInput{
			types.SiafundInput{
				UnlockConditions: loadedKeys[0].UnlockConditions,
			},
		},
	}
	// Loop through and sign the transaction multiple times. All keys will be
	// used at least once by the time the loop terminates.
	var i int
	for i != config.Siakg.TotalKeys {
		// i tracks which key is next to be used. If i + RequiredKeys results
		// in going out-of-bounds, reduce i so that the last key will be used
		// for the final signature.
		if i+config.Siakg.RequiredKeys > config.Siakg.TotalKeys {
			i = config.Siakg.TotalKeys - config.Siakg.RequiredKeys
		}
		var j int
		for j < config.Siakg.RequiredKeys {
			txn.Signatures = append(txn.Signatures, types.TransactionSignature{
				PublicKeyIndex: uint64(i),
				CoveredFields:  types.CoveredFields{WholeTransaction: true},
			})
			sigHash := txn.SigHash(j)
			sig, err := crypto.SignHash(sigHash, loadedKeys[i].SecretKey)
			if err != nil {
				fmt.Println(err)
				return
			}
			txn.Signatures[j].Signature = types.Signature(sig[:])
			i++
			j++
		}
		// Check that the signature is valid.
		err := txn.StandaloneValid(0)
		if err != nil {
			fmt.Println(err)
		}
		// Delete all of the signatures for the next iteration.
		txn.Signatures = nil
	}

	fmt.Printf("Success, the address for this set of keys is: %x\n", uc.UnlockHash())
}

// printKey opens a keyfile and prints the contents.
func printKey(*cobra.Command, []string) {
	var kp KeyPair
	err := encoding.ReadFile(config.Address.Filename, &kp)
	if err != nil {
		fmt.Println(err)
		return
	}

	fmt.Printf("Found a key for a %v of %v address.\n", kp.UnlockConditions.NumSignatures, len(kp.UnlockConditions.PublicKeys))
	fmt.Printf("The address is: %x\n", kp.UnlockConditions.UnlockHash())
}
<|endoftext|>