<commit_before>// Package tracer implements a Dapper-style tracing system. It is
// compatible with the OpenTracing specification.
//
// Sampling
//
// To keep the overhead incurred by tracing low, only a subset of
// requests should be traced. This is achieved by a sampler. Each
// tracer has a sampler that may sample requests based on chance, a
// rate, or possibly other mechanisms. The default sampler samples all
// requests, which is useful for testing, and viable for low-traffic
// systems.
//
// Only root spans make sampling decisions. Child spans will inherit
// the sampling decisions of the root spans.
//
// Errors and logging
//
// The instrumentation is defensive and will never purposefully panic.
// At the same time, most functions do not return errors, because
// they'll be called by automatic instrumentation, hidden from the
// user. Instead, errors will be logged.
package tracer

import (
	"crypto/rand"
	"encoding/binary"
	"encoding/hex"
	"log"
	"reflect"
	"sync"
	"time"

	"github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/ext"
)

// The various flags of a Span.
const (
	// The Span has been sampled.
	FlagSampled = 1 << iota
)

// A Logger logs messages.
type Logger interface {
	// Printf logs a single message, given a format and values. The
	// format is documented in the fmt package.
	Printf(format string, values ...interface{})
}

type defaultLogger struct{}

func (defaultLogger) Printf(format string, values ...interface{}) {
	log.Printf(format, values...)
}

// valueType returns the broad categorization of a value's type and
// whether it is permitted as a payload.
func valueType(v interface{}) (string, bool) {
	if v == nil {
		return "", true
	}
	rv := reflect.ValueOf(v)
	if !rv.IsValid() {
		return "", false
	}
	switch rv.Type().Kind() {
	case reflect.Bool:
		return "boolean", true
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32,
		reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16,
		reflect.Uint32, reflect.Uint64, reflect.Float32, reflect.Float64:
		return "number", true
	case reflect.String:
		return "string", true
	}
	return "", false
}

// A RawTrace contains all the data associated with a trace.
type RawTrace struct {
	TraceID   uint64        `json:"trace_id"`
	Spans     []RawSpan     `json:"spans"`
	Relations []RawRelation `json:"relations"`
}

// A RawRelation represents the relation between two spans.
type RawRelation struct {
	ParentID uint64 `json:"parent_id"`
	ChildID  uint64 `json:"child_id"`
	Kind     string `json:"kind"`
}

// Span is an implementation of the OpenTracing Span interface.
type Span struct {
	mu     sync.RWMutex
	tracer *Tracer
	raw    RawSpan
}

// A RawSpan contains all the data associated with a span.
type RawSpan struct {
	SpanContext
	ServiceName   string    `json:"service_name"`
	OperationName string    `json:"operation_name"`
	StartTime     time.Time `json:"start_time"`
	FinishTime    time.Time `json:"finish_time"`

	Tags map[string]interface{} `json:"tags"`
	Logs []opentracing.LogData  `json:"logs"`
}

// RawSpan returns a deep copy of the span's underlying data.
func (sp *Span) RawSpan() RawSpan {
	sp.mu.Lock()
	defer sp.mu.Unlock()
	raw := sp.raw
	tags := raw.Tags
	raw.Tags = map[string]interface{}{}
	for k, v := range tags {
		raw.Tags[k] = v
	}
	raw.Logs = append([]opentracing.LogData(nil), raw.Logs...)
	baggage := raw.Baggage
	raw.Baggage = map[string]string{}
	for k, v := range baggage {
		raw.Baggage[k] = v
	}
	return raw
}

// Sampled reports whether this span was sampled.
func (sp *Span) Sampled() bool {
	sp.mu.RLock()
	defer sp.mu.RUnlock()
	return sp.sampled()
}

func (sp *Span) sampled() bool {
	return (sp.raw.Flags & FlagSampled) > 0
}

// SetOperationName implements the opentracing.Span interface.
func (sp *Span) SetOperationName(name string) opentracing.Span {
	sp.mu.Lock()
	defer sp.mu.Unlock()
	sp.raw.OperationName = name
	return sp
}

// SetTag implements the opentracing.Span interface.
func (sp *Span) SetTag(key string, value interface{}) opentracing.Span {
	sp.mu.Lock()
	defer sp.mu.Unlock()
	if !sp.sampled() {
		return sp
	}
	if _, ok := valueType(value); !ok {
		sp.tracer.Logger.Printf("unsupported tag value type for tag %q: %T", key, value)
		return sp
	}
	if sp.raw.Tags == nil {
		sp.raw.Tags = map[string]interface{}{}
	}
	sp.raw.Tags[key] = value
	return sp
}

// SetBaggageItem implements the opentracing.Tracer interface.
func (sp *Span) SetBaggageItem(key, value string) opentracing.Span {
	sp.raw.SpanContext.Baggage[key] = value
	return sp
}

// BaggageItem implements the opentracing.Tracer interface.
func (sp *Span) BaggageItem(key string) string {
	return sp.raw.SpanContext.Baggage[key]
}

// Finish implements the opentracing.Span interface.
func (sp *Span) Finish() {
	if !sp.Sampled() {
		return
	}
	sp.FinishWithOptions(opentracing.FinishOptions{})
}

// FinishWithOptions implements the opentracing.Span interface.
func (sp *Span) FinishWithOptions(opts opentracing.FinishOptions) {
	sp.mu.Lock()
	defer sp.mu.Unlock()
	if !sp.sampled() {
		return
	}
	if opts.FinishTime.IsZero() {
		opts.FinishTime = time.Now()
	}
	sp.raw.FinishTime = opts.FinishTime
	for _, log := range opts.BulkLogData {
		sp.log(log)
	}
	if err := sp.tracer.storer.Store(sp.raw); err != nil {
		sp.tracer.Logger.Printf("error while storing tracing span: %s", err)
	}
}

// LogEvent implements the opentracing.Span interface.
func (sp *Span) LogEvent(event string) {
	if !sp.Sampled() {
		return
	}
	sp.Log(opentracing.LogData{
		Event: event,
	})
}

// LogEventWithPayload implements the opentracing.Span interface.
func (sp *Span) LogEventWithPayload(event string, payload interface{}) {
	if !sp.Sampled() {
		return
	}
	sp.Log(opentracing.LogData{
		Event:   event,
		Payload: payload,
	})
}

// Log implements the opentracing.Span interface.
func (sp *Span) Log(data opentracing.LogData) {
	sp.mu.Lock()
	defer sp.mu.Unlock()
	sp.log(data)
}

func (sp *Span) log(data opentracing.LogData) {
	if !sp.sampled() {
		return
	}
	if _, ok := valueType(data.Payload); !ok {
		sp.tracer.Logger.Printf("unsupported log payload type for event %q: %T", data.Event, data.Payload)
		return
	}
	if data.Timestamp.IsZero() {
		data.Timestamp = time.Now()
	}
	sp.raw.Logs = append(sp.raw.Logs, data)
}

// Context implements the opentracing.Span interface.
func (sp *Span) Context() opentracing.SpanContext {
	sp.mu.RLock()
	defer sp.mu.RUnlock()
	return sp.raw.SpanContext
}

// Tracer implements the opentracing.Span interface.
func (sp *Span) Tracer() opentracing.Tracer {
	sp.mu.RLock()
	defer sp.mu.RUnlock()
	return sp.tracer
}

// Tracer is an implementation of the OpenTracing Tracer interface.
type Tracer struct {
	ServiceName string
	Logger      Logger
	Sampler     Sampler

	storer      Storer
	idGenerator IDGenerator
}

// NewTracer returns a new tracer.
func NewTracer(serviceName string, storer Storer, idGenerator IDGenerator) *Tracer {
	return &Tracer{
		ServiceName: serviceName,
		Logger:      defaultLogger{},
		Sampler:     NewConstSampler(true),
		storer:      storer,
		idGenerator: idGenerator,
	}
}

// StartSpan implements the opentracing.Tracer interface.
func (tr *Tracer) StartSpan(operationName string, opts ...opentracing.StartSpanOption) opentracing.Span {
	var sopts opentracing.StartSpanOptions
	for _, opt := range opts {
		opt.Apply(&sopts)
	}
	if sopts.StartTime.IsZero() {
		sopts.StartTime = time.Now()
	}

	id := tr.idGenerator.GenerateID()
	sp := &Span{
		tracer: tr,
		raw: RawSpan{
			SpanContext: SpanContext{
				SpanID:  id,
				TraceID: id,
			},
			ServiceName:   tr.ServiceName,
			OperationName: operationName,
			StartTime:     sopts.StartTime,
		},
	}
	if len(sopts.References) > 0 {
		// TODO(dh): support multiple parents, support ChildOf and
		// FollowsFrom as separate kinds of relations.
		parent, ok := sopts.References[0].ReferencedContext.(SpanContext)
		if !ok {
			panic("parent span must be of type *Span")
		}
		sp.raw.ParentID = parent.SpanID
		sp.raw.TraceID = parent.TraceID
		sp.raw.Flags = parent.Flags
	} else {
		if n, _ := sopts.Tags[string(ext.SamplingPriority)].(uint16); n > 0 {
			sp.raw.Flags |= FlagSampled
		} else if tr.Sampler.Sample(id) {
			sp.raw.Flags |= FlagSampled
		}
	}
	sp.raw.Tags = sopts.Tags
	return sp
}

func (tr *Tracer) Flush() error {
	f, ok := tr.storer.(Flusher)
	if !ok {
		return nil
	}
	return f.Flush()
}

func idToHex(id uint64) string {
	b := make([]byte, 8)
	binary.BigEndian.PutUint64(b, id)
	return hex.EncodeToString(b)
}

func idFromHex(s string) uint64 {
	b, _ := hex.DecodeString(s)
	return binary.BigEndian.Uint64(b)
}

// Inject implements the opentracing.Tracer interface.
func (tr *Tracer) Inject(sm opentracing.SpanContext, format interface{}, carrier interface{}) error {
	context, ok := sm.(SpanContext)
	if !ok {
		return opentracing.ErrInvalidSpanContext
	}
	injecter, ok := injecters[format]
	if !ok {
		return opentracing.ErrUnsupportedFormat
	}
	return injecter(context, carrier)
}

// Extract implements the opentracing.Tracer interface.
func (tr *Tracer) Extract(format interface{}, carrier interface{}) (opentracing.SpanContext, error) {
	extracter, ok := extracters[format]
	if !ok {
		return nil, opentracing.ErrUnsupportedFormat
	}
	context, err := extracter(carrier)
	if err != nil {
		return nil, err
	}
	return context, nil
}

// IDGenerator generates IDs for traces and spans. The ID with value 0
// is reserved to mean "no parent span" and should not be generated.
type IDGenerator interface {
	GenerateID() uint64
}

// A Storer stores a finished span. "Storing" a span may either mean
// saving it in a storage engine, or sending it to a remote
// collector.
//
// If a span with the same ID and the same trace ID already exists,
// the existing and new spans should be merged into one span.
//
// Because spans are only stored once they're done, children will be
// stored before their parents.
type Storer interface {
	Store(sp RawSpan) error
}

// Flusher is an optional interface that when implemented allows a
// Storer to flush buffered spawns.
type Flusher interface {
	Flush() error
}

var _ IDGenerator = RandomID{}

// RandomID generates random IDs by using crypto/rand.
type RandomID struct{}

// GenerateID generates an ID.
func (RandomID) GenerateID() uint64 {
	b := make([]byte, 8)
	for {
		_, _ = rand.Read(b)
		x := binary.BigEndian.Uint64(b)
		if x != 0 {
			return x
		}
	}
}
<commit_msg>add methods LogFields and LogKV for span<commit_after>// Package tracer implements a Dapper-style tracing system. It is
// compatible with the OpenTracing specification.
//
// Sampling
//
// To keep the overhead incurred by tracing low, only a subset of
// requests should be traced. This is achieved by a sampler. Each
// tracer has a sampler that may sample requests based on chance, a
// rate, or possibly other mechanisms. The default sampler samples all
// requests, which is useful for testing, and viable for low-traffic
// systems.
//
// Only root spans make sampling decisions. Child spans will inherit
// the sampling decisions of the root spans.
//
// Errors and logging
//
// The instrumentation is defensive and will never purposefully panic.
// At the same time, most functions do not return errors, because
// they'll be called by automatic instrumentation, hidden from the
// user. Instead, errors will be logged.
package tracer

import (
	"crypto/rand"
	"encoding/binary"
	"encoding/hex"
	"log"
	"reflect"
	"sync"
	"time"

	"github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/ext"
	tracinglog "github.com/opentracing/opentracing-go/log"
)

// The various flags of a Span.
const (
	// The Span has been sampled.
	FlagSampled = 1 << iota
)

// A Logger logs messages.
type Logger interface {
	// Printf logs a single message, given a format and values. The
	// format is documented in the fmt package.
	Printf(format string, values ...interface{})
}

type defaultLogger struct{}

func (defaultLogger) Printf(format string, values ...interface{}) {
	log.Printf(format, values...)
}

// valueType returns the broad categorization of a value's type and
// whether it is permitted as a payload.
func valueType(v interface{}) (string, bool) {
	if v == nil {
		return "", true
	}
	rv := reflect.ValueOf(v)
	if !rv.IsValid() {
		return "", false
	}
	switch rv.Type().Kind() {
	case reflect.Bool:
		return "boolean", true
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32,
		reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16,
		reflect.Uint32, reflect.Uint64, reflect.Float32, reflect.Float64:
		return "number", true
	case reflect.String:
		return "string", true
	}
	return "", false
}

// A RawTrace contains all the data associated with a trace.
type RawTrace struct {
	TraceID   uint64        `json:"trace_id"`
	Spans     []RawSpan     `json:"spans"`
	Relations []RawRelation `json:"relations"`
}

// A RawRelation represents the relation between two spans.
type RawRelation struct {
	ParentID uint64 `json:"parent_id"`
	ChildID  uint64 `json:"child_id"`
	Kind     string `json:"kind"`
}

// Span is an implementation of the OpenTracing Span interface.
type Span struct {
	mu     sync.RWMutex
	tracer *Tracer
	raw    RawSpan
}

// A RawSpan contains all the data associated with a span.
type RawSpan struct {
	SpanContext
	ServiceName   string    `json:"service_name"`
	OperationName string    `json:"operation_name"`
	StartTime     time.Time `json:"start_time"`
	FinishTime    time.Time `json:"finish_time"`

	Tags map[string]interface{} `json:"tags"`
	Logs []opentracing.LogData  `json:"logs"`
}

// RawSpan returns a deep copy of the span's underlying data.
func (sp *Span) RawSpan() RawSpan {
	sp.mu.Lock()
	defer sp.mu.Unlock()
	raw := sp.raw
	tags := raw.Tags
	raw.Tags = map[string]interface{}{}
	for k, v := range tags {
		raw.Tags[k] = v
	}
	raw.Logs = append([]opentracing.LogData(nil), raw.Logs...)
	baggage := raw.Baggage
	raw.Baggage = map[string]string{}
	for k, v := range baggage {
		raw.Baggage[k] = v
	}
	return raw
}

// Sampled reports whether this span was sampled.
func (sp *Span) Sampled() bool {
	sp.mu.RLock()
	defer sp.mu.RUnlock()
	return sp.sampled()
}

func (sp *Span) sampled() bool {
	return (sp.raw.Flags & FlagSampled) > 0
}

// SetOperationName implements the opentracing.Span interface.
func (sp *Span) SetOperationName(name string) opentracing.Span {
	sp.mu.Lock()
	defer sp.mu.Unlock()
	sp.raw.OperationName = name
	return sp
}

// SetTag implements the opentracing.Span interface.
func (sp *Span) SetTag(key string, value interface{}) opentracing.Span {
	sp.mu.Lock()
	defer sp.mu.Unlock()
	if !sp.sampled() {
		return sp
	}
	if _, ok := valueType(value); !ok {
		sp.tracer.Logger.Printf("unsupported tag value type for tag %q: %T", key, value)
		return sp
	}
	if sp.raw.Tags == nil {
		sp.raw.Tags = map[string]interface{}{}
	}
	sp.raw.Tags[key] = value
	return sp
}

// SetBaggageItem implements the opentracing.Tracer interface.
func (sp *Span) SetBaggageItem(key, value string) opentracing.Span {
	sp.raw.SpanContext.Baggage[key] = value
	return sp
}

// BaggageItem implements the opentracing.Tracer interface.
func (sp *Span) BaggageItem(key string) string {
	return sp.raw.SpanContext.Baggage[key]
}

// Finish implements the opentracing.Span interface.
func (sp *Span) Finish() {
	if !sp.Sampled() {
		return
	}
	sp.FinishWithOptions(opentracing.FinishOptions{})
}

// FinishWithOptions implements the opentracing.Span interface.
func (sp *Span) FinishWithOptions(opts opentracing.FinishOptions) {
	sp.mu.Lock()
	defer sp.mu.Unlock()
	if !sp.sampled() {
		return
	}
	if opts.FinishTime.IsZero() {
		opts.FinishTime = time.Now()
	}
	sp.raw.FinishTime = opts.FinishTime
	for _, log := range opts.BulkLogData {
		sp.log(log)
	}
	if err := sp.tracer.storer.Store(sp.raw); err != nil {
		sp.tracer.Logger.Printf("error while storing tracing span: %s", err)
	}
}

// LogEvent implements the opentracing.Span interface.
func (sp *Span) LogEvent(event string) {
	if !sp.Sampled() {
		return
	}
	sp.Log(opentracing.LogData{
		Event: event,
	})
}

// LogEventWithPayload implements the opentracing.Span interface.
func (sp *Span) LogEventWithPayload(event string, payload interface{}) {
	if !sp.Sampled() {
		return
	}
	sp.Log(opentracing.LogData{
		Event:   event,
		Payload: payload,
	})
}

// Log implements the opentracing.Span interface.
func (sp *Span) Log(data opentracing.LogData) {
	sp.mu.Lock()
	defer sp.mu.Unlock()
	sp.log(data)
}

func (sp *Span) LogFields(fields ...tracinglog.Field) {
	for _, f := range fields {
		sp.log(opentracing.LogData{
			Event:     f.Key(),
			Timestamp: time.Now(),
			Payload:   f.Value(),
		})
	}
}

func (sp *Span) LogKV(keyValues ...interface{}) {
	fields, err := tracinglog.InterleavedKVToFields(keyValues)
	if err != nil {
		sp.tracer.Logger.Printf("can't parse keyValues: %s", err)
		return
	}
	sp.LogFields(fields...)
}

func (sp *Span) log(data opentracing.LogData) {
	if !sp.sampled() {
		return
	}
	if _, ok := valueType(data.Payload); !ok {
		sp.tracer.Logger.Printf("unsupported log payload type for event %q: %T", data.Event, data.Payload)
		return
	}
	if data.Timestamp.IsZero() {
		data.Timestamp = time.Now()
	}
	sp.raw.Logs = append(sp.raw.Logs, data)
}

// Context implements the opentracing.Span interface.
func (sp *Span) Context() opentracing.SpanContext {
	sp.mu.RLock()
	defer sp.mu.RUnlock()
	return sp.raw.SpanContext
}

// Tracer implements the opentracing.Span interface.
func (sp *Span) Tracer() opentracing.Tracer {
	sp.mu.RLock()
	defer sp.mu.RUnlock()
	return sp.tracer
}

// Tracer is an implementation of the OpenTracing Tracer interface.
type Tracer struct {
	ServiceName string
	Logger      Logger
	Sampler     Sampler

	storer      Storer
	idGenerator IDGenerator
}

// NewTracer returns a new tracer.
func NewTracer(serviceName string, storer Storer, idGenerator IDGenerator) *Tracer {
	return &Tracer{
		ServiceName: serviceName,
		Logger:      defaultLogger{},
		Sampler:     NewConstSampler(true),
		storer:      storer,
		idGenerator: idGenerator,
	}
}

// StartSpan implements the opentracing.Tracer interface.
func (tr *Tracer) StartSpan(operationName string, opts ...opentracing.StartSpanOption) opentracing.Span {
	var sopts opentracing.StartSpanOptions
	for _, opt := range opts {
		opt.Apply(&sopts)
	}
	if sopts.StartTime.IsZero() {
		sopts.StartTime = time.Now()
	}

	id := tr.idGenerator.GenerateID()
	sp := &Span{
		tracer: tr,
		raw: RawSpan{
			SpanContext: SpanContext{
				SpanID:  id,
				TraceID: id,
			},
			ServiceName:   tr.ServiceName,
			OperationName: operationName,
			StartTime:     sopts.StartTime,
		},
	}
	if len(sopts.References) > 0 {
		// TODO(dh): support multiple parents, support ChildOf and
		// FollowsFrom as separate kinds of relations.
		parent, ok := sopts.References[0].ReferencedContext.(SpanContext)
		if !ok {
			panic("parent span must be of type *Span")
		}
		sp.raw.ParentID = parent.SpanID
		sp.raw.TraceID = parent.TraceID
		sp.raw.Flags = parent.Flags
	} else {
		if n, _ := sopts.Tags[string(ext.SamplingPriority)].(uint16); n > 0 {
			sp.raw.Flags |= FlagSampled
		} else if tr.Sampler.Sample(id) {
			sp.raw.Flags |= FlagSampled
		}
	}
	sp.raw.Tags = sopts.Tags
	return sp
}

func (tr *Tracer) Flush() error {
	f, ok := tr.storer.(Flusher)
	if !ok {
		return nil
	}
	return f.Flush()
}

func idToHex(id uint64) string {
	b := make([]byte, 8)
	binary.BigEndian.PutUint64(b, id)
	return hex.EncodeToString(b)
}

func idFromHex(s string) uint64 {
	b, _ := hex.DecodeString(s)
	return binary.BigEndian.Uint64(b)
}

// Inject implements the opentracing.Tracer interface.
func (tr *Tracer) Inject(sm opentracing.SpanContext, format interface{}, carrier interface{}) error {
	context, ok := sm.(SpanContext)
	if !ok {
		return opentracing.ErrInvalidSpanContext
	}
	injecter, ok := injecters[format]
	if !ok {
		return opentracing.ErrUnsupportedFormat
	}
	return injecter(context, carrier)
}

// Extract implements the opentracing.Tracer interface.
func (tr *Tracer) Extract(format interface{}, carrier interface{}) (opentracing.SpanContext, error) {
	extracter, ok := extracters[format]
	if !ok {
		return nil, opentracing.ErrUnsupportedFormat
	}
	context, err := extracter(carrier)
	if err != nil {
		return nil, err
	}
	return context, nil
}

// IDGenerator generates IDs for traces and spans. The ID with value 0
// is reserved to mean "no parent span" and should not be generated.
type IDGenerator interface {
	GenerateID() uint64
}

// A Storer stores a finished span. "Storing" a span may either mean
// saving it in a storage engine, or sending it to a remote
// collector.
//
// If a span with the same ID and the same trace ID already exists,
// the existing and new spans should be merged into one span.
//
// Because spans are only stored once they're done, children will be
// stored before their parents.
type Storer interface {
	Store(sp RawSpan) error
}

// Flusher is an optional interface that when implemented allows a
// Storer to flush buffered spawns.
type Flusher interface {
	Flush() error
}

var _ IDGenerator = RandomID{}

// RandomID generates random IDs by using crypto/rand.
type RandomID struct{}

// GenerateID generates an ID.
func (RandomID) GenerateID() uint64 {
	b := make([]byte, 8)
	for {
		_, _ = rand.Read(b)
		x := binary.BigEndian.Uint64(b)
		if x != 0 {
			return x
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Dorival de Moraes Pedroso. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package goga

import (
	"encoding/json"

	"github.com/cpmech/gosl/chk"
	"github.com/cpmech/gosl/io"
	"github.com/cpmech/gosl/rnd"
)

// Parameters hold all configuration parameters
type Parameters struct {

	// sizes
	Nova int // number of objective values
	Noor int // number of out-of-range values
	Nsol int // total number of solutions
	Ncpu int // number of cpus

	// time
	Tf    int // final time
	DtExc int // delta time for exchange
	DtOut int // delta time for output

	// options
	DEC      float64 // C-coefficient for differential evolution
	Pll      bool    // parallel
	Seed     int     // seed for random numbers generator
	GenType  string  // generation type: "latin", "halton", "rnd"
	LatinDup int     // Latin Hypercube duplicates number
	EpsH     float64 // minimum value for 'h' constraints
	Verbose  bool    // show messages
	GenAll   bool    // generate all solutions together; i.e. not within each group/CPU
	Nsamples int     // run many samples
	BinInt   int     // flag that integers represent binary numbers if BinInt > 0; thus Nint=BinInt
	ClearFlt bool    // clear flt if corresponding int is 0
	ExcTour  bool    // use exchange via tournament
	ExcOne   bool    // use exchange one randomly

	// crossover and mutation of integers
	IntPc       float64 // probability of crossover for ints
	IntNcuts    int     // number of cuts in crossover of ints
	IntPm       float64 // probability of mutation for ints
	IntNchanges int     // number of changes during mutation of ints

	// range
	FltMin []float64 // minimum float allowed
	FltMax []float64 // maximum float allowed
	IntMin []int     // minimum int allowed
	IntMax []int     // maximum int allowed

	// derived
	Nflt   int       // number of floats
	Nint   int       // number of integers
	DelFlt []float64 // max float range
	DelInt []int     // max int range
}

// Default sets default parameters
func (o *Parameters) Default() {

	// sizes
	o.Nova = 1
	o.Noor = 0
	o.Nsol = 40
	o.Ncpu = 4

	// time
	o.Tf = 100
	o.DtExc = -1
	o.DtOut = -1

	// options
	o.DEC = 0.8
	o.Pll = true
	o.Seed = 0
	o.GenType = "latin"
	o.LatinDup = 2
	o.EpsH = 0.1
	o.Verbose = true
	o.GenAll = false
	o.Nsamples = 10
	o.BinInt = 0
	o.ClearFlt = false
	o.ExcTour = true
	o.ExcOne = true

	// crossover and mutation of integers
	o.IntPc = 0.8
	o.IntNcuts = 1
	o.IntPm = 0.01
	o.IntNchanges = 1
}

// Read reads configuration parameters from JSON file
func (o *Parameters) Read(filenamepath string) {
	o.Default()
	b, err := io.ReadFile(filenamepath)
	if err != nil {
		chk.Panic("cannot read parameters file %q", filenamepath)
	}
	err = json.Unmarshal(b, o)
	if err != nil {
		chk.Panic("cannot unmarshal parameters file %q", filenamepath)
	}
	return
}

// CalcDerived computes derived variables and checks consistency
func (o *Parameters) CalcDerived() {

	// check
	if o.Nova < 1 {
		chk.Panic("number of objective values (nova) must be greater than 0")
	}
	if o.Nsol < 6 {
		chk.Panic("number of solutions must greater than 6. Nsol = %d is invalid", o.Nsol)
	}
	if o.Ncpu < 2 {
		o.Ncpu = 1
		o.Pll = false
		o.DtExc = 1
	}
	if o.Ncpu > o.Nsol/2 {
		chk.Panic("number of CPU must be smaller than or equal to half the number of solutions. Ncpu=%d > Nsol/2=%d", o.Ncpu, o.Nsol/2)
	}
	if o.Tf < 1 {
		o.Tf = 1
	}
	if o.DtExc < 1 {
		o.DtExc = o.Tf / 10
	}
	if o.DtOut < 1 {
		o.DtOut = o.Tf / 5
	}

	// derived
	o.Nflt = len(o.FltMin)
	o.Nint = len(o.IntMin)
	if o.BinInt > 0 {
		o.Nint = o.BinInt
	}
	if o.Nflt == 0 && o.Nint == 0 {
		chk.Panic("either floats and ints must be set (via FltMin/Max or IntMin/Max)")
	}

	// floats
	if o.Nflt > 0 {
		chk.IntAssert(len(o.FltMax), o.Nflt)
		o.DelFlt = make([]float64, o.Nflt)
		for i := 0; i < o.Nflt; i++ {
			o.DelFlt[i] = o.FltMax[i] - o.FltMin[i]
		}
	}

	// generic ints
	if o.BinInt == 0 && o.Nint > 0 {
		chk.IntAssert(len(o.IntMax), o.Nint)
		o.DelInt = make([]int, o.Nint)
		for i := 0; i < o.Nint; i++ {
			o.DelInt[i] = o.IntMax[i] - o.IntMin[i]
		}
	}
	if o.Nint != o.Nflt {
		o.ClearFlt = false
	}

	// number of cuts and changes in ints
	if o.Nint > 0 {
		if o.IntNcuts > o.Nint {
			o.IntNcuts = o.Nint
		}
		if o.IntNchanges > o.Nint {
			o.IntNchanges = o.Nint
		}
	}

	// initialise random numbers generator
	rnd.Init(o.Seed)
}

// EnforceRange makes sure x is within given range
func (o *Parameters) EnforceRange(i int, x float64) float64 {
	if x < o.FltMin[i] {
		return o.FltMin[i]
	}
	if x > o.FltMax[i] {
		return o.FltMax[i]
	}
	return x
}

// LogParams returns a log with current parameters
func (o *Parameters) LogParams() (l string) {

	// sizes
	l += io.ArgsTable("SIZES",
		"number of objective values", "Nova", o.Nova,
		"number of out-of-range values", "Noor", o.Noor,
		"total number of solutions", "Nsol", o.Nsol,
		"number of cpus", "Ncpu", o.Ncpu,
	)

	// time
	l += "\n"
	l += io.ArgsTable("TIME",
		"final time", "Tf", o.Tf,
		"delta time for exchange", "DtExc", o.DtExc,
		"delta time for output", "DtOut", o.DtOut,
	)

	// options
	l += "\n"
	l += io.ArgsTable("OPTIONS",
		"C-coefficient for differential evolution", "DEC", o.DEC,
		"parallel", "Pll", o.Pll,
		"seed for random numbers generator", "Seed", o.Seed,
		"generation type: 'latin', 'halton', 'rnd'", "GenType", o.GenType,
		"Latin Hypercube duplicates number", "LatinDup", o.LatinDup,
		"minimum value for 'h' constraints", "EpsH", o.EpsH,
		"show messages", "Verbose", o.Verbose,
		"generate all solutions together", "GenAll", o.GenAll,
		"run many trials", "Nsamples", o.Nsamples,
		"integers represent binary numbers", "BinInt", o.BinInt,
		"clear flt if corresponding int is 0", "ClearFlt", o.ClearFlt,
		"use exchange via tournament", "ExcTour", o.ExcTour,
		"use exchange one randomly", "ExcOne", o.ExcOne,
	)

	// crossover and mutation of integers
	l += "\n"
	l += io.ArgsTable("CROSSOVER AND MUTATION OF INTS",
		"probability of crossover for ints", "IntPc", o.IntPc,
		"number of cuts in crossover of ints", "IntNcuts", o.IntNcuts,
		"probability of mutation for ints", "IntPm", o.IntPm,
		"number of changes during mutation of ints", "IntNchanges", o.IntNchanges,
	)

	// derived
	l += "\n"
	l += io.ArgsTable("DERIVED",
		"number of floats", "Nflt", o.Nflt,
		"number of integers", "Nint", o.Nint,
	)
	return
}
<commit_msg>extra parameters (not for goga) added<commit_after>// Copyright 2015 Dorival de Moraes Pedroso. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package goga

import (
	"encoding/json"

	"github.com/cpmech/gosl/chk"
	"github.com/cpmech/gosl/io"
	"github.com/cpmech/gosl/rnd"
)

// Parameters hold all configuration parameters
type Parameters struct {

	// sizes
	Nova int // number of objective values
	Noor int // number of out-of-range values
	Nsol int // total number of solutions
	Ncpu int // number of cpus

	// time
	Tf    int // final time
	DtExc int // delta time for exchange
	DtOut int // delta time for output

	// options
	DEC      float64 // C-coefficient for differential evolution
	Pll      bool    // parallel
	Seed     int     // seed for random numbers generator
	GenType  string  // generation type: "latin", "halton", "rnd"
	LatinDup int     // Latin Hypercube duplicates number
	EpsH     float64 // minimum value for 'h' constraints
	Verbose  bool    // show messages
	GenAll   bool    // generate all solutions together; i.e. not within each group/CPU
	Nsamples int     // run many samples
	BinInt   int     // flag that integers represent binary numbers if BinInt > 0; thus Nint=BinInt
	ClearFlt bool    // clear flt if corresponding int is 0
	ExcTour  bool    // use exchange via tournament
	ExcOne   bool    // use exchange one randomly

	// crossover and mutation of integers
	IntPc       float64 // probability of crossover for ints
	IntNcuts    int     // number of cuts in crossover of ints
	IntPm       float64 // probability of mutation for ints
	IntNchanges int     // number of changes during mutation of ints

	// range
	FltMin []float64 // minimum float allowed
	FltMax []float64 // maximum float allowed
	IntMin []int     // minimum int allowed
	IntMax []int     // maximum int allowed

	// derived
	Nflt   int       // number of floats
	Nint   int       // number of integers
	DelFlt []float64 // max float range
	DelInt []int     // max int range

	// extra variables not direclty related to GOGA (for convenience of having a reader already)
	Strategy int  // strategy
	PlotSet1 bool // plot set of graphs 1
	PlotSet2 bool // plot set of graphs 2
}

// Default sets default parameters
func (o *Parameters) Default() {

	// sizes
	o.Nova = 1
	o.Noor = 0
	o.Nsol = 40
	o.Ncpu = 4

	// time
	o.Tf = 100
	o.DtExc = -1
	o.DtOut = -1

	// options
	o.DEC = 0.8
	o.Pll = true
	o.Seed = 0
	o.GenType = "latin"
	o.LatinDup = 2
	o.EpsH = 0.1
	o.Verbose = true
	o.GenAll = false
	o.Nsamples = 10
	o.BinInt = 0
	o.ClearFlt = false
	o.ExcTour = true
	o.ExcOne = true

	// crossover and mutation of integers
	o.IntPc = 0.8
	o.IntNcuts = 1
	o.IntPm = 0.01
	o.IntNchanges = 1
}

// Read reads configuration parameters from JSON file
func (o *Parameters) Read(filenamepath string) {
	o.Default()
	b, err := io.ReadFile(filenamepath)
	if err != nil {
		chk.Panic("cannot read parameters file %q", filenamepath)
	}
	err = json.Unmarshal(b, o)
	if err != nil {
		chk.Panic("cannot unmarshal parameters file %q", filenamepath)
	}
	return
}

// CalcDerived computes derived variables and checks consistency
func (o *Parameters) CalcDerived() {

	// check
	if o.Nova < 1 {
		chk.Panic("number of objective values (nova) must be greater than 0")
	}
	if o.Nsol < 6 {
		chk.Panic("number of solutions must greater than 6. Nsol = %d is invalid", o.Nsol)
	}
	if o.Ncpu < 2 {
		o.Ncpu = 1
		o.Pll = false
		o.DtExc = 1
	}
	if o.Ncpu > o.Nsol/2 {
		chk.Panic("number of CPU must be smaller than or equal to half the number of solutions. Ncpu=%d > Nsol/2=%d", o.Ncpu, o.Nsol/2)
	}
	if o.Tf < 1 {
		o.Tf = 1
	}
	if o.DtExc < 1 {
		o.DtExc = o.Tf / 10
	}
	if o.DtOut < 1 {
		o.DtOut = o.Tf / 5
	}

	// derived
	o.Nflt = len(o.FltMin)
	o.Nint = len(o.IntMin)
	if o.BinInt > 0 {
		o.Nint = o.BinInt
	}
	if o.Nflt == 0 && o.Nint == 0 {
		chk.Panic("either floats and ints must be set (via FltMin/Max or IntMin/Max)")
	}

	// floats
	if o.Nflt > 0 {
		chk.IntAssert(len(o.FltMax), o.Nflt)
		o.DelFlt = make([]float64, o.Nflt)
		for i := 0; i < o.Nflt; i++ {
			o.DelFlt[i] = o.FltMax[i] - o.FltMin[i]
		}
	}

	// generic ints
	if o.BinInt == 0 && o.Nint > 0 {
		chk.IntAssert(len(o.IntMax), o.Nint)
		o.DelInt = make([]int, o.Nint)
		for i := 0; i < o.Nint; i++ {
			o.DelInt[i] = o.IntMax[i] - o.IntMin[i]
		}
	}
	if o.Nint != o.Nflt {
		o.ClearFlt = false
	}

	// number of cuts and changes in ints
	if o.Nint > 0 {
		if o.IntNcuts > o.Nint {
			o.IntNcuts = o.Nint
		}
		if o.IntNchanges > o.Nint {
			o.IntNchanges = o.Nint
		}
	}

	// initialise random numbers generator
	rnd.Init(o.Seed)
}

// EnforceRange makes sure x is within given range
func (o *Parameters) EnforceRange(i int, x float64) float64 {
	if x < o.FltMin[i] {
		return o.FltMin[i]
	}
	if x > o.FltMax[i] {
		return o.FltMax[i]
	}
	return x
}

// LogParams returns a log with current parameters
func (o *Parameters) LogParams() (l string) {

	// sizes
	l += io.ArgsTable("SIZES",
		"number of objective values", "Nova", o.Nova,
		"number of out-of-range values", "Noor", o.Noor,
		"total number of solutions", "Nsol", o.Nsol,
		"number of cpus", "Ncpu", o.Ncpu,
	)

	// time
	l += "\n"
	l += io.ArgsTable("TIME",
		"final time", "Tf", o.Tf,
		"delta time for exchange", "DtExc", o.DtExc,
		"delta time for output", "DtOut", o.DtOut,
	)

	// options
	l += "\n"
	l += io.ArgsTable("OPTIONS",
		"C-coefficient for differential evolution", "DEC", o.DEC,
		"parallel", "Pll", o.Pll,
		"seed for random numbers generator", "Seed", o.Seed,
		"generation type: 'latin', 'halton', 'rnd'", "GenType", o.GenType,
		"Latin Hypercube duplicates number", "LatinDup", o.LatinDup,
		"minimum value for 'h' constraints", "EpsH", o.EpsH,
		"show messages", "Verbose", o.Verbose,
		"generate all solutions together", "GenAll", o.GenAll,
		"run many trials", "Nsamples", o.Nsamples,
		"integers represent binary numbers", "BinInt", o.BinInt,
		"clear flt if corresponding int is 0", "ClearFlt", o.ClearFlt,
		"use exchange via tournament", "ExcTour", o.ExcTour,
		"use exchange one randomly", "ExcOne", o.ExcOne,
	)

	// crossover and mutation of integers
	l += "\n"
	l += io.ArgsTable("CROSSOVER AND MUTATION OF INTS",
		"probability of crossover for ints", "IntPc", o.IntPc,
		"number of cuts in crossover of ints", "IntNcuts", o.IntNcuts,
		"probability of mutation for ints", "IntPm", o.IntPm,
		"number of changes during mutation of ints", "IntNchanges", o.IntNchanges,
	)

	// derived
	l += "\n"
	l += io.ArgsTable("DERIVED",
		"number of floats", "Nflt", o.Nflt,
		"number of integers", "Nint", o.Nint,
	)

	// extra
	l += "\n"
	l += io.ArgsTable("EXTRA",
		"strategy", "Strategy", o.Strategy,
		"plot set of graphs 1", "PlotSet1", o.PlotSet1,
		"plot set of graphs 2", "PlotSet2", o.PlotSet2,
	)
	return
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"io"
	"regexp"
	"strconv"
)

const (
	GCRegexpGo14 = `gc\d+\(\d+\): ([\d.]+\+?)+ us, \d+ -> (?P<Heap1>\d+) MB, \d+ \(\d+-\d+\) objects,( \d+ goroutines,)? \d+\/\d+\/\d+ sweeps, \d+\(\d+\) handoff, \d+\(\d+\) steal, \d+\/\d+\/\d+ yields`
	GCRegexpGo15 = `gc \d+ @[\d.]+s \d+%: [\d.+/]+ ms clock, [\d.+/]+ ms cpu, \d+->\d+->\d+ MB, (?P<Heap1>\d+) MB goal, \d+ P`
	SCVGRegexp   = `scvg\d+: inuse: (?P<inuse>\d+), idle: (?P<idle>\d+), sys: (?P<sys>\d+), released: (?P<released>\d+), consumed: (?P<consumed>\d+) \(MB\)`
)

var (
	gcrego14 = regexp.MustCompile(GCRegexpGo14)
	gcrego15 = regexp.MustCompile(GCRegexpGo15)
	scvgre   = regexp.MustCompile(SCVGRegexp)
)

type Parser struct {
	reader      io.Reader
	GcChan      chan *gctrace
	ScvgChan    chan *scvgtrace
	NoMatchChan chan string
	done        chan bool

	Err error

	scvgRegexp *regexp.Regexp
}

func NewParser(r io.Reader) *Parser {
	return &Parser{
		reader:      r,
		GcChan:      make(chan *gctrace, 1),
		ScvgChan:    make(chan *scvgtrace, 1),
		NoMatchChan: make(chan string, 1),
		done:        make(chan bool),
	}
}

func (p *Parser) Run() {
	sc := bufio.NewScanner(p.reader)

	for sc.Scan() {
		line := sc.Text()

		if result := gcrego15.FindStringSubmatch(line); result != nil {
			p.GcChan <- parseGCTrace(gcrego15, result)
			continue
		}

		if result := gcrego14.FindStringSubmatch(line); result != nil {
			p.GcChan <- parseGCTrace(gcrego14, result)
			continue
		}

		if result := scvgre.FindStringSubmatch(line); result != nil {
			p.ScvgChan <- parseSCVGTrace(result)
			continue
		}

		p.NoMatchChan <- line
	}

	p.Err = sc.Err()

	close(p.done)
}

func parseGCTrace(gcre *regexp.Regexp, matches []string) *gctrace {
	matchMap := getMatchMap(gcre, matches)

	return &gctrace{
		Heap1: matchMap["Heap1"],
	}
}

func parseSCVGTrace(matches []string) *scvgtrace {
	matchMap := getMatchMap(scvgre, matches)

	return &scvgtrace{
		inuse:    matchMap["inuse"],
		idle:     matchMap["idle"],
		sys:      matchMap["sys"],
		released: matchMap["released"],
		consumed: matchMap["consumed"],
	}
}

// Transform our matches in a readable hash map.
//
// The resulting hash map will be something like { "Heap1": 123 }
func getMatchMap(re *regexp.Regexp, matches []string) map[string]int64 {
	matchingNames := re.SubexpNames()
	matchMap := map[string]int64{}
	for i, value := range matches {
		intVal, err := strconv.ParseInt(value, 10, 64)
		if err != nil {
			// Happens on first element of range and any matching parenthesis
			// that includes non-parseable string
			//
			// For example a matching array would contain:
			// [ "scvg1: inuse:3 ..." "3" ]
			continue
		}
		matchMap[matchingNames[i]] = intVal
	}
	return matchMap
}
<commit_msg>fix gc15 regex to be compatible with go 1.6 (tip)<commit_after>package main

import (
	"bufio"
	"io"
	"regexp"
	"strconv"
)

const (
	GCRegexpGo14 = `gc\d+\(\d+\): ([\d.]+\+?)+ us, \d+ -> (?P<Heap1>\d+) MB, \d+ \(\d+-\d+\) objects,( \d+ goroutines,)? \d+\/\d+\/\d+ sweeps, \d+\(\d+\) handoff, \d+\(\d+\) steal, \d+\/\d+\/\d+ yields`
	GCRegexpGo15 = `^gc (#)\d+ @[\d.]+s \d+%: [\d.+/]+ ms clock, [\d.+/]+ ms cpu, \d+->\d+->\d+ MB, (?P<Heap1>\d+) MB goal, \d+ P`
	SCVGRegexp   = `scvg\d+: inuse: (?P<inuse>\d+), idle: (?P<idle>\d+), sys: (?P<sys>\d+), released: (?P<released>\d+), consumed: (?P<consumed>\d+) \(MB\)`
)

var (
	gcrego14 = regexp.MustCompile(GCRegexpGo14)
	gcrego15 = regexp.MustCompile(GCRegexpGo15)
	scvgre   = regexp.MustCompile(SCVGRegexp)
)

type Parser struct {
	reader      io.Reader
	GcChan      chan *gctrace
	ScvgChan    chan *scvgtrace
	NoMatchChan chan string
	done        chan bool

	Err error

	scvgRegexp *regexp.Regexp
}

func NewParser(r io.Reader) *Parser {
	return &Parser{
		reader:      r,
		GcChan:      make(chan *gctrace, 1),
		ScvgChan:    make(chan *scvgtrace, 1),
		NoMatchChan: make(chan string, 1),
		done:        make(chan bool),
	}
}

func (p *Parser) Run() {
	sc := bufio.NewScanner(p.reader)

	for sc.Scan() {
		line := sc.Text()

		if result := gcrego15.FindStringSubmatch(line); result != nil {
			p.GcChan <- parseGCTrace(gcrego15, result)
			continue
		}

		if result := gcrego14.FindStringSubmatch(line); result != nil {
			p.GcChan <- parseGCTrace(gcrego14, result)
			continue
		}

		if result := scvgre.FindStringSubmatch(line); result != nil {
			p.ScvgChan <- parseSCVGTrace(result)
			continue
		}

		p.NoMatchChan <- line
	}

	p.Err = sc.Err()

	close(p.done)
}

func parseGCTrace(gcre *regexp.Regexp, matches []string) *gctrace {
	matchMap := getMatchMap(gcre, matches)

	return &gctrace{
		Heap1: matchMap["Heap1"],
	}
}

func parseSCVGTrace(matches []string) *scvgtrace {
	matchMap := getMatchMap(scvgre, matches)

	return &scvgtrace{
		inuse:    matchMap["inuse"],
		idle:     matchMap["idle"],
		sys:      matchMap["sys"],
		released: matchMap["released"],
		consumed: matchMap["consumed"],
	}
}

// Transform our matches in a readable hash map.
//
// The resulting hash map will be something like { "Heap1": 123 }
func getMatchMap(re *regexp.Regexp, matches []string) map[string]int64 {
	matchingNames := re.SubexpNames()
	matchMap := map[string]int64{}
	for i, value := range matches {
		intVal, err := strconv.ParseInt(value, 10, 64)
		if err != nil {
			// Happens on first element of range and any matching parenthesis
			// that includes non-parseable string
			//
			// For example a matching array would contain:
			// [ "scvg1: inuse:3 ..." "3" ]
			continue
		}
		matchMap[matchingNames[i]] = intVal
	}
	return matchMap
}
<|endoftext|>


<commit_before>// Copyright 2012 Apcera Inc. All rights reserved.

// A Go client for the NATS messaging system (https://github.com/derekcollison/nats).
package nats

import (
	"fmt"
)

type msgArg struct {
	subject []byte
	reply   []byte
	sid     int64
	size    int
}

const MAX_CONTROL_LINE_SIZE = 1024

type parseState struct {
	state   int
	as      int
	drop    int
	ma      msgArg
	argBuf  []byte
	msgBuf  []byte
	scratch [MAX_CONTROL_LINE_SIZE]byte
}

const (
	OP_START = iota
	OP_PLUS
	OP_PLUS_O
	OP_PLUS_OK
	OP_MINUS
	OP_MINUS_E
	OP_MINUS_ER
	OP_MINUS_ERR
	OP_MINUS_ERR_SPC
	MINUS_ERR_ARG
	OP_C
	OP_CO
	OP_CON
	OP_CONN
	OP_CONNE
	OP_CONNEC
	OP_CONNECT
	CONNECT_ARG
	OP_M
	OP_MS
	OP_MSG
	OP_MSG_SPC
	MSG_ARG
	MSG_PAYLOAD
	MSG_END
	OP_P
	OP_PI
	OP_PIN
	OP_PING
	OP_PO
	OP_PON
	OP_PONG
)

// parse is the fast protocol parser engine.
func (nc *Conn) parse(buf []byte) error {
	var i int
	var b byte

	for i, b = range buf {
		switch nc.ps.state {
		case OP_START:
			switch b {
			case 'M', 'm':
				nc.ps.state = OP_M
			case 'C', 'c':
				nc.ps.state = OP_C
			case 'P', 'p':
				nc.ps.state = OP_P
			case '+':
				nc.ps.state = OP_PLUS
			case '-':
				nc.ps.state = OP_MINUS
			default:
				goto parseErr
			}
		case OP_M:
			switch b {
			case 'S', 's':
				nc.ps.state = OP_MS
			default:
				goto parseErr
			}
		case OP_MS:
			switch b {
			case 'G', 'g':
				nc.ps.state = OP_MSG
			default:
				goto parseErr
			}
		case OP_MSG:
			switch b {
			case ' ', '\t':
				nc.ps.state = OP_MSG_SPC
			default:
				goto parseErr
			}
		case OP_MSG_SPC:
			switch b {
			case ' ', '\t':
				continue
			default:
				nc.ps.state = MSG_ARG
				nc.ps.as = i
			}
		case MSG_ARG:
			switch b {
			case '\r':
				nc.ps.drop = 1
			case '\n':
				var arg []byte
				if nc.ps.argBuf != nil {
					arg = nc.ps.argBuf
				} else {
					arg = buf[nc.ps.as : i-nc.ps.drop]
				}
				if err := nc.processMsgArgs(arg); err != nil {
					return err
				}
				nc.ps.drop, nc.ps.as, nc.ps.state = 0, i+1, MSG_PAYLOAD
			default:
				if nc.ps.argBuf != nil {
					nc.ps.argBuf = append(nc.ps.argBuf, b)
				}
			}
		case MSG_PAYLOAD:
			if nc.ps.msgBuf != nil {
				if len(nc.ps.msgBuf) >= nc.ps.ma.size {
					nc.processMsg(nc.ps.msgBuf)
					nc.ps.argBuf, nc.ps.msgBuf, nc.ps.state = nil, nil, MSG_END
				} else {
					nc.ps.msgBuf = append(nc.ps.msgBuf, b)
				}
			} else if i-nc.ps.as >= nc.ps.ma.size {
				nc.processMsg(buf[nc.ps.as:i])
				nc.ps.argBuf, nc.ps.msgBuf, nc.ps.state = nil, nil, MSG_END
			}
		case MSG_END:
			switch b {
			case '\n':
				nc.ps.drop, nc.ps.as, nc.ps.state = 0, i+1, OP_START
			default:
				continue
			}
		case OP_PLUS:
			switch b {
			case 'O', 'o':
				nc.ps.state = OP_PLUS_O
			default:
				goto parseErr
			}
		case OP_PLUS_O:
			switch b {
			case 'K', 'k':
				nc.ps.state = OP_PLUS_OK
			default:
				goto parseErr
			}
		case OP_PLUS_OK:
			switch b {
			case '\n':
				nc.processOK()
				nc.ps.drop, nc.ps.state = 0, OP_START
			}
		case OP_MINUS:
			switch b {
			case 'E', 'e':
				nc.ps.state = OP_MINUS_E
			default:
				goto parseErr
			}
		case OP_MINUS_E:
			switch b {
			case 'R', 'r':
				nc.ps.state = OP_MINUS_ER
			default:
				goto parseErr
			}
		case OP_MINUS_ER:
			switch b {
			case 'R', 'r':
				nc.ps.state = OP_MINUS_ERR
			default:
				goto parseErr
			}
		case OP_MINUS_ERR:
			switch b {
			case ' ', '\t':
				nc.ps.state = OP_MINUS_ERR_SPC
			default:
				goto parseErr
			}
		case OP_MINUS_ERR_SPC:
			switch b {
			case ' ', '\t':
				continue
			default:
				nc.ps.state = MINUS_ERR_ARG
				nc.ps.as = i
			}
		case MINUS_ERR_ARG:
			switch b {
			case '\r':
				nc.ps.drop = 1
			case '\n':
				var arg []byte
				if nc.ps.argBuf != nil {
					arg = nc.ps.argBuf
					nc.ps.argBuf = nil
				} else {
					arg = buf[nc.ps.as : i-nc.ps.drop]
				}
				nc.processErr(string(arg))
				nc.ps.drop, nc.ps.as, nc.ps.state = 0, i+1, OP_START
			default:
				if nc.ps.argBuf != nil {
					nc.ps.argBuf = append(nc.ps.argBuf, b)
				}
			}
		case OP_P:
			switch b {
			case 'I', 'i':
				nc.ps.state = OP_PI
			case 'O', 'o':
				nc.ps.state = OP_PO
			default:
				goto parseErr
			}
		case OP_PO:
			switch b {
			case 'N', 'n':
				nc.ps.state = OP_PON
			default:
				goto parseErr
			}
		case OP_PON:
			switch b {
			case 'G', 'g':
				nc.ps.state = OP_PONG
			default:
				goto parseErr
			}
		case OP_PONG:
			switch b {
			case '\n':
				nc.processPong()
				nc.ps.drop, nc.ps.state = 0, OP_START
			}
		case OP_PI:
			switch b {
			case 'N', 'n':
				nc.ps.state = OP_PIN
			default:
				goto parseErr
			}
		case OP_PIN:
			switch b {
			case 'G', 'g':
				nc.ps.state = OP_PING
			default:
				goto parseErr
			}
		case OP_PING:
			switch b {
			case '\n':
				nc.processPing()
				nc.ps.drop, nc.ps.state = 0, OP_START
			}
		default:
			goto parseErr
		}
	}
	// Check for split buffer scenarios
	if (nc.ps.state == MSG_ARG || nc.ps.state == MINUS_ERR_ARG) && nc.ps.argBuf == nil {
		nc.ps.argBuf = nc.ps.scratch[:0]
		nc.ps.argBuf = append(nc.ps.argBuf, buf[nc.ps.as:(i+1)-nc.ps.drop]...)
		// FIXME, check max len
	}
	// Check for split msg
	if nc.ps.state == MSG_PAYLOAD && nc.ps.msgBuf == nil {
		// We need to clone the msgArg if it is still referencing the
		// read buffer and we are not able to process the msg.
		if nc.ps.argBuf == nil {
			nc.cloneMsgArg()
		}
		// FIXME: copy better here? Make whole buf if large?
		nc.ps.msgBuf = nc.ps.scratch[len(nc.ps.argBuf):len(nc.ps.argBuf)]
		nc.ps.msgBuf = append(nc.ps.msgBuf, (buf[nc.ps.as:])...)
	}

	return nil

parseErr:
	return fmt.Errorf("nats: Parse Error [%d]: '%s'", nc.ps.state, buf[i:])
}

// clonePubArg is used when the split buffer scenario has the pubArg in the existing read buffer, but
// we need to hold onto it into the next read.
func (nc *Conn) cloneMsgArg() {
	nc.ps.argBuf = nc.ps.scratch[:0]
	nc.ps.argBuf = append(nc.ps.argBuf, nc.ps.ma.subject...)
	nc.ps.argBuf = append(nc.ps.argBuf, nc.ps.ma.reply...)
	nc.ps.ma.subject = nc.ps.argBuf[:len(nc.ps.ma.subject)]
	if nc.ps.ma.reply != nil {
		nc.ps.ma.reply = nc.ps.argBuf[len(nc.ps.ma.subject):]
	}
}

const argsLenMax = 4

func (nc *Conn) processMsgArgs(arg []byte) error {
	// Unroll splitArgs to avoid runtime/heap issues
	a := [argsLenMax][]byte{}
	args := a[:0]
	start := -1
	for i, b := range arg {
		switch b {
		case ' ', '\t', '\r', '\n':
			if start >= 0 {
				args = append(args, arg[start:i])
				start = -1
			}
		default:
			if start < 0 {
				start = i
			}
		}
	}
	if start >= 0 {
		args = append(args, arg[start:])
	}

	switch len(args) {
	case 3:
		nc.ps.ma.subject = args[0]
		nc.ps.ma.sid = parseInt64(args[1])
		nc.ps.ma.reply = nil
		nc.ps.ma.size = int(parseInt64(args[2]))
	case 4:
		nc.ps.ma.subject = args[0]
		nc.ps.ma.sid = parseInt64(args[1])
		nc.ps.ma.reply = args[2]
		nc.ps.ma.size = int(parseInt64(args[3]))
	default:
		return fmt.Errorf("nats: processMsgArgs Parse Error: '%s'", arg)
	}
	if nc.ps.ma.size < 0 {
		return fmt.Errorf("nats: processMsgArgs Bad or Missing Size: '%s'", arg)
	}
	return nil
}

// Ascii numbers 0-9
const (
	ascii_0 = 48
	ascii_9 = 57
)

// parseInt64 expects decimal positive numbers. We
// return -1 to signal error
func parseInt64(d []byte) (n int64) {
	if len(d) == 0 {
		return -1
	}
	for _, dec := range d {
		if dec < ascii_0 || dec > ascii_9 {
			return -1
		}
		n = n*10 + (int64(dec) - ascii_0)
	}
	return n
}
<commit_msg>Bad comment<commit_after>// Copyright 2012 Apcera Inc. All rights reserved.

// A Go client for the NATS messaging system (https://github.com/derekcollison/nats).
package nats

import (
	"fmt"
)

type msgArg struct {
	subject []byte
	reply   []byte
	sid     int64
	size    int
}

const MAX_CONTROL_LINE_SIZE = 1024

type parseState struct {
	state   int
	as      int
	drop    int
	ma      msgArg
	argBuf  []byte
	msgBuf  []byte
	scratch [MAX_CONTROL_LINE_SIZE]byte
}

const (
	OP_START = iota
	OP_PLUS
	OP_PLUS_O
	OP_PLUS_OK
	OP_MINUS
	OP_MINUS_E
	OP_MINUS_ER
	OP_MINUS_ERR
	OP_MINUS_ERR_SPC
	MINUS_ERR_ARG
	OP_C
	OP_CO
	OP_CON
	OP_CONN
	OP_CONNE
	OP_CONNEC
	OP_CONNECT
	CONNECT_ARG
	OP_M
	OP_MS
	OP_MSG
	OP_MSG_SPC
	MSG_ARG
	MSG_PAYLOAD
	MSG_END
	OP_P
	OP_PI
	OP_PIN
	OP_PING
	OP_PO
	OP_PON
	OP_PONG
)

// parse is the fast protocol parser engine.
func (nc *Conn) parse(buf []byte) error {
	var i int
	var b byte

	for i, b = range buf {
		switch nc.ps.state {
		case OP_START:
			switch b {
			case 'M', 'm':
				nc.ps.state = OP_M
			case 'C', 'c':
				nc.ps.state = OP_C
			case 'P', 'p':
				nc.ps.state = OP_P
			case '+':
				nc.ps.state = OP_PLUS
			case '-':
				nc.ps.state = OP_MINUS
			default:
				goto parseErr
			}
		case OP_M:
			switch b {
			case 'S', 's':
				nc.ps.state = OP_MS
			default:
				goto parseErr
			}
		case OP_MS:
			switch b {
			case 'G', 'g':
				nc.ps.state = OP_MSG
			default:
				goto parseErr
			}
		case OP_MSG:
			switch b {
			case ' ', '\t':
				nc.ps.state = OP_MSG_SPC
			default:
				goto parseErr
			}
		case OP_MSG_SPC:
			switch b {
			case ' ', '\t':
				continue
			default:
				nc.ps.state = MSG_ARG
				nc.ps.as = i
			}
		case MSG_ARG:
			switch b {
			case '\r':
				nc.ps.drop = 1
			case '\n':
				var arg []byte
				if nc.ps.argBuf != nil {
					arg = nc.ps.argBuf
				} else {
					arg = buf[nc.ps.as : i-nc.ps.drop]
				}
				if err := nc.processMsgArgs(arg); err != nil {
					return err
				}
				nc.ps.drop, nc.ps.as, nc.ps.state = 0, i+1, MSG_PAYLOAD
			default:
				if nc.ps.argBuf != nil {
					nc.ps.argBuf = append(nc.ps.argBuf, b)
				}
			}
		case MSG_PAYLOAD:
			if nc.ps.msgBuf != nil {
				if len(nc.ps.msgBuf) >= nc.ps.ma.size {
					nc.processMsg(nc.ps.msgBuf)
					nc.ps.argBuf, nc.ps.msgBuf, nc.ps.state = nil, nil, MSG_END
				} else {
					nc.ps.msgBuf = append(nc.ps.msgBuf, b)
				}
			} else if i-nc.ps.as >= nc.ps.ma.size {
				nc.processMsg(buf[nc.ps.as:i])
				nc.ps.argBuf, nc.ps.msgBuf, nc.ps.state = nil, nil, MSG_END
			}
		case MSG_END:
			switch b {
			case '\n':
				nc.ps.drop, nc.ps.as, nc.ps.state = 0, i+1, OP_START
			default:
				continue
			}
		case OP_PLUS:
			switch b {
			case 'O', 'o':
				nc.ps.state = OP_PLUS_O
			default:
				goto parseErr
			}
		case OP_PLUS_O:
			switch b {
			case 'K', 'k':
				nc.ps.state = OP_PLUS_OK
			default:
				goto parseErr
			}
		case OP_PLUS_OK:
			switch b {
			case '\n':
				nc.processOK()
				nc.ps.drop, nc.ps.state = 0, OP_START
			}
		case OP_MINUS:
			switch b {
			case 'E', 'e':
				nc.ps.state = OP_MINUS_E
			default:
				goto parseErr
			}
		case OP_MINUS_E:
			switch b {
			case 'R', 'r':
				nc.ps.state = OP_MINUS_ER
			default:
				goto parseErr
			}
		case OP_MINUS_ER:
			switch b {
			case 'R', 'r':
				nc.ps.state = OP_MINUS_ERR
			default:
				goto parseErr
			}
		case OP_MINUS_ERR:
			switch b {
			case ' ', '\t':
				nc.ps.state = OP_MINUS_ERR_SPC
			default:
				goto parseErr
			}
		case OP_MINUS_ERR_SPC:
			switch b {
			case ' ', '\t':
				continue
			default:
				nc.ps.state = MINUS_ERR_ARG
				nc.ps.as = i
			}
		case MINUS_ERR_ARG:
			switch b {
			case '\r':
				nc.ps.drop = 1
			case '\n':
				var arg []byte
				if nc.ps.argBuf != nil {
					arg = nc.ps.argBuf
					nc.ps.argBuf = nil
				} else {
					arg = buf[nc.ps.as : i-nc.ps.drop]
				}
				nc.processErr(string(arg))
				nc.ps.drop, nc.ps.as, nc.ps.state = 0, i+1, OP_START
			default:
				if nc.ps.argBuf != nil {
					nc.ps.argBuf = append(nc.ps.argBuf, b)
				}
			}
		case OP_P:
			switch b {
			case 'I', 'i':
				nc.ps.state = OP_PI
			case 'O', 'o':
				nc.ps.state = OP_PO
			default:
				goto parseErr
			}
		case OP_PO:
			switch b {
			case 'N', 'n':
				nc.ps.state = OP_PON
			default:
				goto parseErr
			}
		case OP_PON:
			switch b {
			case 'G', 'g':
				nc.ps.state = OP_PONG
			default:
				goto parseErr
			}
		case OP_PONG:
			switch b {
			case '\n':
				nc.processPong()
				nc.ps.drop, nc.ps.state = 0, OP_START
			}
		case OP_PI:
			switch b {
			case 'N', 'n':
				nc.ps.state = OP_PIN
			default:
				goto parseErr
			}
		case OP_PIN:
			switch b {
			case 'G', 'g':
				nc.ps.state = OP_PING
			default:
				goto parseErr
			}
		case OP_PING:
			switch b {
			case '\n':
				nc.processPing()
				nc.ps.drop, nc.ps.state = 0, OP_START
			}
		default:
			goto parseErr
		}
	}
	// Check for split buffer scenarios
	if (nc.ps.state == MSG_ARG || nc.ps.state == MINUS_ERR_ARG) && nc.ps.argBuf == nil {
		nc.ps.argBuf = nc.ps.scratch[:0]
		nc.ps.argBuf = append(nc.ps.argBuf, buf[nc.ps.as:(i+1)-nc.ps.drop]...)
		// FIXME, check max len
	}
	// Check for split msg
	if nc.ps.state == MSG_PAYLOAD && nc.ps.msgBuf == nil {
		// We need to clone the msgArg if it is still referencing the
		// read buffer and we are not able to process the msg.
		if nc.ps.argBuf == nil {
			nc.cloneMsgArg()
		}
		// FIXME: copy better here? Make whole buf if large?
		nc.ps.msgBuf = nc.ps.scratch[len(nc.ps.argBuf):len(nc.ps.argBuf)]
		nc.ps.msgBuf = append(nc.ps.msgBuf, (buf[nc.ps.as:])...)
	}

	return nil

parseErr:
	return fmt.Errorf("nats: Parse Error [%d]: '%s'", nc.ps.state, buf[i:])
}

// cloneMsgArg is used when the split buffer scenario has the pubArg in the existing read buffer, but
// we need to hold onto it into the next read.
func (nc *Conn) cloneMsgArg() {
	nc.ps.argBuf = nc.ps.scratch[:0]
	nc.ps.argBuf = append(nc.ps.argBuf, nc.ps.ma.subject...)
	nc.ps.argBuf = append(nc.ps.argBuf, nc.ps.ma.reply...)
	nc.ps.ma.subject = nc.ps.argBuf[:len(nc.ps.ma.subject)]
	if nc.ps.ma.reply != nil {
		nc.ps.ma.reply = nc.ps.argBuf[len(nc.ps.ma.subject):]
	}
}

const argsLenMax = 4

func (nc *Conn) processMsgArgs(arg []byte) error {
	// Unroll splitArgs to avoid runtime/heap issues
	a := [argsLenMax][]byte{}
	args := a[:0]
	start := -1
	for i, b := range arg {
		switch b {
		case ' ', '\t', '\r', '\n':
			if start >= 0 {
				args = append(args, arg[start:i])
				start = -1
			}
		default:
			if start < 0 {
				start = i
			}
		}
	}
	if start >= 0 {
		args = append(args, arg[start:])
	}

	switch len(args) {
	case 3:
		nc.ps.ma.subject = args[0]
		nc.ps.ma.sid = parseInt64(args[1])
		nc.ps.ma.reply = nil
		nc.ps.ma.size = int(parseInt64(args[2]))
	case 4:
		nc.ps.ma.subject = args[0]
		nc.ps.ma.sid = parseInt64(args[1])
		nc.ps.ma.reply = args[2]
		nc.ps.ma.size = int(parseInt64(args[3]))
	default:
		return fmt.Errorf("nats: processMsgArgs Parse Error: '%s'", arg)
	}
	if nc.ps.ma.size < 0 {
		return fmt.Errorf("nats: processMsgArgs Bad or Missing Size: '%s'", arg)
	}
	return nil
}

// Ascii numbers 0-9
const (
	ascii_0 = 48
	ascii_9 = 57
)

// parseInt64 expects decimal positive numbers. We
// return -1 to signal error
func parseInt64(d []byte) (n int64) {
	if len(d) == 0 {
		return -1
	}
	for _, dec := range d {
		if dec < ascii_0 || dec > ascii_9 {
			return -1
		}
		n = n*10 + (int64(dec) - ascii_0)
	}
	return n
}
<|endoftext|>


<commit_before>package mark

import (
	fmt "github.com/k0kubun/pp"
	"regexp"
	"strings"
	"unicode"
	"unicode/utf8"
)

type Tree struct {
	text  string
	lex   *lexer
	Nodes []Node
	// Parsing only
	token     [3]item // three-token lookahead for parser
	peekCount int
	output    string
	links     map[string]*DefLinkNode
}

// Parse convert the raw text to NodeTree.
func (t *Tree) parse() {
Loop:
	for {
		var n Node
		switch p := t.peek(); p.typ {
		case eof, itemError:
			break Loop
		case itemNewLine:
			n = t.newLine(t.next().pos)
		case itemBr:
			n = t.newBr(t.next().pos)
		case itemHr:
			n = t.newHr(t.next().pos)
		case itemHTML:
			p = t.next()
			n = t.newHTML(p.pos, p.val)
		case itemText:
			tmp := t.newParagraph(p.pos)
			tmp.Nodes = t.parseText(t.next().val)
			n = tmp
		case itemDefLink:
			n = t.parseDefLink()
		case itemHeading, itemLHeading:
			n = t.parseHeading()
		case itemCodeBlock, itemGfmCodeBlock:
			n = t.parseCodeBlock()
		case itemList:
			n = t.parseList()
		case itemTable, itemLpTable:
			n = t.parseTable()
		case itemBlockQuote:
			n = t.parseBlockQuote()
		default:
			t.next()
		}
		if n != nil {
			t.append(n)
		}
	}
}

// Render parse nodes to the wanted output
func (t *Tree) render() {
	var last Node
	last = t.newLine(0)
	for _, node := range t.Nodes {
		if last.Type() != NodeNewLine || node.Type() != last.Type() {
			t.output += node.Render()
		}
		last = node
	}
}

// append new node to nodes-list
func (t *Tree) append(n Node) {
	t.Nodes = append(t.Nodes, n)
}

// next returns the next token
func (t *Tree) next() item {
	if t.peekCount > 0 {
		t.peekCount--
	} else {
		t.token[0] = t.lex.nextItem()
	}
	return t.token[t.peekCount]
}

// peek returns but does not consume the next token.
func (t *Tree) peek() item {
	if t.peekCount > 0 {
		return t.token[t.peekCount-1]
	}
	t.peekCount = 1
	t.token[0] = t.lex.nextItem()
	return t.token[0]
}

// backup backs the input stream tp one token
func (t *Tree) backup() {
	t.peekCount++
}

// backup2 backs the input stream up two tokens.
// The zeroth token is already there.
func (t *Tree) backup2(t1 item) {
	t.token[1] = t1
	t.peekCount = 2
}

// parseText scan until itemBr occur.
func (t *Tree) parseText(input string) (nodes []Node) {
	// HACK: if there's more 'itemText' in the way, make it one.
	for {
		tkn := t.next()
		if tkn.typ == itemText {
			input += tkn.val
		} else if tkn.typ == itemNewLine {
			if t.peek().typ != itemText {
				t.backup2(tkn)
				break
			}
			input += tkn.val
		} else {
			t.backup()
			break
		}
	}
	l := lexInline(input)
	for token := range l.items {
		var node Node
		switch token.typ {
		case itemNewLine:
			node = t.newLine(token.pos)
		case itemBr:
			node = t.newBr(token.pos)
		case itemStrong, itemItalic, itemStrike, itemCode:
			node = t.parseEmphasis(token.typ, token.pos, token.val)
		case itemLink, itemAutoLink, itemGfmLink:
			var title, text, href string
			match := span[token.typ].FindStringSubmatch(token.val)
			if token.typ == itemLink {
				text, href, title = match[1], match[2], match[3]
			} else {
				text, href = match[1], match[1]
			}
			node = t.newLink(token.pos, title, href, text)
		case itemImage:
			match := span[token.typ].FindStringSubmatch(token.val)
			node = t.newImage(token.pos, match[3], match[2], match[1])
		case itemText:
			node = t.newText(token.pos, token.val)
		default:
			fmt.Println("Matching not found for this token:", token)
		}
		nodes = append(nodes, node)
	}
	return nodes
}

// Parse inline emphasis
func (t *Tree) parseEmphasis(typ itemType, pos Pos, val string) *EmphasisNode {
	node := t.newEmphasis(pos, typ)
	match := span[typ].FindStringSubmatch(val)
	text := match[len(match)-1]
	if text == "" {
		text = match[1]
	}
	node.Nodes = t.parseText(text)
	return node
}

// parse heading block
func (t *Tree) parseHeading() (node *HeadingNode) {
	token := t.next()
	match := block[token.typ].FindStringSubmatch(token.val)
	if token.typ == itemHeading {
		node = t.newHeading(token.pos, len(match[1]), match[2])
	} else {
		// using equal signs for first-level, and dashes for second-level.
		level := 1
		if match[2] == "-" {
			level = 2
		}
		node = t.newHeading(token.pos, level, match[1])
	}
	return
}

func (t *Tree) parseDefLink() *DefLinkNode {
	token := t.next()
	match := block[itemDefLink].FindStringSubmatch(token.val)
	name := strings.ToLower(match[1])
	// name(lowercase), href, title
	n := t.newDefLink(token.pos, name, match[2], match[3])
	// store in links
	t.links[name] = n
	return n
}

// parse codeBlock
func (t *Tree) parseCodeBlock() *CodeNode {
	var lang, text string
	token := t.next()
	if token.typ == itemGfmCodeBlock {
		match := block[itemGfmCodeBlock].FindStringSubmatch(token.val)
		if text = match[2]; text == "" {
			text = match[4]
		}
		if lang = match[1]; lang == "" {
			lang = match[3]
		}
	} else {
		text = regexp.MustCompile("(?m)^( {4})").ReplaceAllLiteralString(token.val, "")
	}
	return t.newCode(token.pos, lang, text)
}

func (t *Tree) parseBlockQuote() (n *BlockQuoteNode) {
	token := t.next()
	// replacer
	re := regexp.MustCompile(`(?m)^> ?`)
	raw := re.ReplaceAllString(token.val, "")
	// TODO(Ariel): not work right now with defLink(inside the blockQuote)
	tr := &Tree{lex: lex(raw)}
	tr.parse()
	n = t.newBlockQuote(token.pos)
	n.Nodes = tr.Nodes
	return
}

// parse list
func (t *Tree) parseList() *ListNode {
	token := t.next()
	list := t.newList(token.pos, isDigit(token.val))
Loop:
	for {
		switch token = t.peek(); token.typ {
		case itemLooseItem, itemListItem:
			list.append(t.parseListItem())
		default:
			break Loop
		}
	}
	return list
}

// parse listItem
// Add ignore list(e.g: table should parse as a text)
func (t *Tree) parseListItem() *ListItemNode {
	token := t.next()
	item := t.newListItem(token.pos)
	tr := &Tree{lex: lex(strings.TrimSpace(token.val))}
	tr.parse()
	for _, node := range tr.Nodes {
		// wrap with paragraph only when it's loose item
		if n, ok := node.(*ParagraphNode); ok && token.typ == itemListItem {
			item.Nodes = append(item.Nodes, n.Nodes...)
		} else {
			item.append(node)
		}
	}
	return item
}

// parse table
func (t *Tree) parseTable() *TableNode {
	// End of table
	done := t.lex.eot
	token := t.next()
	// ignore the first and last one...
	//lp := token.val == "|"
	table := t.newTable(token.pos)
	// Align	[ None, Left, Right, ... ]
	// Header	[ Cels: [token, token, ... ] ]
	// Data:	[ Row: [Cells: [token, ... ] ] ]
	rows := struct {
		Align  []AlignType
		Header [][]item
		Data   [][][]item
	}{}
	var cell []item
	var row [][]item
	// Collect items
Loop:
	for i := 0; ; {
		switch token := t.next(); token.typ {
		case eof, itemError:
			break Loop
		case itemNewLine:
			// If we done with this table
			if t.peek().pos >= done {
				break Loop
			}
			fallthrough
		case itemPipe:
			// Test if cell non-empty before appending to current row
			if len(cell) > 0 {
				// Header
				if i == 0 {
					rows.Header = append(rows.Header, cell)
					// Alignment
				} else if i == 1 {
					align := cell[0].val
					if len(cell) > 1 {
						for i := 1; i < len(cell); i++ {
							align += cell[i].val
						}
					}
					// Trim spaces
					rows.Align = append(rows.Align, parseAlign(align))
					// Data
				} else {
					row = append(row, cell)
				}
			}
			if token.typ == itemNewLine {
				i++
				// test if there's an elemnts to append to tbody.
				// we want to avoid situations like `appending empty rows`, etc..
				if i > 2 && len(row) > 0 {
					rows.Data = append(rows.Data, row)
					row = [][]item{}
				}
			}
			cell = []item{}
		default:
			cell = append(cell, token)
		}
	}
	// Drain cell/row
	if len(cell) > 0 {
		row = append(row, cell)
	}
	if len(row) > 0 {
		rows.Data = append(rows.Data, row)
	}
	// Tranform to nodes
	// Add an average mechanisem, that ignore empty(or " ") in end of cell
	//	rowLen := len(rows.Align)
	// Table head
	table.append(t.parseCells(Header, rows.Header, rows.Align))
	// Table body
	for _, row := range rows.Data {
		table.append(t.parseCells(Data, row, rows.Align))
	}
	return table
}

// Should return typ []CellNode
func (t *Tree) parseCells(kind int, items [][]item, align []AlignType) *RowNode {
	// TODO(Ariel): Add real position
	row := t.newRow(1)
	for i, item := range items {
		// Cell contain nodes
		var s string
		cell := t.newCell(item[0].pos, kind, align[i])
		for _, tkn := range item {
			s += tkn.val
		}
		// Used before: `^[ |\t]*|[ |\t]*$`
		cell.Nodes = t.parseText(strings.TrimSpace(s))
		row.append(cell)
	}
	return row
}

// get align-string and return the align type of it
// e.g: ":---", "---:", ":---:", "---"
func parseAlign(s string) (typ AlignType) {
	// Trim spaces before
	s = strings.TrimSpace(s)
	sfx, pfx := strings.HasSuffix(s, ":"), strings.HasPrefix(s, ":")
	switch {
	case sfx && pfx:
		typ = Center
	case sfx:
		typ = Right
	case pfx:
		typ = Left
	}
	return
}

// test if given string is digit
func isDigit(s string) bool {
	r, _ := utf8.DecodeRuneInString(s)
	return unicode.IsDigit(r)
}

// test if given token is type block
func isBlock(item itemType) (b bool) {
	switch item {
	case itemHeading, itemLHeading, itemCodeBlock, itemBlockQuote,
		itemList, itemTable, itemGfmCodeBlock, itemHr:
		b = true
	}
	return
}
<commit_msg>feat(parser): rootGetter + parse refernceItem<commit_after>package mark

import (
	fmt "github.com/k0kubun/pp"
	"regexp"
	"strings"
	"unicode"
	"unicode/utf8"
)

type Tree struct {
	lex   *lexer
	tr    *Tree
	Nodes []Node
	// Parsing only
	token     [3]item // three-token lookahead for parser
	peekCount int
	output    string
	links     map[string]*DefLinkNode
}

// Parse convert the raw text to NodeTree.
func (t *Tree) parse() {
Loop:
	for {
		var n Node
		switch p := t.peek(); p.typ {
		case eof, itemError:
			break Loop
		case itemNewLine:
			n = t.newLine(t.next().pos)
		case itemBr:
			n = t.newBr(t.next().pos)
		case itemHr:
			n = t.newHr(t.next().pos)
		case itemHTML:
			p = t.next()
			n = t.newHTML(p.pos, p.val)
		case itemText:
			tmp := t.newParagraph(p.pos)
			tmp.Nodes = t.parseText(t.next().val)
			n = tmp
		case itemDefLink:
			n = t.parseDefLink()
		case itemHeading, itemLHeading:
			n = t.parseHeading()
		case itemCodeBlock, itemGfmCodeBlock:
			n = t.parseCodeBlock()
		case itemList:
			n = t.parseList()
		case itemTable, itemLpTable:
			n = t.parseTable()
		case itemBlockQuote:
			n = t.parseBlockQuote()
		default:
			t.next()
		}
		if n != nil {
			t.append(n)
		}
	}
}

// Root getter
func (t *Tree) root() *Tree {
	if t.tr == nil {
		return t
	}
	return t.tr.root()
}

// Render parse nodes to the wanted output
func (t *Tree) render() {
	var last Node
	last = t.newLine(0)
	for _, node := range t.Nodes {
		if last.Type() != NodeNewLine || node.Type() != last.Type() {
			t.output += node.Render()
		}
		last = node
	}
}

// append new node to nodes-list
func (t *Tree) append(n Node) {
	t.Nodes = append(t.Nodes, n)
}

// next returns the next token
func (t *Tree) next() item {
	if t.peekCount > 0 {
		t.peekCount--
	} else {
		t.token[0] = t.lex.nextItem()
	}
	return t.token[t.peekCount]
}

// peek returns but does not consume the next token.
func (t *Tree) peek() item {
	if t.peekCount > 0 {
		return t.token[t.peekCount-1]
	}
	t.peekCount = 1
	t.token[0] = t.lex.nextItem()
	return t.token[0]
}

// backup backs the input stream tp one token
func (t *Tree) backup() {
	t.peekCount++
}

// backup2 backs the input stream up two tokens.
// The zeroth token is already there.
func (t *Tree) backup2(t1 item) {
	t.token[1] = t1
	t.peekCount = 2
}

// parseText scan until itemBr occur.
func (t *Tree) parseText(input string) (nodes []Node) {
	// HACK: if there's more 'itemText' in the way, make it one.
	for {
		tkn := t.next()
		if tkn.typ == itemText {
			input += tkn.val
		} else if tkn.typ == itemNewLine {
			if t.peek().typ != itemText {
				t.backup2(tkn)
				break
			}
			input += tkn.val
		} else {
			t.backup()
			break
		}
	}
	l := lexInline(input)
	for token := range l.items {
		var node Node
		switch token.typ {
		case itemNewLine:
			node = t.newLine(token.pos)
		case itemBr:
			node = t.newBr(token.pos)
		case itemStrong, itemItalic, itemStrike, itemCode:
			node = t.parseEmphasis(token.typ, token.pos, token.val)
		case itemLink, itemAutoLink, itemGfmLink:
			var title, text, href string
			match := span[token.typ].FindStringSubmatch(token.val)
			if token.typ == itemLink {
				text, href, title = match[1], match[2], match[3]
			} else {
				text, href = match[1], match[1]
			}
			node = t.newLink(token.pos, title, href, text)
		case itemImage:
			match := span[token.typ].FindStringSubmatch(token.val)
			node = t.newImage(token.pos, match[3], match[2], match[1])
		case itemRefLink, itemRefImage:
			match := span[itemRefLink].FindStringSubmatch(token.val)
			node = t.newRef(token.typ, token.pos, token.val, match[1], match[2])
		case itemText:
			node = t.newText(token.pos, token.val)
		default:
			fmt.Println("Matching not found for this token:", token)
		}
		nodes = append(nodes, node)
	}
	return nodes
}

// Parse inline emphasis
func (t *Tree) parseEmphasis(typ itemType, pos Pos, val string) *EmphasisNode {
	node := t.newEmphasis(pos, typ)
	match := span[typ].FindStringSubmatch(val)
	text := match[len(match)-1]
	if text == "" {
		text = match[1]
	}
	node.Nodes = t.parseText(text)
	return node
}

// parse heading block
func (t *Tree) parseHeading() (node *HeadingNode) {
	token := t.next()
	match := block[token.typ].FindStringSubmatch(token.val)
	if token.typ == itemHeading {
		node = t.newHeading(token.pos, len(match[1]), match[2])
	} else {
		// using equal signs for first-level, and dashes for second-level.
		level := 1
		if match[2] == "-" {
			level = 2
		}
		node = t.newHeading(token.pos, level, match[1])
	}
	return
}

func (t *Tree) parseDefLink() *DefLinkNode {
	token := t.next()
	match := block[itemDefLink].FindStringSubmatch(token.val)
	name := strings.ToLower(match[1])
	// name(lowercase), href, title
	n := t.newDefLink(token.pos, name, match[2], match[3])
	// store in links
	t.links[name] = n
	return n
}

// parse codeBlock
func (t *Tree) parseCodeBlock() *CodeNode {
	var lang, text string
	token := t.next()
	if token.typ == itemGfmCodeBlock {
		match := block[itemGfmCodeBlock].FindStringSubmatch(token.val)
		if text = match[2]; text == "" {
			text = match[4]
		}
		if lang = match[1]; lang == "" {
			lang = match[3]
		}
	} else {
		text = regexp.MustCompile("(?m)^( {4})").ReplaceAllLiteralString(token.val, "")
	}
	return t.newCode(token.pos, lang, text)
}

func (t *Tree) parseBlockQuote() (n *BlockQuoteNode) {
	token := t.next()
	// replacer
	re := regexp.MustCompile(`(?m)^> ?`)
	raw := re.ReplaceAllString(token.val, "")
	// TODO(Ariel): not work right now with defLink(inside the blockQuote)
	tr := &Tree{lex: lex(raw)}
	tr.parse()
	n = t.newBlockQuote(token.pos)
	n.Nodes = tr.Nodes
	return
}

// parse list
func (t *Tree) parseList() *ListNode {
	token := t.next()
	list := t.newList(token.pos, isDigit(token.val))
Loop:
	for {
		switch token = t.peek(); token.typ {
		case itemLooseItem, itemListItem:
			list.append(t.parseListItem())
		default:
			break Loop
		}
	}
	return list
}

// parse listItem
// Add ignore list(e.g: table should parse as a text)
func (t *Tree) parseListItem() *ListItemNode {
	token := t.next()
	item := t.newListItem(token.pos)
	tr := &Tree{lex: lex(strings.TrimSpace(token.val))}
	tr.parse()
	for _, node := range tr.Nodes {
		// wrap with paragraph only when it's loose item
		if n, ok := node.(*ParagraphNode); ok && token.typ == itemListItem {
			item.Nodes = append(item.Nodes, n.Nodes...)
		} else {
			item.append(node)
		}
	}
	return item
}

// parse table
func (t *Tree) parseTable() *TableNode {
	// End of table
	done := t.lex.eot
	token := t.next()
	// ignore the first and last one...
	//lp := token.val == "|"
	table := t.newTable(token.pos)
	// Align	[ None, Left, Right, ... ]
	// Header	[ Cels: [token, token, ... ] ]
	// Data:	[ Row: [Cells: [token, ... ] ] ]
	rows := struct {
		Align  []AlignType
		Header [][]item
		Data   [][][]item
	}{}
	var cell []item
	var row [][]item
	// Collect items
Loop:
	for i := 0; ; {
		switch token := t.next(); token.typ {
		case eof, itemError:
			break Loop
		case itemNewLine:
			// If we done with this table
			if t.peek().pos >= done {
				break Loop
			}
			fallthrough
		case itemPipe:
			// Test if cell non-empty before appending to current row
			if len(cell) > 0 {
				// Header
				if i == 0 {
					rows.Header = append(rows.Header, cell)
					// Alignment
				} else if i == 1 {
					align := cell[0].val
					if len(cell) > 1 {
						for i := 1; i < len(cell); i++ {
							align += cell[i].val
						}
					}
					// Trim spaces
					rows.Align = append(rows.Align, parseAlign(align))
					// Data
				} else {
					row = append(row, cell)
				}
			}
			if token.typ == itemNewLine {
				i++
				// test if there's an elemnts to append to tbody.
				// we want to avoid situations like `appending empty rows`, etc..
				if i > 2 && len(row) > 0 {
					rows.Data = append(rows.Data, row)
					row = [][]item{}
				}
			}
			cell = []item{}
		default:
			cell = append(cell, token)
		}
	}
	// Drain cell/row
	if len(cell) > 0 {
		row = append(row, cell)
	}
	if len(row) > 0 {
		rows.Data = append(rows.Data, row)
	}
	// Tranform to nodes
	// Add an average mechanisem, that ignore empty(or " ") in end of cell
	//	rowLen := len(rows.Align)
	// Table head
	table.append(t.parseCells(Header, rows.Header, rows.Align))
	// Table body
	for _, row := range rows.Data {
		table.append(t.parseCells(Data, row, rows.Align))
	}
	return table
}

// Should return typ []CellNode
func (t *Tree) parseCells(kind int, items [][]item, align []AlignType) *RowNode {
	// TODO(Ariel): Add real position
	row := t.newRow(1)
	for i, item := range items {
		// Cell contain nodes
		var s string
		cell := t.newCell(item[0].pos, kind, align[i])
		for _, tkn := range item {
			s += tkn.val
		}
		// Used before: `^[ |\t]*|[ |\t]*$`
		cell.Nodes = t.parseText(strings.TrimSpace(s))
		row.append(cell)
	}
	return row
}

// get align-string and return the align type of it
// e.g: ":---", "---:", ":---:", "---"
func parseAlign(s string) (typ AlignType) {
	// Trim spaces before
	s = strings.TrimSpace(s)
	sfx, pfx := strings.HasSuffix(s, ":"), strings.HasPrefix(s, ":")
	switch {
	case sfx && pfx:
		typ = Center
	case sfx:
		typ = Right
	case pfx:
		typ = Left
	}
	return
}

// test if given string is digit
func isDigit(s string) bool {
	r, _ := utf8.DecodeRuneInString(s)
	return unicode.IsDigit(r)
}

// test if given token is type block
func isBlock(item itemType) (b bool) {
	switch item {
	case itemHeading, itemLHeading, itemCodeBlock, itemBlockQuote,
		itemList, itemTable, itemGfmCodeBlock, itemHr:
		b = true
	}
	return
}
<|endoftext|>


<commit_before>package iniparser

import (
	"bufio"
	"errors"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

//Represents the configurations in the ini file
type Config struct {
	FileName string
	Sections []*Section
}

//Represents a section
type Section struct {
	Name string
	Keys map[string]string
}

//Loads and parses an ini file
func (conf *Config) LoadFile(fileName string) error {
	if filepath.Dir(fileName) == "." {
		fullPath, _ := filepath.Abs(os.Args[0])
		pos := strings.LastIndex(fullPath, string(filepath.Separator))
		path := fullPath[0 : pos + 1]

		fileName = filepath.Join(path, fileName)
	}

	conf.FileName = fileName

	file, err := os.Open(fileName)
	if err != nil {
		return err
	}

	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line := scanner.Text()
		line = strings.TrimSpace(line)

		err = conf.parseLine(line)
		if err != nil {
			return err
		}
	}

	if err := scanner.Err(); err != nil {
		return err
	}

	return nil
}

func (conf *Config) parseLine(line string) error {
	if len(line) == 0 || line[0] == ';' || line[0] == '#' || line[0] == '\n' || line[0] == '\r' {
		return nil
	}

	if line[0] == '[' {
		pos := strings.Index(line, "]")
		if pos < 1 {
			return errors.New("Failed to parse section header: " + line)
		}

		section := new(Section)
		section.Name = line[1:pos]
		section.Keys = make(map[string]string)

		conf.Sections = append(conf.Sections, section)

		return nil
	}

	if conf.Sections == nil || len(conf.Sections) == 0 {
		return errors.New("Key is not under any section: " + line)
	}

	pos := strings.Index(line, "=")
	if pos < 1 {
		return errors.New("Failed to parse key line: " + line)
	}

	name := line[0:pos]
	val := line[pos + 1 : len(line)]

	conf.Sections[len(conf.Sections) - 1].Keys[name] = val

	return nil
}

//Returns a section
func (conf *Config) GetSection(name string) (*Section, error) {
	for _, section := range conf.Sections {
		if section.Name == name {
			return section, nil
		}
	}

	return nil, errors.New("Section not found: " + name)
}

//Returns a list of all sections
func (conf *Config) GetSections() []*Section {
	return conf.Sections
}

//Returns a string value of a key in a section
func (conf *Config) GetString(section string, key string) (string, error) {
	sec, err := conf.GetSection(section)
	if err != nil {
		return "", err
	}

	val := sec.Keys[key]
	if val == "" {
		return "", errors.New("Key " + key + " not found")
	}

	return val, nil
}

//Returns a boolean value of a key in a section
func (conf *Config) GetBool(section string, key string) (bool, error) {
	value, err := conf.GetString(section, key)
	if err != nil {
		return false, err
	}

	return strconv.ParseBool(value)
}

//Returns an integer value of a key in a section
func (conf *Config) GetInt(section string, key string) (int, error) {
	value, err := conf.GetString(section, key)
	if err != nil {
		return 0, err
	}

	return strconv.Atoi(value)
}

//Returns a 64-bit integer value of a key in a section
func (conf *Config) GetInt64(section string, key string) (int64, error) {
	value, err := conf.GetString(section, key)
	if err != nil {
		return 0, err
	}

	return strconv.ParseInt(value, 10, 64)
}

//Returns a 64-bit float value of a key in a section
func (conf *Config) GetFloat64(section string, key string) (float64, error) {
	value, err := conf.GetString(section, key)
	if err != nil {
		return 0, err
	}

	return strconv.ParseFloat(value, 64)
}
<commit_msg>Reworded comments; fixed white space trimming<commit_after>package iniparser

import (
	"bufio"
	"errors"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

//Config represents the configurations in the ini file
type Config struct {
	FileName string
	Sections []*Section
}

//Section represents a section
type Section struct {
	Name string
	Keys map[string]string
}

//LoadFile loads and parses an ini file
func (conf *Config) LoadFile(fileName string) error {
	if filepath.Dir(fileName) == "." {
		fullPath, _ := filepath.Abs(os.Args[0])
		pos := strings.LastIndex(fullPath, string(filepath.Separator))
		path := fullPath[0 : pos + 1]

		fileName = filepath.Join(path, fileName)
	}

	conf.FileName = fileName

	file, err := os.Open(fileName)
	if err != nil {
		return err
	}

	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line := scanner.Text()
		line = strings.TrimSpace(line)

		err = conf.parseLine(line)
		if err != nil {
			return err
		}
	}

	if err := scanner.Err(); err != nil {
		return err
	}

	return nil
}

func (conf *Config) parseLine(line string) error {
	if len(line) == 0 || line[0] == ';' || line[0] == '#' || line[0] == '\n' || line[0] == '\r' {
		return nil
	}

	if line[0] == '[' {
		pos := strings.Index(line, "]")
		if pos < 1 {
			return errors.New("Failed to parse section header: " + line)
		}

		section := new(Section)
		section.Name = line[1:pos]
		section.Keys = make(map[string]string)

		conf.Sections = append(conf.Sections, section)

		return nil
	}

	if conf.Sections == nil || len(conf.Sections) == 0 {
		return errors.New("Key is not under any section: " + line)
	}

	pos := strings.Index(line, "=")
	if pos < 1 {
		return errors.New("Failed to parse key line: " + line)
	}

	name := strings.TrimSpace(line[0:pos])
	val := strings.TrimSpace(line[pos + 1 : len(line)])

	conf.Sections[len(conf.Sections) - 1].Keys[name] = val

	return nil
}

//GetSection returns a section
func (conf *Config) GetSection(name string) (*Section, error) {
	for _, section := range conf.Sections {
		if section.Name == name {
			return section, nil
		}
	}

	return nil, errors.New("Section not found: " + name)
}

//GetSections returns a list of all sections
func (conf *Config) GetSections() []*Section {
	return conf.Sections
}

//GetString returns a string value of a key in a section
func (conf *Config) GetString(section string, key string) (string, error) {
	sec, err := conf.GetSection(section)
	if err != nil {
		return "", err
	}

	val := sec.Keys[key]
	if val == "" {
		return "", errors.New("Key " + key + " not found")
	}

	return val, nil
}

//GetBool returns a boolean value of a key in a section
func (conf *Config) GetBool(section string, key string) (bool, error) {
	value, err := conf.GetString(section, key)
	if err != nil {
		return false, err
	}

	return strconv.ParseBool(value)
}

//GetInt returns an integer value of a key in a section
func (conf *Config) GetInt(section string, key string) (int, error) {
	value, err := conf.GetString(section, key)
	if err != nil {
		return 0, err
	}

	return strconv.Atoi(value)
}

//GetInt64 returns a 64-bit integer value of a key in a section
func (conf *Config) GetInt64(section string, key string) (int64, error) {
	value, err := conf.GetString(section, key)
	if err != nil {
		return 0, err
	}

	return strconv.ParseInt(value, 10, 64)
}

//GetFloat64 returns a 64-bit float value of a key in a section
func (conf *Config) GetFloat64(section string, key string) (float64, error) {
	value, err := conf.GetString(section, key)
	if err != nil {
		return 0, err
	}

	return strconv.ParseFloat(value, 64)
}
<|endoftext|>


<commit_before>package main

import (
	"./stat"
	"fmt"
	"github.com/droundy/goopt"
	"github.com/satyrius/gonx"
	"io"
	"os"
	"path/filepath"
	"sort"
	"strconv"
)

var debug *bool
var format *string
var nginxConfig *string
var nginxFormat *string
var aggField *string
var groupBy *string
var groupByReqexp *string

func init() {
	format = goopt.String([]string{"--fmt", "--format"}, "",
		"Log format (e.g. '$remote_addr [$time_local] \"$request\"')")
	nginxConfig = goopt.String([]string{"--nginx"}, "",
		"Nginx config to look for 'log_format' directive. You also should specify --nginx-format")
	nginxFormat = goopt.String([]string{"--nginx-format"}, "",
		"Name of nginx 'log_format', should be passed with --nginx option")
	aggField = goopt.String([]string{"-a", "--aggregate"}, "request_time",
		"Nginx access log variable to aggregate")
	groupBy = goopt.String([]string{"-g", "--group-by"}, "request",
		"Nginx access log variable to group by")
	groupByReqexp = goopt.String([]string{"-r", "--regexp"}, "",
		"You can specify regular expression to extract exact data from group by data. "+
			"For example, you might want to group by a path inside $request, so you should "+
			"set this option to '^\\S+\\s(.*)(?:\\?.*)?$'.")
	debug = goopt.Flag([]string{"--debug"}, []string{"--no-debug"},
		"Log debug information", "Do not log debug information")
}

func badUsage() {
	fmt.Println(goopt.Usage())
	os.Exit(1)
}

func getReader(file io.Reader) (reader *gonx.Reader, err error) {
	if *format != "" {
		reader = gonx.NewReader(file, *format)
	} else {
		if *nginxConfig == "" || *nginxFormat == "" {
			badUsage()
		}
		cfg, err := os.Open(*nginxConfig)
		if err != nil {
			return nil, err
		}
		defer cfg.Close()
		reader, err = gonx.NewNginxReader(file, cfg, *nginxFormat)
	}
	return
}

func main() {
	goopt.Parse(nil)

	var logs []*os.File
	if len(goopt.Args) != 0 {
		fmt.Println("We are going to parse those files:")
		for _, log := range goopt.Args {
			log, _ := filepath.Abs(log)
			fmt.Println(log)
			file, err := os.Open(log)
			if err != nil {
				panic(err)
			}
			defer file.Close()
			logs = append(logs, file)
		}
	} else {
		fmt.Println("Read from STDIN")
		logs = append(logs, os.Stdin)
	}

	aggregator := func(i *stat.Item, entry *gonx.Entry) (val float64, err error) {
		if strVal, ok := (*entry)[*aggField]; ok {
			v, err := strconv.ParseFloat(strVal, 64)
			if err != nil {
				return 0, err
			}
			if i.Count == 1 {
				val = v
			} else {
				val = (i.AggValue*float64(i.Count-1) + v) / float64(i.Count)
			}
		} else {
			err = fmt.Errorf("Invalid entry data")
		}
		return
	}
	st := stat.NewStat(aggregator, *groupBy, *groupByReqexp)

	for _, file := range logs {
		st.AddLog(file.Name())
		reader, err := getReader(file)
		if err != nil {
			panic(err)
		}
		for {
			record, err := reader.Read()
			if err == io.EOF {
				break
			} else if err != nil {
				panic(err)
			}
			if *debug {
				fmt.Println("==============================")
				for name, value := range record {
					fmt.Printf("LR $%v = '%v'\n", name, value)
				}
			}
			if err := st.Add(&record); err != nil {
				panic(err)
			}
		}
	}
	fmt.Printf("Gratz! You've parsed %v log entries, it took %v\n", st.EntriesParsed, st.Stop())
	sort.Sort(st)
	for _, item := range st.Data {
		fmt.Printf("%7.3f %6d %v\n", item.AggValue, item.Count, item.Name)
	}
}
<commit_msg>save json encoded result<commit_after>package main

import (
	"./stat"
	"encoding/json"
	"fmt"
	"github.com/droundy/goopt"
	"github.com/satyrius/gonx"
	"io"
	"os"
	"path/filepath"
	"sort"
	"strconv"
)

var debug *bool
var format *string
var nginxConfig *string
var nginxFormat *string
var aggField *string
var groupBy *string
var groupByReqexp *string
var jsonOutput *string

func init() {
	format = goopt.String([]string{"--fmt", "--format"}, "",
		"Log format (e.g. '$remote_addr [$time_local] \"$request\"')")
	nginxConfig = goopt.String([]string{"--nginx"}, "",
		"Nginx config to look for 'log_format' directive. You also should specify --nginx-format")
	nginxFormat = goopt.String([]string{"--nginx-format"}, "",
		"Name of nginx 'log_format', should be passed with --nginx option")
	aggField = goopt.String([]string{"-a", "--aggregate"}, "request_time",
		"Nginx access log variable to aggregate")
	groupBy = goopt.String([]string{"-g", "--group-by"}, "request",
		"Nginx access log variable to group by")
	groupByReqexp = goopt.String([]string{"-r", "--regexp"}, "",
		"You can specify regular expression to extract exact data from group by data. "+
			"For example, you might want to group by a path inside $request, so you should "+
			"set this option to '^\\S+\\s(.*)(?:\\?.*)?$'.")
	debug = goopt.Flag([]string{"--debug"}, []string{"--no-debug"},
		"Log debug information", "Do not log debug information")
	jsonOutput = goopt.String([]string{"-o", "--json"}, "",
		"Save result as json encoded file")
}

func badUsage() {
	fmt.Println(goopt.Usage())
	os.Exit(1)
}

func getReader(file io.Reader) (reader *gonx.Reader, err error) {
	if *format != "" {
		reader = gonx.NewReader(file, *format)
	} else {
		if *nginxConfig == "" || *nginxFormat == "" {
			badUsage()
		}
		cfg, err := os.Open(*nginxConfig)
		if err != nil {
			return nil, err
		}
		defer cfg.Close()
		reader, err = gonx.NewNginxReader(file, cfg, *nginxFormat)
	}
	return
}

func main() {
	goopt.Parse(nil)

	var logs []*os.File
	if len(goopt.Args) != 0 {
		fmt.Println("We are going to parse those files:")
		for _, log := range goopt.Args {
			log, _ := filepath.Abs(log)
			fmt.Println(log)
			file, err := os.Open(log)
			if err != nil {
				panic(err)
			}
			defer file.Close()
			logs = append(logs, file)
		}
	} else {
		fmt.Println("Read from STDIN")
		logs = append(logs, os.Stdin)
	}

	aggregator := func(i *stat.Item, entry *gonx.Entry) (val float64, err error) {
		if strVal, ok := (*entry)[*aggField]; ok {
			v, err := strconv.ParseFloat(strVal, 64)
			if err != nil {
				return 0, err
			}
			if i.Count == 1 {
				val = v
			} else {
				val = (i.AggValue*float64(i.Count-1) + v) / float64(i.Count)
			}
		} else {
			err = fmt.Errorf("Invalid entry data")
		}
		return
	}
	st := stat.NewStat(aggregator, *groupBy, *groupByReqexp)

	for _, file := range logs {
		st.AddLog(file.Name())
		reader, err := getReader(file)
		if err != nil {
			panic(err)
		}
		for {
			record, err := reader.Read()
			if err == io.EOF {
				break
			} else if err != nil {
				panic(err)
			}
			if *debug {
				fmt.Println("==============================")
				for name, value := range record {
					fmt.Printf("LR $%v = '%v'\n", name, value)
				}
			}
			if err := st.Add(&record); err != nil {
				panic(err)
			}
		}
	}
	fmt.Printf("Gratz! You've parsed %v log entries, it took %v\n", st.EntriesParsed, st.Stop())
	sort.Sort(st)
	for _, item := range st.Data {
		fmt.Printf("%7.3f %6d %v\n", item.AggValue, item.Count, item.Name)
	}

	if *jsonOutput != "" {
		jsonFile, err := os.OpenFile(*jsonOutput, os.O_WRONLY|os.O_CREATE, 0644)
		if err != nil {
			panic(err)
		}
		defer jsonFile.Close()
		jsData, err := json.MarshalIndent(st.Data, "", "  ")
		if err != nil {
			panic(err)
		}
		jsonFile.Write(jsData)
		fmt.Printf("Result was saved to JSON encoded file '%v'\n", jsonFile.Name())
	}
}
<|endoftext|>