<commit_before>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package buildbucket

import (
	"encoding/json"
	"fmt"
	"strconv"
	"strings"
	"time"

	"go.chromium.org/luci/common/api/buildbucket/buildbucket/v1"
	"go.chromium.org/luci/common/auth/identity"
	"go.chromium.org/luci/common/data/strpair"
	"go.chromium.org/luci/common/errors"
)

// TagBuilder is the key of builder name tag.
const TagBuilder = "builder"

// Build is a buildbucket build.
// It is a more type-safe version of buildbucket.ApiCommonBuildMessage.
type Build struct {
	// fields set at the build creation time, immutable.

	ID int64
	// Address is an alternative identifier of the build.
	// If the build has build number, it is "<bucket>/<builder>/<number>".
	// Otherwise it is "<id>".
	Address      string
	CreationTime time.Time
	CreatedBy    identity.Identity
	Bucket       string
	Builder      string
	// Number identifies the build within the builder.
	// Build numbers are monotonically increasing, mostly contiguous.
	//
	// The type is *int to prevent accidental confusion
	// of valid build number 0 with absence of the number (zero value).
	Number *int
	// BuildSets is parsed "buildset" tag values.
	//
	// If a buildset is present in tags, but not recognized
	// it won't be included here.
	BuildSets        []BuildSet
	Tags             strpair.Map
	Input            Input
	CanaryPreference CanaryPreference

	// fields that can change during build lifetime

	Status           Status
	StatusChangeTime time.Time
	URL              string
	StartTime        time.Time
	UpdateTime       time.Time
	Canary           bool

	// fields set on build completion

	CompletionTime time.Time
	Output         Output
}

// RunDuration returns duration between build start and completion.
func (b *Build) RunDuration() (duration time.Duration, ok bool) {
	if b.StartTime.IsZero() || b.CompletionTime.IsZero() {
		return 0, false
	}
	return b.CompletionTime.Sub(b.StartTime), true
}

// SchedulingDuration returns duration between build creation and start.
func (b *Build) SchedulingDuration() (duration time.Duration, ok bool) {
	if b.CreationTime.IsZero() || b.StartTime.IsZero() {
		return 0, false
	}
	return b.StartTime.Sub(b.CreationTime), true
}

// Properties is data provided by users, opaque to LUCI services.
// The value must be JSON marshalable/unmarshalable into/out from a
// JSON object.
//
// When using an unmarshaling function, such as (*Build).ParseMessage,
// if the user knows the properties they need, they may set the
// value to a json-compatible struct. The unmarshaling function will try
// to unmarshal the properties into the struct. Otherwise, the unmarshaling
// function will use a generic type, e.g. map[string]interface{}.
//
// Example:
//
//   var props struct {
//     A string
//   }
//   var build buildbucket.Build
//   build.Input.Properties = &props
//   if err := build.ParseMessage(msg); err != nil {
//     return err
//   }
//   println(props.A)
type Properties interface{}

// Input is the input to the builder.
type Input struct {
	// Properties is opaque data passed to the build.
	// For recipe-based builds, this is build properties.
	Properties Properties
}

// Output is build output.
type Output struct {
	Properties Properties

	// TODO(nodir, iannucci): replace type "error" with a new type that
	// represents a stack of errors emitted by different layers of the system,
	// where each error has
	// - domain string, e.g. "kitchen"
	// - reason string, e.g. kitchen-specific error code
	// - message string: human readable error
	// - meta: a proto.Struct with random data provided by the layer
	// The new type must implement error so that the change is
	// backward compatible.
	Err error // populated in builds with status StatusError
}

// ParseMessage parses a build message to Build.
//
// Numeric values in JSON-formatted fields, e.g. property values, are parsed as
// json.Number.
//
// If an error is returned, the state of b is undefined.
func (b *Build) ParseMessage(msg *buildbucket.ApiCommonBuildMessage) error {
	status, err := ParseStatus(msg)
	if err != nil {
		return err
	}

	createdBy, err := identity.MakeIdentity(msg.CreatedBy)
	if err != nil {
		return err
	}

	tags := strpair.ParseMap(msg.Tags)
	builder := tags.Get(TagBuilder)

	canaryPref, err := parseEndpointsCanaryPreference(msg.CanaryPreference)
	if err != nil {
		return err
	}

	address := tags.Get("build_address")
	var number *int
	if address == "" {
		address = strconv.FormatInt(msg.Id, 10)
	} else {
		parts := strings.Split(address, "/")
		if len(parts) != 3 || parts[0] != msg.Bucket || parts[1] != builder {
			return fmt.Errorf("invalid build_address %q", address)
		}
		num, err := strconv.Atoi(parts[2])
		if err != nil {
			return fmt.Errorf("invalid build_address %q", address)
		}
		number = &num
	}

	input := struct{ Properties interface{} }{b.Input.Properties}
	if err := parseJSON(msg.ParametersJson, &input); err != nil {
		return errors.Annotate(err, "invalid msg.ParametersJson").Err()
	}

	output := struct {
		Properties interface{}
		Error      struct {
			Message string
		}
	}{Properties: b.Output.Properties}
	if err := parseJSON(msg.ResultDetailsJson, &output); err != nil {
		return errors.Annotate(err, "invalid msg.ResultDetailsJson").Err()
	}
	var outErr error
	if output.Error.Message != "" {
		outErr = errors.New(output.Error.Message)
	}

	var bs []BuildSet
	for _, t := range tags[TagBuildSet] {
		if parsed := ParseBuildSet(t); parsed != nil {
			bs = append(bs, parsed)
		}
	}

	*b = Build{
		ID:               msg.Id,
		Address:          address,
		CreationTime:     ParseTimestamp(msg.CreatedTs),
		CreatedBy:        createdBy,
		Bucket:           msg.Bucket,
		Builder:          builder,
		Number:           number,
		BuildSets:        bs,
		Tags:             tags,
		CanaryPreference: canaryPref,
		Input: Input{
			Properties: input.Properties,
		},

		Status:           status,
		StatusChangeTime: ParseTimestamp(msg.StatusChangedTs),
		URL:              msg.Url,
		StartTime:        ParseTimestamp(msg.StartedTs),
		UpdateTime:       ParseTimestamp(msg.UpdatedTs),
		Canary:           msg.Canary,

		CompletionTime: ParseTimestamp(msg.CompletedTs),
		Output: Output{
			Properties: output.Properties,
			Err:        outErr,
		},
	}
	return nil
}

// PutRequest converts b to a build creation request.
//
// If a buildset is present in both b.BuildSets and b.Map, it is deduped.
// Returned value has zero ClientOperationId.
// Returns an error if properties could not be marshaled to JSON.
func (b *Build) PutRequest() (*buildbucket.ApiPutRequestMessage, error) {
	tags := b.Tags.Copy()
	tags.Del(TagBuilder) // buildbucket adds it automatically
	for _, bs := range b.BuildSets {
		s := bs.String()
		if !tags.Contains(TagBuildSet, s) {
			tags.Add(TagBuildSet, s)
		}
	}

	canaryPref, err := b.CanaryPreference.endpointsString()
	if err != nil {
		return nil, err
	}
	msg := &buildbucket.ApiPutRequestMessage{
		Bucket:           b.Bucket,
		Tags:             tags.Format(),
		CanaryPreference: canaryPref,
	}

	parameters := map[string]interface{}{
		"builder_name": b.Builder,
		"properties":   b.Input.Properties,
		// keep this synced with marshaling error annotation
	}
	if data, err := json.Marshal(parameters); err != nil {
		// realistically, only properties may cause this.
		return nil, errors.Annotate(err, "marshaling properties").Err()
	} else {
		msg.ParametersJson = string(data)
	}

	return msg, nil
}

func parseJSON(data string, v interface{}) error {
	if data == "" {
		return nil
	}
	dec := json.NewDecoder(strings.NewReader(data))
	dec.UseNumber()
	return dec.Decode(v)
}

// ParseTimestamp parses a buildbucket timestamp.
func ParseTimestamp(usec int64) time.Time {
	if usec == 0 {
		return time.Time{}
	}
	// TODO(nodir): will start to overflow in year ~2250+, fix it then.
	return time.Unix(0, usec*1000).UTC()
}

// FormatTimestamp converts t to a buildbucket timestamp.
func FormatTimestamp(t time.Time) int64 {
	return t.UnixNano() / 1000
}
<commit_msg>[buildbucket] tolerate empty createdBy<commit_after>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package buildbucket

import (
	"encoding/json"
	"fmt"
	"strconv"
	"strings"
	"time"

	"go.chromium.org/luci/common/api/buildbucket/buildbucket/v1"
	"go.chromium.org/luci/common/auth/identity"
	"go.chromium.org/luci/common/data/strpair"
	"go.chromium.org/luci/common/errors"
)

// TagBuilder is the key of builder name tag.
const TagBuilder = "builder"

// Build is a buildbucket build.
// It is a more type-safe version of buildbucket.ApiCommonBuildMessage.
type Build struct {
	// fields set at the build creation time, immutable.

	ID int64
	// Address is an alternative identifier of the build.
	// If the build has build number, it is "<bucket>/<builder>/<number>".
	// Otherwise it is "<id>".
	Address      string
	CreationTime time.Time
	CreatedBy    identity.Identity
	Bucket       string
	Builder      string
	// Number identifies the build within the builder.
	// Build numbers are monotonically increasing, mostly contiguous.
	//
	// The type is *int to prevent accidental confusion
	// of valid build number 0 with absence of the number (zero value).
	Number *int
	// BuildSets is parsed "buildset" tag values.
	//
	// If a buildset is present in tags, but not recognized
	// it won't be included here.
	BuildSets        []BuildSet
	Tags             strpair.Map
	Input            Input
	CanaryPreference CanaryPreference

	// fields that can change during build lifetime

	Status           Status
	StatusChangeTime time.Time
	URL              string
	StartTime        time.Time
	UpdateTime       time.Time
	Canary           bool

	// fields set on build completion

	CompletionTime time.Time
	Output         Output
}

// RunDuration returns duration between build start and completion.
func (b *Build) RunDuration() (duration time.Duration, ok bool) {
	if b.StartTime.IsZero() || b.CompletionTime.IsZero() {
		return 0, false
	}
	return b.CompletionTime.Sub(b.StartTime), true
}

// SchedulingDuration returns duration between build creation and start.
func (b *Build) SchedulingDuration() (duration time.Duration, ok bool) {
	if b.CreationTime.IsZero() || b.StartTime.IsZero() {
		return 0, false
	}
	return b.StartTime.Sub(b.CreationTime), true
}

// Properties is data provided by users, opaque to LUCI services.
// The value must be JSON marshalable/unmarshalable into/out from a
// JSON object.
//
// When using an unmarshaling function, such as (*Build).ParseMessage,
// if the user knows the properties they need, they may set the
// value to a json-compatible struct. The unmarshaling function will try
// to unmarshal the properties into the struct. Otherwise, the unmarshaling
// function will use a generic type, e.g. map[string]interface{}.
//
// Example:
//
//   var props struct {
//     A string
//   }
//   var build buildbucket.Build
//   build.Input.Properties = &props
//   if err := build.ParseMessage(msg); err != nil {
//     return err
//   }
//   println(props.A)
type Properties interface{}

// Input is the input to the builder.
type Input struct {
	// Properties is opaque data passed to the build.
	// For recipe-based builds, this is build properties.
	Properties Properties
}

// Output is build output.
type Output struct {
	Properties Properties

	// TODO(nodir, iannucci): replace type "error" with a new type that
	// represents a stack of errors emitted by different layers of the system,
	// where each error has
	// - domain string, e.g. "kitchen"
	// - reason string, e.g. kitchen-specific error code
	// - message string: human readable error
	// - meta: a proto.Struct with random data provided by the layer
	// The new type must implement error so that the change is
	// backward compatible.
	Err error // populated in builds with status StatusError
}

// ParseMessage parses a build message to Build.
//
// Numeric values in JSON-formatted fields, e.g. property values, are parsed as
// json.Number.
//
// If an error is returned, the state of b is undefined.
func (b *Build) ParseMessage(msg *buildbucket.ApiCommonBuildMessage) error {
	status, err := ParseStatus(msg)
	if err != nil {
		return err
	}

	var createdBy identity.Identity
	if msg.CreatedBy != "" {
		createdBy, err = identity.MakeIdentity(msg.CreatedBy)
		if err != nil {
			return err
		}
	}

	tags := strpair.ParseMap(msg.Tags)
	builder := tags.Get(TagBuilder)

	canaryPref, err := parseEndpointsCanaryPreference(msg.CanaryPreference)
	if err != nil {
		return err
	}

	address := tags.Get("build_address")
	var number *int
	if address == "" {
		address = strconv.FormatInt(msg.Id, 10)
	} else {
		parts := strings.Split(address, "/")
		if len(parts) != 3 || parts[0] != msg.Bucket || parts[1] != builder {
			return fmt.Errorf("invalid build_address %q", address)
		}
		num, err := strconv.Atoi(parts[2])
		if err != nil {
			return fmt.Errorf("invalid build_address %q", address)
		}
		number = &num
	}

	input := struct{ Properties interface{} }{b.Input.Properties}
	if err := parseJSON(msg.ParametersJson, &input); err != nil {
		return errors.Annotate(err, "invalid msg.ParametersJson").Err()
	}

	output := struct {
		Properties interface{}
		Error      struct {
			Message string
		}
	}{Properties: b.Output.Properties}
	if err := parseJSON(msg.ResultDetailsJson, &output); err != nil {
		return errors.Annotate(err, "invalid msg.ResultDetailsJson").Err()
	}
	var outErr error
	if output.Error.Message != "" {
		outErr = errors.New(output.Error.Message)
	}

	var bs []BuildSet
	for _, t := range tags[TagBuildSet] {
		if parsed := ParseBuildSet(t); parsed != nil {
			bs = append(bs, parsed)
		}
	}

	*b = Build{
		ID:               msg.Id,
		Address:          address,
		CreationTime:     ParseTimestamp(msg.CreatedTs),
		CreatedBy:        createdBy,
		Bucket:           msg.Bucket,
		Builder:          builder,
		Number:           number,
		BuildSets:        bs,
		Tags:             tags,
		CanaryPreference: canaryPref,
		Input: Input{
			Properties: input.Properties,
		},

		Status:           status,
		StatusChangeTime: ParseTimestamp(msg.StatusChangedTs),
		URL:              msg.Url,
		StartTime:        ParseTimestamp(msg.StartedTs),
		UpdateTime:       ParseTimestamp(msg.UpdatedTs),
		Canary:           msg.Canary,

		CompletionTime: ParseTimestamp(msg.CompletedTs),
		Output: Output{
			Properties: output.Properties,
			Err:        outErr,
		},
	}
	return nil
}

// PutRequest converts b to a build creation request.
//
// If a buildset is present in both b.BuildSets and b.Map, it is deduped.
// Returned value has zero ClientOperationId.
// Returns an error if properties could not be marshaled to JSON.
func (b *Build) PutRequest() (*buildbucket.ApiPutRequestMessage, error) {
	tags := b.Tags.Copy()
	tags.Del(TagBuilder) // buildbucket adds it automatically
	for _, bs := range b.BuildSets {
		s := bs.String()
		if !tags.Contains(TagBuildSet, s) {
			tags.Add(TagBuildSet, s)
		}
	}

	canaryPref, err := b.CanaryPreference.endpointsString()
	if err != nil {
		return nil, err
	}
	msg := &buildbucket.ApiPutRequestMessage{
		Bucket:           b.Bucket,
		Tags:             tags.Format(),
		CanaryPreference: canaryPref,
	}

	parameters := map[string]interface{}{
		"builder_name": b.Builder,
		"properties":   b.Input.Properties,
		// keep this synced with marshaling error annotation
	}
	if data, err := json.Marshal(parameters); err != nil {
		// realistically, only properties may cause this.
		return nil, errors.Annotate(err, "marshaling properties").Err()
	} else {
		msg.ParametersJson = string(data)
	}

	return msg, nil
}

func parseJSON(data string, v interface{}) error {
	if data == "" {
		return nil
	}
	dec := json.NewDecoder(strings.NewReader(data))
	dec.UseNumber()
	return dec.Decode(v)
}

// ParseTimestamp parses a buildbucket timestamp.
func ParseTimestamp(usec int64) time.Time {
	if usec == 0 {
		return time.Time{}
	}
	// TODO(nodir): will start to overflow in year ~2250+, fix it then.
	return time.Unix(0, usec*1000).UTC()
}

// FormatTimestamp converts t to a buildbucket timestamp.
func FormatTimestamp(t time.Time) int64 {
	return t.UnixNano() / 1000
}
<|endoftext|>


<commit_before>package parse

import (
	"fmt"
	"strconv"
	"time"

	"github.com/looyun/feedall/controllers"
	"github.com/looyun/feedall/models"
	"github.com/mmcdole/gofeed"
	"gopkg.in/mgo.v2/bson"
)

func Parse() {
	for {
		timer := time.NewTimer(600 * time.Second)
		fmt.Println("start parse!")
		feeds := make([]*models.Feed, 0)
		err := models.FindAll(models.Feeds, nil, &feeds)
		if err != nil {
			fmt.Println(<-timer.C)
			continue
		} else {
			Finish := make(chan string)
			fb := gofeed.NewParser()
			for _, u := range feeds {
				go func(u *models.Feed) {
					origin_feed, err := fb.ParseURL(u.FeedLink)
					if err != nil {
						fmt.Println("Parse err: ", err)
						Finish <- u.FeedLink
					} else {
						data, err := bson.Marshal(origin_feed)
						if err != nil {
							fmt.Println(err)
						}
						var items struct {
							Items []*models.Item `bson:"items"`
						}
						err = bson.Unmarshal(data, &items)
						if err != nil {
							fmt.Println(err)
						}
						feed := models.Feed{}
						models.FindOne(models.Feeds,
							bson.M{"feedLink": u.FeedLink},
							&feed)
						for _, v := range items.Items {
							if v.Content == "" {
								if v.Extensions != nil && v.Extensions["content"] != nil {
									v.Content = v.Extensions["content"]["encoded"][0].Value
								} else {
									v.Content = v.Description
								}
							}
							v.Content = controllers.DecodeImg(v.Content, u.Link)
							if v.Published == "" {
								v.Published = v.Updated
							}
							publishedParsed := controllers.ParseDate(v.Published)
							v.PublishedParsed = strconv.FormatInt(publishedParsed.Unix(), 10)
							v.FeedID = feed.ID

							info, err := models.Upsert(models.Items,
								bson.M{"link": v.Link},
								v)
							if err != nil {
								fmt.Println(err)
							}
							if info == nil {
								continue
							}
						}
						Finish <- u.FeedLink
					}
				}(u)
			}
			for _, _ = range feeds {
				fmt.Println(<-Finish)
			}
		}
		fmt.Println("OK!")
		fmt.Println(<-timer.C)
	}
}
<commit_msg>Generate id for item<commit_after>package parse

import (
	"fmt"
	"strconv"
	"time"

	"github.com/looyun/feedall/controllers"
	"github.com/looyun/feedall/models"
	"github.com/mmcdole/gofeed"
	"gopkg.in/mgo.v2/bson"
)

func Parse() {
	for {
		timer := time.NewTimer(600 * time.Second)
		fmt.Println("start parse!")
		feeds := make([]*models.Feed, 0)
		err := models.FindAll(models.Feeds, nil, &feeds)
		if err != nil {
			fmt.Println(<-timer.C)
			continue
		} else {
			Finish := make(chan string)
			fb := gofeed.NewParser()
			for _, u := range feeds {
				go func(u *models.Feed) {
					origin_feed, err := fb.ParseURL(u.FeedLink)
					if err != nil {
						fmt.Println("Parse err: ", err)
						Finish <- u.FeedLink
					} else {
						data, err := bson.Marshal(origin_feed)
						if err != nil {
							fmt.Println(err)
						}
						var items struct {
							Items []*models.Item `bson:"items"`
						}
						err = bson.Unmarshal(data, &items)
						if err != nil {
							fmt.Println(err)
						}
						feed := models.Feed{}
						models.FindOne(models.Feeds,
							bson.M{"feedLink": u.FeedLink},
							&feed)
						for _, v := range items.Items {
							if v.Content == "" {
								if v.Extensions != nil && v.Extensions["content"] != nil {
									v.Content = v.Extensions["content"]["encoded"][0].Value
								} else {
									v.Content = v.Description
								}
							}
							v.Content = controllers.DecodeImg(v.Content, u.Link)
							if v.Published == "" {
								v.Published = v.Updated
							}
							publishedParsed := controllers.ParseDate(v.Published)
							v.PublishedParsed = strconv.FormatInt(publishedParsed.Unix(), 10)
							v.FeedID = feed.ID
							v.ID = bson.NewObjectId()

							info, err := models.Upsert(models.Items,
								bson.M{"link": v.Link},
								v)
							if err != nil {
								fmt.Println(err)
							}
							if info == nil {
								continue
							}
						}
						Finish <- u.FeedLink
					}
				}(u)
			}
			for _, _ = range feeds {
				fmt.Println(<-Finish)
			}
		}
		fmt.Println("OK!")
		fmt.Println(<-timer.C)
	}
}
<|endoftext|>


<commit_before>// go-rst - A reStructuredText parser for Go
// 2014 (c) The go-rst Authors
// MIT Licensed. See LICENSE for details.

package parse

import (
	"fmt"
	"github.com/demizer/go-elog"
	"github.com/demizer/go-spew/spew"
	"os"
	"reflect"
)

var spd = spew.ConfigState{Indent: "\t", DisableMethods: true}

type systemMessageLevel int

const (
	levelInfo systemMessageLevel = iota
	levelWarning
	levelError
	levelSevere
)

var systemMessageLevels = [...]string{
	"INFO",
	"WARNING",
	"ERROR",
	"SEVERE",
}

func (s systemMessageLevel) String() string {
	return systemMessageLevels[s]
}

type systemMessage struct {
	level  systemMessageLevel
	line   int
	source string
	items  []item
}

type sectionLevels []*SectionNode

func (s *sectionLevels) String() string {
	var out string
	for _, sec := range *s {
		out += fmt.Sprintf("level: %d, rune: %q, overline: %t, length: %d\n",
			sec.Level, sec.UnderLine.Rune, sec.OverLine != nil, sec.Length)
	}
	return out
}

// Returns nil if not found
func (s *sectionLevels) FindByRune(adornChar rune) *SectionNode {
	for _, sec := range *s {
		if sec.UnderLine.Rune == adornChar {
			return sec
		}
	}
	return nil
}

// If exists == true, a section node with the same text and underline has been found in
// sectionLevels, sec is the matching SectionNode. If exists == false, then the sec return value is
// the similarly leveled SectionNode. If exists == false and sec == nil, then the SectionNode added
// to sectionLevels is a new Node.
func (s *sectionLevels) Add(section *SectionNode) (exists bool, sec *SectionNode) {
	sec = s.FindByRune(section.UnderLine.Rune)
	if sec != nil {
		if sec.Text == section.Text {
			return true, sec
		} else if sec.Text != section.Text {
			section.Level = sec.Level
		}
	} else {
		section.Level = len(*s) + 1
	}
	exists = false
	*s = append(*s, section)
	return
}

func (s *sectionLevels) Level() int {
	return len(*s)
}

// Parse is the entry point for the reStructuredText parser.
func Parse(name, text string) (t *Tree, errors []error) {
	t = New(name)
	t.text = text
	_, errors = t.Parse(text, t)
	return
}

func New(name string) *Tree {
	return &Tree{Name: name, Nodes: newList(), nodeTarget: newList(), sectionLevels: new(sectionLevels)}
}

var tokenPos = 2

type Tree struct {
	Name             string
	Nodes            *NodeList // The root node list
	nodeTarget       *NodeList // Used by the parser to add nodes to a target NodeList
	Errors           []error
	text             string
	lex              *lexer
	tokenBackupCount int
	tokenPeekCount   int
	token            [5]*item
	sectionLevels    *sectionLevels // Encountered section levels
	id               int            // The unique id of the node in the tree
}

func (t *Tree) errorf(format string, args ...interface{}) {
	format = fmt.Sprintf("go-rst: %s:%d: %s\n", t.Name, t.lex.lineNumber(), format)
	t.Errors = append(t.Errors, fmt.Errorf(format, args...))
}

func (t *Tree) error(err error) {
	t.errorf("%s\n", err)
}

// startParse initializes the parser, using the lexer.
func (t *Tree) startParse(lex *lexer) {
	t.lex = lex
}

// stopParse terminates parsing.
func (t *Tree) stopParse() {
	t.Nodes = nil
	t.nodeTarget = nil
	t.lex = nil
}

func (t *Tree) Parse(text string, treeSet *Tree) (tree *Tree, errors []error) {
	log.Debugln("Start")
	t.startParse(lex(t.Name, text))
	t.text = text
	t.parse(treeSet)
	log.Debugln("End")
	return t, t.Errors
}

func (t *Tree) parse(tree *Tree) {
	log.Debugln("Start")

	t.nodeTarget = t.Nodes

	for t.peek(1).Type != itemEOF {
		var n Node
		token := t.next()
		log.Infof("Got token: %#+v\n", token)

		switch token.Type {
		case itemTitle: // Section includes overline/underline
			n = t.section(token)
		case itemBlankLine:
			n = newBlankLine(token)
		case itemParagraph:
			n = newParagraph(token)
		case itemSpace:
			n = newSpace(token)
		case itemSectionAdornment:
			// Section adornments should be consumed with itemTitle
			panic("Parser should not find itemSectionAdornment!")
		default:
			t.errorf("%q Not implemented!", token.Type)
			continue
		}

		t.nodeTarget.append(n)
		if n.NodeType() == NodeSection {
			t.nodeTarget =
				reflect.ValueOf(n).Elem().FieldByName("NodeList").Addr().Interface().(*NodeList)
		}
	}

	log.Debugln("End")
}

func (t *Tree) backup() *item {
	t.tokenBackupCount++
	// log.Debugln("t.tokenBackupCount:", t.tokenPeekCount)
	if t.tokenBackupCount > 2 {
		panic("t.backup() can only be used twice consecutively.")
	}
	for i := 4; i > 0; i-- {
		t.token[i] = t.token[i-1]
		t.token[i-1] = nil
	}
	// log.Debugf("\n##### backup() aftermath #####\n\n")
	// spd.Dump(t.token)
	return t.token[tokenPos-t.tokenBackupCount]
}

func (t *Tree) peekBack(pos int) *item {
	if pos > 2 {
		panic("Cannot peek back more than two positions!")
	}
	return t.token[tokenPos-pos]
}

func (t *Tree) peek(pos int) *item {
	// log.Debugln("t.tokenPeekCount:", t.tokenPeekCount)
	if pos > 2 {
		panic("It is only possible to peek ahead two positions!")
	}
	for i := pos; i <= pos; i++ {
		t.tokenPeekCount++
		if t.token[tokenPos+t.tokenPeekCount] == nil {
			t.token[tokenPos+t.tokenPeekCount] = t.lex.nextItem()
		}
	}
	// log.Debugf("\n##### peek() aftermath #####\n\n")
	// spd.Dump(t.token)
	return t.token[tokenPos+t.tokenPeekCount]
}

func (t *Tree) next() *item {
	// log.Debugln("t.tokenPeekCount:", t.tokenPeekCount)
	// shifts the pointers left in t.token, pos is the amount to shift
	shift := func(pos int) {
		for i := pos; i > 0; i-- {
			for x := 0; x < 4; x++ {
				t.token[x] = t.token[x+1]
				t.token[x+1] = nil
			}
		}
	}
	if t.tokenPeekCount > 0 {
		shift(t.tokenPeekCount)
	} else {
		shift(1)
		t.token[tokenPos] = t.lex.nextItem()
	}
	t.tokenBackupCount, t.tokenPeekCount = 0, 0
	// log.Debugf("\n##### next() aftermath #####\n\n")
	// spd.Dump(t.token)
	return t.token[tokenPos]
}

func (t *Tree) section(i *item) Node {
	log.Debugln("Start")
	var overAdorn, title, underAdorn *item
	var overline bool

	peekBack := t.peekBack(1)
	if peekBack != nil {
		switch peekBack.Type {
		case itemSectionAdornment:
			overline = true
			overAdorn = peekBack
		case itemSpace:
			// TODO: Handle indented titles here!
			log.Debugln("FOUND ITEMSPACE BEFORE TITLE")
		}
	}

	title = i
	underAdorn = t.next() // Grab the section underline

	// Check adornment for proper syntax
	if title.Length != underAdorn.Length {
		t.errorf("Section under line  not equal to title length!")
	} else if overline && title.Length != overAdorn.Length {
		t.errorf("Section over line not equal to title length!")
	} else if overline && overAdorn.Text != underAdorn.Text {
		t.errorf("Section title over line does not match section title under line.")
	}

	sec := newSection(title, overAdorn, underAdorn)
	exists, eSec := t.sectionLevels.Add(sec)
	if exists && eSec != nil {
		t.errorf("SectionNode using Text \"%s\" and Rune '%s' was previously parsed!",
			sec.Text, string(sec.UnderLine.Rune))
	} else if !exists && eSec != nil {
		// There is a matching level in sectionLevels
		t.nodeTarget = &(*t.sectionLevels)[sec.Level - 2].NodeList
	}

	log.Debugln("End")
	return sec
}
<commit_msg>parse.go: Fix loop condition in peek()<commit_after>// go-rst - A reStructuredText parser for Go
// 2014 (c) The go-rst Authors
// MIT Licensed. See LICENSE for details.

package parse

import (
	"fmt"
	"github.com/demizer/go-elog"
	"github.com/demizer/go-spew/spew"
	"os"
	"reflect"
)

var spd = spew.ConfigState{Indent: "\t", DisableMethods: true}

type systemMessageLevel int

const (
	levelInfo systemMessageLevel = iota
	levelWarning
	levelError
	levelSevere
)

var systemMessageLevels = [...]string{
	"INFO",
	"WARNING",
	"ERROR",
	"SEVERE",
}

func (s systemMessageLevel) String() string {
	return systemMessageLevels[s]
}

type systemMessage struct {
	level  systemMessageLevel
	line   int
	source string
	items  []item
}

type sectionLevels []*SectionNode

func (s *sectionLevels) String() string {
	var out string
	for _, sec := range *s {
		out += fmt.Sprintf("level: %d, rune: %q, overline: %t, length: %d\n",
			sec.Level, sec.UnderLine.Rune, sec.OverLine != nil, sec.Length)
	}
	return out
}

// Returns nil if not found
func (s *sectionLevels) FindByRune(adornChar rune) *SectionNode {
	for _, sec := range *s {
		if sec.UnderLine.Rune == adornChar {
			return sec
		}
	}
	return nil
}

// If exists == true, a section node with the same text and underline has been found in
// sectionLevels, sec is the matching SectionNode. If exists == false, then the sec return value is
// the similarly leveled SectionNode. If exists == false and sec == nil, then the SectionNode added
// to sectionLevels is a new Node.
func (s *sectionLevels) Add(section *SectionNode) (exists bool, sec *SectionNode) {
	sec = s.FindByRune(section.UnderLine.Rune)
	if sec != nil {
		if sec.Text == section.Text {
			return true, sec
		} else if sec.Text != section.Text {
			section.Level = sec.Level
		}
	} else {
		section.Level = len(*s) + 1
	}
	exists = false
	*s = append(*s, section)
	return
}

func (s *sectionLevels) Level() int {
	return len(*s)
}

// Parse is the entry point for the reStructuredText parser.
func Parse(name, text string) (t *Tree, errors []error) {
	t = New(name)
	t.text = text
	_, errors = t.Parse(text, t)
	return
}

func New(name string) *Tree {
	return &Tree{Name: name, Nodes: newList(), nodeTarget: newList(), sectionLevels: new(sectionLevels)}
}

var tokenPos = 2

type Tree struct {
	Name             string
	Nodes            *NodeList // The root node list
	nodeTarget       *NodeList // Used by the parser to add nodes to a target NodeList
	Errors           []error
	text             string
	lex              *lexer
	tokenBackupCount int
	tokenPeekCount   int
	token            [5]*item
	sectionLevels    *sectionLevels // Encountered section levels
	id               int            // The unique id of the node in the tree
}

func (t *Tree) errorf(format string, args ...interface{}) {
	format = fmt.Sprintf("go-rst: %s:%d: %s\n", t.Name, t.lex.lineNumber(), format)
	t.Errors = append(t.Errors, fmt.Errorf(format, args...))
}

func (t *Tree) error(err error) {
	t.errorf("%s\n", err)
}

// startParse initializes the parser, using the lexer.
func (t *Tree) startParse(lex *lexer) {
	t.lex = lex
}

// stopParse terminates parsing.
func (t *Tree) stopParse() {
	t.Nodes = nil
	t.nodeTarget = nil
	t.lex = nil
}

func (t *Tree) Parse(text string, treeSet *Tree) (tree *Tree, errors []error) {
	log.Debugln("Start")
	t.startParse(lex(t.Name, text))
	t.text = text
	t.parse(treeSet)
	log.Debugln("End")
	return t, t.Errors
}

func (t *Tree) parse(tree *Tree) {
	log.Debugln("Start")

	t.nodeTarget = t.Nodes

	for t.peek(1).Type != itemEOF {
		var n Node
		token := t.next()
		log.Infof("Got token: %#+v\n", token)

		switch token.Type {
		case itemTitle: // Section includes overline/underline
			n = t.section(token)
		case itemBlankLine:
			n = newBlankLine(token)
		case itemParagraph:
			n = newParagraph(token)
		case itemSpace:
			n = newSpace(token)
		case itemSectionAdornment:
			// Section adornments should be consumed with itemTitle
			panic("Parser should not find itemSectionAdornment!")
		default:
			t.errorf("%q Not implemented!", token.Type)
			continue
		}

		t.nodeTarget.append(n)
		if n.NodeType() == NodeSection {
			t.nodeTarget =
				reflect.ValueOf(n).Elem().FieldByName("NodeList").Addr().Interface().(*NodeList)
		}
	}

	log.Debugln("End")
}

func (t *Tree) backup() *item {
	t.tokenBackupCount++
	// log.Debugln("t.tokenBackupCount:", t.tokenPeekCount)
	if t.tokenBackupCount > 2 {
		panic("t.backup() can only be used twice consecutively.")
	}
	for i := 4; i > 0; i-- {
		t.token[i] = t.token[i-1]
		t.token[i-1] = nil
	}
	// log.Debugf("\n##### backup() aftermath #####\n\n")
	// spd.Dump(t.token)
	return t.token[tokenPos-t.tokenBackupCount]
}

func (t *Tree) peekBack(pos int) *item {
	if pos > 2 {
		panic("Cannot peek back more than two positions!")
	}
	return t.token[tokenPos-pos]
}

func (t *Tree) peek(pos int) *item {
	// log.Debugln("t.tokenPeekCount:", t.tokenPeekCount)
	if pos > 2 {
		panic("It is only possible to peek ahead two positions!")
	}
	for i := 0; i < pos; i++ {
		t.tokenPeekCount++
		if t.token[tokenPos+t.tokenPeekCount] == nil {
			t.token[tokenPos+t.tokenPeekCount] = t.lex.nextItem()
		}
	}
	// log.Debugf("\n##### peek() aftermath #####\n\n")
	// spd.Dump(t.token)
	return t.token[tokenPos+t.tokenPeekCount]
}

func (t *Tree) next() *item {
	// log.Debugln("t.tokenPeekCount:", t.tokenPeekCount)
	// shifts the pointers left in t.token, pos is the amount to shift
	shift := func(pos int) {
		for i := pos; i > 0; i-- {
			for x := 0; x < 4; x++ {
				t.token[x] = t.token[x+1]
				t.token[x+1] = nil
			}
		}
	}
	if t.tokenPeekCount > 0 {
		shift(t.tokenPeekCount)
	} else {
		shift(1)
		t.token[tokenPos] = t.lex.nextItem()
	}
	t.tokenBackupCount, t.tokenPeekCount = 0, 0
	// log.Debugf("\n##### next() aftermath #####\n\n")
	// spd.Dump(t.token)
	return t.token[tokenPos]
}

func (t *Tree) section(i *item) Node {
	log.Debugln("Start")
	var overAdorn, title, underAdorn *item
	var overline bool

	peekBack := t.peekBack(1)
	if peekBack != nil {
		switch peekBack.Type {
		case itemSectionAdornment:
			overline = true
			overAdorn = peekBack
		case itemSpace:
			// TODO: Handle indented titles here!
			log.Debugln("FOUND ITEMSPACE BEFORE TITLE")
		}
	}

	title = i
	underAdorn = t.next() // Grab the section underline

	// Check adornment for proper syntax
	if title.Length != underAdorn.Length {
		t.errorf("Section under line  not equal to title length!")
	} else if overline && title.Length != overAdorn.Length {
		t.errorf("Section over line not equal to title length!")
	} else if overline && overAdorn.Text != underAdorn.Text {
		t.errorf("Section title over line does not match section title under line.")
	}

	sec := newSection(title, overAdorn, underAdorn)
	exists, eSec := t.sectionLevels.Add(sec)
	if exists && eSec != nil {
		t.errorf("SectionNode using Text \"%s\" and Rune '%s' was previously parsed!",
			sec.Text, string(sec.UnderLine.Rune))
	} else if !exists && eSec != nil {
		// There is a matching level in sectionLevels
		t.nodeTarget = &(*t.sectionLevels)[sec.Level - 2].NodeList
	}

	log.Debugln("End")
	return sec
}
<|endoftext|>


<commit_before>// go-rst - A reStructuredText parser for Go
// 2014 (c) The go-rst Authors
// MIT Licensed. See LICENSE for details.

package parse

import (
	"fmt"
	"github.com/demizer/go-elog"
)

type systemMessageLevel int

const (
	levelInfo systemMessageLevel = iota
	levelWarning
	levelError
	levelSevere
)

var systemMessageLevels = [...]string{
	"INFO",
	"WARNING",
	"ERROR",
	"SEVERE",
}

func (s systemMessageLevel) String() string {
	return systemMessageLevels[s]
}

type systemMessage struct {
	level  systemMessageLevel
	line   int
	source string
	items  []item
}

type sectionLevel struct {
	char     rune // The adornment character used to describe the section
	overline bool // The section contains an overline
	length   int  // The length of the adornment lines
}

type sectionLevels []sectionLevel

func (s *sectionLevels) String() string {
	var out string
	for lvl, sec := range *s {
		out += fmt.Sprintf("level: %d, rune: %q, overline: %t, length: %d\n",
			lvl+1, sec.char, sec.overline, sec.length)
	}
	return out
}

func (s *sectionLevels) Add(adornChar rune, overline bool, length int) int {
	lvl := s.Find(adornChar)
	if lvl > 0 {
		return lvl
	}
	*s = append(*s, sectionLevel{char: adornChar, overline: overline, length: length})
	return len(*s)
}

// Returns -1 if not found
func (s *sectionLevels) Find(adornChar rune) int {
	for lvl, sec := range *s {
		if sec.char == adornChar {
			return lvl + 1
		}
	}
	return -1
}

func (s *sectionLevels) Level() int {
	return len(*s)
}

// Parse is the entry point for the reStructuredText parser.
func Parse(name, text string) (t *Tree, errors []error) {
	t = New(name)
	t.text = text
	_, errors = t.Parse(text, t)
	return
}

func New(name string) *Tree {
	return &Tree{Name: name, sectionLevels: new(sectionLevels)}
}

type Tree struct {
	Name          string
	Nodes         *NodeList // The root node list
	Errors        []error
	text          string
	branch        *NodeList // The current branch to add nodes to
	lex           *lexer
	peekCount     int
	token         [3]item        // three-token look-ahead for parser.
	sectionLevel  int            // The current section level of parsing
	sectionLevels *sectionLevels // Encountered section levels
	id            int            // The unique id of the node in the tree
}

func (t *Tree) errorf(format string, args ...interface{}) {
	format = fmt.Sprintf("go-rst: %s:%d: %s\n", t.Name, t.lex.lineNumber(), format)
	t.Errors = append(t.Errors, fmt.Errorf(format, args...))
}

func (t *Tree) error(err error) {
	t.errorf("%s\n", err)
}

// startParse initializes the parser, using the lexer.
func (t *Tree) startParse(lex *lexer) {
	t.Nodes = nil
	t.branch = nil
	t.lex = lex
}

// stopParse terminates parsing.
func (t *Tree) stopParse() {
	t.lex = nil
}

func (t *Tree) Parse(text string, treeSet *Tree) (tree *Tree, errors []error) {
	log.Debugln("Start")
	t.startParse(lex(t.Name, text))
	t.text = text
	t.parse(treeSet)
	log.Debugln("End")
	return t, t.Errors
}

func (t *Tree) parse(tree *Tree) {
	log.Debugln("Start")

	t.Nodes = newList()
	t.branch = newList()

	for t.peek().Type != itemEOF {
		var n Node
		switch token := t.next(); token.Type {
		case itemTitle: // Section includes overline/underline
			n = t.section(token)
		case itemBlankLine:
			n = newBlankLine(token, &t.id)
		case itemParagraph:
			n = newParagraph(token, &t.id)
		}

		if len([]Node(*t.Nodes)) == 0 {
			t.Nodes.append(n)
		} else {
			t.branch.append(n)
		}
	}

	log.Debugln("End")
}

func (t *Tree) backup() {
	t.peekCount++
}

// peekBack returns the last item sent from the lexer.
func (t *Tree) peekBack() item {
	return *t.lex.lastItem
}

// peek returns but does not consume the next token.
func (t *Tree) peek() item {
	if t.peekCount > 0 {
		return t.token[t.peekCount-1]

	}
	t.peekCount = 1
	t.token[0] = t.lex.nextItem()
	return t.token[0]
}

func (t *Tree) next() item {
	if t.peekCount > 0 {
		t.peekCount--
	} else {
		t.token[0] = t.lex.nextItem()
	}
	return t.token[t.peekCount]
}

func (t *Tree) section(i item) Node {
	log.Debugln("Start")
	var overAdorn, title, underAdorn item
	var overline bool

	if t.peekBack().Type == itemSectionAdornment {
		overline = true
		overAdorn = t.peekBack()
	}

	title = i
	underAdorn = t.next() // Grab the section underline

	// Check adornment for proper syntax
	if title.Length != underAdorn.Length {
		t.errorf("Section under line  not equal to title length!")
	} else if overline && title.Length != overAdorn.Length {
		t.errorf("Section over line not equal to title length!")
	} else if overline && overAdorn.Text != underAdorn.Text {
		t.errorf("Section title over line does not match section title under line.")
	}

	// Check section levels to make sure the order of sections seen has not been violated
	if level := t.sectionLevels.Find(rune(underAdorn.Text.(string)[0])); level > 0 {
		if t.sectionLevel == t.sectionLevels.Level() {
			t.sectionLevel++
		} else {
			// The current section level of the parser does not match the previously
			// found section level. This means the user has used incorrect section
			// syntax.
			t.errorf("Incorrect section adornment \"%q\" for section level %d",
				underAdorn.Text.(string)[0], t.sectionLevel)
		}
	} else {
		t.sectionLevel++
	}

	t.sectionLevels.Add(rune(underAdorn.Text.(string)[0]), overline, len(underAdorn.Text.(string)))
	ret := newSection(title, &t.id, t.sectionLevel, overAdorn, underAdorn)
	t.branch = &ret.NodeList

	log.Debugln("End")
	return ret
}
<commit_msg>parse.go: Add default case to switch<commit_after>// go-rst - A reStructuredText parser for Go
// 2014 (c) The go-rst Authors
// MIT Licensed. See LICENSE for details.

package parse

import (
	"fmt"
	"github.com/demizer/go-elog"
)

type systemMessageLevel int

const (
	levelInfo systemMessageLevel = iota
	levelWarning
	levelError
	levelSevere
)

var systemMessageLevels = [...]string{
	"INFO",
	"WARNING",
	"ERROR",
	"SEVERE",
}

func (s systemMessageLevel) String() string {
	return systemMessageLevels[s]
}

type systemMessage struct {
	level  systemMessageLevel
	line   int
	source string
	items  []item
}

type sectionLevel struct {
	char     rune // The adornment character used to describe the section
	overline bool // The section contains an overline
	length   int  // The length of the adornment lines
}

type sectionLevels []sectionLevel

func (s *sectionLevels) String() string {
	var out string
	for lvl, sec := range *s {
		out += fmt.Sprintf("level: %d, rune: %q, overline: %t, length: %d\n",
			lvl+1, sec.char, sec.overline, sec.length)
	}
	return out
}

func (s *sectionLevels) Add(adornChar rune, overline bool, length int) int {
	lvl := s.Find(adornChar)
	if lvl > 0 {
		return lvl
	}
	*s = append(*s, sectionLevel{char: adornChar, overline: overline, length: length})
	return len(*s)
}

// Returns -1 if not found
func (s *sectionLevels) Find(adornChar rune) int {
	for lvl, sec := range *s {
		if sec.char == adornChar {
			return lvl + 1
		}
	}
	return -1
}

func (s *sectionLevels) Level() int {
	return len(*s)
}

// Parse is the entry point for the reStructuredText parser.
func Parse(name, text string) (t *Tree, errors []error) {
	t = New(name)
	t.text = text
	_, errors = t.Parse(text, t)
	return
}

func New(name string) *Tree {
	return &Tree{Name: name, sectionLevels: new(sectionLevels)}
}

type Tree struct {
	Name          string
	Nodes         *NodeList // The root node list
	Errors        []error
	text          string
	branch        *NodeList // The current branch to add nodes to
	lex           *lexer
	peekCount     int
	token         [3]item        // three-token look-ahead for parser.
	sectionLevel  int            // The current section level of parsing
	sectionLevels *sectionLevels // Encountered section levels
	id            int            // The unique id of the node in the tree
}

func (t *Tree) errorf(format string, args ...interface{}) {
	format = fmt.Sprintf("go-rst: %s:%d: %s\n", t.Name, t.lex.lineNumber(), format)
	t.Errors = append(t.Errors, fmt.Errorf(format, args...))
}

func (t *Tree) error(err error) {
	t.errorf("%s\n", err)
}

// startParse initializes the parser, using the lexer.
func (t *Tree) startParse(lex *lexer) {
	t.Nodes = nil
	t.branch = nil
	t.lex = lex
}

// stopParse terminates parsing.
func (t *Tree) stopParse() {
	t.lex = nil
}

func (t *Tree) Parse(text string, treeSet *Tree) (tree *Tree, errors []error) {
	log.Debugln("Start")
	t.startParse(lex(t.Name, text))
	t.text = text
	t.parse(treeSet)
	log.Debugln("End")
	return t, t.Errors
}

func (t *Tree) parse(tree *Tree) {
	log.Debugln("Start")

	t.Nodes = newList()
	t.branch = newList()

	for t.peek().Type != itemEOF {
		var n Node
		switch token := t.next(); token.Type {
		case itemTitle: // Section includes overline/underline
			n = t.section(token)
		case itemBlankLine:
			n = newBlankLine(token, &t.id)
		case itemParagraph:
			n = newParagraph(token, &t.id)
		default:
			t.errorf("%q Not implemented!", token.Type)
			continue
		}

		if len([]Node(*t.Nodes)) == 0 {
			t.Nodes.append(n)
		} else {
			t.branch.append(n)
		}
	}

	log.Debugln("End")
}

func (t *Tree) backup() {
	t.peekCount++
}

// peekBack returns the last item sent from the lexer.
func (t *Tree) peekBack() item {
	return *t.lex.lastItem
}

// peek returns but does not consume the next token.
func (t *Tree) peek() item {
	if t.peekCount > 0 {
		return t.token[t.peekCount-1]

	}
	t.peekCount = 1
	t.token[0] = t.lex.nextItem()
	return t.token[0]
}

func (t *Tree) next() item {
	if t.peekCount > 0 {
		t.peekCount--
	} else {
		t.token[0] = t.lex.nextItem()
	}
	return t.token[t.peekCount]
}

func (t *Tree) section(i item) Node {
	log.Debugln("Start")
	var overAdorn, title, underAdorn item
	var overline bool

	if t.peekBack().Type == itemSectionAdornment {
		overline = true
		overAdorn = t.peekBack()
	}

	title = i
	underAdorn = t.next() // Grab the section underline

	// Check adornment for proper syntax
	if title.Length != underAdorn.Length {
		t.errorf("Section under line  not equal to title length!")
	} else if overline && title.Length != overAdorn.Length {
		t.errorf("Section over line not equal to title length!")
	} else if overline && overAdorn.Text != underAdorn.Text {
		t.errorf("Section title over line does not match section title under line.")
	}

	// Check section levels to make sure the order of sections seen has not been violated
	if level := t.sectionLevels.Find(rune(underAdorn.Text.(string)[0])); level > 0 {
		if t.sectionLevel == t.sectionLevels.Level() {
			t.sectionLevel++
		} else {
			// The current section level of the parser does not match the previously
			// found section level. This means the user has used incorrect section
			// syntax.
			t.errorf("Incorrect section adornment \"%q\" for section level %d",
				underAdorn.Text.(string)[0], t.sectionLevel)
		}
	} else {
		t.sectionLevel++
	}

	t.sectionLevels.Add(rune(underAdorn.Text.(string)[0]), overline, len(underAdorn.Text.(string)))
	ret := newSection(title, &t.id, t.sectionLevel, overAdorn, underAdorn)
	t.branch = &ret.NodeList

	log.Debugln("End")
	return ret
}
<|endoftext|>


<commit_before>package victor

import (
    "github.com/brettbuddin/victor/utils/google"
    "strconv"
)

func registerDefaultAbilities(brain *Brain) {
    brain.Respond("me id", func(ctx *Context) {
        id := strconv.Itoa(ctx.Message().UserId())
        ctx.Reply(id)
    })

    brain.Respond("ping", func(ctx *Context) {
        ctx.Reply("pong!")
    })

    brain.Respond("(image|img|gif|animate) (.*)", func(ctx *Context) {
        gifOnly := (ctx.Matches()[0] == "gif" || ctx.Matches()[0] == "animate")

        result, err := google.ImageSearch(ctx.Matches()[1], gifOnly)

        if err != nil {
            ctx.Send("There was error making the request.")
            return
        }

        if result == "" {
            ctx.Send("I didn't find anything.")
            return
        }

        ctx.Send(result)
    })

    brain.Respond("users (list|show)", func(ctx *Context) {
        list := ""

        for _, user := range brain.Users() {
            list += user.Name() + "\n"
        }

        ctx.Paste(list)
    })
}
<commit_msg>wrong spots<commit_after>package victor

import (
    "github.com/brettbuddin/victor/utils/google"
    "strconv"
)

func registerDefaultAbilities(brain *Brain) {
    brain.Respond("me id", func(ctx *Context) {
        id := strconv.Itoa(ctx.Message().UserId())
        ctx.Reply(id)
    })

    brain.Respond("ping", func(ctx *Context) {
        ctx.Reply("pong!")
    })

    brain.Respond("(image|img|gif|animate) (.*)", func(ctx *Context) {
        gifOnly := (ctx.Matches()[1] == "gif" || ctx.Matches()[1] == "animate")

        result, err := google.ImageSearch(ctx.Matches()[2], gifOnly)

        if err != nil {
            ctx.Send("There was error making the request.")
            return
        }

        if result == "" {
            ctx.Send("I didn't find anything.")
            return
        }

        ctx.Send(result)
    })

    brain.Respond("users (list|show)", func(ctx *Context) {
        list := ""

        for _, user := range brain.Users() {
            list += user.Name() + "\n"
        }

        ctx.Paste(list)
    })
}
<|endoftext|>


<commit_before>package buildkite

import "fmt"
import "strings"

// You can overriden buildVersion at compile time by using:
//
//  go run -ldflags "-X github.com/buildkite/agent/buildkite.buildVersion abc" *.go --version
//
// On CI, the binaries are always build with the buildVersion variable set.

var baseVersion string = "1.0-beta.29"
var buildVersion string = ""

func Version() string {
	// Only output the build version if a pre-release
	if strings.Contains(baseVersion, "beta") || strings.Contains(baseVersion, "alpha") {
		// Use a default buildVersion if one doesn't exist
		actualBuildVersion := buildVersion
		if actualBuildVersion == "" {
			actualBuildVersion = "x"
		}

		return fmt.Sprintf("%s.%s", baseVersion, actualBuildVersion)
	} else {
		return baseVersion
	}
}
<commit_msg>Bumped to 1.0-beta.30<commit_after>package buildkite

import "fmt"
import "strings"

// You can overriden buildVersion at compile time by using:
//
//  go run -ldflags "-X github.com/buildkite/agent/buildkite.buildVersion abc" *.go --version
//
// On CI, the binaries are always build with the buildVersion variable set.

var baseVersion string = "1.0-beta.30"
var buildVersion string = ""

func Version() string {
	// Only output the build version if a pre-release
	if strings.Contains(baseVersion, "beta") || strings.Contains(baseVersion, "alpha") {
		// Use a default buildVersion if one doesn't exist
		actualBuildVersion := buildVersion
		if actualBuildVersion == "" {
			actualBuildVersion = "x"
		}

		return fmt.Sprintf("%s.%s", baseVersion, actualBuildVersion)
	} else {
		return baseVersion
	}
}
<|endoftext|>


<commit_before>package paystack

import "testing"

func TestBankList(t *testing.T) {
	// retrieve the bank list
	banks, err := c.Bank.List()

	if err != nil || !(len(banks.Values) > 0) {
		t.Errorf("Expected Bank list, got %d, returned error %v", len(banks.Values), err)
	}
}

func TestResolveBVN(t *testing.T) {
	resp, err := c.Bank.ResolveBVN(21212917741)
	// Should fail. Server response: Your balance is not enough to fulfill this request
	if err == nil {
		t.Errorf("Expected error, got %+v'", resp)
	}
	//if len(resp.BVN) == 0 {
	//	t.Errorf("Expected response to contain bvn")
	//}
}

func TestResolveAccountNumber(t *testing.T) {
	resp, err := c.Bank.ResolveAccountNumber("0022728151", "063")
	if err == nil {
		t.Errorf("Expected error, got %+v'", resp)
	}

	/*
		if _, ok := resp["account_number"]; !ok {
			t.Errorf("Expected response to contain 'account_number'")
		}

		if _, ok := resp["account_name"]; !ok {
			t.Errorf("Expected response to contain 'account_name'")
		}
	*/
}
<commit_msg>Test BVN invalid code and free calls limit<commit_after>package paystack

import "testing"

func TestBankList(t *testing.T) {
	// retrieve the bank list
	banks, err := c.Bank.List()

	if err != nil || !(len(banks.Values) > 0) {
		t.Errorf("Expected Bank list, got %d, returned error %v", len(banks.Values), err)
	}
}

func TestResolveBVN(t *testing.T) {
	// Test invlaid BVN.
	// Err not nill. Resp status code is 400
	resp, err := c.Bank.ResolveBVN(21212917)
	if err == nil {
		t.Errorf("Expected error for invalid BVN, got %+v'", resp)
	}

	// Test free calls limit
	// Error is nil
	// &{Meta:{CallsThisMonth:0 FreeCallsLeft:0} BVN:cZ+MKrsLAqJCUi+hxIdQqw==}’
	resp, err = c.Bank.ResolveBVN(21212917741)
	if resp.Meta.FreeCallsLeft != 0 {
		t.Errorf("Expected free calls limit exceeded, got %+v'", resp)
	}
	// TODO(yao): Reproduce error: Your balance is not enough to fulfill this request
}

func TestResolveAccountNumber(t *testing.T) {
	resp, err := c.Bank.ResolveAccountNumber("0022728151", "063")
	if err == nil {
		t.Errorf("Expected error, got %+v'", resp)
	}

	/*
		if _, ok := resp["account_number"]; !ok {
			t.Errorf("Expected response to contain 'account_number'")
		}

		if _, ok := resp["account_name"]; !ok {
			t.Errorf("Expected response to contain 'account_name'")
		}
	*/
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/hex"
	"encoding/json"
	"fmt"
	"os"
	"strings"

	"github.com/bazooka-ci/bazooka/commons/matrix"

	log "github.com/Sirupsen/logrus"
	lib "github.com/bazooka-ci/bazooka/commons"
	bzklog "github.com/bazooka-ci/bazooka/commons/logs"
)

const (
	BazookaConfigFile       = ".bazooka.yml"
	TravisConfigFile        = ".travis.yml"
	MX_ENV_PREFIX           = "env::"
	BazookaEnvJobParameters = "BZK_JOB_PARAMETERS"
)

func init() {
	log.SetFormatter(&bzklog.BzkFormatter{})
	err := lib.LoadCryptoKeyFromFile("/bazooka-cryptokey")
	if err != nil {
		log.Fatal(err)
	}

}

func main() {
	log.Info("Starting Parsing Phase")
	// Find either .travis.yml or .bazooka.yml file in the project
	configFile, err := lib.ResolveConfigFile(paths.container.source)
	if err != nil {
		log.Fatal(err)
	}

	envParamString := os.Getenv(BazookaEnvJobParameters)
	var envParams []lib.BzkString
	err = json.Unmarshal([]byte(envParamString), &envParams)
	if err != nil {
		log.Fatal(err)
	}

	jobParameters := groupByName(envParams, MX_ENV_PREFIX)

	// parse the configuration
	config := &lib.Config{}
	err = lib.Parse(configFile, config)
	if err != nil {
		log.Fatal(err)
	}
	log.WithFields(log.Fields{
		"config": config,
	}).Debug("Configuration parsed")

	var variants []*variantData

	if len(config.Language) == 0 {
		if len(config.Image) == 0 {
			log.Fatal("One of 'language' or 'image' needs to be set")
		}
		variants = generateImageVariants(config)
	} else {
		// resolve the docker image corresponding to this particular language parser
		image, err := resolveLanguageParser(config.Language)
		if err != nil {
			log.Fatal(err)
		}

		// run the parser image
		langParser := &LanguageParser{
			Image: image,
		}
		variants, err = langParser.Parse()
		if err != nil {
			log.Fatal(err)
		}
	}

	log.Info("Starting Matrix generation")

	for _, variant := range variants {
		// create a matrix from the environment variables
		// a matrix has N variables (dimensions) where each variable has M values
		// config.Env is a list of key=value strings
		// groupByName transforms that into a map[string][]string
		// for example ["A=1", "A=2", B="3"]
		// when grouped by name gets transformed into
		// {A: [1, 2], B: [3]}
		// this matches the matrix layout, so we could store them directly into the matrix
		// but since we would like to be able to extract them later, and to avoid mixing them with a language specific variables (like jdk, go, etc.)
		// explode prefixes the env variables names with a prefix defined in the constant MX_ENV_PREFIX
		// Hence, our matrix is more like: {"env::A": [1, 2], "env::B": [3]}
		variantVariables := groupByName(variant.config.Env, MX_ENV_PREFIX)

		// insert or replace environment variables defined by the job parameters
		for k, v := range jobParameters {
			variantVariables[k] = v
		}

		// check if all environment variables are defined
		for k, v := range variantVariables {
			if len(v) == 1 && len(v[0]) == 0 {
				log.Fatalf("Missing required parameter %v", k)
			}
		}

		mx := matrix.Matrix(variantVariables)

		// and then add the new language specific variables parsed from the meta file to the build matrix (which already contains the env variables)
		err = feedMatrix(variant.meta, &mx)
		if err != nil {
			log.Fatal(err)
		}

		// we're not done yet: we need to also handle the matrix exclusions
		// we parse them into a list of matrices
		exclusions, err := exclusionsMatrices(variant.config.Matrix.Exclude)
		if err != nil {
			log.Fatal(err)
		}

		// and finally, we iterate over all the permutations of the build matrix
		// these permutations are the list of all the combinations of env variables and language specific variables, minux the exclusions
		mx.IterAll(func(permutation map[string]string, counter string) {
			// we get called for every non-excluded permutations with the different variables values for this permutations and a unique permutation counter
			// handlePermutation will start from the .bazooka.*.yml file, which should already contain a single language specific permutation
			// and enrich it with the env variables combination
			// the same goes for the meta file
			if err := handlePermutation(permutation, variant.config, variant.meta, counter, variant.counter); err != nil {
				log.Fatal(fmt.Errorf("Error while generating the permutations: %v", err))
			}
		}, exclusions)
	}
	log.Info("Matrix generated")

	log.Info("Starting generating Dockerfiles from Matrix")
	// Now we're left with the final build files
	files, err := lib.ListFilesWithPrefix(paths.container.output, ".bazooka")
	if err != nil {
		log.Fatal(fmt.Errorf("Error while listing .bazooka* files: %v", err))
	}

	for _, file := range files {
		config := &lib.Config{}
		err = lib.Parse(file, config)
		if err != nil {
			log.Fatal(fmt.Errorf("Error while parsing config file %s: %v", file, err))
		}

		// transform the .bazooka.x.yml file into a set of dockerfile + shell scripts who perform the actual build
		g := &Generator{
			Config:       config,
			OutputFolder: paths.container.output,
			Index:        parseCounter(file),
		}
		err = g.GenerateDockerfile()
		if err != nil {
			log.Fatal("Error while generating a dockerfile: %v", err)
		}
	}
	log.Info("Dockerfiles all created successfully")

}

func handlePermutation(permutation map[string]string, config *lib.Config, meta map[string]interface{}, counter, rootCounter string) error {
	// start from the language-spcecific permutation
	newConfig := *config

	// and replace its env variables with this unique permutation
	envMap := extractEnv(permutation, config.Env)

	// Insert BZK_BUILD_DIR if not present
	if _, ok := envMap["BZK_BUILD_DIR"]; !ok {
		envMap["BZK_BUILD_DIR"] = lib.BzkString{"BZK_BUILD_DIR", "/bazooka", false}
	}

	newConfig.Env = mapValues(envMap)
	if err := lib.Flush(newConfig, fmt.Sprintf("%s/.bazooka.%s%s.yml", paths.container.output, rootCounter, counter)); err != nil {
		return err
	}

	// do the same for the meta file
	// start from the language specific permutation meta file
	// and add this permutation's env map
	metaEnv, err:=generateEnvForMeta(newConfig.Env)
	if err != nil {
		return err
	}

	meta["env"] = metaEnv
	metaFile := fmt.Sprintf("%s/%s%s", paths.container.meta, rootCounter, counter)
	lib.Flush(meta, metaFile)

	return nil
}

func generateImageVariants(conf *lib.Config) []*variantData {
	res := make([]*variantData, len(conf.Image))
	for i, im := range conf.Image {
		imageConf := *conf
		imageConf.FromImage = im
		imageConf.Image = nil

		res[i] = &variantData{
			counter: fmt.Sprintf("%d", i),
			config:  &imageConf,
			meta: map[string]interface{}{
				"image": im,
			},
		}
	}
	return res
}

func feedMatrix(extra map[string]interface{}, mx *matrix.Matrix) error {
	for k, v := range extra {
		switch k {
		case "env":
			switch converted:=v.(type) {
			case []interface{}:
				envVars := []lib.BzkString{}
				for _, envVar := range converted {
					if strEnvVar, ok := envVar.(string); ok {
						n, v:=lib.SplitNameValue(strEnvVar)
						envVars = append(envVars, lib.BzkString{n, v, false})
					} else {
						return fmt.Errorf("Invalid config: env should contain a sequence of strings: found a non string value %v:%T", envVar, envVar)

					}
				}
				mx.Merge(groupByName(envVars, MX_ENV_PREFIX))
			case string:
						n, v:=lib.SplitNameValue(converted)
				mx.Merge(groupByName([]lib.BzkString{lib.BzkString{n, v, false}}, MX_ENV_PREFIX))
			default:
				return fmt.Errorf("Invalid config: env should contain a sequence of strings: %v:%T", v, v)
			}
		default:
			mx.AddVar(k, fmt.Sprintf("%v", v))
		}
	}
	return nil
}

func exclusionsMatrices(xs []map[string]interface{}) ([]*matrix.Matrix, error) {
	res := make([]*matrix.Matrix, len(xs))
	for i, x := range xs {
		mx := matrix.Matrix{}
		if err := feedMatrix(x, &mx); err != nil {
			return nil, err
		}
		res[i] = &mx
	}
	return res, nil
}

// parseCounter extract the * part from a .bazooka.*.yml file name
func parseCounter(filePath string) string {
	splits := strings.Split(filePath, "/")
	file := splits[len(splits)-1]
	return strings.Split(file, ".")[2]
}

// groupByName starts from a list of key=valye strings and stores them into a map
// it also handles repeated values, so ["A=1", "A=2", B="3"] gets transformed into {A: [1, 2], B: [3]}
func groupByName(props []lib.BzkString, keyPrefix string) map[string][]string {
	res := make(map[string][]string)
	for _, env := range props {
		res[keyPrefix+env.Name] = append(res[keyPrefix+env.Name], env.Value)
	}
	return res
}

// prefixMapKeys returns a new map where all keys are prefixed with prefix
func prefixMapKeys(m map[string][]string, prefix string) map[string][]string {
	res := make(map[string][]string)
	for k, v := range m {
		res[prefix+k] = v
	}
	return res
}

// extractEnv extracts back the env variables from the permutation values
func extractEnv(from map[string]string, originalEnv []lib.BzkString) map[string]lib.BzkString {
	res := make(map[string]lib.BzkString)
	for k, v := range from {
		if strings.HasPrefix(k, MX_ENV_PREFIX) {
			name:=strings.TrimPrefix(k, MX_ENV_PREFIX)
			res[name] = findOrCreateBzkString(name, v, originalEnv)
		}
	}
	return res
}

func findOrCreateBzkString(name, value string, env []lib.BzkString) lib.BzkString {
	for _, e:=range env {
		if name==e.Name && value==e.Value {
			return e
		}
	}
	return lib.BzkString{name, value, false}
}

func mapValues(m map[string]lib.BzkString) []lib.BzkString {
	res :=make([]lib.BzkString, 0, len(m))
	for _, v:=range m {
		res=append(res, v)
	}
	return res
}

func generateEnvForMeta(env []lib.BzkString) ([]string, error) {
	key, err:=lib.ReadCryptoKey(paths.container.cryptoKey)
	if err != nil {
		return nil, err
	}

	res := make([]string, 0, len(env))
	for _, e:=range env {
		value := e.Value
		if e.Secured {
			eValue, err:= lib.Encrypt(key, []byte(value))
			if err != nil {
				return nil, err
			}
			value = hex.EncodeToString(eValue)
		}
		res = append(res, fmt.Sprintf("%s=%s", e.Name, value))
	}
	return res, nil
}
func resolveLanguageParser(language string) (string, error) {
	parserMap := map[string]string{
		"golang":  "bazooka/parser-golang",
		"go":      "bazooka/parser-golang",
		"java":    "bazooka/parser-java",
		"python":  "bazooka/parser-python",
		"node_js": "bazooka/parser-nodejs",
		"nodejs":  "bazooka/parser-nodejs",
	}
	if val, ok := parserMap[language]; ok {
		return val, nil
	}
	return "", fmt.Errorf("Unable to find Bazooka Docker Image for Language Parser %s\n", language)
}
<commit_msg>Add php parser<commit_after>package main

import (
	"encoding/hex"
	"encoding/json"
	"fmt"
	"os"
	"strings"

	"github.com/bazooka-ci/bazooka/commons/matrix"

	log "github.com/Sirupsen/logrus"
	lib "github.com/bazooka-ci/bazooka/commons"
	bzklog "github.com/bazooka-ci/bazooka/commons/logs"
)

const (
	BazookaConfigFile       = ".bazooka.yml"
	TravisConfigFile        = ".travis.yml"
	MX_ENV_PREFIX           = "env::"
	BazookaEnvJobParameters = "BZK_JOB_PARAMETERS"
)

func init() {
	log.SetFormatter(&bzklog.BzkFormatter{})
	err := lib.LoadCryptoKeyFromFile("/bazooka-cryptokey")
	if err != nil {
		log.Fatal(err)
	}

}

func main() {
	log.Info("Starting Parsing Phase")
	// Find either .travis.yml or .bazooka.yml file in the project
	configFile, err := lib.ResolveConfigFile(paths.container.source)
	if err != nil {
		log.Fatal(err)
	}

	envParamString := os.Getenv(BazookaEnvJobParameters)
	var envParams []lib.BzkString
	err = json.Unmarshal([]byte(envParamString), &envParams)
	if err != nil {
		log.Fatal(err)
	}

	jobParameters := groupByName(envParams, MX_ENV_PREFIX)

	// parse the configuration
	config := &lib.Config{}
	err = lib.Parse(configFile, config)
	if err != nil {
		log.Fatal(err)
	}
	log.WithFields(log.Fields{
		"config": config,
	}).Debug("Configuration parsed")

	var variants []*variantData

	if len(config.Language) == 0 {
		if len(config.Image) == 0 {
			log.Fatal("One of 'language' or 'image' needs to be set")
		}
		variants = generateImageVariants(config)
	} else {
		// resolve the docker image corresponding to this particular language parser
		image, err := resolveLanguageParser(config.Language)
		if err != nil {
			log.Fatal(err)
		}

		// run the parser image
		langParser := &LanguageParser{
			Image: image,
		}
		variants, err = langParser.Parse()
		if err != nil {
			log.Fatal(err)
		}
	}

	log.Info("Starting Matrix generation")

	for _, variant := range variants {
		// create a matrix from the environment variables
		// a matrix has N variables (dimensions) where each variable has M values
		// config.Env is a list of key=value strings
		// groupByName transforms that into a map[string][]string
		// for example ["A=1", "A=2", B="3"]
		// when grouped by name gets transformed into
		// {A: [1, 2], B: [3]}
		// this matches the matrix layout, so we could store them directly into the matrix
		// but since we would like to be able to extract them later, and to avoid mixing them with a language specific variables (like jdk, go, etc.)
		// explode prefixes the env variables names with a prefix defined in the constant MX_ENV_PREFIX
		// Hence, our matrix is more like: {"env::A": [1, 2], "env::B": [3]}
		variantVariables := groupByName(variant.config.Env, MX_ENV_PREFIX)

		// insert or replace environment variables defined by the job parameters
		for k, v := range jobParameters {
			variantVariables[k] = v
		}

		// check if all environment variables are defined
		for k, v := range variantVariables {
			if len(v) == 1 && len(v[0]) == 0 {
				log.Fatalf("Missing required parameter %v", k)
			}
		}

		mx := matrix.Matrix(variantVariables)

		// and then add the new language specific variables parsed from the meta file to the build matrix (which already contains the env variables)
		err = feedMatrix(variant.meta, &mx)
		if err != nil {
			log.Fatal(err)
		}

		// we're not done yet: we need to also handle the matrix exclusions
		// we parse them into a list of matrices
		exclusions, err := exclusionsMatrices(variant.config.Matrix.Exclude)
		if err != nil {
			log.Fatal(err)
		}

		// and finally, we iterate over all the permutations of the build matrix
		// these permutations are the list of all the combinations of env variables and language specific variables, minux the exclusions
		mx.IterAll(func(permutation map[string]string, counter string) {
			// we get called for every non-excluded permutations with the different variables values for this permutations and a unique permutation counter
			// handlePermutation will start from the .bazooka.*.yml file, which should already contain a single language specific permutation
			// and enrich it with the env variables combination
			// the same goes for the meta file
			if err := handlePermutation(permutation, variant.config, variant.meta, counter, variant.counter); err != nil {
				log.Fatal(fmt.Errorf("Error while generating the permutations: %v", err))
			}
		}, exclusions)
	}
	log.Info("Matrix generated")

	log.Info("Starting generating Dockerfiles from Matrix")
	// Now we're left with the final build files
	files, err := lib.ListFilesWithPrefix(paths.container.output, ".bazooka")
	if err != nil {
		log.Fatal(fmt.Errorf("Error while listing .bazooka* files: %v", err))
	}

	for _, file := range files {
		config := &lib.Config{}
		err = lib.Parse(file, config)
		if err != nil {
			log.Fatal(fmt.Errorf("Error while parsing config file %s: %v", file, err))
		}

		// transform the .bazooka.x.yml file into a set of dockerfile + shell scripts who perform the actual build
		g := &Generator{
			Config:       config,
			OutputFolder: paths.container.output,
			Index:        parseCounter(file),
		}
		err = g.GenerateDockerfile()
		if err != nil {
			log.Fatal("Error while generating a dockerfile: %v", err)
		}
	}
	log.Info("Dockerfiles all created successfully")

}

func handlePermutation(permutation map[string]string, config *lib.Config, meta map[string]interface{}, counter, rootCounter string) error {
	// start from the language-spcecific permutation
	newConfig := *config

	// and replace its env variables with this unique permutation
	envMap := extractEnv(permutation, config.Env)

	// Insert BZK_BUILD_DIR if not present
	if _, ok := envMap["BZK_BUILD_DIR"]; !ok {
		envMap["BZK_BUILD_DIR"] = lib.BzkString{"BZK_BUILD_DIR", "/bazooka", false}
	}

	newConfig.Env = mapValues(envMap)
	if err := lib.Flush(newConfig, fmt.Sprintf("%s/.bazooka.%s%s.yml", paths.container.output, rootCounter, counter)); err != nil {
		return err
	}

	// do the same for the meta file
	// start from the language specific permutation meta file
	// and add this permutation's env map
	metaEnv, err:=generateEnvForMeta(newConfig.Env)
	if err != nil {
		return err
	}

	meta["env"] = metaEnv
	metaFile := fmt.Sprintf("%s/%s%s", paths.container.meta, rootCounter, counter)
	lib.Flush(meta, metaFile)

	return nil
}

func generateImageVariants(conf *lib.Config) []*variantData {
	res := make([]*variantData, len(conf.Image))
	for i, im := range conf.Image {
		imageConf := *conf
		imageConf.FromImage = im
		imageConf.Image = nil

		res[i] = &variantData{
			counter: fmt.Sprintf("%d", i),
			config:  &imageConf,
			meta: map[string]interface{}{
				"image": im,
			},
		}
	}
	return res
}

func feedMatrix(extra map[string]interface{}, mx *matrix.Matrix) error {
	for k, v := range extra {
		switch k {
		case "env":
			switch converted:=v.(type) {
			case []interface{}:
				envVars := []lib.BzkString{}
				for _, envVar := range converted {
					if strEnvVar, ok := envVar.(string); ok {
						n, v:=lib.SplitNameValue(strEnvVar)
						envVars = append(envVars, lib.BzkString{n, v, false})
					} else {
						return fmt.Errorf("Invalid config: env should contain a sequence of strings: found a non string value %v:%T", envVar, envVar)

					}
				}
				mx.Merge(groupByName(envVars, MX_ENV_PREFIX))
			case string:
						n, v:=lib.SplitNameValue(converted)
				mx.Merge(groupByName([]lib.BzkString{lib.BzkString{n, v, false}}, MX_ENV_PREFIX))
			default:
				return fmt.Errorf("Invalid config: env should contain a sequence of strings: %v:%T", v, v)
			}
		default:
			mx.AddVar(k, fmt.Sprintf("%v", v))
		}
	}
	return nil
}

func exclusionsMatrices(xs []map[string]interface{}) ([]*matrix.Matrix, error) {
	res := make([]*matrix.Matrix, len(xs))
	for i, x := range xs {
		mx := matrix.Matrix{}
		if err := feedMatrix(x, &mx); err != nil {
			return nil, err
		}
		res[i] = &mx
	}
	return res, nil
}

// parseCounter extract the * part from a .bazooka.*.yml file name
func parseCounter(filePath string) string {
	splits := strings.Split(filePath, "/")
	file := splits[len(splits)-1]
	return strings.Split(file, ".")[2]
}

// groupByName starts from a list of key=valye strings and stores them into a map
// it also handles repeated values, so ["A=1", "A=2", B="3"] gets transformed into {A: [1, 2], B: [3]}
func groupByName(props []lib.BzkString, keyPrefix string) map[string][]string {
	res := make(map[string][]string)
	for _, env := range props {
		res[keyPrefix+env.Name] = append(res[keyPrefix+env.Name], env.Value)
	}
	return res
}

// prefixMapKeys returns a new map where all keys are prefixed with prefix
func prefixMapKeys(m map[string][]string, prefix string) map[string][]string {
	res := make(map[string][]string)
	for k, v := range m {
		res[prefix+k] = v
	}
	return res
}

// extractEnv extracts back the env variables from the permutation values
func extractEnv(from map[string]string, originalEnv []lib.BzkString) map[string]lib.BzkString {
	res := make(map[string]lib.BzkString)
	for k, v := range from {
		if strings.HasPrefix(k, MX_ENV_PREFIX) {
			name:=strings.TrimPrefix(k, MX_ENV_PREFIX)
			res[name] = findOrCreateBzkString(name, v, originalEnv)
		}
	}
	return res
}

func findOrCreateBzkString(name, value string, env []lib.BzkString) lib.BzkString {
	for _, e:=range env {
		if name==e.Name && value==e.Value {
			return e
		}
	}
	return lib.BzkString{name, value, false}
}

func mapValues(m map[string]lib.BzkString) []lib.BzkString {
	res :=make([]lib.BzkString, 0, len(m))
	for _, v:=range m {
		res=append(res, v)
	}
	return res
}

func generateEnvForMeta(env []lib.BzkString) ([]string, error) {
	key, err:=lib.ReadCryptoKey(paths.container.cryptoKey)
	if err != nil {
		return nil, err
	}

	res := make([]string, 0, len(env))
	for _, e:=range env {
		value := e.Value
		if e.Secured {
			eValue, err:= lib.Encrypt(key, []byte(value))
			if err != nil {
				return nil, err
			}
			value = hex.EncodeToString(eValue)
		}
		res = append(res, fmt.Sprintf("%s=%s", e.Name, value))
	}
	return res, nil
}
func resolveLanguageParser(language string) (string, error) {
	parserMap := map[string]string{
		"golang":  "bazooka/parser-golang",
		"go":      "bazooka/parser-golang",
		"java":    "bazooka/parser-java",
		"python":  "bazooka/parser-python",
		"node_js": "bazooka/parser-nodejs",
		"nodejs":  "bazooka/parser-nodejs",
		"php":     "bazooka/parser-php",
	}
	if val, ok := parserMap[language]; ok {
		return val, nil
	}
	return "", fmt.Errorf("Unable to find Bazooka Docker Image for Language Parser %s\n", language)
}
<|endoftext|>


<commit_before>package main

import (
	"reflect"
	"strings"
	"testing"
)

func TestParser_Scan(t *testing.T) {
	var tests = []struct {
		s    string
		expr Expression
		err  string
	}{
		{
			s:    `a = 1`,
			expr: Int(1),
		},
		{
			s: `1 + 2`,
			expr: Binary{
				Left:     Int(1),
				Right:    Int(2),
				Operator: "+",
			},
		},
		{
			s: `1 - 2`,
			expr: Binary{
				Left:     Int(1),
				Right:    Int(2),
				Operator: "-",
			},
		},
		{
			s: `1* 2`,
			expr: Binary{
				Left:     Int(1),
				Right:    Int(2),
				Operator: "*",
			},
		},
		{
			s: `a + 2`,
			expr: Binary{
				Left:     Int(1),
				Right:    Int(2),
				Operator: "+",
			},
		},
		{
			s: `2 + a`,
			expr: Binary{
				Left:     Int(2),
				Right:    Int(1),
				Operator: "+",
			},
		},
		{
			s: `a + a`,
			expr: Binary{
				Left:     Int(1),
				Right:    Int(1),
				Operator: "+",
			},
		},
		{
			s:   `b`,
			err: `ERROR`,
		},
		{
			s:    `a + a + a + a`,
			expr: Int(4),
		},
		{
			s:    `a + a - a + a`,
			expr: Int(2),
		},
		{
			s:    `b = 42`,
			expr: Int(42),
		},
		{
			s:    `a = b`,
			expr: Int(42),
		},
	}

	for i, tt := range tests {
		expr, err := NewParser(strings.NewReader(tt.s)).Parse()
		if !strings.Contains(errstring(err), tt.err) {
			t.Errorf("%d. %q: error mismatch:\n  exp=%s\n  got=%s\n\n", i, tt.s, tt.err, err)
		} else if tt.err == "" && !reflect.DeepEqual(tt.expr.Evaluate(), (*expr).Evaluate()) {
			t.Errorf("%d. %q\n\nstmt mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.s, tt.expr.Evaluate(), (*expr).Evaluate())
		}
	}
}

// errstring returns the string representation of an error.
func errstring(err error) string {
	if err != nil {
		return err.Error()
	}
	return ""
}
<commit_msg>reformat test.<commit_after>package main

import (
	"reflect"
	"strings"
	"testing"
)

func TestParser_Scan(t *testing.T) {
	var tests = []struct {
		s    string
		expr Expression
		err  string
	}{
		{s: `a = 1`, expr: Int(1)},
		{s: `1 + 2`, expr: Binary{Left: Int(1), Right: Int(2), Operator: "+"}},
		{s: `1 - 2`, expr: Binary{Left: Int(1), Right: Int(2), Operator: "-"}},
		{s: `1* 2`, expr: Binary{Left: Int(1), Right: Int(2), Operator: "*"}},
		{s: `a + 2`, expr: Binary{Left: Int(1), Right: Int(2), Operator: "+"}},
		{s: `2 + a`, expr: Binary{Left: Int(2), Right: Int(1), Operator: "+"}},
		{s: `a + a`, expr: Binary{Left: Int(1), Right: Int(1), Operator: "+"}},
		{s: `b`, err: `ERROR`}, {s: `a + a + a + a`, expr: Int(4)},
		{s: `a + a - a + a`, expr: Int(2)},
		{s: `b = 42`, expr: Int(42)},
		{s: `a = b`, expr: Int(42)},
	}

	for i, tt := range tests {
		expr, err := NewParser(strings.NewReader(tt.s)).Parse()
		if !strings.Contains(errstring(err), tt.err) {
			t.Errorf("%d. %q: error mismatch:\n  exp=%s\n  got=%s\n\n", i, tt.s, tt.err, err)
		} else if tt.err == "" && !reflect.DeepEqual(tt.expr.Evaluate(), (*expr).Evaluate()) {
			t.Errorf("%d. %q\n\nstmt mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.s, tt.expr.Evaluate(), (*expr).Evaluate())
		}
	}
}

// errstring returns the string representation of an error.
func errstring(err error) string {
	if err != nil {
		return err.Error()
	}
	return ""
}
<|endoftext|>


<commit_before>/*

The Rep type that models a replay (and everything in it).

*/

package rep

import (
	"encoding/json"
	"errors"
	"io"

	"github.com/icza/mpq"
	"github.com/icza/s2prot"
)

const (
	// ParserVersion is a Semver2 compatible version of the parser.
	ParserVersion = "v1.2.0-dev"
)

var (
	// ErrInvalidRepFile means invalid replay file.
	ErrInvalidRepFile = errors.New("Invalid SC2Replay file")

	// ErrUnsupportedRepVersion means the replay file is valid but its version is not supported.
	ErrUnsupportedRepVersion = errors.New("Unsupported replay version")

	// ErrDecoding means decoding the replay file failed,
	// Most likely because replay file is invalid, but also might be due to an implementation bug
	ErrDecoding = errors.New("Decoding error")
)

// Rep describes a replay.
type Rep struct {
	m *mpq.MPQ // MPQ parser for reading the file

	protocol *s2prot.Protocol // Protocol to decode the replay

	Header   Header   // Replay header (replay game version and length)
	Details  Details  // Game details (overall replay details)
	InitData InitData // Replay init data (the initial lobby)
	AttrEvts AttrEvts // Attributes events

	Metadata Metadata // Game metadata (calculated, confirmed results)

	GameEvts    []s2prot.Event // Game events
	MessageEvts []s2prot.Event // Message events
	TrackerEvts *TrackerEvts   // Tracker events

	GameEvtsErr    bool // Tells if decoding game events had errors
	MessageEvtsErr bool // Tells if decoding message events had errors
	TrackerEvtsErr bool // Tells if decoding tracker events had errors
}

// NewFromFile returns a new Rep constructed from a file.
// All types of events are decoded from the replay.
// The returned Rep must be closed with the Close method!
//
// ErrInvalidRepFile is returned if the specified name does not denote a valid SC2Replay file.
//
// ErrUnsupportedRepVersion is returned if the file exists and is a valid SC2Replay file but its version is not supported.
//
// ErrDecoding is returned if decoding the replay fails. This is most likely because the replay file is invalid, but also might be due to an implementation bug.
func NewFromFile(name string) (*Rep, error) {
	return NewFromFileEvts(name, true, true, true)
}

// NewFromFileEvts returns a new Rep constructed from a file, only the specified types of events decoded.
// The game, message and tracker tells if game events, message events and tracker events are to be decoded.
// Replay header, init data, details, attributes events and game metadata are always decoded.
// The returned Rep must be closed with the Close method!
//
// ErrInvalidRepFile is returned if the specified name does not denote a valid SC2Replay file.
//
// ErrUnsupportedRepVersion is returned if the file exists and is a valid SC2Replay file but its version is not supported.
//
// ErrDecoding is returned if decoding the replay fails. This is most likely because the replay file is invalid, but also might be due to an implementation bug.
func NewFromFileEvts(name string, game, message, tracker bool) (*Rep, error) {
	m, err := mpq.NewFromFile(name)
	if err != nil {
		return nil, ErrInvalidRepFile
	}
	return newRep(m, game, message, tracker)
}

// New returns a new Rep using the specified io.ReadSeeker as the SC2Replay file source.
// All types of events are decoded from the replay.
// The returned Rep must be closed with the Close method!
//
// ErrInvalidRepFile is returned if the input is not a valid SC2Replay file content.
//
// ErrUnsupportedRepVersion is returned if the input is a valid SC2Replay file but its version is not supported.
//
// ErrDecoding is returned if decoding the replay fails. This is most likely because the input is invalid, but also might be due to an implementation bug.
func New(input io.ReadSeeker) (*Rep, error) {
	return NewEvts(input, true, true, true)
}

// NewEvts returns a new Rep using the specified io.ReadSeeker as the SC2Replay file source, only the specified types of events decoded.
// The game, message and tracker tells if game events, message events and tracker events are to be decoded.
// Replay header, init data, details, attributes events and game metadata are always decoded.
// The returned Rep must be closed with the Close method!
//
// ErrInvalidRepFile is returned if the input is not a valid SC2Replay file content.
//
// ErrUnsupportedRepVersion is returned if the input is a valid SC2Replay file but its version is not supported.
//
// ErrDecoding is returned if decoding the replay fails. This is most likely because the input is invalid, but also might be due to an implementation bug.
func NewEvts(input io.ReadSeeker, game, message, tracker bool) (*Rep, error) {
	m, err := mpq.New(input)
	if err != nil {
		return nil, ErrInvalidRepFile
	}
	return newRep(m, game, message, tracker)
}

// newRep returns a new Rep constructed using the specified mpq.MPQ handler of the SC2Replay file, only the specified types of events decoded.
// The game, message and tracker tells if game events, message events and tracker events are to be decoded.
// Replay header, init data, details, attributes events and game metadata are always decoded.
// The returned Rep must be closed with the Close method!
//
// ErrInvalidRepFile is returned if the specified name does not denote a valid SC2Replay file.
//
// ErrUnsupportedRepVersion is returned if the input is a valid SC2Replay file but its version is not supported.
//
// ErrDecoding is returned if decoding the replay fails. This is most likely because the input is invalid, but also might be due to an implementation bug.
func newRep(m *mpq.MPQ, game, message, tracker bool) (parsedRep *Rep, errRes error) {
	closeMPQ := true
	defer func() {
		// If returning due to an error, MPQ must be closed!
		if closeMPQ {
			m.Close()
		}

		// The input is completely untrusted and the decoding implementation omits error checks for efficiency:
		// Protect replay decoding:
		if r := recover(); r != nil {
			errRes = ErrDecoding
		}
	}()

	rep := Rep{m: m}

	rep.Header = Header{Struct: s2prot.DecodeHeader(m.UserData())}
	if rep.Header.Struct == nil {
		return nil, ErrInvalidRepFile
	}

	bb := rep.Header.BaseBuild()
	p := s2prot.GetProtocol(int(bb))
	if p == nil {
		return nil, ErrUnsupportedRepVersion
	}
	rep.protocol = p

	data, err := m.FileByHash(620083690, 3548627612, 4013960850) // "replay.details"
	if err != nil {
		return nil, ErrInvalidRepFile
	}
	rep.Details = Details{Struct: p.DecodeDetails(data)}

	data, err = m.FileByHash(3544165653, 1518242780, 4280631132) // "replay.initData"
	if err != nil {
		return nil, ErrInvalidRepFile
	}
	rep.InitData = newInitData(p.DecodeInitData(data))

	data, err = m.FileByHash(1306016990, 497594575, 2731474728) // "replay.attributes.events"
	if err != nil {
		return nil, ErrInvalidRepFile
	}
	rep.AttrEvts = newAttrEvts(p.DecodeAttributesEvts(data))

	data, err = m.FileByHash(3675439372, 3912155403, 1108615308) // "replay.gamemetadata.json"
	if err != nil {
		return nil, ErrInvalidRepFile
	}
	if data != nil { // Might not be present, was added around 3.7
		if err = json.Unmarshal(data, &rep.Metadata.Struct); err != nil {
			return nil, ErrInvalidRepFile
		}
	}

	if game {
		data, err = m.FileByHash(496563520, 2864883019, 4101385109) // "replay.game.events"
		if err != nil {
			return nil, ErrInvalidRepFile
		}
		rep.GameEvts, err = p.DecodeGameEvts(data)
		rep.GameEvtsErr = err != nil
	}

	if message {
		data, err = m.FileByHash(1089231967, 831857289, 1784674979) // "replay.message.events"
		if err != nil {
			return nil, ErrInvalidRepFile
		}
		rep.MessageEvts, err = p.DecodeMessageEvts(data)
		rep.MessageEvtsErr = err != nil
	}

	if tracker {
		data, err = m.FileByHash(1501940595, 4263103390, 1648390237) // "replay.tracker.events"
		if err != nil {
			return nil, ErrInvalidRepFile
		}
		evts, err := p.DecodeTrackerEvts(data)
		rep.TrackerEvts = &TrackerEvts{Evts: evts}
		rep.TrackerEvts.init(&rep)
		rep.TrackerEvtsErr = err != nil
	}

	// Everything went well, Rep is about to be returned, do not close MPQ
	// (it will be the caller's responsibility, done via Rep.Close()).
	closeMPQ = false

	return &rep, nil
}

// Close closes the Rep and its resources.
func (r *Rep) Close() error {
	return r.m.Close()
}
<commit_msg>Added MPQ.Close() is only called if it's not nil.<commit_after>/*

The Rep type that models a replay (and everything in it).

*/

package rep

import (
	"encoding/json"
	"errors"
	"io"

	"github.com/icza/mpq"
	"github.com/icza/s2prot"
)

const (
	// ParserVersion is a Semver2 compatible version of the parser.
	ParserVersion = "v1.2.0-dev"
)

var (
	// ErrInvalidRepFile means invalid replay file.
	ErrInvalidRepFile = errors.New("Invalid SC2Replay file")

	// ErrUnsupportedRepVersion means the replay file is valid but its version is not supported.
	ErrUnsupportedRepVersion = errors.New("Unsupported replay version")

	// ErrDecoding means decoding the replay file failed,
	// Most likely because replay file is invalid, but also might be due to an implementation bug
	ErrDecoding = errors.New("Decoding error")
)

// Rep describes a replay.
type Rep struct {
	m *mpq.MPQ // MPQ parser for reading the file

	protocol *s2prot.Protocol // Protocol to decode the replay

	Header   Header   // Replay header (replay game version and length)
	Details  Details  // Game details (overall replay details)
	InitData InitData // Replay init data (the initial lobby)
	AttrEvts AttrEvts // Attributes events

	Metadata Metadata // Game metadata (calculated, confirmed results)

	GameEvts    []s2prot.Event // Game events
	MessageEvts []s2prot.Event // Message events
	TrackerEvts *TrackerEvts   // Tracker events

	GameEvtsErr    bool // Tells if decoding game events had errors
	MessageEvtsErr bool // Tells if decoding message events had errors
	TrackerEvtsErr bool // Tells if decoding tracker events had errors
}

// NewFromFile returns a new Rep constructed from a file.
// All types of events are decoded from the replay.
// The returned Rep must be closed with the Close method!
//
// ErrInvalidRepFile is returned if the specified name does not denote a valid SC2Replay file.
//
// ErrUnsupportedRepVersion is returned if the file exists and is a valid SC2Replay file but its version is not supported.
//
// ErrDecoding is returned if decoding the replay fails. This is most likely because the replay file is invalid, but also might be due to an implementation bug.
func NewFromFile(name string) (*Rep, error) {
	return NewFromFileEvts(name, true, true, true)
}

// NewFromFileEvts returns a new Rep constructed from a file, only the specified types of events decoded.
// The game, message and tracker tells if game events, message events and tracker events are to be decoded.
// Replay header, init data, details, attributes events and game metadata are always decoded.
// The returned Rep must be closed with the Close method!
//
// ErrInvalidRepFile is returned if the specified name does not denote a valid SC2Replay file.
//
// ErrUnsupportedRepVersion is returned if the file exists and is a valid SC2Replay file but its version is not supported.
//
// ErrDecoding is returned if decoding the replay fails. This is most likely because the replay file is invalid, but also might be due to an implementation bug.
func NewFromFileEvts(name string, game, message, tracker bool) (*Rep, error) {
	m, err := mpq.NewFromFile(name)
	if err != nil {
		return nil, ErrInvalidRepFile
	}
	return newRep(m, game, message, tracker)
}

// New returns a new Rep using the specified io.ReadSeeker as the SC2Replay file source.
// All types of events are decoded from the replay.
// The returned Rep must be closed with the Close method!
//
// ErrInvalidRepFile is returned if the input is not a valid SC2Replay file content.
//
// ErrUnsupportedRepVersion is returned if the input is a valid SC2Replay file but its version is not supported.
//
// ErrDecoding is returned if decoding the replay fails. This is most likely because the input is invalid, but also might be due to an implementation bug.
func New(input io.ReadSeeker) (*Rep, error) {
	return NewEvts(input, true, true, true)
}

// NewEvts returns a new Rep using the specified io.ReadSeeker as the SC2Replay file source, only the specified types of events decoded.
// The game, message and tracker tells if game events, message events and tracker events are to be decoded.
// Replay header, init data, details, attributes events and game metadata are always decoded.
// The returned Rep must be closed with the Close method!
//
// ErrInvalidRepFile is returned if the input is not a valid SC2Replay file content.
//
// ErrUnsupportedRepVersion is returned if the input is a valid SC2Replay file but its version is not supported.
//
// ErrDecoding is returned if decoding the replay fails. This is most likely because the input is invalid, but also might be due to an implementation bug.
func NewEvts(input io.ReadSeeker, game, message, tracker bool) (*Rep, error) {
	m, err := mpq.New(input)
	if err != nil {
		return nil, ErrInvalidRepFile
	}
	return newRep(m, game, message, tracker)
}

// newRep returns a new Rep constructed using the specified mpq.MPQ handler of the SC2Replay file, only the specified types of events decoded.
// The game, message and tracker tells if game events, message events and tracker events are to be decoded.
// Replay header, init data, details, attributes events and game metadata are always decoded.
// The returned Rep must be closed with the Close method!
//
// ErrInvalidRepFile is returned if the specified name does not denote a valid SC2Replay file.
//
// ErrUnsupportedRepVersion is returned if the input is a valid SC2Replay file but its version is not supported.
//
// ErrDecoding is returned if decoding the replay fails. This is most likely because the input is invalid, but also might be due to an implementation bug.
func newRep(m *mpq.MPQ, game, message, tracker bool) (parsedRep *Rep, errRes error) {
	closeMPQ := true
	defer func() {
		// If returning due to an error, MPQ must be closed!
		if closeMPQ {
			m.Close()
		}

		// The input is completely untrusted and the decoding implementation omits error checks for efficiency:
		// Protect replay decoding:
		if r := recover(); r != nil {
			errRes = ErrDecoding
		}
	}()

	rep := Rep{m: m}

	rep.Header = Header{Struct: s2prot.DecodeHeader(m.UserData())}
	if rep.Header.Struct == nil {
		return nil, ErrInvalidRepFile
	}

	bb := rep.Header.BaseBuild()
	p := s2prot.GetProtocol(int(bb))
	if p == nil {
		return nil, ErrUnsupportedRepVersion
	}
	rep.protocol = p

	data, err := m.FileByHash(620083690, 3548627612, 4013960850) // "replay.details"
	if err != nil {
		return nil, ErrInvalidRepFile
	}
	rep.Details = Details{Struct: p.DecodeDetails(data)}

	data, err = m.FileByHash(3544165653, 1518242780, 4280631132) // "replay.initData"
	if err != nil {
		return nil, ErrInvalidRepFile
	}
	rep.InitData = newInitData(p.DecodeInitData(data))

	data, err = m.FileByHash(1306016990, 497594575, 2731474728) // "replay.attributes.events"
	if err != nil {
		return nil, ErrInvalidRepFile
	}
	rep.AttrEvts = newAttrEvts(p.DecodeAttributesEvts(data))

	data, err = m.FileByHash(3675439372, 3912155403, 1108615308) // "replay.gamemetadata.json"
	if err != nil {
		return nil, ErrInvalidRepFile
	}
	if data != nil { // Might not be present, was added around 3.7
		if err = json.Unmarshal(data, &rep.Metadata.Struct); err != nil {
			return nil, ErrInvalidRepFile
		}
	}

	if game {
		data, err = m.FileByHash(496563520, 2864883019, 4101385109) // "replay.game.events"
		if err != nil {
			return nil, ErrInvalidRepFile
		}
		rep.GameEvts, err = p.DecodeGameEvts(data)
		rep.GameEvtsErr = err != nil
	}

	if message {
		data, err = m.FileByHash(1089231967, 831857289, 1784674979) // "replay.message.events"
		if err != nil {
			return nil, ErrInvalidRepFile
		}
		rep.MessageEvts, err = p.DecodeMessageEvts(data)
		rep.MessageEvtsErr = err != nil
	}

	if tracker {
		data, err = m.FileByHash(1501940595, 4263103390, 1648390237) // "replay.tracker.events"
		if err != nil {
			return nil, ErrInvalidRepFile
		}
		evts, err := p.DecodeTrackerEvts(data)
		rep.TrackerEvts = &TrackerEvts{Evts: evts}
		rep.TrackerEvts.init(&rep)
		rep.TrackerEvtsErr = err != nil
	}

	// Everything went well, Rep is about to be returned, do not close MPQ
	// (it will be the caller's responsibility, done via Rep.Close()).
	closeMPQ = false

	return &rep, nil
}

// Close closes the Rep and its resources.
func (r *Rep) Close() error {
	if r.m == nil {
		return nil
	}
	return r.m.Close()
}
<|endoftext|>


<commit_before>package sarama

import (
	"hash"
	"hash/fnv"
	"math/rand"
	"time"
)

// Partitioner is anything that, given a Kafka message key and a number of partitions indexed [0...numPartitions-1],
// decides to which partition to send the message. RandomPartitioner and RoundRobinPartitioner are the
// two simple default implementations.
type Partitioner interface {
	Partition(key Encoder, numPartitions int) int
}

// RandomPartitioner implements the Partitioner interface by choosing a random partition each time.
type RandomPartitioner struct {
	generator *rand.Rand
}

func NewRandomPartitioner() *RandomPartitioner {
	p := new(RandomPartitioner)
	p.generator = rand.New(rand.NewSource(time.Now().UTC().UnixNano()))
	return p
}

func (p *RandomPartitioner) Partition(key Encoder, numPartitions int) int {
	return p.generator.Intn(numPartitions)
}

// RoundRobinPartitioner implements the Partitioner interface by walking through the available partitions one at a time.
type RoundRobinPartitioner struct {
	partition int
}

func (p *RoundRobinPartitioner) Partition(key Encoder, numPartitions int) int {
	if p.partition >= numPartitions {
		p.partition = 0
	}
	ret := p.partition
	p.partition++
	return ret
}

// HashPartitioner implements the Partitioner interface. If the key is nil, or fails to encode, then a random partition
// is chosen. Otherwise the FNV-1a hash of the encoded bytes is used modulus the number of partitions. This ensures that messages
// with the same key always end up on the same partition.
type HashPartitioner struct {
	random *RandomPartitioner
	hasher hash.Hash32
}

func NewHashPartitioner() *HashPartitioner {
	p := new(HashPartitioner)
	p.random = NewRandomPartitioner()
	p.hasher = fnv.New32a()
	return p
}

func (p *HashPartitioner) Partition(key Encoder, numPartitions int) int {
	if key == nil {
		return p.random.Partition(key, numPartitions)
	}
	bytes, err := key.Encode()
	if err != nil {
		return p.random.Partition(key, numPartitions)
	}
	p.hasher.Reset()
	p.hasher.Write(bytes)
	return int(p.hasher.Sum32()) % numPartitions
}
<commit_msg>Update godoc<commit_after>package sarama

import (
	"hash"
	"hash/fnv"
	"math/rand"
	"time"
)

// Partitioner is anything that, given a Kafka message key and a number of partitions indexed [0...numPartitions-1],
// decides to which partition to send the message. RandomPartitioner, RoundRobinPartitioner and HashPartitioner are provided
// as simple default implementations.
type Partitioner interface {
	Partition(key Encoder, numPartitions int) int
}

// RandomPartitioner implements the Partitioner interface by choosing a random partition each time.
type RandomPartitioner struct {
	generator *rand.Rand
}

func NewRandomPartitioner() *RandomPartitioner {
	p := new(RandomPartitioner)
	p.generator = rand.New(rand.NewSource(time.Now().UTC().UnixNano()))
	return p
}

func (p *RandomPartitioner) Partition(key Encoder, numPartitions int) int {
	return p.generator.Intn(numPartitions)
}

// RoundRobinPartitioner implements the Partitioner interface by walking through the available partitions one at a time.
type RoundRobinPartitioner struct {
	partition int
}

func (p *RoundRobinPartitioner) Partition(key Encoder, numPartitions int) int {
	if p.partition >= numPartitions {
		p.partition = 0
	}
	ret := p.partition
	p.partition++
	return ret
}

// HashPartitioner implements the Partitioner interface. If the key is nil, or fails to encode, then a random partition
// is chosen. Otherwise the FNV-1a hash of the encoded bytes is used modulus the number of partitions. This ensures that messages
// with the same key always end up on the same partition.
type HashPartitioner struct {
	random *RandomPartitioner
	hasher hash.Hash32
}

func NewHashPartitioner() *HashPartitioner {
	p := new(HashPartitioner)
	p.random = NewRandomPartitioner()
	p.hasher = fnv.New32a()
	return p
}

func (p *HashPartitioner) Partition(key Encoder, numPartitions int) int {
	if key == nil {
		return p.random.Partition(key, numPartitions)
	}
	bytes, err := key.Encode()
	if err != nil {
		return p.random.Partition(key, numPartitions)
	}
	p.hasher.Reset()
	p.hasher.Write(bytes)
	return int(p.hasher.Sum32()) % numPartitions
}
<|endoftext|>