<commit_before>package kubernetes

import (
	"fmt"

	"github.com/pkg/errors"

	"github.com/grafana/tanka/pkg/kubernetes/client"
	"github.com/grafana/tanka/pkg/kubernetes/manifest"
	"github.com/grafana/tanka/pkg/kubernetes/util"
)

// Diff takes the desired state and returns the differences from the cluster
func (k *Kubernetes) Diff(state manifest.List, opts DiffOpts) (*string, error) {
	// required for separating
	namespaces, err := k.ctl.Namespaces()
	if err != nil {
		return nil, errors.Wrap(err, "listing namespaces")
	}
	resources, err := k.ctl.Resources()
	if err != nil {
		return nil, errors.Wrap(err, "listing known api-resources")
	}

	// separate resources in groups
	//
	// soon: resources that have unmet dependencies that will be met during
	// apply. These will be diffed statically, because checking with the cluster
	// would cause an error
	//
	// live: all other resources
	live, soon := separate(state, k.Spec.Namespace, separateOpts{
		namespaces: namespaces,
		resources:  resources,
	})

	// differ for live resources
	liveDiff, err := k.differ(opts.Strategy)
	if err != nil {
		return nil, err
	}

	// reports all resources as new
	staticDiff := StaticDiffer(true)

	// run the diff
	d, err := multiDiff{
		{differ: liveDiff, state: live},
		{differ: staticDiff, state: soon},
	}.diff()

	switch {
	case err != nil:
		return nil, err
	case d == nil:
		return nil, nil
	}

	if opts.Summarize {
		return util.Diffstat(*d)
	}

	return d, nil
}

type separateOpts struct {
	namespaces map[string]bool
	resources  client.Resources
}

func separate(state manifest.List, defaultNs string, opts separateOpts) (live manifest.List, soon manifest.List) {
	soonNs := make(map[string]bool)
	for _, m := range state {
		if m.Kind() != "Namespace" {
			continue
		}
		soonNs[m.Metadata().Name()] = true
	}

	for _, m := range state {
		// non-namespaced always live
		if !opts.resources.Namespaced(m) {
			live = append(live, m)
			continue
		}

		// handle implicit default
		ns := m.Metadata().Namespace()
		if ns == "" {
			ns = defaultNs
		}

		// special case: namespace missing, BUT will be created during apply
		if !opts.namespaces[ns] && soonNs[ns] {
			soon = append(soon, m)
			continue
		}

		// everything else
		live = append(live, m)
	}

	return live, soon
}

// ErrorDiffStrategyUnknown occurs when a diff-strategy is requested that does
// not exist.
type ErrorDiffStrategyUnknown struct {
	Requested string
	differs   map[string]Differ
}

func (e ErrorDiffStrategyUnknown) Error() string {
	strats := []string{}
	for s := range e.differs {
		strats = append(strats, s)
	}
	return fmt.Sprintf("diff strategy `%s` does not exist. Pick one of: %v", e.Requested, strats)
}

func (k *Kubernetes) differ(override string) (Differ, error) {
	strategy := k.Spec.DiffStrategy
	if override != "" {
		strategy = override
	}

	d, ok := k.differs[strategy]
	if !ok {
		return nil, ErrorDiffStrategyUnknown{
			Requested: strategy,
			differs:   k.differs,
		}
	}

	return d, nil
}

// StaticDiffer returns a differ that reports all resources as either created or
// deleted.
func StaticDiffer(create bool) Differ {
	return func(state manifest.List) (*string, error) {
		s := ""
		for _, m := range state {
			is, should := m.String(), ""
			if create {
				is, should = should, is
			}

			d, err := util.DiffStr(util.DiffName(m), is, should)
			if err != nil {
				return nil, err
			}
			s += d
		}

		if s == "" {
			return nil, nil
		}

		return &s, nil
	}
}

// multiDiff runs multiple differs (in series). In the future it might be worth
// parallelizing this.
type multiDiff []struct {
	differ Differ
	state  manifest.List
}

func (m multiDiff) diff() (*string, error) {
	diff := ""
	for _, d := range m {
		s, err := d.differ(d.state)
		if err != nil {
			return nil, err
		}

		if s == nil {
			continue
		}
		diff += *s
	}

	if diff == "" {
		return nil, nil
	}
	return &diff, nil
}
<commit_msg>fix(diff): refuse to diff on kubectl 1.18 (#254)<commit_after>package kubernetes

import (
	"fmt"

	"github.com/Masterminds/semver"
	"github.com/pkg/errors"

	"github.com/grafana/tanka/pkg/kubernetes/client"
	"github.com/grafana/tanka/pkg/kubernetes/manifest"
	"github.com/grafana/tanka/pkg/kubernetes/util"
)

// Diff takes the desired state and returns the differences from the cluster
func (k *Kubernetes) Diff(state manifest.List, opts DiffOpts) (*string, error) {
	// prevent https://github.com/kubernetes/kubernetes/issues/89762 until fixed
	if k.ctl.Info().ClientVersion.Equal(semver.MustParse("1.18.0")) {
		return nil, fmt.Errorf(`You seem to be using kubectl 1.18.0, which contains an unfixed issue
that makes 'kubectl diff' modify resources in your cluster.
Please downgrade kubectl until https://github.com/kubernetes/kubernetes/issues/89762 is fixed.`)
	}

	// required for separating
	namespaces, err := k.ctl.Namespaces()
	if err != nil {
		return nil, errors.Wrap(err, "listing namespaces")
	}
	resources, err := k.ctl.Resources()
	if err != nil {
		return nil, errors.Wrap(err, "listing known api-resources")
	}

	// separate resources in groups
	//
	// soon: resources that have unmet dependencies that will be met during
	// apply. These will be diffed statically, because checking with the cluster
	// would cause an error
	//
	// live: all other resources
	live, soon := separate(state, k.Spec.Namespace, separateOpts{
		namespaces: namespaces,
		resources:  resources,
	})

	// differ for live resources
	liveDiff, err := k.differ(opts.Strategy)
	if err != nil {
		return nil, err
	}

	// reports all resources as new
	staticDiff := StaticDiffer(true)

	// run the diff
	d, err := multiDiff{
		{differ: liveDiff, state: live},
		{differ: staticDiff, state: soon},
	}.diff()

	switch {
	case err != nil:
		return nil, err
	case d == nil:
		return nil, nil
	}

	if opts.Summarize {
		return util.Diffstat(*d)
	}

	return d, nil
}

type separateOpts struct {
	namespaces map[string]bool
	resources  client.Resources
}

func separate(state manifest.List, defaultNs string, opts separateOpts) (live manifest.List, soon manifest.List) {
	soonNs := make(map[string]bool)
	for _, m := range state {
		if m.Kind() != "Namespace" {
			continue
		}
		soonNs[m.Metadata().Name()] = true
	}

	for _, m := range state {
		// non-namespaced always live
		if !opts.resources.Namespaced(m) {
			live = append(live, m)
			continue
		}

		// handle implicit default
		ns := m.Metadata().Namespace()
		if ns == "" {
			ns = defaultNs
		}

		// special case: namespace missing, BUT will be created during apply
		if !opts.namespaces[ns] && soonNs[ns] {
			soon = append(soon, m)
			continue
		}

		// everything else
		live = append(live, m)
	}

	return live, soon
}

// ErrorDiffStrategyUnknown occurs when a diff-strategy is requested that does
// not exist.
type ErrorDiffStrategyUnknown struct {
	Requested string
	differs   map[string]Differ
}

func (e ErrorDiffStrategyUnknown) Error() string {
	strats := []string{}
	for s := range e.differs {
		strats = append(strats, s)
	}
	return fmt.Sprintf("diff strategy `%s` does not exist. Pick one of: %v", e.Requested, strats)
}

func (k *Kubernetes) differ(override string) (Differ, error) {
	strategy := k.Spec.DiffStrategy
	if override != "" {
		strategy = override
	}

	d, ok := k.differs[strategy]
	if !ok {
		return nil, ErrorDiffStrategyUnknown{
			Requested: strategy,
			differs:   k.differs,
		}
	}

	return d, nil
}

// StaticDiffer returns a differ that reports all resources as either created or
// deleted.
func StaticDiffer(create bool) Differ {
	return func(state manifest.List) (*string, error) {
		s := ""
		for _, m := range state {
			is, should := m.String(), ""
			if create {
				is, should = should, is
			}

			d, err := util.DiffStr(util.DiffName(m), is, should)
			if err != nil {
				return nil, err
			}
			s += d
		}

		if s == "" {
			return nil, nil
		}

		return &s, nil
	}
}

// multiDiff runs multiple differs (in series). In the future it might be worth
// parallelizing this.
type multiDiff []struct {
	differ Differ
	state  manifest.List
}

func (m multiDiff) diff() (*string, error) {
	diff := ""
	for _, d := range m {
		s, err := d.differ(d.state)
		if err != nil {
			return nil, err
		}

		if s == nil {
			continue
		}
		diff += *s
	}

	if diff == "" {
		return nil, nil
	}
	return &diff, nil
}
<|endoftext|>


<commit_before>// +build go1.11

package mutagen

import (
	"encoding/binary"
	"fmt"
	"io"
)

const (
	// VersionMajor represents the current major version of Mutagen.
	VersionMajor = 0
	// VersionMinor represents the current minor version of Mutagen.
	VersionMinor = 8
	// VersionPatch represents the current patch version of Mutagen.
	VersionPatch = 0
	// VersionTag represents a tag to be appended to the Mutagen version string.
	// It must not contain spaces. If empty, no tag is appended to the version
	// string.
	VersionTag = "beta2"
)

// Version provides a stringified version of the current Mutagen version.
var Version string

// init performs global initialization.
func init() {
	// Compute the stringified version.
	if VersionTag != "" {
		Version = fmt.Sprintf("%d.%d.%d-%s", VersionMajor, VersionMinor, VersionPatch, VersionTag)
	} else {
		Version = fmt.Sprintf("%d.%d.%d", VersionMajor, VersionMinor, VersionPatch)
	}
}

// versionBytes is a type that can be used to send and receive version
// information over the wire.
type versionBytes [12]byte

// SendVersion writes the current Mutagen version to the specified writer.
func SendVersion(writer io.Writer) error {
	// Compute the version bytes.
	var data versionBytes
	binary.BigEndian.PutUint32(data[:4], VersionMajor)
	binary.BigEndian.PutUint32(data[4:8], VersionMinor)
	binary.BigEndian.PutUint32(data[8:], VersionPatch)

	// Transmit the bytes.
	_, err := writer.Write(data[:])
	return err
}

// ReceiveVersion reads version information from the specified reader.
func ReceiveVersion(reader io.Reader) (uint32, uint32, uint32, error) {
	// Read the bytes.
	var data versionBytes
	if _, err := io.ReadFull(reader, data[:]); err != nil {
		return 0, 0, 0, err
	}

	// Decode components.
	major := binary.BigEndian.Uint32(data[:4])
	minor := binary.BigEndian.Uint32(data[4:8])
	patch := binary.BigEndian.Uint32(data[8:])

	// Done.
	return major, minor, patch, nil
}

// ReceiveAndCompareVersion reads version information from the specified reader
// and ensures that it matches the current Mutagen version.
func ReceiveAndCompareVersion(reader io.Reader) (bool, error) {
	// Receive the version.
	major, minor, patch, err := ReceiveVersion(reader)
	if err != nil {
		return false, err
	}

	// Compare the version.
	return major == VersionMajor &&
		minor == VersionMinor &&
		patch == VersionPatch, nil
}
<commit_msg>Bumped version to v0.8.0-beta3.<commit_after>// +build go1.11

package mutagen

import (
	"encoding/binary"
	"fmt"
	"io"
)

const (
	// VersionMajor represents the current major version of Mutagen.
	VersionMajor = 0
	// VersionMinor represents the current minor version of Mutagen.
	VersionMinor = 8
	// VersionPatch represents the current patch version of Mutagen.
	VersionPatch = 0
	// VersionTag represents a tag to be appended to the Mutagen version string.
	// It must not contain spaces. If empty, no tag is appended to the version
	// string.
	VersionTag = "beta3"
)

// Version provides a stringified version of the current Mutagen version.
var Version string

// init performs global initialization.
func init() {
	// Compute the stringified version.
	if VersionTag != "" {
		Version = fmt.Sprintf("%d.%d.%d-%s", VersionMajor, VersionMinor, VersionPatch, VersionTag)
	} else {
		Version = fmt.Sprintf("%d.%d.%d", VersionMajor, VersionMinor, VersionPatch)
	}
}

// versionBytes is a type that can be used to send and receive version
// information over the wire.
type versionBytes [12]byte

// SendVersion writes the current Mutagen version to the specified writer.
func SendVersion(writer io.Writer) error {
	// Compute the version bytes.
	var data versionBytes
	binary.BigEndian.PutUint32(data[:4], VersionMajor)
	binary.BigEndian.PutUint32(data[4:8], VersionMinor)
	binary.BigEndian.PutUint32(data[8:], VersionPatch)

	// Transmit the bytes.
	_, err := writer.Write(data[:])
	return err
}

// ReceiveVersion reads version information from the specified reader.
func ReceiveVersion(reader io.Reader) (uint32, uint32, uint32, error) {
	// Read the bytes.
	var data versionBytes
	if _, err := io.ReadFull(reader, data[:]); err != nil {
		return 0, 0, 0, err
	}

	// Decode components.
	major := binary.BigEndian.Uint32(data[:4])
	minor := binary.BigEndian.Uint32(data[4:8])
	patch := binary.BigEndian.Uint32(data[8:])

	// Done.
	return major, minor, patch, nil
}

// ReceiveAndCompareVersion reads version information from the specified reader
// and ensures that it matches the current Mutagen version.
func ReceiveAndCompareVersion(reader io.Reader) (bool, error) {
	// Receive the version.
	major, minor, patch, err := ReceiveVersion(reader)
	if err != nil {
		return false, err
	}

	// Compare the version.
	return major == VersionMajor &&
		minor == VersionMinor &&
		patch == VersionPatch, nil
}
<|endoftext|>


<commit_before>// +build go1.13

package mutagen

import (
	"encoding/binary"
	"fmt"
	"io"
	"net"

	"github.com/pkg/errors"
)

const (
	// VersionMajor represents the current major version of Mutagen.
	VersionMajor = 0
	// VersionMinor represents the current minor version of Mutagen.
	VersionMinor = 11
	// VersionPatch represents the current patch version of Mutagen.
	VersionPatch = 0
	// VersionTag represents a tag to be appended to the Mutagen version string.
	// It must not contain spaces. If empty, no tag is appended to the version
	// string.
	VersionTag = "rc1"
)

// Version provides a stringified version of the current Mutagen version.
var Version string

// init performs global initialization.
func init() {
	// Compute the stringified version.
	if VersionTag != "" {
		Version = fmt.Sprintf("%d.%d.%d-%s", VersionMajor, VersionMinor, VersionPatch, VersionTag)
	} else {
		Version = fmt.Sprintf("%d.%d.%d", VersionMajor, VersionMinor, VersionPatch)
	}
}

// versionBytes is a type that can be used to send and receive version
// information over the wire.
type versionBytes [12]byte

// sendVersion writes the current version to the specified writer. Version tag
// components are neither transmitted nor received.
func sendVersion(writer io.Writer) error {
	// Compute the version bytes.
	var data versionBytes
	binary.BigEndian.PutUint32(data[:4], VersionMajor)
	binary.BigEndian.PutUint32(data[4:8], VersionMinor)
	binary.BigEndian.PutUint32(data[8:], VersionPatch)

	// Transmit the bytes.
	_, err := writer.Write(data[:])
	return err
}

// receiveVersion reads version information from the specified reader. Version
// tag components are neither transmitted nor received.
func receiveVersion(reader io.Reader) (uint32, uint32, uint32, error) {
	// Read the bytes.
	var data versionBytes
	if _, err := io.ReadFull(reader, data[:]); err != nil {
		return 0, 0, 0, err
	}

	// Decode components.
	major := binary.BigEndian.Uint32(data[:4])
	minor := binary.BigEndian.Uint32(data[4:8])
	patch := binary.BigEndian.Uint32(data[8:])

	// Done.
	return major, minor, patch, nil
}

// ClientVersionHandshake performs the client side of a version handshake,
// returning an error if the received server version is not compatible with the
// client version.
//
// TODO: Add some ability to support version skew in this function.
func ClientVersionHandshake(connection net.Conn) error {
	// Receive the server's version.
	serverMajor, serverMinor, _, err := receiveVersion(connection)
	if err != nil {
		return errors.Wrap(err, "unable to receive server version")
	}

	// Send our version to the server.
	if err := sendVersion(connection); err != nil {
		return errors.Wrap(err, "unable to send client version")
	}

	// Ensure that our Mutagen versions are compatible. For now, we enforce that
	// they're equal at the minor release level.
	// TODO: Once we lock-in an internal protocol that we're going to support
	// for some time, we can allow some version skew. On the client side in
	// particular, we'll probably want to look out for the specific "locked-in"
	// server protocol that we support and instantiate some frozen client
	// implementation from that version.
	versionMatch := serverMajor == VersionMajor && serverMinor == VersionMinor
	if !versionMatch {
		return errors.New("version mismatch")
	}

	// Success.
	return nil
}

// ServerVersionHandshake performs the server side of a version handshake,
// returning an error if the received client version is not compatible with the
// server version.
//
// TODO: Add some ability to support version skew in this function.
func ServerVersionHandshake(connection net.Conn) error {
	// Send our version to the client.
	if err := sendVersion(connection); err != nil {
		return errors.Wrap(err, "unable to send server version")
	}

	// Receive the client's version.
	clientMajor, clientMinor, _, err := receiveVersion(connection)
	if err != nil {
		return errors.Wrap(err, "unable to receive client version")
	}

	// Ensure that our versions are compatible. For now, we enforce that they're
	// equal at the minor release level.
	versionMatch := clientMajor == VersionMajor && clientMinor == VersionMinor
	if !versionMatch {
		return errors.New("version mismatch")
	}

	// Success.
	return nil
}
<commit_msg>Bumped version to v0.11.0.<commit_after>// +build go1.13

package mutagen

import (
	"encoding/binary"
	"fmt"
	"io"
	"net"

	"github.com/pkg/errors"
)

const (
	// VersionMajor represents the current major version of Mutagen.
	VersionMajor = 0
	// VersionMinor represents the current minor version of Mutagen.
	VersionMinor = 11
	// VersionPatch represents the current patch version of Mutagen.
	VersionPatch = 0
	// VersionTag represents a tag to be appended to the Mutagen version string.
	// It must not contain spaces. If empty, no tag is appended to the version
	// string.
	VersionTag = ""
)

// Version provides a stringified version of the current Mutagen version.
var Version string

// init performs global initialization.
func init() {
	// Compute the stringified version.
	if VersionTag != "" {
		Version = fmt.Sprintf("%d.%d.%d-%s", VersionMajor, VersionMinor, VersionPatch, VersionTag)
	} else {
		Version = fmt.Sprintf("%d.%d.%d", VersionMajor, VersionMinor, VersionPatch)
	}
}

// versionBytes is a type that can be used to send and receive version
// information over the wire.
type versionBytes [12]byte

// sendVersion writes the current version to the specified writer. Version tag
// components are neither transmitted nor received.
func sendVersion(writer io.Writer) error {
	// Compute the version bytes.
	var data versionBytes
	binary.BigEndian.PutUint32(data[:4], VersionMajor)
	binary.BigEndian.PutUint32(data[4:8], VersionMinor)
	binary.BigEndian.PutUint32(data[8:], VersionPatch)

	// Transmit the bytes.
	_, err := writer.Write(data[:])
	return err
}

// receiveVersion reads version information from the specified reader. Version
// tag components are neither transmitted nor received.
func receiveVersion(reader io.Reader) (uint32, uint32, uint32, error) {
	// Read the bytes.
	var data versionBytes
	if _, err := io.ReadFull(reader, data[:]); err != nil {
		return 0, 0, 0, err
	}

	// Decode components.
	major := binary.BigEndian.Uint32(data[:4])
	minor := binary.BigEndian.Uint32(data[4:8])
	patch := binary.BigEndian.Uint32(data[8:])

	// Done.
	return major, minor, patch, nil
}

// ClientVersionHandshake performs the client side of a version handshake,
// returning an error if the received server version is not compatible with the
// client version.
//
// TODO: Add some ability to support version skew in this function.
func ClientVersionHandshake(connection net.Conn) error {
	// Receive the server's version.
	serverMajor, serverMinor, _, err := receiveVersion(connection)
	if err != nil {
		return errors.Wrap(err, "unable to receive server version")
	}

	// Send our version to the server.
	if err := sendVersion(connection); err != nil {
		return errors.Wrap(err, "unable to send client version")
	}

	// Ensure that our Mutagen versions are compatible. For now, we enforce that
	// they're equal at the minor release level.
	// TODO: Once we lock-in an internal protocol that we're going to support
	// for some time, we can allow some version skew. On the client side in
	// particular, we'll probably want to look out for the specific "locked-in"
	// server protocol that we support and instantiate some frozen client
	// implementation from that version.
	versionMatch := serverMajor == VersionMajor && serverMinor == VersionMinor
	if !versionMatch {
		return errors.New("version mismatch")
	}

	// Success.
	return nil
}

// ServerVersionHandshake performs the server side of a version handshake,
// returning an error if the received client version is not compatible with the
// server version.
//
// TODO: Add some ability to support version skew in this function.
func ServerVersionHandshake(connection net.Conn) error {
	// Send our version to the client.
	if err := sendVersion(connection); err != nil {
		return errors.Wrap(err, "unable to send server version")
	}

	// Receive the client's version.
	clientMajor, clientMinor, _, err := receiveVersion(connection)
	if err != nil {
		return errors.Wrap(err, "unable to receive client version")
	}

	// Ensure that our versions are compatible. For now, we enforce that they're
	// equal at the minor release level.
	versionMatch := clientMajor == VersionMajor && clientMinor == VersionMinor
	if !versionMatch {
		return errors.New("version mismatch")
	}

	// Success.
	return nil
}
<|endoftext|>


<commit_before>package project

var (
	description string = "The azure-operator manages Kubernetes clusters on Azure."
	gitSHA             = "n/a"
	name        string = "azure-operator"
	source      string = "https://github.com/giantswarm/azure-operator"
	version            = "5.0.0-alpha1"
)

func Description() string {
	return description
}

func GitSHA() string {
	return gitSHA
}

func Name() string {
	return name
}

func Source() string {
	return source
}

func Version() string {
	return version
}
<commit_msg>Bump version to 5.0.1-dev (#1089)<commit_after>package project

var (
	description string = "The azure-operator manages Kubernetes clusters on Azure."
	gitSHA             = "n/a"
	name        string = "azure-operator"
	source      string = "https://github.com/giantswarm/azure-operator"
	version            = "5.0.1-dev"
)

func Description() string {
	return description
}

func GitSHA() string {
	return gitSHA
}

func Name() string {
	return name
}

func Source() string {
	return source
}

func Version() string {
	return version
}
<|endoftext|>


<commit_before>// Copyright 2017 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package rulefmt

import (
	"context"
	"io/ioutil"
	"strings"
	"time"

	"github.com/pkg/errors"
	"github.com/prometheus/common/model"
	yaml "gopkg.in/yaml.v3"

	"github.com/prometheus/prometheus/pkg/timestamp"
	"github.com/prometheus/prometheus/promql/parser"
	"github.com/prometheus/prometheus/template"
)

// Error represents semantic errors on parsing rule groups.
type Error struct {
	Group    string
	Rule     int
	RuleName string
	Err      WrappedError
}

// WrappedError wraps error with the yaml node which can be used to represent
// the line and column numbers of the error.
type WrappedError struct {
	err     error
	node    *yaml.Node
	nodeAlt *yaml.Node
}

func (err *Error) Error() string {
	if err.Err.nodeAlt != nil {
		return errors.Wrapf(err.Err.err, "%d:%d: %d:%d: group %q, rule %d, %q", err.Err.node.Line, err.Err.node.Column, err.Err.nodeAlt.Line, err.Err.nodeAlt.Column, err.Group, err.Rule, err.RuleName).Error()
	} else if err.Err.node != nil {
		return errors.Wrapf(err.Err.err, "%d:%d: group %q, rule %d, %q", err.Err.node.Line, err.Err.node.Column, err.Group, err.Rule, err.RuleName).Error()
	}
	return errors.Wrapf(err.Err.err, "group %q, rule %d, %q", err.Group, err.Rule, err.RuleName).Error()
}

// RuleGroups is a set of rule groups that are typically exposed in a file.
type RuleGroups struct {
	Groups []RuleGroup `yaml:"groups"`
}

type ruleGroups struct {
	Groups []yaml.Node `yaml:"groups"`
}

// Validate validates all rules in the rule groups.
func (g *RuleGroups) Validate(node ruleGroups) (errs []error) {
	set := map[string]struct{}{}

	for j, g := range g.Groups {
		if g.Name == "" {
			errs = append(errs, errors.Errorf("%d:%d: Groupname should not be empty", node.Groups[j].Line, node.Groups[j].Column))
		}

		if _, ok := set[g.Name]; ok {
			errs = append(
				errs,
				errors.Errorf("%d:%d: groupname: \"%s\" is repeated in the same file", node.Groups[j].Line, node.Groups[j].Column, g.Name),
			)
		}

		set[g.Name] = struct{}{}

		for i, r := range g.Rules {
			for _, node := range r.Validate() {
				var ruleName yaml.Node
				if r.Alert.Value != "" {
					ruleName = r.Alert
				} else {
					ruleName = r.Record
				}
				errs = append(errs, &Error{
					Group:    g.Name,
					Rule:     i + 1,
					RuleName: ruleName.Value,
					Err:      node,
				})
			}
		}
	}

	return errs
}

// RuleGroup is a list of sequentially evaluated recording and alerting rules.
type RuleGroup struct {
	Name     string         `yaml:"name"`
	Interval model.Duration `yaml:"interval,omitempty"`
	Rules    []RuleNode     `yaml:"rules"`
}

// Rule describes an alerting or recording rule.
type Rule struct {
	Record      string            `yaml:"record,omitempty"`
	Alert       string            `yaml:"alert,omitempty"`
	Expr        string            `yaml:"expr"`
	For         model.Duration    `yaml:"for,omitempty"`
	Labels      map[string]string `yaml:"labels,omitempty"`
	Annotations map[string]string `yaml:"annotations,omitempty"`
}

// RuleNode adds yaml.v3 layer to support line and column outputs for invalid rules.
type RuleNode struct {
	Record      yaml.Node         `yaml:"record,omitempty"`
	Alert       yaml.Node         `yaml:"alert,omitempty"`
	Expr        yaml.Node         `yaml:"expr"`
	For         model.Duration    `yaml:"for,omitempty"`
	Labels      map[string]string `yaml:"labels,omitempty"`
	Annotations map[string]string `yaml:"annotations,omitempty"`
}

// Validate the rule and return a list of encountered errors.
func (r *RuleNode) Validate() (nodes []WrappedError) {
	if r.Record.Value != "" && r.Alert.Value != "" {
		nodes = append(nodes, WrappedError{
			err:     errors.Errorf("only one of 'record' and 'alert' must be set"),
			node:    &r.Record,
			nodeAlt: &r.Alert,
		})
	}
	if r.Record.Value == "" && r.Alert.Value == "" {
		if r.Record.Value == "0" {
			nodes = append(nodes, WrappedError{
				err:  errors.Errorf("one of 'record' or 'alert' must be set"),
				node: &r.Alert,
			})
		} else {
			nodes = append(nodes, WrappedError{
				err:  errors.Errorf("one of 'record' or 'alert' must be set"),
				node: &r.Record,
			})
		}
	}

	if r.Expr.Value == "" {
		nodes = append(nodes, WrappedError{
			err:  errors.Errorf("field 'expr' must be set in rule"),
			node: &r.Expr,
		})
	} else if _, err := parser.ParseExpr(r.Expr.Value); err != nil {
		nodes = append(nodes, WrappedError{
			err:  errors.Wrapf(err, "could not parse expression"),
			node: &r.Expr,
		})
	}
	if r.Record.Value != "" {
		if len(r.Annotations) > 0 {
			nodes = append(nodes, WrappedError{
				err:  errors.Errorf("invalid field 'annotations' in recording rule"),
				node: &r.Record,
			})
		}
		if r.For != 0 {
			nodes = append(nodes, WrappedError{
				err:  errors.Errorf("invalid field 'for' in recording rule"),
				node: &r.Record,
			})
		}
		if !model.IsValidMetricName(model.LabelValue(r.Record.Value)) {
			nodes = append(nodes, WrappedError{
				err:  errors.Errorf("invalid recording rule name: %s", r.Record.Value),
				node: &r.Record,
			})
		}
	}

	for k, v := range r.Labels {
		if !model.LabelName(k).IsValid() {
			nodes = append(nodes, WrappedError{
				err: errors.Errorf("invalid label name: %s", k),
			})
		}

		if !model.LabelValue(v).IsValid() {
			nodes = append(nodes, WrappedError{
				err: errors.Errorf("invalid label value: %s", v),
			})
		}
	}

	for k := range r.Annotations {
		if !model.LabelName(k).IsValid() {
			nodes = append(nodes, WrappedError{
				err: errors.Errorf("invalid annotation name: %s", k),
			})
		}
	}

	for _, err := range testTemplateParsing(r) {
		nodes = append(nodes, WrappedError{err: err})
	}

	return
}

// testTemplateParsing checks if the templates used in labels and annotations
// of the alerting rules are parsed correctly.
func testTemplateParsing(rl *RuleNode) (errs []error) {
	if rl.Alert.Value == "" {
		// Not an alerting rule.
		return errs
	}

	// Trying to parse templates.
	tmplData := template.AlertTemplateData(map[string]string{}, map[string]string{}, 0)
	defs := []string{
		"{{$labels := .Labels}}",
		"{{$externalLabels := .ExternalLabels}}",
		"{{$value := .Value}}",
	}
	parseTest := func(text string) error {
		tmpl := template.NewTemplateExpander(
			context.TODO(),
			strings.Join(append(defs, text), ""),
			"__alert_"+rl.Alert.Value,
			tmplData,
			model.Time(timestamp.FromTime(time.Now())),
			nil,
			nil,
		)
		return tmpl.ParseTest()
	}

	// Parsing Labels.
	for k, val := range rl.Labels {
		err := parseTest(val)
		if err != nil {
			errs = append(errs, errors.Wrapf(err, "label %q", k))
		}
	}

	// Parsing Annotations.
	for k, val := range rl.Annotations {
		err := parseTest(val)
		if err != nil {
			errs = append(errs, errors.Wrapf(err, "annotation %q", k))
		}
	}

	return errs
}

// Parse parses and validates a set of rules.
func Parse(content []byte) (*RuleGroups, []error) {
	var (
		groups RuleGroups
		node   ruleGroups
		errs   []error
	)

	err := yaml.Unmarshal(content, &groups)
	if err != nil {
		errs = append(errs, err)
	}
	err = yaml.Unmarshal(content, &node)
	if err != nil {
		errs = append(errs, err)
	}

	if len(errs) > 0 {
		return nil, errs
	}

	return &groups, groups.Validate(node)
}

// ParseFile reads and parses rules from a file.
func ParseFile(file string) (*RuleGroups, []error) {
	b, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, []error{errors.Wrap(err, file)}
	}
	rgs, errs := Parse(b)
	for i := range errs {
		errs[i] = errors.Wrap(errs[i], file)
	}
	return rgs, errs
}
<commit_msg>Clarify error messaging, group names must be provided. (#7529)<commit_after>// Copyright 2017 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package rulefmt

import (
	"context"
	"io/ioutil"
	"strings"
	"time"

	"github.com/pkg/errors"
	"github.com/prometheus/common/model"
	yaml "gopkg.in/yaml.v3"

	"github.com/prometheus/prometheus/pkg/timestamp"
	"github.com/prometheus/prometheus/promql/parser"
	"github.com/prometheus/prometheus/template"
)

// Error represents semantic errors on parsing rule groups.
type Error struct {
	Group    string
	Rule     int
	RuleName string
	Err      WrappedError
}

// WrappedError wraps error with the yaml node which can be used to represent
// the line and column numbers of the error.
type WrappedError struct {
	err     error
	node    *yaml.Node
	nodeAlt *yaml.Node
}

func (err *Error) Error() string {
	if err.Err.nodeAlt != nil {
		return errors.Wrapf(err.Err.err, "%d:%d: %d:%d: group %q, rule %d, %q", err.Err.node.Line, err.Err.node.Column, err.Err.nodeAlt.Line, err.Err.nodeAlt.Column, err.Group, err.Rule, err.RuleName).Error()
	} else if err.Err.node != nil {
		return errors.Wrapf(err.Err.err, "%d:%d: group %q, rule %d, %q", err.Err.node.Line, err.Err.node.Column, err.Group, err.Rule, err.RuleName).Error()
	}
	return errors.Wrapf(err.Err.err, "group %q, rule %d, %q", err.Group, err.Rule, err.RuleName).Error()
}

// RuleGroups is a set of rule groups that are typically exposed in a file.
type RuleGroups struct {
	Groups []RuleGroup `yaml:"groups"`
}

type ruleGroups struct {
	Groups []yaml.Node `yaml:"groups"`
}

// Validate validates all rules in the rule groups.
func (g *RuleGroups) Validate(node ruleGroups) (errs []error) {
	set := map[string]struct{}{}

	for j, g := range g.Groups {
		if g.Name == "" {
			errs = append(errs, errors.Errorf("%d:%d: Groupname must not be empty", node.Groups[j].Line, node.Groups[j].Column))
		}

		if _, ok := set[g.Name]; ok {
			errs = append(
				errs,
				errors.Errorf("%d:%d: groupname: \"%s\" is repeated in the same file", node.Groups[j].Line, node.Groups[j].Column, g.Name),
			)
		}

		set[g.Name] = struct{}{}

		for i, r := range g.Rules {
			for _, node := range r.Validate() {
				var ruleName yaml.Node
				if r.Alert.Value != "" {
					ruleName = r.Alert
				} else {
					ruleName = r.Record
				}
				errs = append(errs, &Error{
					Group:    g.Name,
					Rule:     i + 1,
					RuleName: ruleName.Value,
					Err:      node,
				})
			}
		}
	}

	return errs
}

// RuleGroup is a list of sequentially evaluated recording and alerting rules.
type RuleGroup struct {
	Name     string         `yaml:"name"`
	Interval model.Duration `yaml:"interval,omitempty"`
	Rules    []RuleNode     `yaml:"rules"`
}

// Rule describes an alerting or recording rule.
type Rule struct {
	Record      string            `yaml:"record,omitempty"`
	Alert       string            `yaml:"alert,omitempty"`
	Expr        string            `yaml:"expr"`
	For         model.Duration    `yaml:"for,omitempty"`
	Labels      map[string]string `yaml:"labels,omitempty"`
	Annotations map[string]string `yaml:"annotations,omitempty"`
}

// RuleNode adds yaml.v3 layer to support line and column outputs for invalid rules.
type RuleNode struct {
	Record      yaml.Node         `yaml:"record,omitempty"`
	Alert       yaml.Node         `yaml:"alert,omitempty"`
	Expr        yaml.Node         `yaml:"expr"`
	For         model.Duration    `yaml:"for,omitempty"`
	Labels      map[string]string `yaml:"labels,omitempty"`
	Annotations map[string]string `yaml:"annotations,omitempty"`
}

// Validate the rule and return a list of encountered errors.
func (r *RuleNode) Validate() (nodes []WrappedError) {
	if r.Record.Value != "" && r.Alert.Value != "" {
		nodes = append(nodes, WrappedError{
			err:     errors.Errorf("only one of 'record' and 'alert' must be set"),
			node:    &r.Record,
			nodeAlt: &r.Alert,
		})
	}
	if r.Record.Value == "" && r.Alert.Value == "" {
		if r.Record.Value == "0" {
			nodes = append(nodes, WrappedError{
				err:  errors.Errorf("one of 'record' or 'alert' must be set"),
				node: &r.Alert,
			})
		} else {
			nodes = append(nodes, WrappedError{
				err:  errors.Errorf("one of 'record' or 'alert' must be set"),
				node: &r.Record,
			})
		}
	}

	if r.Expr.Value == "" {
		nodes = append(nodes, WrappedError{
			err:  errors.Errorf("field 'expr' must be set in rule"),
			node: &r.Expr,
		})
	} else if _, err := parser.ParseExpr(r.Expr.Value); err != nil {
		nodes = append(nodes, WrappedError{
			err:  errors.Wrapf(err, "could not parse expression"),
			node: &r.Expr,
		})
	}
	if r.Record.Value != "" {
		if len(r.Annotations) > 0 {
			nodes = append(nodes, WrappedError{
				err:  errors.Errorf("invalid field 'annotations' in recording rule"),
				node: &r.Record,
			})
		}
		if r.For != 0 {
			nodes = append(nodes, WrappedError{
				err:  errors.Errorf("invalid field 'for' in recording rule"),
				node: &r.Record,
			})
		}
		if !model.IsValidMetricName(model.LabelValue(r.Record.Value)) {
			nodes = append(nodes, WrappedError{
				err:  errors.Errorf("invalid recording rule name: %s", r.Record.Value),
				node: &r.Record,
			})
		}
	}

	for k, v := range r.Labels {
		if !model.LabelName(k).IsValid() {
			nodes = append(nodes, WrappedError{
				err: errors.Errorf("invalid label name: %s", k),
			})
		}

		if !model.LabelValue(v).IsValid() {
			nodes = append(nodes, WrappedError{
				err: errors.Errorf("invalid label value: %s", v),
			})
		}
	}

	for k := range r.Annotations {
		if !model.LabelName(k).IsValid() {
			nodes = append(nodes, WrappedError{
				err: errors.Errorf("invalid annotation name: %s", k),
			})
		}
	}

	for _, err := range testTemplateParsing(r) {
		nodes = append(nodes, WrappedError{err: err})
	}

	return
}

// testTemplateParsing checks if the templates used in labels and annotations
// of the alerting rules are parsed correctly.
func testTemplateParsing(rl *RuleNode) (errs []error) {
	if rl.Alert.Value == "" {
		// Not an alerting rule.
		return errs
	}

	// Trying to parse templates.
	tmplData := template.AlertTemplateData(map[string]string{}, map[string]string{}, 0)
	defs := []string{
		"{{$labels := .Labels}}",
		"{{$externalLabels := .ExternalLabels}}",
		"{{$value := .Value}}",
	}
	parseTest := func(text string) error {
		tmpl := template.NewTemplateExpander(
			context.TODO(),
			strings.Join(append(defs, text), ""),
			"__alert_"+rl.Alert.Value,
			tmplData,
			model.Time(timestamp.FromTime(time.Now())),
			nil,
			nil,
		)
		return tmpl.ParseTest()
	}

	// Parsing Labels.
	for k, val := range rl.Labels {
		err := parseTest(val)
		if err != nil {
			errs = append(errs, errors.Wrapf(err, "label %q", k))
		}
	}

	// Parsing Annotations.
	for k, val := range rl.Annotations {
		err := parseTest(val)
		if err != nil {
			errs = append(errs, errors.Wrapf(err, "annotation %q", k))
		}
	}

	return errs
}

// Parse parses and validates a set of rules.
func Parse(content []byte) (*RuleGroups, []error) {
	var (
		groups RuleGroups
		node   ruleGroups
		errs   []error
	)

	err := yaml.Unmarshal(content, &groups)
	if err != nil {
		errs = append(errs, err)
	}
	err = yaml.Unmarshal(content, &node)
	if err != nil {
		errs = append(errs, err)
	}

	if len(errs) > 0 {
		return nil, errs
	}

	return &groups, groups.Validate(node)
}

// ParseFile reads and parses rules from a file.
func ParseFile(file string) (*RuleGroups, []error) {
	b, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, []error{errors.Wrap(err, file)}
	}
	rgs, errs := Parse(b)
	for i := range errs {
		errs[i] = errors.Wrap(errs[i], file)
	}
	return rgs, errs
}
<|endoftext|>


<commit_before>// read rdf file

package main

import (
	"bytes"
	"strconv"
	"encoding/binary"
	//"unsafe"
	"fmt"
	"os"
	"strings"
	"io/ioutil"
	"container/list"
	//"bytes"
)
//import "encoding/binary"
//import "github.com/wallix/triplestore"
import "github.com/iand/ntriples"

const (
	intval = iota  // c0 == 0
	strval = iota  // c1 == 1
	sizeofint = 8
)
// details on the node itself
type SAttributeValBlob struct {
	BytePosition uintptr // pos in file
	CardinalPos int // pos in hash
//	Size uintptr // size of object always sizeofint
	Bytes []byte // bytes of object
	Value int
	Incount int //  incoming count- how many times has this be referenced
	Datatype int // the type of the field
}

type SAttributeValStrBlob struct {
	BytePosition uintptr // pos in file
	CardinalPos int // pos in hash
	Size int // size of object
	Bytes []byte // bytes of object
	Value string
	Incount int //  incoming count- how many times has this be referenced
	Datatype int // the type of the field
}

func (t * SAttributeValBlob) Val(val int) {
	t.Incount++
}

func (t * SAttributeValStrBlob) Val(val string) {
	t.Incount++
}

func (t * SAttributeValBlob) Report() {
	fmt.Printf("%d %d %d %x %d %d %d\n",t.BytePosition,
		t.CardinalPos,
		sizeofint,
		t.Bytes,
		t.Value,
		t.Incount,
		t.Datatype)
}

// depending on the values create a blob object for storage
type IntAttributeVals struct {
	TheVals map[int]*SAttributeValBlob
	Count int
	Max int
	NextPos uintptr // the next position int the block
}

func (t * IntAttributeVals) Report(name string) {
	fmt.Printf("Int Name %s, Count %d, Max %d, Next %d\n",name, t.Count, t.Max, t.NextPos)
	d := make([]byte,t.NextPos)

	for _, v := range t.TheVals {
		for i:=0; i < sizeofint; i++{
			d[v.BytePosition+uintptr(i)]=v.Bytes[i]
		}
	}
	fn := fmt.Sprintf("data/%s.dat",name)
	err := ioutil.WriteFile(fn, d, 0644)
	if err != nil {
		panic(err)
	}
	//fmt.Printf("%x\n",d)
}

func (t * IntAttributeVals) Val(val int) {
	if (t.TheVals == nil){
		t.TheVals = make(map[int]*SAttributeValBlob)
	}
	if valo, ok := t.TheVals[val]; ok {
		valo.Val(val)
	} else {
		valo := &SAttributeValBlob{
			BytePosition: t.NextPos,
			CardinalPos : t.Count,
			Datatype : intval,
//			Size : sizeofint,
			Value : val,
			Incount : 0, // will be incremented
		}
		valo.Bytes = make([]byte,sizeofint)
		binary.LittleEndian.PutUint32(valo.Bytes,uint32(val))
		t.Count++
		if val > t.Max{
			t.Max = val
		}
		valo.Val(val)
		t.NextPos=t.NextPos+uintptr(sizeofint)
		t.TheVals[val] = valo
	}
}

type StrAttributeVals struct {
	TheVals map[string]*SAttributeValStrBlob
	Count int
	Max int
	NextPos uintptr // the next position int the block

}

func (t * StrAttributeVals) Report(name string) {
	fmt.Printf("Str Name %s, Count %d, Max %d, Next %d\n",name, t.Count, t.Max, t.NextPos)
	d := make([]byte,t.NextPos)
	for _, v := range t.TheVals {
		for i:=0; i <v.Size; i++{
			d[v.BytePosition+uintptr(i)]=v.Bytes[i]
		}
	}
	fn := fmt.Sprintf("data/%s_str.dat",name)
	err := ioutil.WriteFile(fn, d, 0644)
	if err != nil {
		panic(err)
	}
	//fmt.Printf("%x\n",d)
}

func (t * StrAttributeVals) ReportSizes(name string) {
	fmt.Printf("Str Size %s, Count %d, Max %d, Next %d\n",name, t.Count, t.Max, t.NextPos)
	d := make([]byte,t.Count * sizeofint)

	bp := 0

	for _, v := range t.TheVals {
		b2 := make([]byte,sizeofint)
		binary.LittleEndian.PutUint32(b2,uint32(v.Size))
		for i:=0; i < sizeofint; i++{
			d[bp+i]=b2[i]
		}
		bp += sizeofint
	}
	fn := fmt.Sprintf("data/%s_str_sizes.dat",name)
	err := ioutil.WriteFile(fn, d, 0644)
	if err != nil {
		panic(err)
	}

}

func (t * StrAttributeVals) Val(val string) {
	if (t.TheVals == nil){
		t.TheVals = make(map[string]*SAttributeValStrBlob)
	}
	if valo, ok := t.TheVals[val]; ok {
		valo.Val(val)
	} else {
		valo := &SAttributeValStrBlob{
			BytePosition: t.NextPos,
			CardinalPos : t.Count,
			Datatype : strval,
			Size : len(val),
			Value : val,
			Incount : 0, // will be incremented
		}
		valo.Bytes = make([]byte,valo.Size)
		valo.Bytes = []byte(val)
		t.Count++
		if len(val) > t.Max{ // max len
			t.Max = len(val)
		}
		valo.Val(val)
		t.NextPos=t.NextPos+uintptr(valo.Size)
		t.TheVals[val] = valo
	}
}

type NodePair struct {
	From int
	To int
}

type NodePairArray struct {
	Count int
	Pairs * list.List
}

func CreateNodePairArray() (* NodePairArray){
	return &NodePairArray{	Count :0, Pairs : list.New()	}
}

func( t * NodePairArray) Report(p string){
	fmt.Printf("Pairs Name %s, Count %d\n",p, t.Count)
	//bytes = make([]byte,t.Count * unsafe.Sizeof(NodePair))
	var b bytes.Buffer

	b.Grow(int(uintptr(t.Count) * sizeofint * 2))

	for e := t.Pairs.Front(); e != nil; e = e.Next() {
		switch t := e.Value.(type) {
		case NodePair :
			b2 := make([]byte,sizeofint)
			binary.LittleEndian.PutUint32(b2,uint32(t.From))
			b.Write(b2)
			binary.LittleEndian.PutUint32(b2,uint32(t.To))
			b.Write(b2)

		default:
			panic(t)
		}
	}
	fn := fmt.Sprintf("data/%s_pairs.dat",p)
	err := ioutil.WriteFile(fn, b.Bytes(), 0644)
	if err != nil {
		panic(err)
	}
}

func( t * NodePairArray) Add(s int, o int){
	t.Count ++
	t.Pairs.PushBack(NodePair{From:s, To:o})
}

// for each attribute name, if int or string instanciate one of these
type SAttributeNames struct {
	IntVals map[string]*IntAttributeVals

	Pairs map[string]*NodePairArray  // one array per field for node ref fields

	//IntValTotal IntAttributeVals
	StrVals map[string]*StrAttributeVals
}

func (t * SAttributeNames) Report() {

	for k, v := range t.StrVals {
		v.Report(k)
		v.ReportSizes(k)
	}

	for k, v := range t.Pairs {
		v.Report(k)
	}
	for k, v := range t.IntVals {
		//fmt.Printf("Name:%s\t",k)
		v.Report(k)
	}

}

func (t * SAttributeNames) Pair(p string, s int, o int) {

	if (t.Pairs == nil){
		t.Pairs = make(map[string]*NodePairArray)
	}

	if valo, ok := t.Pairs[p]; ok {
		valo.Add(s,o)
	} else {
		valo := CreateNodePairArray()
		t.Pairs[p] = valo
		valo.Add(s,o)
	}

}

func (t * SAttributeNames) IntVal(key string, val int) {

	if (t.IntVals == nil){
		t.IntVals = make(map[string]*IntAttributeVals)
	}

	if valo, ok := t.IntVals[key]; ok {
		valo.Val(val)
	} else {
		valo := &IntAttributeVals{
			Count:0,
			Max : 0,
			NextPos : 0,
		}
		t.IntVals[key] = valo
		valo.Val(val)
	}

}

func (t * SAttributeNames) StrVal(key string, val string) {

	if (t.StrVals == nil){
		t.StrVals = make(map[string]*StrAttributeVals)
	}

	if valo, ok := t.StrVals[key]; ok {
		valo.Val(val)
	} else {
		valo := &StrAttributeVals{
			Count:0,
			Max : 0,
			NextPos : 0,
		}
		t.StrVals[key] = valo
		valo.Val(val)
	}

}

func main(){
	//var buff bytes.Buffer
	ntfile, err := os.Open("test.ntriples")
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %s", err.Error())
		os.Exit(1)
	}
	defer ntfile.Close()

	datamap := SAttributeNames{}

	prefix := "https://h4ck3rm1k3.github.io/gogccintro/gcc/ontology/2017/05/20/gcc_compiler.owl#"
	stype := "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"
	named := "http://www.w3.org/2002/07/owl#NamedIndividual"
	count := 0
	r := ntriples.NewReader(ntfile)

	for triple, err := r.Read(); err == nil;  triple, err = r.Read() {
		count++
		s := triple.Subject()
		s = strings.Replace(s,prefix,"",1)
		si, err := strconv.Atoi(s)
		if err != nil {
			panic(err)
		}

		o := triple.Object()
		if o == named {
			continue
		}
		p := triple.Predicate()
		o = strings.Replace(o,prefix,"",1)
		p = strings.Replace(p,prefix,"",1)

		oi, err := strconv.Atoi(o)
		if err != nil {
			datamap.IntVal("id",si)
			//panic(err)
			if p == stype {
				//fmt.Printf("%d TYPE:%s\n", si,o)
				datamap.IntVal(o,si)// peg node type as predicate...
			} else {
				//fmt.Printf("%d %s OTHER:%s\n", si,p,o)
				datamap.StrVal(p,o) // other string value
			}
		} else {
			datamap.IntVal("id",si)
			datamap.IntVal(p,oi)
			datamap.Pair(p, si,oi)
			//fmt.Printf("%d %s %d\n", si,p,oi)
		}
	}
	datamap.Report()
}
<commit_msg>now ready to test<commit_after>// read rdf file

package main

import (
	"bytes"
	"strconv"
	"encoding/binary"
	//"unsafe"
	"fmt"
	"os"
	"strings"
	"io/ioutil"
	"container/list"
	//"bytes"
)
//import "encoding/binary"
//import "github.com/wallix/triplestore"
import "github.com/iand/ntriples"

const (
	intval = iota  // c0 == 0
	strval = iota  // c1 == 1
	sizeofint = 8
)
// details on the node itself
type SAttributeValBlob struct {
	BytePosition uintptr // pos in file
	CardinalPos int // pos in hash
//	Size uintptr // size of object always sizeofint
	Bytes []byte // bytes of object
	Value int
	Incount int //  incoming count- how many times has this be referenced
	Datatype int // the type of the field
}

type SAttributeValStrBlob struct {
	BytePosition uintptr // pos in file
	CardinalPos int // pos in hash
	Size int // size of object
	Bytes []byte // bytes of object
	Value string
	Incount int //  incoming count- how many times has this be referenced
	Datatype int // the type of the field
}

func (t * SAttributeValBlob) Val(val int) {
	t.Incount++
}

func (t * SAttributeValStrBlob) Val(val string) {
	t.Incount++
}

func (t * SAttributeValBlob) Report() {
	fmt.Printf("%d %d %d %x %d %d %d\n",t.BytePosition,
		t.CardinalPos,
		sizeofint,
		t.Bytes,
		t.Value,
		t.Incount,
		t.Datatype)
}

// depending on the values create a blob object for storage
type IntAttributeVals struct {
	TheVals map[int]*SAttributeValBlob
	Count int
	Max int
	NextPos uintptr // the next position int the block
}

func (t * IntAttributeVals) Report(name string) {
	fmt.Printf("Int Name %s, Count %d, Max %d, Next %d\n",name, t.Count, t.Max, t.NextPos)
	d := make([]byte,t.NextPos)

	for _, v := range t.TheVals {
		for i:=0; i < sizeofint; i++{
			d[v.BytePosition+uintptr(i)]=v.Bytes[i]
		}
	}
	fn := fmt.Sprintf("data/%s.dat",name)
	err := ioutil.WriteFile(fn, d, 0644)
	if err != nil {
		panic(err)
	}
	//fmt.Printf("%x\n",d)
}

func (t * IntAttributeVals) Val(val int) {
	if (t.TheVals == nil){
		t.TheVals = make(map[int]*SAttributeValBlob)
	}
	if valo, ok := t.TheVals[val]; ok {
		valo.Val(val)
	} else {
		valo := &SAttributeValBlob{
			BytePosition: t.NextPos,
			CardinalPos : t.Count,
			Datatype : intval,
//			Size : sizeofint,
			Value : val,
			Incount : 0, // will be incremented
		}
		valo.Bytes = make([]byte,sizeofint)
		binary.LittleEndian.PutUint32(valo.Bytes,uint32(val))
		t.Count++
		if val > t.Max{
			t.Max = val
		}
		valo.Val(val)
		t.NextPos=t.NextPos+uintptr(sizeofint)
		t.TheVals[val] = valo
	}
}

type StrAttributeVals struct {
	TheVals map[string]*SAttributeValStrBlob
	Count int
	Max int
	NextPos uintptr // the next position int the block

}

func (t * StrAttributeVals) Report(name string) {
	fmt.Printf("Str Name %s, Count %d, Max %d, Next %d\n",name, t.Count, t.Max, t.NextPos)
	d := make([]byte,t.NextPos)
	for _, v := range t.TheVals {
		for i:=0; i <v.Size; i++{
			d[v.BytePosition+uintptr(i)]=v.Bytes[i]
		}
	}
	fn := fmt.Sprintf("data/%s_str.dat",name)
	err := ioutil.WriteFile(fn, d, 0644)
	if err != nil {
		panic(err)
	}
	//fmt.Printf("%x\n",d)
}

func (t * StrAttributeVals) ReportSizes(name string) {
	fmt.Printf("Str Size %s, Count %d, Max %d, Next %d\n",name, t.Count, t.Max, t.NextPos)
	d := make([]byte,t.Count * sizeofint)

	bp := 0

	for _, v := range t.TheVals {
		b2 := make([]byte,sizeofint)
		binary.LittleEndian.PutUint32(b2,uint32(v.Size))
		for i:=0; i < sizeofint; i++{
			d[bp+i]=b2[i]
		}
		bp += sizeofint
	}
	fn := fmt.Sprintf("data/%s_str_sizes.dat",name)
	err := ioutil.WriteFile(fn, d, 0644)
	if err != nil {
		panic(err)
	}

}

func (t * StrAttributeVals) Val(val string) (int) {
	if (t.TheVals == nil){
		t.TheVals = make(map[string]*SAttributeValStrBlob)
	}
	if valo, ok := t.TheVals[val]; ok {
		valo.Val(val)
		return valo.CardinalPos
	} else {
		valo := &SAttributeValStrBlob{
			BytePosition: t.NextPos,
			CardinalPos : t.Count,
			Datatype : strval,
			Size : len(val),
			Value : val,
			Incount : 0, // will be incremented
		}
		valo.Bytes = make([]byte,valo.Size)
		valo.Bytes = []byte(val)
		t.Count++
		if len(val) > t.Max{ // max len
			t.Max = len(val)
		}
		valo.Val(val)
		t.NextPos=t.NextPos+uintptr(valo.Size)
		t.TheVals[val] = valo
		return valo.CardinalPos
	}
}

type NodePair struct {
	From int
	To int
}

type NodePairArray struct {
	Count int
	Pairs * list.List
}

func CreateNodePairArray() (* NodePairArray){
	return &NodePairArray{	Count :0, Pairs : list.New()	}
}

func( t * NodePairArray) Report(p string){
	fmt.Printf("Pairs Name %s, Count %d\n",p, t.Count)
	//bytes = make([]byte,t.Count * unsafe.Sizeof(NodePair))
	var b bytes.Buffer

	b.Grow(int(uintptr(t.Count) * sizeofint * 2))

	for e := t.Pairs.Front(); e != nil; e = e.Next() {
		switch t := e.Value.(type) {
		case NodePair :
			b2 := make([]byte,sizeofint)
			binary.LittleEndian.PutUint32(b2,uint32(t.From))
			b.Write(b2)
			binary.LittleEndian.PutUint32(b2,uint32(t.To))
			b.Write(b2)

		default:
			panic(t)
		}
	}
	fn := fmt.Sprintf("data/%s_pairs.dat",p)
	err := ioutil.WriteFile(fn, b.Bytes(), 0644)
	if err != nil {
		panic(err)
	}
}

func( t * NodePairArray) Add(s int, o int){
	t.Count ++
	t.Pairs.PushBack(NodePair{From:s, To:o})
}

// for each attribute name, if int or string instanciate one of these
type SAttributeNames struct {
	IntVals map[string]*IntAttributeVals

	Pairs map[string]*NodePairArray  // one array per field for node ref fields

	//IntValTotal IntAttributeVals
	StrVals map[string]*StrAttributeVals

	//PredicateIds map[string]int
}

func (t * SAttributeNames) Report() {

	for k, v := range t.StrVals {
		v.Report(k)
		v.ReportSizes(k)
	}

	for k, v := range t.Pairs {
		v.Report(k)
	}
	for k, v := range t.IntVals {
		//fmt.Printf("Name:%s\t",k)
		v.Report(k)
	}

}

func (t * SAttributeNames) Pair(p string, s int, o int) {

	if (t.Pairs == nil){
		t.Pairs = make(map[string]*NodePairArray)
	}

	if valo, ok := t.Pairs[p]; ok {
		valo.Add(s,o)
	} else {
		valo := CreateNodePairArray()
		t.Pairs[p] = valo
		valo.Add(s,o)
	}

}

func (t * SAttributeNames) IntVal(key string, val int) {

	if (t.IntVals == nil){
		t.IntVals = make(map[string]*IntAttributeVals)
	}

	if valo, ok := t.IntVals[key]; ok {
		valo.Val(val)
	} else {
		valo := &IntAttributeVals{
			Count:0,
			Max : 0,
			NextPos : 0,
		}
		t.IntVals[key] = valo
		valo.Val(val)
	}

}

// func (t * SAttributeNames) PredVal(key string) (int){
// /*
// assign each predicate an id
// */
// 	if (t.PredicateIds == nil){
// 		t.PredicateIds = make(map[string]int)
// 	}
// 	if valo, ok := t.PredicateIds[key]; ok {
// 		return valo
// 	} else {
// 		valo := len(t.PredicateIds)
// 		t.PredicateIds[key] = valo
// 		return valo
// 	}

// }

func (t * SAttributeNames) StrVal(key string, val string) (int){

	if (t.StrVals == nil){
		t.StrVals = make(map[string]*StrAttributeVals)
	}

	if valo, ok := t.StrVals[key]; ok {
		return valo.Val(val)
	} else {
		valo := &StrAttributeVals{
			Count:0,
			Max : 0,
			NextPos : 0,
		}
		t.StrVals[key] = valo
		return valo.Val(val)
	}

}

func main(){
	//var buff bytes.Buffer
	ntfile, err := os.Open("test.ntriples")
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %s", err.Error())
		os.Exit(1)
	}
	defer ntfile.Close()

	datamap := SAttributeNames{}

	prefix := "https://h4ck3rm1k3.github.io/gogccintro/gcc/ontology/2017/05/20/gcc_compiler.owl#"
	stype := "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"
	named := "http://www.w3.org/2002/07/owl#NamedIndividual"
	count := 0
	r := ntriples.NewReader(ntfile)

	for triple, err := r.Read(); err == nil;  triple, err = r.Read() {
		count++
		s := triple.Subject()
		s = strings.Replace(s,prefix,"",1)
		si, err := strconv.Atoi(s)
		if err != nil {
			panic(err)
		}

		o := triple.Object()
		if o == named {
			continue
		}
		p := triple.Predicate()
		o = strings.Replace(o,prefix,"",1)
		p = strings.Replace(p,prefix,"",1)

		oi, err := strconv.Atoi(o)
		if err != nil {
			datamap.IntVal("id",si)
			//panic(err)
			if p == stype {
				//fmt.Printf("%d TYPE:%s\n", si,o)
				datamap.IntVal(o,si)// peg node type as predicate...

				datamap.StrVal("object_type",o) // other string value
			} else {
				//fmt.Printf("%d %s OTHER:%s\n", si,p,o)
				oid := datamap.StrVal(p,o) // other string value
				// the  object id are the indexes to the file where these will be output
				sp2 := fmt.Sprintf("s_%s",p)
				datamap.Pair( sp2, si,oid)  // use the cardinality of the string for the index

				datamap.StrVal("string_predicate",p) // other string value
				datamap.StrVal("string_predicate2",sp2) // other string value
			}
		} else {
			datamap.StrVal("int_predicate",p) // other string value
			datamap.IntVal("id",si)

			//datamap.IntVal(p,oi)
			datamap.Pair(p, si,oi)
			//fmt.Printf("%d %s %d\n", si,p,oi)
		}
	}
	datamap.Report()
}
<|endoftext|>


<commit_before>// Copyright 2014 John DeWyze. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package bettercsv

import (
	"reflect"
	"strings"
	"testing"
)

var readTests = []struct {
	Name               string
	Input              string
	Output             [][]string
	OutputMap          []map[string]string
	UseFieldsPerRecord bool // false (default) means FieldsPerRecord is -1

	// These fields are copied into the Reader
	Comma            rune
	Comment          rune
	FieldsPerRecord  int
	LazyQuotes       bool
	TrailingComma    bool
	TrimLeadingSpace bool
	SkipLineOnErr    bool

	Error  string
	Line   int // Expected error line if != 0
	Column int // Expected error column if line != 0
	Errors []string
}{
	{
		Name:   "Simple",
		Input:  "a,b,c\n",
		Output: [][]string{{"a", "b", "c"}},
	},
	{
		Name:   "CRLF",
		Input:  "a,b\r\nc,d\r\n",
		Output: [][]string{{"a", "b"}, {"c", "d"}},
	},
	{
		Name:   "BareCR",
		Input:  "a,b\rc,d\r\n",
		Output: [][]string{{"a", "b\rc", "d"}},
	},
	{
		Name:               "RFC4180test",
		UseFieldsPerRecord: true,
		Input: `#field1,field2,field3
"aaa","bb
b","ccc"
"a,a","b""bb","ccc"
zzz,yyy,xxx
`,
		Output: [][]string{
			{"#field1", "field2", "field3"},
			{"aaa", "bb\nb", "ccc"},
			{"a,a", `b"bb`, "ccc"},
			{"zzz", "yyy", "xxx"},
		},
	},
	{
		Name:   "NoEOLTest",
		Input:  "a,b,c",
		Output: [][]string{{"a", "b", "c"}},
	},
	{
		Name:   "Semicolon",
		Comma:  ';',
		Input:  "a;b;c\n",
		Output: [][]string{{"a", "b", "c"}},
	},
	{
		Name: "MultiLine",
		Input: `"two
line","one line","three
line
field"`,
		Output: [][]string{{"two\nline", "one line", "three\nline\nfield"}},
	},
	{
		Name:  "BlankLine",
		Input: "a,b,c\n\nd,e,f\n\n",
		Output: [][]string{
			{"a", "b", "c"},
			{"d", "e", "f"},
		},
	},
	{
		Name:             "TrimSpace",
		Input:            " a,  b,   c\n",
		TrimLeadingSpace: true,
		Output:           [][]string{{"a", "b", "c"}},
	},
	{
		Name:   "LeadingSpace",
		Input:  " a,  b,   c\n",
		Output: [][]string{{" a", "  b", "   c"}},
	},
	{
		Name:    "Comment",
		Comment: '#',
		Input:   "#1,2,3\na,b,c\n#comment",
		Output:  [][]string{{"a", "b", "c"}},
	},
	{
		Name:   "NoComment",
		Input:  "#1,2,3\na,b,c",
		Output: [][]string{{"#1", "2", "3"}, {"a", "b", "c"}},
	},
	{
		Name:       "LazyQuotes",
		LazyQuotes: true,
		Input:      `a "word","1"2",a","b`,
		Output:     [][]string{{`a "word"`, `1"2`, `a"`, `b`}},
	},
	{
		Name:       "BareQuotes",
		LazyQuotes: true,
		Input:      `a "word","1"2",a"`,
		Output:     [][]string{{`a "word"`, `1"2`, `a"`}},
	},
	{
		Name:       "BareDoubleQuotes",
		LazyQuotes: true,
		Input:      `a""b,c`,
		Output:     [][]string{{`a""b`, `c`}},
	},
	{
		Name:  "BadDoubleQuotes",
		Input: `a""b,c`,
		Error: `bare " in non-quoted-field`, Line: 1, Column: 1,
	},
	{
		Name:             "TrimQuote",
		Input:            ` "a"," b",c`,
		TrimLeadingSpace: true,
		Output:           [][]string{{"a", " b", "c"}},
	},
	{
		Name:  "BadBareQuote",
		Input: `a "word","b"`,
		Error: `bare " in non-quoted-field`, Line: 1, Column: 2,
	},
	{
		Name:  "BadTrailingQuote",
		Input: `"a word",b"`,
		Error: `bare " in non-quoted-field`, Line: 1, Column: 10,
	},
	{
		Name:  "ExtraneousQuote",
		Input: `"a "word","b"`,
		Error: `extraneous " in field`, Line: 1, Column: 3,
	},
	{
		Name:               "BadFieldCount",
		UseFieldsPerRecord: true,
		Input:              "a,b,c\nd,e",
		Error:              "wrong number of fields", Line: 2,
	},
	{
		Name:               "BadFieldCount1",
		UseFieldsPerRecord: true,
		FieldsPerRecord:    2,
		Input:              `a,b,c`,
		Error:              "wrong number of fields", Line: 1,
	},
	{
		Name:   "FieldCount",
		Input:  "a,b,c\nd,e",
		Output: [][]string{{"a", "b", "c"}, {"d", "e"}},
	},
	{
		Name:   "TrailingCommaEOF",
		Input:  "a,b,c,",
		Output: [][]string{{"a", "b", "c", ""}},
	},
	{
		Name:   "TrailingCommaEOL",
		Input:  "a,b,c,\n",
		Output: [][]string{{"a", "b", "c", ""}},
	},
	{
		Name:             "TrailingCommaSpaceEOF",
		TrimLeadingSpace: true,
		Input:            "a,b,c, ",
		Output:           [][]string{{"a", "b", "c", ""}},
	},
	{
		Name:             "TrailingCommaSpaceEOL",
		TrimLeadingSpace: true,
		Input:            "a,b,c, \n",
		Output:           [][]string{{"a", "b", "c", ""}},
	},
	{
		Name:             "TrailingCommaLine3",
		TrimLeadingSpace: true,
		Input:            "a,b,c\nd,e,f\ng,hi,",
		Output:           [][]string{{"a", "b", "c"}, {"d", "e", "f"}, {"g", "hi", ""}},
	},
	{
		Name:   "NotTrailingComma3",
		Input:  "a,b,c, \n",
		Output: [][]string{{"a", "b", "c", " "}},
	},
	{
		Name:          "CommaFieldTest",
		TrailingComma: true,
		Input: `x,y,z,w
x,y,z,
x,y,,
x,,,
,,,
"x","y","z","w"
"x","y","z",""
"x","y","",""
"x","","",""
"","","",""
`,
		Output: [][]string{
			{"x", "y", "z", "w"},
			{"x", "y", "z", ""},
			{"x", "y", "", ""},
			{"x", "", "", ""},
			{"", "", "", ""},
			{"x", "y", "z", "w"},
			{"x", "y", "z", ""},
			{"x", "y", "", ""},
			{"x", "", "", ""},
			{"", "", "", ""},
		},
	},
	{
		Name:             "TrailingCommaIneffective1",
		TrailingComma:    true,
		TrimLeadingSpace: true,
		Input:            "a,b,\nc,d,e",
		Output: [][]string{
			{"a", "b", ""},
			{"c", "d", "e"},
		},
	},
	{
		Name:             "TrailingCommaIneffective2",
		TrailingComma:    false,
		TrimLeadingSpace: true,
		Input:            "a,b,\nc,d,e",
		Output: [][]string{
			{"a", "b", ""},
			{"c", "d", "e"},
		},
	},
	{
		Name:          "SkipLine1DoubleQuote",
		SkipLineOnErr: true,
		Input:         "a\nb\"\nc",
		Output:        [][]string{{"a"}, {"c"}},
		Errors:        []string{"line 2, column 2: bare \" in non-quoted-field"},
	},
	{
		Name:          "SkipLine2DoubleQuote",
		SkipLineOnErr: true,
		Input:         "a\nb\"b\"\nc",
		Output:        [][]string{{"a"}, {"c"}},
		Errors:        []string{"line 2, column 4: bare \" in non-quoted-field"},
	},
	{
		Name:               "SkipLineNoOfArgs",
		SkipLineOnErr:      true,
		UseFieldsPerRecord: true,
		Input:              "a,b,c\nd,e,f,g\nh,i,j",
		Output:             [][]string{{"a", "b", "c"}, {"h", "i", "j"}},
		Errors:             []string{"line 2, column 0: wrong number of fields in line"},
	},
	{
		Name:          "SkipLineExtraneousQuote",
		SkipLineOnErr: true,
		Input:         "a,b,c\nd,\"e\"e\",f\ng,h,i",
		Output:        [][]string{{"a", "b", "c"}, {"g", "h", "i"}},
		Errors:        []string{"line 2, column 8: extraneous \" in field"},
	},
	{
		Name:               "SkipLineMultilineFieldWithErrors",
		SkipLineOnErr:      true,
		UseFieldsPerRecord: true,
		Input:              "a,b,c\nd,\"e\"\nf\",g\nh,i,j",
		Output:             [][]string{{"a", "b", "c"}, {"h", "i", "j"}},
		Errors:             []string{"line 2, column 0: wrong number of fields in line", "line 3, column 4: bare \" in non-quoted-field"},
	},
	{
		Name:               "ReadAllWithHeaders",
		UseFieldsPerRecord: true,
		Input:              "a,b,c\n1,2,3\n4,5,6",
		OutputMap: []map[string]string{
			{"a": "a", "b": "b", "c": "c"},
			{"a": "1", "b": "2", "c": "3"},
			{"a": "4", "b": "5", "c": "6"}},
	},
	{
		Name:               "ReadAllWithHeadersAndErrors",
		UseFieldsPerRecord: true,
		Input:              "a,b,c\n1,2\",3\n4,5,6\n7,8,9,10\n11,12,13",
		Errors:             []string{"line 2, column 6: bare \" in non-quoted-field", "line 4, column 0: wrong number of fields in line"},
		OutputMap: []map[string]string{
			{"a": "a", "b": "b", "c": "c"},
			{"a": "4", "b": "5", "c": "6"},
			{"a": "11", "b": "12", "c": "13"}},
	},
}

func TestRead(t *testing.T) {
	for _, tt := range readTests {
		r := NewReader(strings.NewReader(tt.Input))
		r.Comment = tt.Comment
		if tt.UseFieldsPerRecord {
			r.FieldsPerRecord = tt.FieldsPerRecord
		} else {
			r.FieldsPerRecord = -1
		}
		r.LazyQuotes = tt.LazyQuotes
		r.TrailingComma = tt.TrailingComma
		r.TrimLeadingSpace = tt.TrimLeadingSpace
		r.SkipLineOnErr = tt.SkipLineOnErr
		if tt.Comma != 0 {
			r.Comma = tt.Comma
		}
		if tt.Name == "ReadAllWithHeaders" {
			out, err := r.ReadAllWithHeaders()
			if err != nil {
				t.Errorf("%s: unexpected error %v", tt.Name, err)
			} else if !reflect.DeepEqual(out, tt.OutputMap) {
				t.Errorf("%s: out=%q want %q", tt.Name, out, tt.OutputMap)
			}
		} else if tt.Name == "ReadAllWithHeadersAndErrors" {
			out, errs := r.ReadAllWithHeadersAndErrors()
			if !reflect.DeepEqual(out, tt.OutputMap) {
				t.Errorf("%s: out=%q want %q", tt.Name, out, tt.OutputMap)
			}
			var errorStrings []string
			for _, err := range errs {
				errorStrings = append(errorStrings, err.Error())
			}
			if !reflect.DeepEqual(errs, tt.Errors) {
				t.Errorf("%s: errors=%q want %q", tt.Name, errorStrings, tt.Errors)
			}
		} else {
			out, err := r.ReadAll()
			perr, _ := err.(*ParseError)
			if tt.Error != "" && !tt.SkipLineOnErr {
				if err == nil || !strings.Contains(err.Error(), tt.Error) {
					t.Errorf("%s: error %v, want error %q", tt.Name, err, tt.Error)
				} else if tt.Line != 0 && (tt.Line != perr.Line || tt.Column != perr.Column) {
					t.Errorf("%s: error at %d:%d expected %d:%d", tt.Name, perr.Line, perr.Column, tt.Line, tt.Column)
				}
			} else if err != nil && !tt.SkipLineOnErr {
				t.Errorf("%s: unexpected error %v", tt.Name, err)
			} else if !reflect.DeepEqual(out, tt.Output) {
				t.Errorf("%s: out=%q want %q", tt.Name, out, tt.Output)
			}
			if tt.SkipLineOnErr {
				r := NewReader(strings.NewReader(tt.Input))
				_, errors := r.ReadAllWithErrors()
				var errorStrings []string
				for _, err := range errors {
					errorStrings = append(errorStrings, err.Error())
				}
				if !reflect.DeepEqual(errorStrings, tt.Errors) {
					t.Errorf("%s: errors=%q want %q", tt.Name, errors, tt.Errors)
				}
			}
		}
	}
}
<commit_msg>Refactor tests based on Read method flags<commit_after>// Copyright 2014 John DeWyze. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package bettercsv

import (
	"reflect"
	"strings"
	"testing"
)

var readTests = []struct {
	Name               string
	Input              string
	Output             [][]string
	OutputMap          []map[string]string
	UseFieldsPerRecord bool // false (default) means FieldsPerRecord is -1
	UseHeaders         bool // true means use Headers methods for reading
	UseHeadersAndErrs  bool // true means use HeadersAndErrors methods for reading

	// These fields are copied into the Reader
	Comma            rune
	Comment          rune
	FieldsPerRecord  int
	LazyQuotes       bool
	TrailingComma    bool
	TrimLeadingSpace bool
	SkipLineOnErr    bool

	Error  string
	Line   int // Expected error line if != 0
	Column int // Expected error column if line != 0
	Errors []string
}{
	{
		Name:   "Simple",
		Input:  "a,b,c\n",
		Output: [][]string{{"a", "b", "c"}},
	},
	{
		Name:   "CRLF",
		Input:  "a,b\r\nc,d\r\n",
		Output: [][]string{{"a", "b"}, {"c", "d"}},
	},
	{
		Name:   "BareCR",
		Input:  "a,b\rc,d\r\n",
		Output: [][]string{{"a", "b\rc", "d"}},
	},
	{
		Name:               "RFC4180test",
		UseFieldsPerRecord: true,
		Input: `#field1,field2,field3
"aaa","bb
b","ccc"
"a,a","b""bb","ccc"
zzz,yyy,xxx
`,
		Output: [][]string{
			{"#field1", "field2", "field3"},
			{"aaa", "bb\nb", "ccc"},
			{"a,a", `b"bb`, "ccc"},
			{"zzz", "yyy", "xxx"},
		},
	},
	{
		Name:   "NoEOLTest",
		Input:  "a,b,c",
		Output: [][]string{{"a", "b", "c"}},
	},
	{
		Name:   "Semicolon",
		Comma:  ';',
		Input:  "a;b;c\n",
		Output: [][]string{{"a", "b", "c"}},
	},
	{
		Name: "MultiLine",
		Input: `"two
line","one line","three
line
field"`,
		Output: [][]string{{"two\nline", "one line", "three\nline\nfield"}},
	},
	{
		Name:  "BlankLine",
		Input: "a,b,c\n\nd,e,f\n\n",
		Output: [][]string{
			{"a", "b", "c"},
			{"d", "e", "f"},
		},
	},
	{
		Name:             "TrimSpace",
		Input:            " a,  b,   c\n",
		TrimLeadingSpace: true,
		Output:           [][]string{{"a", "b", "c"}},
	},
	{
		Name:   "LeadingSpace",
		Input:  " a,  b,   c\n",
		Output: [][]string{{" a", "  b", "   c"}},
	},
	{
		Name:    "Comment",
		Comment: '#',
		Input:   "#1,2,3\na,b,c\n#comment",
		Output:  [][]string{{"a", "b", "c"}},
	},
	{
		Name:   "NoComment",
		Input:  "#1,2,3\na,b,c",
		Output: [][]string{{"#1", "2", "3"}, {"a", "b", "c"}},
	},
	{
		Name:       "LazyQuotes",
		LazyQuotes: true,
		Input:      `a "word","1"2",a","b`,
		Output:     [][]string{{`a "word"`, `1"2`, `a"`, `b`}},
	},
	{
		Name:       "BareQuotes",
		LazyQuotes: true,
		Input:      `a "word","1"2",a"`,
		Output:     [][]string{{`a "word"`, `1"2`, `a"`}},
	},
	{
		Name:       "BareDoubleQuotes",
		LazyQuotes: true,
		Input:      `a""b,c`,
		Output:     [][]string{{`a""b`, `c`}},
	},
	{
		Name:  "BadDoubleQuotes",
		Input: `a""b,c`,
		Error: `bare " in non-quoted-field`, Line: 1, Column: 1,
	},
	{
		Name:             "TrimQuote",
		Input:            ` "a"," b",c`,
		TrimLeadingSpace: true,
		Output:           [][]string{{"a", " b", "c"}},
	},
	{
		Name:  "BadBareQuote",
		Input: `a "word","b"`,
		Error: `bare " in non-quoted-field`, Line: 1, Column: 2,
	},
	{
		Name:  "BadTrailingQuote",
		Input: `"a word",b"`,
		Error: `bare " in non-quoted-field`, Line: 1, Column: 10,
	},
	{
		Name:  "ExtraneousQuote",
		Input: `"a "word","b"`,
		Error: `extraneous " in field`, Line: 1, Column: 3,
	},
	{
		Name:               "BadFieldCount",
		UseFieldsPerRecord: true,
		Input:              "a,b,c\nd,e",
		Error:              "wrong number of fields", Line: 2,
	},
	{
		Name:               "BadFieldCount1",
		UseFieldsPerRecord: true,
		FieldsPerRecord:    2,
		Input:              `a,b,c`,
		Error:              "wrong number of fields", Line: 1,
	},
	{
		Name:   "FieldCount",
		Input:  "a,b,c\nd,e",
		Output: [][]string{{"a", "b", "c"}, {"d", "e"}},
	},
	{
		Name:   "TrailingCommaEOF",
		Input:  "a,b,c,",
		Output: [][]string{{"a", "b", "c", ""}},
	},
	{
		Name:   "TrailingCommaEOL",
		Input:  "a,b,c,\n",
		Output: [][]string{{"a", "b", "c", ""}},
	},
	{
		Name:             "TrailingCommaSpaceEOF",
		TrimLeadingSpace: true,
		Input:            "a,b,c, ",
		Output:           [][]string{{"a", "b", "c", ""}},
	},
	{
		Name:             "TrailingCommaSpaceEOL",
		TrimLeadingSpace: true,
		Input:            "a,b,c, \n",
		Output:           [][]string{{"a", "b", "c", ""}},
	},
	{
		Name:             "TrailingCommaLine3",
		TrimLeadingSpace: true,
		Input:            "a,b,c\nd,e,f\ng,hi,",
		Output:           [][]string{{"a", "b", "c"}, {"d", "e", "f"}, {"g", "hi", ""}},
	},
	{
		Name:   "NotTrailingComma3",
		Input:  "a,b,c, \n",
		Output: [][]string{{"a", "b", "c", " "}},
	},
	{
		Name:          "CommaFieldTest",
		TrailingComma: true,
		Input: `x,y,z,w
x,y,z,
x,y,,
x,,,
,,,
"x","y","z","w"
"x","y","z",""
"x","y","",""
"x","","",""
"","","",""
`,
		Output: [][]string{
			{"x", "y", "z", "w"},
			{"x", "y", "z", ""},
			{"x", "y", "", ""},
			{"x", "", "", ""},
			{"", "", "", ""},
			{"x", "y", "z", "w"},
			{"x", "y", "z", ""},
			{"x", "y", "", ""},
			{"x", "", "", ""},
			{"", "", "", ""},
		},
	},
	{
		Name:             "TrailingCommaIneffective1",
		TrailingComma:    true,
		TrimLeadingSpace: true,
		Input:            "a,b,\nc,d,e",
		Output: [][]string{
			{"a", "b", ""},
			{"c", "d", "e"},
		},
	},
	{
		Name:             "TrailingCommaIneffective2",
		TrailingComma:    false,
		TrimLeadingSpace: true,
		Input:            "a,b,\nc,d,e",
		Output: [][]string{
			{"a", "b", ""},
			{"c", "d", "e"},
		},
	},
	{
		Name:          "SkipLine1DoubleQuote",
		SkipLineOnErr: true,
		Input:         "a\nb\"\nc",
		Output:        [][]string{{"a"}, {"c"}},
		Errors:        []string{"line 2, column 2: bare \" in non-quoted-field"},
	},
	{
		Name:          "SkipLine2DoubleQuote",
		SkipLineOnErr: true,
		Input:         "a\nb\"b\"\nc",
		Output:        [][]string{{"a"}, {"c"}},
		Errors:        []string{"line 2, column 4: bare \" in non-quoted-field"},
	},
	{
		Name:               "SkipLineNoOfArgs",
		SkipLineOnErr:      true,
		UseFieldsPerRecord: true,
		Input:              "a,b,c\nd,e,f,g\nh,i,j",
		Output:             [][]string{{"a", "b", "c"}, {"h", "i", "j"}},
		Errors:             []string{"line 2, column 0: wrong number of fields in line"},
	},
	{
		Name:          "SkipLineExtraneousQuote",
		SkipLineOnErr: true,
		Input:         "a,b,c\nd,\"e\"e\",f\ng,h,i",
		Output:        [][]string{{"a", "b", "c"}, {"g", "h", "i"}},
		Errors:        []string{"line 2, column 8: extraneous \" in field"},
	},
	{
		Name:               "SkipLineMultilineFieldWithErrors",
		SkipLineOnErr:      true,
		UseFieldsPerRecord: true,
		Input:              "a,b,c\nd,\"e\"\nf\",g\nh,i,j",
		Output:             [][]string{{"a", "b", "c"}, {"h", "i", "j"}},
		Errors:             []string{"line 2, column 0: wrong number of fields in line", "line 3, column 4: bare \" in non-quoted-field"},
	},
	{
		Name:               "ReadAllWithHeaders",
		UseFieldsPerRecord: true,
		UseHeaders:         true,
		Input:              "a,b,c\n1,2,3\n4,5,6",
		OutputMap: []map[string]string{
			{"a": "a", "b": "b", "c": "c"},
			{"a": "1", "b": "2", "c": "3"},
			{"a": "4", "b": "5", "c": "6"}},
	},
	{
		Name:               "ReadAllWithHeadersAndErrors",
		UseFieldsPerRecord: true,
		UseHeadersAndErrs:  true,
		Input:              "a,b,c\n1,2\",3\n4,5,6\n7,8,9,10\n11,12,13",
		Errors:             []string{"line 2, column 6: bare \" in non-quoted-field", "line 4, column 0: wrong number of fields in line"},
		OutputMap: []map[string]string{
			{"a": "a", "b": "b", "c": "c"},
			{"a": "4", "b": "5", "c": "6"},
			{"a": "11", "b": "12", "c": "13"}},
	},
}

func TestRead(t *testing.T) {
	for _, tt := range readTests {
		r := NewReader(strings.NewReader(tt.Input))
		r.Comment = tt.Comment
		if tt.UseFieldsPerRecord {
			r.FieldsPerRecord = tt.FieldsPerRecord
		} else {
			r.FieldsPerRecord = -1
		}
		r.LazyQuotes = tt.LazyQuotes
		r.TrailingComma = tt.TrailingComma
		r.TrimLeadingSpace = tt.TrimLeadingSpace
		r.SkipLineOnErr = tt.SkipLineOnErr
		if tt.Comma != 0 {
			r.Comma = tt.Comma
		}
		if tt.SkipLineOnErr {
			out, errors := r.ReadAllWithErrors()
			var errorStrings []string
			for _, err := range errors {
				errorStrings = append(errorStrings, err.Error())
			}
			if !reflect.DeepEqual(errorStrings, tt.Errors) {
				t.Errorf("%s: errors=%q want %q", tt.Name, errors, tt.Errors)
			}
			if !reflect.DeepEqual(out, tt.Output) {
				t.Errorf("%s: out=%q want %q", tt.Name, out, tt.Output)
			}
		} else if tt.UseHeaders {
			out, err := r.ReadAllWithHeaders()
			if err != nil {
				t.Errorf("%s: unexpected error %v", tt.Name, err)
			} else if !reflect.DeepEqual(out, tt.OutputMap) {
				t.Errorf("%s: out=%q want %q", tt.Name, out, tt.OutputMap)
			}
		} else if tt.UseHeadersAndErrs {
			out, errs := r.ReadAllWithHeadersAndErrors()
			if !reflect.DeepEqual(out, tt.OutputMap) {
				t.Errorf("%s: out=%q want %q", tt.Name, out, tt.OutputMap)
			}
			var errorStrings []string
			for _, err := range errs {
				errorStrings = append(errorStrings, err.Error())
			}
			if !reflect.DeepEqual(errorStrings, tt.Errors) {
				t.Errorf("%s: errors=%q want %q", tt.Name, errorStrings, tt.Errors)
			}
		} else {
			out, err := r.ReadAll()
			perr, _ := err.(*ParseError)
			if tt.Error != "" && !tt.SkipLineOnErr {
				if err == nil || !strings.Contains(err.Error(), tt.Error) {
					t.Errorf("%s: error %v, want error %q", tt.Name, err, tt.Error)
				} else if tt.Line != 0 && (tt.Line != perr.Line || tt.Column != perr.Column) {
					t.Errorf("%s: error at %d:%d expected %d:%d", tt.Name, perr.Line, perr.Column, tt.Line, tt.Column)
				}
			} else if err != nil && !tt.SkipLineOnErr {
				t.Errorf("%s: unexpected error %v", tt.Name, err)
			} else if !reflect.DeepEqual(out, tt.Output) {
				t.Errorf("%s: out=%q want %q", tt.Name, out, tt.Output)
			}
		}
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"

	"github.com/freneticmonkey/migrate/go/config"
	"github.com/freneticmonkey/migrate/go/configsetup"
	"github.com/freneticmonkey/migrate/go/git"
	"github.com/freneticmonkey/migrate/go/id"
	"github.com/freneticmonkey/migrate/go/migration"
	"github.com/freneticmonkey/migrate/go/mysql"
	"github.com/freneticmonkey/migrate/go/table"
	"github.com/freneticmonkey/migrate/go/util"
	"github.com/freneticmonkey/migrate/go/yaml"
	"github.com/urfave/cli"
)

// GetCreateCommand Create a new migration for the target project at the version indicated by hash
func GetCreateCommand() (setup cli.Command) {
	setup = cli.Command{
		Name:  "create",
		Usage: "This subcommand is used to create a migration and register it with the management database.",
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "version",
				Value: "",
				Usage: "The target git version",
			},
			cli.BoolFlag{
				Name:  "rollback",
				Usage: "Force a rollback (backward) migration to be created",
			},
			cli.StringFlag{
				Name:  "gitinfo",
				Value: "",
				Usage: "Provide a git info file for the schema in the working path",
			},
			cli.BoolFlag{
				Name:  "no-clone",
				Usage: "Do not clone from git.  Use this when the yaml files have already been checked out.",
			},
		},
		Action: func(ctx *cli.Context) error {
			var version string
			var gitinfo string
			var rollback bool
			var clone bool

			rollback = false
			clone = true

			// Override the project settings with the command line flags
			if ctx.IsSet("version") {
				version = ctx.String("version")
			} else if ctx.IsSet("gitinfo") {
				gitinfo = ctx.String("gitinfo")
			} else {
				cli.ShowSubcommandHelp(ctx)
				return cli.NewExitError("Unable to generate a migration.  Please specify a target version to migrate to.", 1)
			}

			if ctx.IsSet("no-clone") {
				clone = false
			}

			// Parse global flags
			parseGlobalFlags(ctx)

			// Setup the management database and configuration settings
			conf, err := configsetup.ConfigureManagement()

			if err != nil {
				return cli.NewExitError(fmt.Sprintf("Configuration Load failed. Error: %v", err), 1)
			}

			if ctx.IsSet("rollback") {
				rollback = ctx.Bool("rollback")
			}

			return create(version, gitinfo, clone, rollback, conf)

		},
	}
	return setup
}

func create(version string, gitinfo string, clone bool, rollback bool, conf config.Config) *cli.ExitError {
	var problems id.ValidationErrors
	var ts string
	var info string
	var err error

	// extract the contents of the gitinfo file
	if len(gitinfo) > 0 {
		version, info, ts, err = git.GetVersionDetailsFile(gitinfo)

		util.LogInfof("Detected gitinfo file. Parsed:\nVersion: %s\nTime: %s\nInfo:\n>>>\n%s\n<<<", version, ts, info)

		if err != nil {
			return cli.NewExitError(fmt.Sprintf("Creation failed.  Unable to read the gitinfo file: %v", err), 1)
		}
	}

	// Override the project settings with the command line flags
	if version != "" {
		conf.Project.Schema.Version = version
	} else {
		// if the version hasn't been defined
		return cli.NewExitError("Creation failed.  Unable to generate a migration as no version was defined to migrate to", 1)
	}

	util.LogInfo("Creating Migration for Version: " + version)

	// If the schema needs to be cloned from git.  Extract it now
	if clone {
		// Clone the target Git Repo
		git.Clone(conf.Project)

		ts, err = git.GetVersionTime(conf.Project.Name, conf.Project.Schema.Version)
		if util.ErrorCheck(err) {
			return cli.NewExitError("Create failed. Unable to obtain Version Timestamp from Git checkout", 1)
		}
		info, err = git.GetVersionDetails(conf.Project.Name, conf.Project.Schema.Version)
		if util.ErrorCheck(err) {
			return cli.NewExitError("Create failed. Unable to obtain Version Details from Git checkout", 1)
		}
	}

	// Read the YAML files cloned from the repo
	err = yaml.ReadTables(conf)
	if util.ErrorCheck(err) {
		return cli.NewExitError("Creation failed. Unable to read YAML Tables", 1)
	}
	problems, err = id.ValidateSchema(yaml.Schema, "YAML Schema", true)
	if util.ErrorCheck(err) {
		return cli.NewExitError("Creation failed. YAML Validation Errors Detected", problems.Count())
	}

	// Read the MySQL tables from the target database
	err = mysql.ReadTables(conf)
	if util.ErrorCheck(err) {
		return cli.NewExitError("Creation failed. Unable to read MySQL Tables", 1)
	}
	problems, err = id.ValidateSchema(mysql.Schema, "Target Database Schema", true)
	if util.ErrorCheck(err) {
		return cli.NewExitError("Creation failed. Target Database Validation Errors Detected", problems.Count())
	}

	problems, err = id.ValidatePropertyIDs(yaml.Schema, mysql.Schema, true)
	if util.ErrorCheck(err) {
		return cli.NewExitError("Validation failed. Detected YAML PropertyID problems", problems.Count())
	}

	forwardDiff, err := table.DiffTables(yaml.Schema, mysql.Schema, false)
	if util.ErrorCheckf(err, "Diff Failed while generating forward migration") {
		return cli.NewExitError("Create failed. Unable to generate a forward migration", 1)
	}
	forwardOps := mysql.GenerateAlters(forwardDiff)

	backwardDiff, err := table.DiffTables(mysql.Schema, yaml.Schema, false)
	if util.ErrorCheckf(err, "Diff Failed while generating backward migration") {
		return cli.NewExitError("Create failed. Unable to generate a backward migration", 1)
	}
	backwardOps := mysql.GenerateAlters(backwardDiff)

	m, err := migration.New(migration.Param{
		Project:     conf.Project.Name,
		Version:     conf.Project.Schema.Version,
		Timestamp:   ts,
		Description: info,
		Forwards:    forwardOps,
		Backwards:   backwardOps,
		Rollback:    rollback,
	})
	if util.ErrorCheck(err) {
		return cli.NewExitError("Create failed. Unable to create new Migration in the management database", 1)
	}

	success := fmt.Sprintf("Created Migration successfully with ID: [%d]", m.MID)

	return cli.NewExitError(success, 0)
}
<commit_msg>Moved create log statement so that it makes sense if there is an error<commit_after>package cmd

import (
	"fmt"

	"github.com/freneticmonkey/migrate/go/config"
	"github.com/freneticmonkey/migrate/go/configsetup"
	"github.com/freneticmonkey/migrate/go/git"
	"github.com/freneticmonkey/migrate/go/id"
	"github.com/freneticmonkey/migrate/go/migration"
	"github.com/freneticmonkey/migrate/go/mysql"
	"github.com/freneticmonkey/migrate/go/table"
	"github.com/freneticmonkey/migrate/go/util"
	"github.com/freneticmonkey/migrate/go/yaml"
	"github.com/urfave/cli"
)

// GetCreateCommand Create a new migration for the target project at the version indicated by hash
func GetCreateCommand() (setup cli.Command) {
	setup = cli.Command{
		Name:  "create",
		Usage: "This subcommand is used to create a migration and register it with the management database.",
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "version",
				Value: "",
				Usage: "The target git version",
			},
			cli.BoolFlag{
				Name:  "rollback",
				Usage: "Force a rollback (backward) migration to be created",
			},
			cli.StringFlag{
				Name:  "gitinfo",
				Value: "",
				Usage: "Provide a git info file for the schema in the working path",
			},
			cli.BoolFlag{
				Name:  "no-clone",
				Usage: "Do not clone from git.  Use this when the yaml files have already been checked out.",
			},
		},
		Action: func(ctx *cli.Context) error {
			var version string
			var gitinfo string
			var rollback bool
			var clone bool

			rollback = false
			clone = true

			// Override the project settings with the command line flags
			if ctx.IsSet("version") {
				version = ctx.String("version")
			} else if ctx.IsSet("gitinfo") {
				gitinfo = ctx.String("gitinfo")
			} else {
				cli.ShowSubcommandHelp(ctx)
				return cli.NewExitError("Unable to generate a migration.  Please specify a target version to migrate to.", 1)
			}

			if ctx.IsSet("no-clone") {
				clone = false
			}

			// Parse global flags
			parseGlobalFlags(ctx)

			// Setup the management database and configuration settings
			conf, err := configsetup.ConfigureManagement()

			if err != nil {
				return cli.NewExitError(fmt.Sprintf("Configuration Load failed. Error: %v", err), 1)
			}

			if ctx.IsSet("rollback") {
				rollback = ctx.Bool("rollback")
			}

			return create(version, gitinfo, clone, rollback, conf)

		},
	}
	return setup
}

func create(version string, gitinfo string, clone bool, rollback bool, conf config.Config) *cli.ExitError {
	var problems id.ValidationErrors
	var ts string
	var info string
	var err error

	// extract the contents of the gitinfo file
	if len(gitinfo) > 0 {
		version, info, ts, err = git.GetVersionDetailsFile(gitinfo)

		if err != nil {
			return cli.NewExitError(fmt.Sprintf("Creation failed.  Unable to read the gitinfo file: %v", err), 1)
		}

		util.LogInfof("Detected gitinfo file. Parsed:\nVersion: %s\nTime: %s\nInfo:\n>>>\n%s\n<<<", version, ts, info)
	}

	// Override the project settings with the command line flags
	if version != "" {
		conf.Project.Schema.Version = version
	} else {
		// if the version hasn't been defined
		return cli.NewExitError("Creation failed.  Unable to generate a migration as no version was defined to migrate to", 1)
	}

	util.LogInfo("Creating Migration for Version: " + version)

	// If the schema needs to be cloned from git.  Extract it now
	if clone {
		// Clone the target Git Repo
		git.Clone(conf.Project)

		ts, err = git.GetVersionTime(conf.Project.Name, conf.Project.Schema.Version)
		if util.ErrorCheck(err) {
			return cli.NewExitError("Create failed. Unable to obtain Version Timestamp from Git checkout", 1)
		}
		info, err = git.GetVersionDetails(conf.Project.Name, conf.Project.Schema.Version)
		if util.ErrorCheck(err) {
			return cli.NewExitError("Create failed. Unable to obtain Version Details from Git checkout", 1)
		}
	}

	// Read the YAML files cloned from the repo
	err = yaml.ReadTables(conf)
	if util.ErrorCheck(err) {
		return cli.NewExitError("Creation failed. Unable to read YAML Tables", 1)
	}
	problems, err = id.ValidateSchema(yaml.Schema, "YAML Schema", true)
	if util.ErrorCheck(err) {
		return cli.NewExitError("Creation failed. YAML Validation Errors Detected", problems.Count())
	}

	// Read the MySQL tables from the target database
	err = mysql.ReadTables(conf)
	if util.ErrorCheck(err) {
		return cli.NewExitError("Creation failed. Unable to read MySQL Tables", 1)
	}
	problems, err = id.ValidateSchema(mysql.Schema, "Target Database Schema", true)
	if util.ErrorCheck(err) {
		return cli.NewExitError("Creation failed. Target Database Validation Errors Detected", problems.Count())
	}

	problems, err = id.ValidatePropertyIDs(yaml.Schema, mysql.Schema, true)
	if util.ErrorCheck(err) {
		return cli.NewExitError("Validation failed. Detected YAML PropertyID problems", problems.Count())
	}

	forwardDiff, err := table.DiffTables(yaml.Schema, mysql.Schema, false)
	if util.ErrorCheckf(err, "Diff Failed while generating forward migration") {
		return cli.NewExitError("Create failed. Unable to generate a forward migration", 1)
	}
	forwardOps := mysql.GenerateAlters(forwardDiff)

	backwardDiff, err := table.DiffTables(mysql.Schema, yaml.Schema, false)
	if util.ErrorCheckf(err, "Diff Failed while generating backward migration") {
		return cli.NewExitError("Create failed. Unable to generate a backward migration", 1)
	}
	backwardOps := mysql.GenerateAlters(backwardDiff)

	m, err := migration.New(migration.Param{
		Project:     conf.Project.Name,
		Version:     conf.Project.Schema.Version,
		Timestamp:   ts,
		Description: info,
		Forwards:    forwardOps,
		Backwards:   backwardOps,
		Rollback:    rollback,
	})
	if util.ErrorCheck(err) {
		return cli.NewExitError("Create failed. Unable to create new Migration in the management database", 1)
	}

	success := fmt.Sprintf("Created Migration successfully with ID: [%d]", m.MID)

	return cli.NewExitError(success, 0)
}
<|endoftext|>


<commit_before>// Simple mandelbrot generator in Go
// Owain Kenway

package main

import ("pnmmodules")

func main() {
    var pixels[][] int
    var xmin, xmax, ymin, ymax, rmax, x0, y0, x, y, xt float64
    var xres, yres, max_iter, iter, px, py int

    xmax = 2.0
    xmin = -2.0

    ymax = 2.0
    ymin = -2.0

    xres = 2048
    yres = 2048

    max_iter = 256

    rmax = 2.0

// Allocate slice of slices.
// A 2D slice is a slice of slices.
    pixels = make([][]int, xres)
    for px = 0; px < xres; px++ {

// Allocate slice in slice of slices
        pixels[px] = make([]int, yres)

        x0 = ((float64(px+1)/float64(xres)) * float64(xmax - xmin)) + xmin

        for py = 0; py < yres; py++ {
            y0 = ((float64(py+1)/float64(yres)) * float64(ymax - ymin)) + ymin

            iter = 0
            x = 0.0
            y = 0.0

            for (((x*x + y*y) < (rmax * rmax)) && (iter < max_iter)) {
                xt = (x*x) - (y*y) + x0
                y = 2.0 * x * y + y0
                x = xt
                iter = iter + 1
            }
            pixels[px][py] = max_iter - iter
        }

    }

// Call our Writepgm routine from pgmmodules.
    pnmmodules.Writepgm(pixels, max_iter - 1, "mandel.pgm")

}<commit_msg>Remove extra whitespace.<commit_after>// Simple mandelbrot generator in Go
// Owain Kenway

package main

import ("pnmmodules")

func main() {
    var pixels[][]int
    var xmin, xmax, ymin, ymax, rmax, x0, y0, x, y, xt float64
    var xres, yres, max_iter, iter, px, py int

    xmax = 2.0
    xmin = -2.0

    ymax = 2.0
    ymin = -2.0

    xres = 2048
    yres = 2048

    max_iter = 256

    rmax = 2.0

// Allocate slice of slices.
// A 2D slice is a slice of slices.
    pixels = make([][]int, xres)
    for px = 0; px < xres; px++ {

// Allocate slice in slice of slices
        pixels[px] = make([]int, yres)

        x0 = ((float64(px+1)/float64(xres)) * float64(xmax - xmin)) + xmin

        for py = 0; py < yres; py++ {
            y0 = ((float64(py+1)/float64(yres)) * float64(ymax - ymin)) + ymin

            iter = 0
            x = 0.0
            y = 0.0

            for (((x*x + y*y) < (rmax * rmax)) && (iter < max_iter)) {
                xt = (x*x) - (y*y) + x0
                y = 2.0 * x * y + y0
                x = xt
                iter = iter + 1
            }
            pixels[px][py] = max_iter - iter
        }

    }

// Call our Writepgm routine from pgmmodules.
    pnmmodules.Writepgm(pixels, max_iter - 1, "mandel.pgm")

}
<|endoftext|>