<commit_before>package catlistnodes

import (
	"flag"
	"fmt"
	"sort"
	"strings"

	"github.com/hashicorp/consul/api"
	"github.com/hashicorp/consul/command/flags"
	"github.com/mitchellh/cli"
	"github.com/ryanuber/columnize"
)

func New(ui cli.Ui) *cmd {
	c := &cmd{UI: ui}
	c.init()
	return c
}

type cmd struct {
	UI      cli.Ui
	flags   *flag.FlagSet
	http    *flags.HTTPFlags
	helpStr string

	// flags
	detailed bool
	near     string
	nodeMeta map[string]string
	service  string
}

const helpPrefix = `Usage: consul catalog nodes [options]

  Retrieves the list nodes registered in a given datacenter. By default, the
  datacenter of the local agent is queried.

  To retrieve the list of nodes:

      $ consul catalog nodes

  To print detailed information including full node IDs, tagged addresses, and
  metadata information:

      $ consul catalog nodes -detailed

  To list nodes which are running a particular service:

      $ consul catalog nodes -service=web

  To filter by node metadata:

      $ consul catalog nodes -node-meta="foo=bar"

  To sort nodes by estimated round-trip time from node-web:

      $ consul catalog nodes -near=node-web

  For a full list of options and examples, please see the Consul documentation.`

// init sets up command flags and help text
func (c *cmd) init() {
	c.flags = flag.NewFlagSet("", flag.ContinueOnError)
	c.flags.BoolVar(&c.detailed, "detailed", false, "Output detailed information about "+
		"the nodes including their addresses and metadata.")
	c.flags.StringVar(&c.near, "near", "", "Node name to sort the node list in ascending "+
		"order based on estimated round-trip time from that node. "+
		"Passing \"_agent\" will use this agent's node for sorting.")
	c.flags.Var((*flags.FlagMapValue)(&c.nodeMeta), "node-meta", "Metadata to "+
		"filter nodes with the given `key=value` pairs. This flag may be "+
		"specified multiple times to filter on multiple sources of metadata.")
	c.flags.StringVar(&c.service, "service", "", "Service `id or name` to filter nodes. "+
		"Only nodes which are providing the given service will be returned.")

	c.http = &flags.HTTPFlags{}
	flags.Merge(c.flags, c.http.ClientFlags())
	flags.Merge(c.flags, c.http.ServerFlags())

	c.helpStr = flags.Usage(helpPrefix, c.flags, c.http.ClientFlags(), c.http.ServerFlags())
}

func (c *cmd) Run(args []string) int {
	if err := c.flags.Parse(args); err != nil {
		return 1
	}

	if l := len(c.flags.Args()); l > 0 {
		c.UI.Error(fmt.Sprintf("Too many arguments (expected 0, got %d)", l))
		return 1
	}

	// Create and test the HTTP client
	client, err := c.http.APIClient()
	if err != nil {
		c.UI.Error(fmt.Sprintf("Error connecting to Consul agent: %s", err))
		return 1
	}

	var nodes []*api.Node
	if c.service != "" {
		services, _, err := client.Catalog().Service(c.service, "", &api.QueryOptions{
			Near:     c.near,
			NodeMeta: c.nodeMeta,
		})
		if err != nil {
			c.UI.Error(fmt.Sprintf("Error listing nodes for service: %s", err))
			return 1
		}

		nodes = make([]*api.Node, len(services))
		for i, s := range services {
			nodes[i] = &api.Node{
				ID:              s.ID,
				Node:            s.Node,
				Address:         s.Address,
				Datacenter:      s.Datacenter,
				TaggedAddresses: s.TaggedAddresses,
				Meta:            s.NodeMeta,
				CreateIndex:     s.CreateIndex,
				ModifyIndex:     s.ModifyIndex,
			}
		}
	} else {
		nodes, _, err = client.Catalog().Nodes(&api.QueryOptions{
			Near:     c.near,
			NodeMeta: c.nodeMeta,
		})
		if err != nil {
			c.UI.Error(fmt.Sprintf("Error listing nodes: %s", err))
			return 1
		}
	}

	// Handle the edge case where there are no nodes that match the query.
	if len(nodes) == 0 {
		c.UI.Error("No nodes match the given query - try expanding your search.")
		return 0
	}

	output, err := printNodes(nodes, c.detailed)
	if err != nil {
		c.UI.Error(fmt.Sprintf("Error printing nodes: %s", err))
		return 1
	}

	c.UI.Info(output)

	return 0
}

func (c *cmd) Synopsis() string {
	return "Lists all nodes in the given datacenter"
}

func (c *cmd) Help() string {
	return c.helpStr
}

// printNodes accepts a list of nodes and prints information in a tabular
// format about the nodes.
func printNodes(nodes []*api.Node, detailed bool) (string, error) {
	var result []string
	if detailed {
		result = detailedNodes(nodes)
	} else {
		result = simpleNodes(nodes)
	}

	return columnize.SimpleFormat(result), nil
}

func detailedNodes(nodes []*api.Node) []string {
	result := make([]string, 0, len(nodes)+1)
	header := "Node|ID|Address|DC|TaggedAddresses|Meta"
	result = append(result, header)

	for _, node := range nodes {
		result = append(result, fmt.Sprintf("%s|%s|%s|%s|%s|%s",
			node.Node, node.ID, node.Address, node.Datacenter,
			mapToKV(node.TaggedAddresses, ", "), mapToKV(node.Meta, ", ")))
	}

	return result
}

func simpleNodes(nodes []*api.Node) []string {
	result := make([]string, 0, len(nodes)+1)
	header := "Node|ID|Address|DC"
	result = append(result, header)

	for _, node := range nodes {
		// Shorten the ID in non-detailed mode to just the first octet.
		id := node.ID
		idx := strings.Index(id, "-")
		if idx > 0 {
			id = id[0:idx]
		}
		result = append(result, fmt.Sprintf("%s|%s|%s|%s",
			node.Node, id, node.Address, node.Datacenter))
	}

	return result
}

// mapToKV converts a map[string]string into a human-friendly key=value list,
// sorted by name.
func mapToKV(m map[string]string, joiner string) string {
	keys := make([]string, 0, len(m))
	for k := range m {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	r := make([]string, len(keys))
	for i, k := range keys {
		r[i] = fmt.Sprintf("%s=%s", k, m[k])
	}
	return strings.Join(r, joiner)
}
<commit_msg>Better name for usage string and moving constant definition down<commit_after>package catlistnodes

import (
	"flag"
	"fmt"
	"sort"
	"strings"

	"github.com/hashicorp/consul/api"
	"github.com/hashicorp/consul/command/flags"
	"github.com/mitchellh/cli"
	"github.com/ryanuber/columnize"
)

func New(ui cli.Ui) *cmd {
	c := &cmd{UI: ui}
	c.init()
	return c
}

type cmd struct {
	UI    cli.Ui
	flags *flag.FlagSet
	http  *flags.HTTPFlags
	usage string

	// flags
	detailed bool
	near     string
	nodeMeta map[string]string
	service  string
}

// init sets up command flags and help text
func (c *cmd) init() {
	c.flags = flag.NewFlagSet("", flag.ContinueOnError)
	c.flags.BoolVar(&c.detailed, "detailed", false, "Output detailed information about "+
		"the nodes including their addresses and metadata.")
	c.flags.StringVar(&c.near, "near", "", "Node name to sort the node list in ascending "+
		"order based on estimated round-trip time from that node. "+
		"Passing \"_agent\" will use this agent's node for sorting.")
	c.flags.Var((*flags.FlagMapValue)(&c.nodeMeta), "node-meta", "Metadata to "+
		"filter nodes with the given `key=value` pairs. This flag may be "+
		"specified multiple times to filter on multiple sources of metadata.")
	c.flags.StringVar(&c.service, "service", "", "Service `id or name` to filter nodes. "+
		"Only nodes which are providing the given service will be returned.")

	c.http = &flags.HTTPFlags{}
	flags.Merge(c.flags, c.http.ClientFlags())
	flags.Merge(c.flags, c.http.ServerFlags())

	c.usage = flags.Usage(usage, c.flags, c.http.ClientFlags(), c.http.ServerFlags())
}

func (c *cmd) Run(args []string) int {
	if err := c.flags.Parse(args); err != nil {
		return 1
	}

	if l := len(c.flags.Args()); l > 0 {
		c.UI.Error(fmt.Sprintf("Too many arguments (expected 0, got %d)", l))
		return 1
	}

	// Create and test the HTTP client
	client, err := c.http.APIClient()
	if err != nil {
		c.UI.Error(fmt.Sprintf("Error connecting to Consul agent: %s", err))
		return 1
	}

	var nodes []*api.Node
	if c.service != "" {
		services, _, err := client.Catalog().Service(c.service, "", &api.QueryOptions{
			Near:     c.near,
			NodeMeta: c.nodeMeta,
		})
		if err != nil {
			c.UI.Error(fmt.Sprintf("Error listing nodes for service: %s", err))
			return 1
		}

		nodes = make([]*api.Node, len(services))
		for i, s := range services {
			nodes[i] = &api.Node{
				ID:              s.ID,
				Node:            s.Node,
				Address:         s.Address,
				Datacenter:      s.Datacenter,
				TaggedAddresses: s.TaggedAddresses,
				Meta:            s.NodeMeta,
				CreateIndex:     s.CreateIndex,
				ModifyIndex:     s.ModifyIndex,
			}
		}
	} else {
		nodes, _, err = client.Catalog().Nodes(&api.QueryOptions{
			Near:     c.near,
			NodeMeta: c.nodeMeta,
		})
		if err != nil {
			c.UI.Error(fmt.Sprintf("Error listing nodes: %s", err))
			return 1
		}
	}

	// Handle the edge case where there are no nodes that match the query.
	if len(nodes) == 0 {
		c.UI.Error("No nodes match the given query - try expanding your search.")
		return 0
	}

	output, err := printNodes(nodes, c.detailed)
	if err != nil {
		c.UI.Error(fmt.Sprintf("Error printing nodes: %s", err))
		return 1
	}

	c.UI.Info(output)

	return 0
}

func (c *cmd) Synopsis() string {
	return "Lists all nodes in the given datacenter"
}

func (c *cmd) Help() string {
	return c.usage
}

// printNodes accepts a list of nodes and prints information in a tabular
// format about the nodes.
func printNodes(nodes []*api.Node, detailed bool) (string, error) {
	var result []string
	if detailed {
		result = detailedNodes(nodes)
	} else {
		result = simpleNodes(nodes)
	}

	return columnize.SimpleFormat(result), nil
}

func detailedNodes(nodes []*api.Node) []string {
	result := make([]string, 0, len(nodes)+1)
	header := "Node|ID|Address|DC|TaggedAddresses|Meta"
	result = append(result, header)

	for _, node := range nodes {
		result = append(result, fmt.Sprintf("%s|%s|%s|%s|%s|%s",
			node.Node, node.ID, node.Address, node.Datacenter,
			mapToKV(node.TaggedAddresses, ", "), mapToKV(node.Meta, ", ")))
	}

	return result
}

func simpleNodes(nodes []*api.Node) []string {
	result := make([]string, 0, len(nodes)+1)
	header := "Node|ID|Address|DC"
	result = append(result, header)

	for _, node := range nodes {
		// Shorten the ID in non-detailed mode to just the first octet.
		id := node.ID
		idx := strings.Index(id, "-")
		if idx > 0 {
			id = id[0:idx]
		}
		result = append(result, fmt.Sprintf("%s|%s|%s|%s",
			node.Node, id, node.Address, node.Datacenter))
	}

	return result
}

// mapToKV converts a map[string]string into a human-friendly key=value list,
// sorted by name.
func mapToKV(m map[string]string, joiner string) string {
	keys := make([]string, 0, len(m))
	for k := range m {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	r := make([]string, len(keys))
	for i, k := range keys {
		r[i] = fmt.Sprintf("%s=%s", k, m[k])
	}
	return strings.Join(r, joiner)
}

const usage = `Usage: consul catalog nodes [options]

  Retrieves the list nodes registered in a given datacenter. By default, the
  datacenter of the local agent is queried.

  To retrieve the list of nodes:

      $ consul catalog nodes

  To print detailed information including full node IDs, tagged addresses, and
  metadata information:

      $ consul catalog nodes -detailed

  To list nodes which are running a particular service:

      $ consul catalog nodes -service=web

  To filter by node metadata:

      $ consul catalog nodes -node-meta="foo=bar"

  To sort nodes by estimated round-trip time from node-web:

      $ consul catalog nodes -near=node-web

  For a full list of options and examples, please see the Consul documentation.`
<|endoftext|>


<commit_before>// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package release

import (
	"fmt"
	"os"
	"strings"

	"code.gitea.io/gitea/models"
	"code.gitea.io/gitea/modules/git"
	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/notification"
	"code.gitea.io/gitea/modules/repository"
	"code.gitea.io/gitea/modules/timeutil"
)

func createTag(gitRepo *git.Repository, rel *models.Release) error {
	// Only actual create when publish.
	if !rel.IsDraft {
		if !gitRepo.IsTagExist(rel.TagName) {
			commit, err := gitRepo.GetCommit(rel.Target)
			if err != nil {
				return fmt.Errorf("GetCommit: %v", err)
			}

			// Trim '--' prefix to prevent command line argument vulnerability.
			rel.TagName = strings.TrimPrefix(rel.TagName, "--")
			if err = gitRepo.CreateTag(rel.TagName, commit.ID.String()); err != nil {
				if strings.Contains(err.Error(), "is not a valid tag name") {
					return models.ErrInvalidTagName{
						TagName: rel.TagName,
					}
				}
				return err
			}
			rel.LowerTagName = strings.ToLower(rel.TagName)
			// Prepare Notify
			if err := rel.LoadAttributes(); err != nil {
				log.Error("LoadAttributes: %v", err)
				return err
			}
			notification.NotifyPushCommits(
				rel.Publisher, rel.Repo, git.TagPrefix+rel.TagName,
				git.EmptySHA, commit.ID.String(), repository.NewPushCommits())
			notification.NotifyCreateRef(rel.Publisher, rel.Repo, "tag", git.TagPrefix+rel.TagName)
		}
		commit, err := gitRepo.GetTagCommit(rel.TagName)
		if err != nil {
			return fmt.Errorf("GetTagCommit: %v", err)
		}

		rel.Sha1 = commit.ID.String()
		rel.CreatedUnix = timeutil.TimeStamp(commit.Author.When.Unix())
		rel.NumCommits, err = commit.CommitsCount()
		if err != nil {
			return fmt.Errorf("CommitsCount: %v", err)
		}
	} else {
		rel.CreatedUnix = timeutil.TimeStampNow()
	}
	return nil
}

// CreateRelease creates a new release of repository.
func CreateRelease(gitRepo *git.Repository, rel *models.Release, attachmentUUIDs []string) error {
	isExist, err := models.IsReleaseExist(rel.RepoID, rel.TagName)
	if err != nil {
		return err
	} else if isExist {
		return models.ErrReleaseAlreadyExist{
			TagName: rel.TagName,
		}
	}

	if err = createTag(gitRepo, rel); err != nil {
		return err
	}

	rel.LowerTagName = strings.ToLower(rel.TagName)
	if err = models.InsertRelease(rel); err != nil {
		return err
	}

	if err = models.AddReleaseAttachments(rel.ID, attachmentUUIDs); err != nil {
		return err
	}

	if !rel.IsDraft {
		notification.NotifyNewRelease(rel)
	}

	return nil
}

// UpdateRelease updates information of a release.
func UpdateRelease(doer *models.User, gitRepo *git.Repository, rel *models.Release, attachmentUUIDs []string) (err error) {
	if err = createTag(gitRepo, rel); err != nil {
		return err
	}
	rel.LowerTagName = strings.ToLower(rel.TagName)

	if err = models.UpdateRelease(models.DefaultDBContext(), rel); err != nil {
		return err
	}

	if err = models.AddReleaseAttachments(rel.ID, attachmentUUIDs); err != nil {
		log.Error("AddReleaseAttachments: %v", err)
	}

	notification.NotifyUpdateRelease(doer, rel)

	return err
}

// DeleteReleaseByID deletes a release and corresponding Git tag by given ID.
func DeleteReleaseByID(id int64, doer *models.User, delTag bool) error {
	rel, err := models.GetReleaseByID(id)
	if err != nil {
		return fmt.Errorf("GetReleaseByID: %v", err)
	}

	repo, err := models.GetRepositoryByID(rel.RepoID)
	if err != nil {
		return fmt.Errorf("GetRepositoryByID: %v", err)
	}

	if delTag {
		if stdout, err := git.NewCommand("tag", "-d", rel.TagName).
			SetDescription(fmt.Sprintf("DeleteReleaseByID (git tag -d): %d", rel.ID)).
			RunInDir(repo.RepoPath()); err != nil && !strings.Contains(err.Error(), "not found") {
			log.Error("DeleteReleaseByID (git tag -d): %d in %v Failed:\nStdout: %s\nError: %v", rel.ID, repo, stdout, err)
			return fmt.Errorf("git tag -d: %v", err)
		}

		if err := models.DeleteReleaseByID(id); err != nil {
			return fmt.Errorf("DeleteReleaseByID: %v", err)
		}
	} else {
		rel.IsTag = true
		rel.IsDraft = false
		rel.IsPrerelease = false
		rel.Title = ""
		rel.Note = ""

		if err = models.UpdateRelease(models.DefaultDBContext(), rel); err != nil {
			return fmt.Errorf("Update: %v", err)
		}
	}

	rel.Repo = repo
	if err = rel.LoadAttributes(); err != nil {
		return fmt.Errorf("LoadAttributes: %v", err)
	}

	if err := models.DeleteAttachmentsByRelease(rel.ID); err != nil {
		return fmt.Errorf("DeleteAttachments: %v", err)
	}

	for i := range rel.Attachments {
		attachment := rel.Attachments[i]
		if err := os.RemoveAll(attachment.LocalPath()); err != nil {
			log.Error("Delete attachment %s of release %s failed: %v", attachment.UUID, rel.ID, err)
		}
	}

	notification.NotifyDeleteRelease(doer, rel)

	return nil
}
<commit_msg>When using API CreateRelease set created_unix to the tag commit time (#11218)<commit_after>// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package release

import (
	"fmt"
	"os"
	"strings"

	"code.gitea.io/gitea/models"
	"code.gitea.io/gitea/modules/git"
	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/notification"
	"code.gitea.io/gitea/modules/repository"
	"code.gitea.io/gitea/modules/timeutil"
)

func createTag(gitRepo *git.Repository, rel *models.Release) error {
	// Only actual create when publish.
	if !rel.IsDraft {
		if !gitRepo.IsTagExist(rel.TagName) {
			commit, err := gitRepo.GetCommit(rel.Target)
			if err != nil {
				return fmt.Errorf("GetCommit: %v", err)
			}

			// Trim '--' prefix to prevent command line argument vulnerability.
			rel.TagName = strings.TrimPrefix(rel.TagName, "--")
			if err = gitRepo.CreateTag(rel.TagName, commit.ID.String()); err != nil {
				if strings.Contains(err.Error(), "is not a valid tag name") {
					return models.ErrInvalidTagName{
						TagName: rel.TagName,
					}
				}
				return err
			}
			rel.LowerTagName = strings.ToLower(rel.TagName)
			// Prepare Notify
			if err := rel.LoadAttributes(); err != nil {
				log.Error("LoadAttributes: %v", err)
				return err
			}
			notification.NotifyPushCommits(
				rel.Publisher, rel.Repo, git.TagPrefix+rel.TagName,
				git.EmptySHA, commit.ID.String(), repository.NewPushCommits())
			notification.NotifyCreateRef(rel.Publisher, rel.Repo, "tag", git.TagPrefix+rel.TagName)
		}
		commit, err := gitRepo.GetTagCommit(rel.TagName)
		if err != nil {
			return fmt.Errorf("GetTagCommit: %v", err)
		}

		rel.Sha1 = commit.ID.String()
		rel.CreatedUnix = timeutil.TimeStampNow()
		rel.NumCommits, err = commit.CommitsCount()
		if err != nil {
			return fmt.Errorf("CommitsCount: %v", err)
		}
	} else {
		rel.CreatedUnix = timeutil.TimeStampNow()
	}
	return nil
}

// CreateRelease creates a new release of repository.
func CreateRelease(gitRepo *git.Repository, rel *models.Release, attachmentUUIDs []string) error {
	isExist, err := models.IsReleaseExist(rel.RepoID, rel.TagName)
	if err != nil {
		return err
	} else if isExist {
		return models.ErrReleaseAlreadyExist{
			TagName: rel.TagName,
		}
	}

	if err = createTag(gitRepo, rel); err != nil {
		return err
	}

	rel.LowerTagName = strings.ToLower(rel.TagName)
	if err = models.InsertRelease(rel); err != nil {
		return err
	}

	if err = models.AddReleaseAttachments(rel.ID, attachmentUUIDs); err != nil {
		return err
	}

	if !rel.IsDraft {
		notification.NotifyNewRelease(rel)
	}

	return nil
}

// UpdateRelease updates information of a release.
func UpdateRelease(doer *models.User, gitRepo *git.Repository, rel *models.Release, attachmentUUIDs []string) (err error) {
	if err = createTag(gitRepo, rel); err != nil {
		return err
	}
	rel.LowerTagName = strings.ToLower(rel.TagName)

	if err = models.UpdateRelease(models.DefaultDBContext(), rel); err != nil {
		return err
	}

	if err = models.AddReleaseAttachments(rel.ID, attachmentUUIDs); err != nil {
		log.Error("AddReleaseAttachments: %v", err)
	}

	notification.NotifyUpdateRelease(doer, rel)

	return err
}

// DeleteReleaseByID deletes a release and corresponding Git tag by given ID.
func DeleteReleaseByID(id int64, doer *models.User, delTag bool) error {
	rel, err := models.GetReleaseByID(id)
	if err != nil {
		return fmt.Errorf("GetReleaseByID: %v", err)
	}

	repo, err := models.GetRepositoryByID(rel.RepoID)
	if err != nil {
		return fmt.Errorf("GetRepositoryByID: %v", err)
	}

	if delTag {
		if stdout, err := git.NewCommand("tag", "-d", rel.TagName).
			SetDescription(fmt.Sprintf("DeleteReleaseByID (git tag -d): %d", rel.ID)).
			RunInDir(repo.RepoPath()); err != nil && !strings.Contains(err.Error(), "not found") {
			log.Error("DeleteReleaseByID (git tag -d): %d in %v Failed:\nStdout: %s\nError: %v", rel.ID, repo, stdout, err)
			return fmt.Errorf("git tag -d: %v", err)
		}

		if err := models.DeleteReleaseByID(id); err != nil {
			return fmt.Errorf("DeleteReleaseByID: %v", err)
		}
	} else {
		rel.IsTag = true
		rel.IsDraft = false
		rel.IsPrerelease = false
		rel.Title = ""
		rel.Note = ""

		if err = models.UpdateRelease(models.DefaultDBContext(), rel); err != nil {
			return fmt.Errorf("Update: %v", err)
		}
	}

	rel.Repo = repo
	if err = rel.LoadAttributes(); err != nil {
		return fmt.Errorf("LoadAttributes: %v", err)
	}

	if err := models.DeleteAttachmentsByRelease(rel.ID); err != nil {
		return fmt.Errorf("DeleteAttachments: %v", err)
	}

	for i := range rel.Attachments {
		attachment := rel.Attachments[i]
		if err := os.RemoveAll(attachment.LocalPath()); err != nil {
			log.Error("Delete attachment %s of release %s failed: %v", attachment.UUID, rel.ID, err)
		}
	}

	notification.NotifyDeleteRelease(doer, rel)

	return nil
}
<|endoftext|>


<commit_before>package redis

import (
	"errors"
	"github.com/blacklightops/libbeat/common"
	"github.com/blacklightops/libbeat/logp"
	"github.com/blacklightops/turnbeat/inputs"
	"github.com/garyburd/redigo/redis"
	"net"
	"time"
)

type RedisInput struct {
	Config inputs.MothershipConfig
	Host	string	/* the host to connect to */
	Port	int		/* the port to connect to */
	DB		int		/* the database to read from */
	Key		string	/* the key to POP from */
	Type	string	/* the type to add to events */
}

func (l *RedisInput) InputType() string {
	return "RedisInput"
}

func (l *RedisInput) InputVersion() string {
	return "0.0.1"
}

func (l *RedisInput) Init(config inputs.MothershipConfig) error {

	l.Config = config

	if config.Host == "" {
		return errors.New("No Input Host specified")
	}
	l.Host = config.Host

	if config.Port == 0 {
		return errors.New("No Input Port specified")
	}
	l.Port = config.Port

	l.DB = config.DB

	if config.Key == "" {
		return errors.New("No Input Key specified")
	}
	l.Key = config.Key

	if config.Type == "" {
		return errors.New("No Event Type specified")
	}
	l.Type = config.Type

	logp.Debug("redisinput", "Using Host %s", l.Host)
	logp.Debug("redisinput", "Using Port %d", l.Port)
	logp.Debug("redisinput", "Using Database %d", l.DB)
	logp.Debug("redisinput", "Using Key %s", l.Key)
	logp.Debug("redisinput", "Adding Event Type %s", l.Type)

	return nil
}

func (l *RedisInput) GetConfig() inputs.MothershipConfig {
	return l.Config
}

func (l *RedisInput) Run(output chan common.MapStr) error {
	logp.Info("[RedisInput] Running Redis Input")
	redisHostname := fmt.Sprintf("%s:%d", l.Host, l.Port)
	server, err := redis.Dial("tcp", redisHostname)
	if err != nil {
		logp.Err("couldn't start listening: " + err.Error())
		return nil
	}
	logp.Info("[RedisInput] Connected to Redis Server")

	// dispatch the master listen thread
	go func(server redis.Conn) {
		var args []interface{}
		for {
			exists, err := redis.Bool(server.Do("EXISTS", append(args, l.Key)))
			if err != nil {
				logp.Err("An error occured while executing EXISTS command")
				return nil
			}
			if exists != true {
				logp.Err("Key %s does not exist!", l.Key)
				return nil;
			}
			handleConn(server, output)
		}
	}(server)
	return nil
}

func (l *RedisInput) handleConn(client net.Conn, output chan common.MapStr) {
	var offset int64 = 0
	var line uint64 = 0

	logp.Debug("redisinput", "Reading events from %s", l.Key)

	now := func() time.Time {
		t := time.Now()
		return t
	}

	for {
		args = []interface{}
		reply, err := server.Do("LPOP", appends(args, l.Key))
		text, err := redis.String(reply, err)
		bytesread += len(text)

		if err != nil {
			logp.Info("Unexpected state reading from %s; error: %s\n", l.Key, err)
			return
		}

		logp.Debug("redisinputlines", "New Line: %s", &text)

		line++

		event := common.MapStr{}
		event["source"] = l.Key
		event["offset"] = offset
		event["line"] = line
		event["message"] = text
		event["type"] = l.Type

		event.EnsureTimestampField(now)
		event.EnsureCountField()

		offset += int64(bytesread)

		logp.Debug("redisinput", "InputEvent: %v", event)
		output <- event // ship the new event downstream
		client.Write([]byte("OK"))
	}
	logp.Debug("redisinput", "Finished reading from %s", l.Key)
}
<commit_msg>Fixing errors<commit_after>package redis

import (
	"errors"
	"github.com/blacklightops/libbeat/common"
	"github.com/blacklightops/libbeat/logp"
	"github.com/blacklightops/turnbeat/inputs"
	"github.com/garyburd/redigo/redis"
	"net"
	"fmt"
	"time"
)

type RedisInput struct {
	Config inputs.MothershipConfig
	Host	string	/* the host to connect to */
	Port	int		/* the port to connect to */
	DB		int		/* the database to read from */
	Key		string	/* the key to POP from */
	Type	string	/* the type to add to events */
}

func (l *RedisInput) InputType() string {
	return "RedisInput"
}

func (l *RedisInput) InputVersion() string {
	return "0.0.1"
}

func (l *RedisInput) Init(config inputs.MothershipConfig) error {

	l.Config = config

	if config.Host == "" {
		return errors.New("No Input Host specified")
	}
	l.Host = config.Host

	if config.Port == 0 {
		return errors.New("No Input Port specified")
	}
	l.Port = config.Port

	l.DB = config.DB

	if config.Key == "" {
		return errors.New("No Input Key specified")
	}
	l.Key = config.Key

	if config.Type == "" {
		return errors.New("No Event Type specified")
	}
	l.Type = config.Type

	logp.Debug("redisinput", "Using Host %s", l.Host)
	logp.Debug("redisinput", "Using Port %d", l.Port)
	logp.Debug("redisinput", "Using Database %d", l.DB)
	logp.Debug("redisinput", "Using Key %s", l.Key)
	logp.Debug("redisinput", "Adding Event Type %s", l.Type)

	return nil
}

func (l *RedisInput) GetConfig() inputs.MothershipConfig {
	return l.Config
}

func (l *RedisInput) Run(output chan common.MapStr) error {
	logp.Info("[RedisInput] Running Redis Input")
	redisHostname := fmt.Sprintf("%s:%d", l.Host, l.Port)
	server, err := redis.Dial("tcp", redisHostname)
	if err != nil {
		logp.Err("couldn't start listening: " + err.Error())
		return nil
	}
	logp.Info("[RedisInput] Connected to Redis Server")

	// dispatch the master listen thread
	go func(server redis.Conn) {
		var args []interface{}
		for {
			exists, err := redis.Bool(server.Do("EXISTS", append(args, l.Key)))
			if err != nil {
				logp.Err("An error occured while executing EXISTS command")
				return
			}
			if exists != true {
				logp.Err("Key %s does not exist!", l.Key)
				return
			}
			handleConn(server, output)
		}
	}(server)
	return nil
}

func (l *RedisInput) handleConn(server redis.Conn, output chan common.MapStr) {
	var offset int64 = 0
	var line uint64 = 0
	var bytesread uint65 = 0
	var args = []interface{}

	logp.Debug("redisinput", "Reading events from %s", l.Key)

	now := func() time.Time {
		t := time.Now()
		return t
	}

	for {
		args = []interface{}
		reply, err := server.Do("LPOP", append(args, l.Key))
		text, err := redis.String(reply, err)
		bytesread += len(text)

		if err != nil {
			logp.Info("Unexpected state reading from %s; error: %s\n", l.Key, err)
			return
		}

		logp.Debug("redisinputlines", "New Line: %s", &text)

		line++

		event := common.MapStr{}
		event["source"] = l.Key
		event["offset"] = offset
		event["line"] = line
		event["message"] = text
		event["type"] = l.Type

		event.EnsureTimestampField(now)
		event.EnsureCountField()

		offset += int64(bytesread)

		logp.Debug("redisinput", "InputEvent: %v", event)
		output <- event // ship the new event downstream
		client.Write([]byte("OK"))
	}
	logp.Debug("redisinput", "Finished reading from %s", l.Key)
}
<|endoftext|>


<commit_before>package challenge52

import (
	"bytes"
	"fmt"
	"testing"
)

func TestCheapestHashEver(t *testing.T) {
	input := []byte("happiness is ever present in the sandwich")
	hash := CheapestHashEver(input, []byte("hi"))

	if !bytes.Equal(hash, []byte{203, 212}) {
		t.Fail()
	}
}

func TestCheapHash(t *testing.T) {
	input := []byte("happiness is ever present in the sandwich")
	hash := CheapHash(input, []byte("hi"))

	if !bytes.Equal(hash, []byte{88, 39, 89}) {
		t.Fail()
	}
}

func TestBeefyHash(t *testing.T) {
	input := []byte("happiness is ever present in the sandwich")
	hash := BeefyHash(input, []byte("hi"))

	if !bytes.Equal(hash, []byte{203, 212, 88, 39, 89}) {
		t.Fail()
	}
}

func TestCheapestCollisionMachine(t *testing.T) {
	initialState := []byte("hi")
	collision := CheapestCollisionMachine(initialState)
	if !bytes.Equal(CheapestHashEver(collision.a, initialState), CheapestHashEver(collision.b, initialState)) {
		t.Fail()
	}
}

func TestFindCheapestCollisions(t *testing.T) {
	hashMap := make(map[string][]byte)
	collisions := FindCheapestCollisions(3)
	for _, c := range collisions {
		hs := fmt.Sprintf("%0x", c)
		if _, found := hashMap[hs]; found {
			// already in map -- duplicate collision, doesn't count
			t.Fail()
		} else {
			hashMap[hs] = c
		}
	}
	if len(collisions) != 8 || len(hashMap) != 8 {
		t.Fail()
	}
}

func TestFindMultiCollision(t *testing.T) {
	initialState := []byte("hi")
	collision := FindMultiCollision()
	cheapestHa := CheapestHashEver(collision.a, initialState)
	cheapestHb := CheapestHashEver(collision.b, initialState)
	cheapHa := CheapHash(collision.a, initialState)
	cheapHb := CheapHash(collision.b, initialState)
	if bytes.Equal(collision.a, collision.b) || !bytes.Equal(cheapHa, cheapHb) || !bytes.Equal(cheapestHa, cheapestHb) {
		t.Fail()
	}

	// and of course the whole point of this:
	if !bytes.Equal(BeefyHash(collision.a, initialState), BeefyHash(collision.b, initialState)) {
		t.Fail()
	}
}
<commit_msg>Add benchmark<commit_after>package challenge52

import (
	"bytes"
	"fmt"
	"testing"
)

func TestCheapestHashEver(t *testing.T) {
	input := []byte("happiness is ever present in the sandwich")
	hash := CheapestHashEver(input, []byte("hi"))

	if !bytes.Equal(hash, []byte{203, 212}) {
		t.Fail()
	}
}

func TestCheapHash(t *testing.T) {
	input := []byte("happiness is ever present in the sandwich")
	hash := CheapHash(input, []byte("hi"))

	if !bytes.Equal(hash, []byte{88, 39, 89}) {
		t.Fail()
	}
}

func TestBeefyHash(t *testing.T) {
	input := []byte("happiness is ever present in the sandwich")
	hash := BeefyHash(input, []byte("hi"))

	if !bytes.Equal(hash, []byte{203, 212, 88, 39, 89}) {
		t.Fail()
	}
}

func TestCheapestCollisionMachine(t *testing.T) {
	initialState := []byte("hi")
	collision := CheapestCollisionMachine(initialState)
	if !bytes.Equal(CheapestHashEver(collision.a, initialState), CheapestHashEver(collision.b, initialState)) {
		t.Fail()
	}
}

func TestFindCheapestCollisions(t *testing.T) {
	hashMap := make(map[string][]byte)
	collisions := FindCheapestCollisions(3)
	for _, c := range collisions {
		hs := fmt.Sprintf("%0x", c)
		if _, found := hashMap[hs]; found {
			// already in map -- duplicate collision, doesn't count
			t.Fail()
		} else {
			hashMap[hs] = c
		}
	}
	if len(collisions) != 8 || len(hashMap) != 8 {
		t.Fail()
	}
}

func TestFindMultiCollision(t *testing.T) {
	initialState := []byte("hi")
	collision := FindMultiCollision()
	cheapestHa := CheapestHashEver(collision.a, initialState)
	cheapestHb := CheapestHashEver(collision.b, initialState)
	cheapHa := CheapHash(collision.a, initialState)
	cheapHb := CheapHash(collision.b, initialState)
	if bytes.Equal(collision.a, collision.b) || !bytes.Equal(cheapHa, cheapHb) || !bytes.Equal(cheapestHa, cheapestHb) {
		t.Fail()
	}

	// and of course the whole point of this:
	if !bytes.Equal(BeefyHash(collision.a, initialState), BeefyHash(collision.b, initialState)) {
		t.Fail()
	}
}

var benchmarkResult *Collision

func BenchmarkFindMultiCollision(b *testing.B) {
	var collision *Collision
	for i := 0; i < b.N; i++ {
		collision = FindMultiCollision()
	}

	benchmarkResult = collision
}
<|endoftext|>


<commit_before>package bytecode

import (
	"fmt"
	"github.com/goby-lang/goby/compiler/ast"
)

func (g *Generator) compileExpression(is *InstructionSet, exp ast.Expression, scope *scope, table *localTable) {
	// See fsm initialization's comment
	if g.fsm.Is(keepExp) {
		switch exp := exp.(type) {
		case *ast.Constant:
			is.define(GetConstant, exp.Value, fmt.Sprint(exp.IsNamespace))
		case *ast.InstanceVariable:
			is.define(GetInstanceVariable, exp.Value)
		case *ast.IntegerLiteral:
			is.define(PutObject, fmt.Sprint(exp.Value))
		case *ast.StringLiteral:
			is.define(PutString, fmt.Sprintf("\"%s\"", exp.Value))
		case *ast.BooleanExpression:
			is.define(PutObject, fmt.Sprint(exp.Value))
		case *ast.NilExpression:
			is.define(PutNull)
		case *ast.RangeExpression:
			g.compileExpression(is, exp.Start, scope, table)
			g.compileExpression(is, exp.End, scope, table)
			is.define(NewRange, 0)
		case *ast.ArrayExpression:
			for _, elem := range exp.Elements {
				g.compileExpression(is, elem, scope, table)
			}
			is.define(NewArray, len(exp.Elements))
		case *ast.HashExpression:
			for key, value := range exp.Data {
				is.define(PutString, fmt.Sprintf("\"%s\"", key))
				g.compileExpression(is, value, scope, table)
			}
			is.define(NewHash, len(exp.Data)*2)
		case *ast.SelfExpression:
			is.define(PutSelf)
		case *ast.PrefixExpression:
			g.compilePrefixExpression(is, exp, scope, table)
		case *ast.InfixExpression:
			g.compileInfixExpression(is, exp, scope, table)
		}
	}

	switch exp := exp.(type) {
	case *ast.Identifier:
		g.compileIdentifier(is, exp, scope, table)
	case *ast.AssignExpression:
		g.compileAssignExpression(is, exp, scope, table)
	case *ast.IfExpression:
		g.compileIfExpression(is, exp, scope, table)
	case *ast.YieldExpression:
		g.compileYieldExpression(is, exp, scope, table)
	case *ast.CallExpression:
		g.compileCallExpression(is, exp, scope, table)
	}
}

func (g *Generator) compileIdentifier(is *InstructionSet, exp *ast.Identifier, scope *scope, table *localTable) {
	index, depth, ok := table.getLCL(exp.Value, table.depth)

	// This means it's a local variable.
	// But we only define the instruction when we'll need it.
	if ok && g.fsm.Is(keepExp) {
		is.define(GetLocal, depth, index)
		return
	}

	// otherwise it's a method call
	is.define(PutSelf)
	is.define(Send, exp.Value, 0)
}

func (g *Generator) compileYieldExpression(is *InstructionSet, exp *ast.YieldExpression, scope *scope, table *localTable) {
	oldState := g.fsm.Current()
	g.fsm.Event(keepExp)

	is.define(PutSelf)

	for _, arg := range exp.Arguments {
		g.compileExpression(is, arg, scope, table)
	}

	is.define(InvokeBlock, len(exp.Arguments))
	g.fsm.Event(oldState)
}

func (g *Generator) compileCallExpression(is *InstructionSet, exp *ast.CallExpression, scope *scope, table *localTable) {
	oldState := g.fsm.Current()

	// We need the receiver expression and argument expressions
	g.fsm.Event(keepExp)
	g.compileExpression(is, exp.Receiver, scope, table)

	for _, arg := range exp.Arguments {
		g.compileExpression(is, arg, scope, table)
	}

	if exp.Block != nil {
		// Inside block should be one level deeper than outside
		newTable := newLocalTable(table.depth + 1)
		newTable.upper = table
		blockIndex := g.blockCounter
		g.blockCounter++
		g.compileBlockArgExpression(blockIndex, exp, scope, newTable)
		is.define(Send, exp.Method, len(exp.Arguments), fmt.Sprintf("block:%d", blockIndex))
		return
	}
	is.define(Send, exp.Method, len(exp.Arguments))

	if exp.Method == "++" || exp.Method == "--" {
		// ++ and -- are methods with side effect and shouldn't return anything
		is.define(Pop)
	}

	g.fsm.Event(oldState)
}

func (g *Generator) compileAssignExpression(is *InstructionSet, exp *ast.AssignExpression, scope *scope, table *localTable) {
	oldState := g.fsm.Current()
	g.fsm.Event(keepExp)
	g.compileExpression(is, exp.Value, scope, table)
	g.fsm.Event(oldState)

	if len(exp.Variables) > 1 {
		is.define(ExpandArray, len(exp.Variables))
	}

	for _, v := range exp.Variables {
		switch name := v.(type) {
		case *ast.Identifier:
			index, depth := table.setLCL(name.Value, table.depth)

			if exp.Optioned != 0 {
				is.define(SetLocal, depth, index, exp.Optioned)
				return
			}

			is.define(SetLocal, depth, index)
		case *ast.InstanceVariable:
			is.define(SetInstanceVariable, name.Value)
		case *ast.Constant:
			is.define(SetConstant, name.Value)
		}

		if exp.IsStmt {
			is.define(Pop)
		}
	}
}

func (g *Generator) compileBlockArgExpression(index int, exp *ast.CallExpression, scope *scope, table *localTable) {
	oldState := g.fsm.Current()
	// We don't need any unused expression inside block
	g.fsm.Event(removeExp)

	is := &InstructionSet{}
	is.name = fmt.Sprint(index)
	is.isType = Block

	for i := 0; i < len(exp.BlockArguments); i++ {
		table.set(exp.BlockArguments[i].Value)
	}

	g.compileCodeBlock(is, exp.Block, scope, table)
	g.endInstructions(is)
	g.instructionSets = append(g.instructionSets, is)

	g.fsm.Event(oldState)
}

func (g *Generator) compileIfExpression(is *InstructionSet, exp *ast.IfExpression, scope *scope, table *localTable) {
	oldState := g.fsm.Current()

	// Compiles condition so we need every expression
	g.fsm.Event(keepExp)
	g.compileExpression(is, exp.Condition, scope, table)

	anchor1 := &anchor{}
	anchor2 := &anchor{}

	is.define(BranchUnless, anchor1)

	// We don't need unused expression in consequence block
	g.fsm.Event(removeExp)
	g.compileCodeBlock(is, exp.Consequence, scope, table)
	g.fsm.Event(oldState)

	anchor1.line = is.count

	// This and the PutNull bellow is needed when we need the returned result
	if g.fsm.Is(keepExp) {
		// BranchIf needs move one more line because the we'll add jump into instructions
		anchor1.line++
		is.define(Jump, anchor2)
	}

	if exp.Alternative == nil {
		if g.fsm.Is(keepExp) {
			// jump over the `putnil` in false case
			anchor2.line = anchor1.line + 1
			is.define(PutNull)
		}

		return
	}

	// We don't need unused expression in alternative block either
	g.fsm.Event(removeExp)
	g.compileCodeBlock(is, exp.Alternative, scope, table)
	g.fsm.Event(oldState)

	anchor2.line = is.count
}

func (g *Generator) compilePrefixExpression(is *InstructionSet, exp *ast.PrefixExpression, scope *scope, table *localTable) {
	switch exp.Operator {
	case "!":
		g.compileExpression(is, exp.Right, scope, table)
		is.define(Send, exp.Operator, 0)
	case "-":
		is.define(PutObject, 0)
		g.compileExpression(is, exp.Right, scope, table)
		is.define(Send, exp.Operator, 1)
	}
}

func (g *Generator) compileInfixExpression(is *InstructionSet, node *ast.InfixExpression, scope *scope, table *localTable) {
	g.compileExpression(is, node.Left, scope, table)
	g.compileExpression(is, node.Right, scope, table)

	if node.Operator != "::" {
		is.define(Send, node.Operator, "1")
	}
}
<commit_msg>Returns assignment value in REPL.<commit_after>package bytecode

import (
	"fmt"
	"github.com/goby-lang/goby/compiler/ast"
)

func (g *Generator) compileExpression(is *InstructionSet, exp ast.Expression, scope *scope, table *localTable) {
	// See fsm initialization's comment
	if g.fsm.Is(keepExp) {
		switch exp := exp.(type) {
		case *ast.Constant:
			is.define(GetConstant, exp.Value, fmt.Sprint(exp.IsNamespace))
		case *ast.InstanceVariable:
			is.define(GetInstanceVariable, exp.Value)
		case *ast.IntegerLiteral:
			is.define(PutObject, fmt.Sprint(exp.Value))
		case *ast.StringLiteral:
			is.define(PutString, fmt.Sprintf("\"%s\"", exp.Value))
		case *ast.BooleanExpression:
			is.define(PutObject, fmt.Sprint(exp.Value))
		case *ast.NilExpression:
			is.define(PutNull)
		case *ast.RangeExpression:
			g.compileExpression(is, exp.Start, scope, table)
			g.compileExpression(is, exp.End, scope, table)
			is.define(NewRange, 0)
		case *ast.ArrayExpression:
			for _, elem := range exp.Elements {
				g.compileExpression(is, elem, scope, table)
			}
			is.define(NewArray, len(exp.Elements))
		case *ast.HashExpression:
			for key, value := range exp.Data {
				is.define(PutString, fmt.Sprintf("\"%s\"", key))
				g.compileExpression(is, value, scope, table)
			}
			is.define(NewHash, len(exp.Data)*2)
		case *ast.SelfExpression:
			is.define(PutSelf)
		case *ast.PrefixExpression:
			g.compilePrefixExpression(is, exp, scope, table)
		case *ast.InfixExpression:
			g.compileInfixExpression(is, exp, scope, table)
		}
	}

	switch exp := exp.(type) {
	case *ast.Identifier:
		g.compileIdentifier(is, exp, scope, table)
	case *ast.AssignExpression:
		g.compileAssignExpression(is, exp, scope, table)
	case *ast.IfExpression:
		g.compileIfExpression(is, exp, scope, table)
	case *ast.YieldExpression:
		g.compileYieldExpression(is, exp, scope, table)
	case *ast.CallExpression:
		g.compileCallExpression(is, exp, scope, table)
	}
}

func (g *Generator) compileIdentifier(is *InstructionSet, exp *ast.Identifier, scope *scope, table *localTable) {
	index, depth, ok := table.getLCL(exp.Value, table.depth)

	// This means it's a local variable.
	// But we only define the instruction when we'll need it.
	if ok && g.fsm.Is(keepExp) {
		is.define(GetLocal, depth, index)
		return
	}

	// otherwise it's a method call
	is.define(PutSelf)
	is.define(Send, exp.Value, 0)
}

func (g *Generator) compileYieldExpression(is *InstructionSet, exp *ast.YieldExpression, scope *scope, table *localTable) {
	oldState := g.fsm.Current()
	g.fsm.Event(keepExp)

	is.define(PutSelf)

	for _, arg := range exp.Arguments {
		g.compileExpression(is, arg, scope, table)
	}

	is.define(InvokeBlock, len(exp.Arguments))
	g.fsm.Event(oldState)
}

func (g *Generator) compileCallExpression(is *InstructionSet, exp *ast.CallExpression, scope *scope, table *localTable) {
	oldState := g.fsm.Current()

	// We need the receiver expression and argument expressions
	g.fsm.Event(keepExp)
	g.compileExpression(is, exp.Receiver, scope, table)

	for _, arg := range exp.Arguments {
		g.compileExpression(is, arg, scope, table)
	}

	if exp.Block != nil {
		// Inside block should be one level deeper than outside
		newTable := newLocalTable(table.depth + 1)
		newTable.upper = table
		blockIndex := g.blockCounter
		g.blockCounter++
		g.compileBlockArgExpression(blockIndex, exp, scope, newTable)
		is.define(Send, exp.Method, len(exp.Arguments), fmt.Sprintf("block:%d", blockIndex))
		return
	}
	is.define(Send, exp.Method, len(exp.Arguments))

	if exp.Method == "++" || exp.Method == "--" {
		// ++ and -- are methods with side effect and shouldn't return anything
		is.define(Pop)
	}

	g.fsm.Event(oldState)
}

func (g *Generator) compileAssignExpression(is *InstructionSet, exp *ast.AssignExpression, scope *scope, table *localTable) {
	oldState := g.fsm.Current()
	g.fsm.Event(keepExp)
	g.compileExpression(is, exp.Value, scope, table)
	g.fsm.Event(oldState)

	if len(exp.Variables) > 1 {
		is.define(ExpandArray, len(exp.Variables))
	}

	for _, v := range exp.Variables {
		switch name := v.(type) {
		case *ast.Identifier:
			index, depth := table.setLCL(name.Value, table.depth)

			if exp.Optioned != 0 {
				is.define(SetLocal, depth, index, exp.Optioned)
				return
			}

			is.define(SetLocal, depth, index)
		case *ast.InstanceVariable:
			is.define(SetInstanceVariable, name.Value)
		case *ast.Constant:
			is.define(SetConstant, name.Value)
		}

		if exp.IsStmt && !g.REPL {
			is.define(Pop)
		}
	}
}

func (g *Generator) compileBlockArgExpression(index int, exp *ast.CallExpression, scope *scope, table *localTable) {
	oldState := g.fsm.Current()
	// We don't need any unused expression inside block
	g.fsm.Event(removeExp)

	is := &InstructionSet{}
	is.name = fmt.Sprint(index)
	is.isType = Block

	for i := 0; i < len(exp.BlockArguments); i++ {
		table.set(exp.BlockArguments[i].Value)
	}

	g.compileCodeBlock(is, exp.Block, scope, table)
	g.endInstructions(is)
	g.instructionSets = append(g.instructionSets, is)

	g.fsm.Event(oldState)
}

func (g *Generator) compileIfExpression(is *InstructionSet, exp *ast.IfExpression, scope *scope, table *localTable) {
	oldState := g.fsm.Current()

	// Compiles condition so we need every expression
	g.fsm.Event(keepExp)
	g.compileExpression(is, exp.Condition, scope, table)

	anchor1 := &anchor{}
	anchor2 := &anchor{}

	is.define(BranchUnless, anchor1)

	// We don't need unused expression in consequence block
	g.fsm.Event(removeExp)
	g.compileCodeBlock(is, exp.Consequence, scope, table)
	g.fsm.Event(oldState)

	anchor1.line = is.count

	// This and the PutNull bellow is needed when we need the returned result
	if g.fsm.Is(keepExp) {
		// BranchIf needs move one more line because the we'll add jump into instructions
		anchor1.line++
		is.define(Jump, anchor2)
	}

	if exp.Alternative == nil {
		if g.fsm.Is(keepExp) {
			// jump over the `putnil` in false case
			anchor2.line = anchor1.line + 1
			is.define(PutNull)
		}

		return
	}

	// We don't need unused expression in alternative block either
	g.fsm.Event(removeExp)
	g.compileCodeBlock(is, exp.Alternative, scope, table)
	g.fsm.Event(oldState)

	anchor2.line = is.count
}

func (g *Generator) compilePrefixExpression(is *InstructionSet, exp *ast.PrefixExpression, scope *scope, table *localTable) {
	switch exp.Operator {
	case "!":
		g.compileExpression(is, exp.Right, scope, table)
		is.define(Send, exp.Operator, 0)
	case "-":
		is.define(PutObject, 0)
		g.compileExpression(is, exp.Right, scope, table)
		is.define(Send, exp.Operator, 1)
	}
}

func (g *Generator) compileInfixExpression(is *InstructionSet, node *ast.InfixExpression, scope *scope, table *localTable) {
	g.compileExpression(is, node.Left, scope, table)
	g.compileExpression(is, node.Right, scope, table)

	if node.Operator != "::" {
		is.define(Send, node.Operator, "1")
	}
}
<|endoftext|>


<commit_before>package directory

import (
	"gob"
	"os"
	"path"
	"rand"
	"log"
	"storage"
	"strconv"
	"sync"
)

const (
	FileIdSaveInterval = 10000
)

type MachineInfo struct {
	Url       string //<server name/ip>[:port]
	PublicUrl string
}
type Machine struct {
	Server   MachineInfo
	Volumes  []storage.VolumeInfo
}

type Mapper struct {
	dir      string
	fileName string

	lock          sync.Mutex
	Machines      []*Machine
	vid2machineId map[uint32]int
	Writers       []int // transient array of Writers volume id

	FileIdSequence uint64
	fileIdCounter  uint64

	volumeSizeLimit uint32
}

func NewMachine(server, publicUrl string, volumes []storage.VolumeInfo) *Machine {
	return &Machine{Server: MachineInfo{Url: server, PublicUrl: publicUrl}, Volumes: volumes}
}

func NewMapper(dirname string, filename string, volumeSizeLimit uint32) (m *Mapper) {
	m = &Mapper{dir: dirname, fileName: filename}
	m.vid2machineId = make(map[uint32]int)
	m.volumeSizeLimit = volumeSizeLimit
	m.Writers = *new([]int)
	m.Machines = *new([]*Machine)

	seqFile, se := os.OpenFile(path.Join(m.dir, m.fileName+".seq"), os.O_RDONLY, 0644)
	if se != nil {
		m.FileIdSequence = FileIdSaveInterval
		log.Println("Setting file id sequence", m.FileIdSequence)
	} else {
		decoder := gob.NewDecoder(seqFile)
		defer seqFile.Close()
		decoder.Decode(&m.FileIdSequence)
		log.Println("Loading file id sequence", m.FileIdSequence, "=>", m.FileIdSequence+FileIdSaveInterval)
		//in case the server stops between intervals
		m.FileIdSequence += FileIdSaveInterval
	}
	return
}
func (m *Mapper) PickForWrite() (string, MachineInfo, os.Error) {
    len_writers := len(m.Writers)
    if len_writers<=0 {
        log.Println("No more writable volumes!")
        return "",m.Machines[rand.Intn(len(m.Machines))].Server, os.NewError("No more writable volumes!")
    }
	machine := m.Machines[m.Writers[rand.Intn(len_writers)]]
	vid := machine.Volumes[rand.Intn(len(machine.Volumes))].Id
	return NewFileId(vid, m.NextFileId(), rand.Uint32()).String(), machine.Server,nil
}
func (m *Mapper) NextFileId() uint64 {
	if m.fileIdCounter <= 0 {
		m.fileIdCounter = FileIdSaveInterval
		m.FileIdSequence += FileIdSaveInterval
		m.saveSequence()
	}
	m.fileIdCounter--
	return m.FileIdSequence - m.fileIdCounter
}
func (m *Mapper) Get(vid uint32) (*Machine, os.Error) {
    machineId := m.vid2machineId[vid]
    if machineId <=0{
        return nil, os.NewError("invalid volume id " + strconv.Uitob64(uint64(vid),10))
    }
	return m.Machines[machineId-1],nil
}
func (m *Mapper) Add(machine Machine){
	//check existing machine, linearly
	m.lock.Lock()
	foundExistingMachineId := -1
	for index, entry := range m.Machines {
		if machine.Server.Url == entry.Server.Url {
			foundExistingMachineId = index
			break
		}
	}
	machineId := foundExistingMachineId
	if machineId < 0 {
		machineId = len(m.Machines)
		m.Machines = append(m.Machines, &machine)
	}else{
	    m.Machines[machineId] = &machine
	}
	m.lock.Unlock()

	//add to vid2machineId map, and Writers array
	for _, v := range machine.Volumes {
		//log.Println("Setting volume", v.Id, "to", machine.Server.Url)
		m.vid2machineId[v.Id] = machineId+1 //use base 1 indexed, to detect not found cases
	}
	//setting Writers, copy-on-write because of possible updating
	var writers []int
	for machine_index, machine_entry := range m.Machines {
		for _, v := range machine_entry.Volumes {
			if v.Size < int64(m.volumeSizeLimit) {
				writers = append(writers, machine_index)
			}
		}
	}
	m.Writers = writers
}
func (m *Mapper) saveSequence() {
	log.Println("Saving file id sequence", m.FileIdSequence, "to", path.Join(m.dir, m.fileName+".seq"))
	seqFile, e := os.OpenFile(path.Join(m.dir, m.fileName+".seq"), os.O_CREATE|os.O_WRONLY, 0644)
	if e != nil {
		log.Fatalf("Sequence File Save [ERROR] %s\n", e)
	}
	defer seqFile.Close()
	encoder := gob.NewEncoder(seqFile)
	encoder.Encode(m.FileIdSequence)
}
<commit_msg>fix enforcing volume size limit<commit_after>package directory

import (
	"gob"
	"os"
	"path"
	"rand"
	"log"
	"storage"
	"strconv"
	"sync"
)

const (
	FileIdSaveInterval = 10000
)

type MachineInfo struct {
	Url       string //<server name/ip>[:port]
	PublicUrl string
}
type Machine struct {
	Server  MachineInfo
	Volumes []storage.VolumeInfo
}

type Mapper struct {
	dir      string
	fileName string

	lock          sync.Mutex
	Machines      []*Machine
	vid2machineId map[uint32]int //machineId is +1 of the index of []*Machine, to detect not found entries
	Writers       []uint32 // transient array of Writers volume id

	FileIdSequence uint64
	fileIdCounter  uint64

	volumeSizeLimit uint32
}

func NewMachine(server, publicUrl string, volumes []storage.VolumeInfo) *Machine {
	return &Machine{Server: MachineInfo{Url: server, PublicUrl: publicUrl}, Volumes: volumes}
}

func NewMapper(dirname string, filename string, volumeSizeLimit uint32) (m *Mapper) {
	m = &Mapper{dir: dirname, fileName: filename}
	m.vid2machineId = make(map[uint32]int)
	m.volumeSizeLimit = volumeSizeLimit
	m.Writers = *new([]uint32)
	m.Machines = *new([]*Machine)

	seqFile, se := os.OpenFile(path.Join(m.dir, m.fileName+".seq"), os.O_RDONLY, 0644)
	if se != nil {
		m.FileIdSequence = FileIdSaveInterval
		log.Println("Setting file id sequence", m.FileIdSequence)
	} else {
		decoder := gob.NewDecoder(seqFile)
		defer seqFile.Close()
		decoder.Decode(&m.FileIdSequence)
		log.Println("Loading file id sequence", m.FileIdSequence, "=>", m.FileIdSequence+FileIdSaveInterval)
		//in case the server stops between intervals
		m.FileIdSequence += FileIdSaveInterval
	}
	return
}
func (m *Mapper) PickForWrite() (string, MachineInfo, os.Error) {
	len_writers := len(m.Writers)
	if len_writers <= 0 {
		log.Println("No more writable volumes!")
		return "", m.Machines[rand.Intn(len(m.Machines))].Server, os.NewError("No more writable volumes!")
	}
	vid := m.Writers[rand.Intn(len_writers)]
	machine_id := m.vid2machineId[vid]
	if machine_id > 0 {
		machine := m.Machines[machine_id-1]
		return NewFileId(vid, m.NextFileId(), rand.Uint32()).String(), machine.Server, nil
	}
	return "", m.Machines[rand.Intn(len(m.Machines))].Server, os.NewError("Strangely vid " + strconv.Uitoa64(uint64(vid)) + " is on no machine!")
}
func (m *Mapper) NextFileId() uint64 {
	if m.fileIdCounter <= 0 {
		m.fileIdCounter = FileIdSaveInterval
		m.FileIdSequence += FileIdSaveInterval
		m.saveSequence()
	}
	m.fileIdCounter--
	return m.FileIdSequence - m.fileIdCounter
}
func (m *Mapper) Get(vid uint32) (*Machine, os.Error) {
	machineId := m.vid2machineId[vid]
	if machineId <= 0 {
		return nil, os.NewError("invalid volume id " + strconv.Uitob64(uint64(vid), 10))
	}
	return m.Machines[machineId-1], nil
}
func (m *Mapper) Add(machine Machine) {
	//check existing machine, linearly
	m.lock.Lock()
	foundExistingMachineId := -1
	for index, entry := range m.Machines {
		if machine.Server.Url == entry.Server.Url {
			foundExistingMachineId = index
			break
		}
	}
	machineId := foundExistingMachineId
	if machineId < 0 {
		machineId = len(m.Machines)
		m.Machines = append(m.Machines, &machine)
	} else {
		m.Machines[machineId] = &machine
	}
	m.lock.Unlock()

	//add to vid2machineId map, and Writers array
	for _, v := range machine.Volumes {
		//log.Println("Setting volume", v.Id, "to", machine.Server.Url)
		m.vid2machineId[v.Id] = machineId + 1 //use base 1 indexed, to detect not found cases
	}
	//setting Writers, copy-on-write because of possible updating
	var writers []uint32
	for _, machine_entry := range m.Machines {
		for _, v := range machine_entry.Volumes {
			if v.Size < int64(m.volumeSizeLimit) {
				writers = append(writers, v.Id)
			}
		}
	}
	m.Writers = writers
}
func (m *Mapper) saveSequence() {
	log.Println("Saving file id sequence", m.FileIdSequence, "to", path.Join(m.dir, m.fileName+".seq"))
	seqFile, e := os.OpenFile(path.Join(m.dir, m.fileName+".seq"), os.O_CREATE|os.O_WRONLY, 0644)
	if e != nil {
		log.Fatalf("Sequence File Save [ERROR] %s\n", e)
	}
	defer seqFile.Close()
	encoder := gob.NewEncoder(seqFile)
	encoder.Encode(m.FileIdSequence)
}
<|endoftext|>


<commit_before>package ogdatv21

import (
	"encoding/csv"
	"fmt"
	"github.com/the42/ogdat"
	"io"
	"log"
	"os"
	"reflect"
	"regexp"
	"unicode/utf8"
)

type ISO6392Lang struct {
	Code, Identifier string
}

const iso639file = "ISO-639-2_utf-8.txt"
const schema_langauge = "ger"
const schema_characterset = "utf8"

var isolangfilemap map[string]*ISO6392Lang

func loadisolanguagefile(filename string) (isolangfilemap map[string]*ISO6392Lang) {
	reader, err := os.Open(iso639file)

	if err == nil {
		defer reader.Close()
		isolangfilemap = make(map[string]*ISO6392Lang)
		csvreader := csv.NewReader(reader)
		csvreader.Comma = '|'

		for record, err := csvreader.Read(); err != io.EOF; record, err = csvreader.Read() {
			isorecord := &ISO6392Lang{Code: record[0], Identifier: record[3]}
			isolangfilemap[record[0]] = isorecord
			if len(record[1]) > 0 {
				isorecord = &ISO6392Lang{Code: record[1], Identifier: record[3]}
				isolangfilemap[record[1]] = isorecord
			}
		}
		log.Printf("Info: Read %d ISO language records", len(isolangfilemap))
	} else {
		log.Printf("Warning: Can not read ISO language records")
	}
	return
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

var regexphtmlcodecheck = regexp.MustCompile(`<\w+.*('|"|)>`)
var regexphtmlescape = regexp.MustCompile(`&\w{1,10};|&#\d{1,6};`)
var regexpurlencode = regexp.MustCompile(`%[0-9a-fA-F][0-9a-fA-F]`)
var regexpposixescape = regexp.MustCompile(`\\n|\\b|\\v|\\t`)

// return values are:
// status: <> 0 indicates sthg. was wrong, // 1 = Info, 2 = Warning, 3 = Error
// position > -1: position of offending input in string, only set if status <> 0
// message: clear text of reason why the input string failes to be a correct OGD string
func CheckOGDTextStringForSaneCharacters(str string) (status, position int, message string) {
	if !utf8.ValidString(str) {
		return 3, -1, "Zeichenfolge ist nicht als UTF8 kodiert"
	}
	if idx := regexphtmlcodecheck.FindIndex([]byte(str)); idx != nil {
		return 2, idx[0], fmt.Sprintf("Mögliche HTML-Sequenz: '%s'", str[idx[0]:min(20, idx[1]-idx[0])])
	}
	if idx := regexphtmlescape.FindIndex([]byte(str)); idx != nil {
		return 2, idx[0], fmt.Sprintf("Mögliche HTML-Escapes: '%s'", str[idx[0]:min(15, idx[1]-idx[0])])
	}
	if idx := regexpurlencode.FindIndex([]byte(str)); idx != nil {
		return 2, idx[0], fmt.Sprintf("Mögliche Url-Escapes: '%s'", str[idx[0]:min(8, idx[1]-idx[0])])
	}
	if idx := regexpposixescape.FindIndex([]byte(str)); idx != nil {
		return 2, idx[0], fmt.Sprintf("Mögliche Posix-Escapes: '%s'", str[idx[0]:min(5, idx[1]-idx[0])])
	}
	return
}

func (md *MetaData) Check() (message []ogdat.CheckMessage, err error) {
	const pflichtfeldfehlt = "Pflichtfeld nicht gesetzt"

	ogdset := ogdat.GetOGDSetForVersion(Version)
	if ogdset == nil {
		return nil, fmt.Errorf("Beschreibung für OGD Version %s ist nicht vorhanden, check kann nicht durchgeführt werden", Version)
	}

nextbeschreibung:
	for _, elm := range ogdset.Beschreibung {
		if elm.IsRequired() && elm.Anzahl != "N" { // check required fields for their existence. However, if the cardinality is defined
			// as 'N', it may be ok that the field is not present, in which case we check explicitely and issue a warning
			ielements := reflect.TypeOf(md).Elem().NumField()
			for i := 0; i < ielements; i++ {
				f := reflect.TypeOf(md).Elem().Field(i)
				if ogdat.GetIDFromMetaDataStructField(f) == elm.ID && ogdat.MetaDataStructFieldIsNil(f) {
					message = append(message, ogdat.CheckMessage{Type: 3, OGDID: ogdat.GetIDFromMetaDataStructField(f), Text: pflichtfeldfehlt})
					break nextbeschreibung // required field is not present - nothing more to check
				}
			}
		}

		switch elm.OGD_Kurzname {
		case "metadata_identifier":
			if md.Extras.Metadata_Identifier.UUID == nil {
				message = append(message, ogdat.CheckMessage{
					Type:  3,
					OGDID: elm.ID,
					Text:  fmt.Sprintf("Feldwert vom Typ UUID erwartet, Wert ist aber keine UUID: '%s'", md.Extras.Metadata_Identifier.Raw)})
			}
		case "metadata_modified":
			if md.Extras.Metadata_Modified.Format != CustomTimeSpecifier2 {
				message = append(message, ogdat.CheckMessage{
					Type:  3,
					OGDID: elm.ID,
					Text:  fmt.Sprintf("Feldwert vom Typ ÖNORM ISO 8601 YYYY-MM-DD erwartet, Wert entspricht aber nicht diesem Typ: '%s'", md.Extras.Metadata_Modified.Raw)})
			}
		case "title":
			if err, idx, msg := CheckOGDTextStringForSaneCharacters(*md.Title); err > 0 {
				message = append(message, ogdat.CheckMessage{
					Type:  err,
					OGDID: elm.ID,
					Text:  fmt.Sprintf("Zeichenfolge enthält potentiell ungeeignete Zeichen ab Position %d: %s", idx, msg)})
			}
		case "description":
			if err, idx, msg := CheckOGDTextStringForSaneCharacters(*md.Description); err > 0 {
				message = append(message, ogdat.CheckMessage{
					Type:  err,
					OGDID: elm.ID,
					Text:  fmt.Sprintf("Zeichenfolge enthält potentiell ungeeignete Zeichen ab Position %d: %s", idx, msg)})
			}
		case "categorization":
			if cat := md.Extras.Categorization; cat == nil {
				message = append(message, ogdat.CheckMessage{
					Type:  2,
					OGDID: elm.ID,
					Text:  "Kategorisierung darf zwar mit Karidnalität 'N' auftreten, jedoch sollte zumindest eine Zuordnung getroffen werden"})

			} else {
				for _, element := range cat {
					if element.NumID == -1 {
						message = append(message, ogdat.CheckMessage{
							Type:  3,
							OGDID: elm.ID,
							Text:  fmt.Sprintf("Die Kategorie '%s' ist keine normierte OGD-Kategorie", element.ID)})
					}
				}
			}
		}
	}
	return
}

func init() {
	isolangfilemap = loadisolanguagefile(iso639file)
}
<commit_msg>extend Check<commit_after>package ogdatv21

import (
	"encoding/csv"
	"fmt"
	"github.com/the42/ogdat"
	"io"
	"log"
	"os"
	"reflect"
	"regexp"
	"unicode/utf8"
)

type ISO6392Lang struct {
	Code, Identifier string
}

const iso639file = "ISO-639-2_utf-8.txt"
const schema_langauge = "ger"
const schema_characterset = "utf8"

var isolangfilemap map[string]*ISO6392Lang

func loadisolanguagefile(filename string) (isolangfilemap map[string]*ISO6392Lang) {
	reader, err := os.Open(iso639file)

	if err == nil {
		defer reader.Close()
		isolangfilemap = make(map[string]*ISO6392Lang)
		csvreader := csv.NewReader(reader)
		csvreader.Comma = '|'

		for record, err := csvreader.Read(); err != io.EOF; record, err = csvreader.Read() {
			isorecord := &ISO6392Lang{Code: record[0], Identifier: record[3]}
			isolangfilemap[record[0]] = isorecord
			if len(record[1]) > 0 {
				isorecord = &ISO6392Lang{Code: record[1], Identifier: record[3]}
				isolangfilemap[record[1]] = isorecord
			}
		}
		log.Printf("Info: Read %d ISO language records", len(isolangfilemap))
	} else {
		log.Printf("Warning: Can not read ISO language records")
	}
	return
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

var regexphtmlcodecheck = regexp.MustCompile(`<\w+.*('|"|)>`)
var regexphtmlescape = regexp.MustCompile(`&\w{1,10};|&#\d{1,6};`)
var regexpurlencode = regexp.MustCompile(`%[0-9a-fA-F][0-9a-fA-F]`)
var regexpposixescape = regexp.MustCompile(`\\n|\\b|\\v|\\t`)
var regexpbboxWKT = regexp.MustCompile(`POLYGON\s{0,1}\({1,2}\s{0,2}[-+]?[0-9]*\.?[0-9]+\s{1,2}[-+]?[0-9]*\.?[0-9]+,\s{0,2}[-+]?[0-9]*\.?[0-9]+\s{1,2}[-+]?[0-9]*\.?[0-9]+\s{0,2}\){1,2}`)

// return values are:
// status: <> 0 indicates sthg. was wrong, // 1 = Info, 2 = Warning, 3 = Error
// position > -1: position of offending input in string, only set if status <> 0
// message: clear text of reason why the input string failes to be a correct OGD string
func CheckOGDTextStringForSaneCharacters(str string) (status, position int, message string) {
	if !utf8.ValidString(str) {
		return 3, -1, "Zeichenfolge ist nicht durchgängig gültig als UTF8 kodiert"
	}
	if idx := regexphtmlcodecheck.FindIndex([]byte(str)); idx != nil {
		return 2, idx[0], fmt.Sprintf("Mögliche HTML-Sequenz: '%s'", str[idx[0]:min(20, idx[1]-idx[0])])
	}
	if idx := regexphtmlescape.FindIndex([]byte(str)); idx != nil {
		return 2, idx[0], fmt.Sprintf("Mögliche HTML-Escapes: '%s'", str[idx[0]:min(15, idx[1]-idx[0])])
	}
	if idx := regexpurlencode.FindIndex([]byte(str)); idx != nil {
		return 2, idx[0], fmt.Sprintf("Mögliche Url-Escapes: '%s'", str[idx[0]:min(8, idx[1]-idx[0])])
	}
	if idx := regexpposixescape.FindIndex([]byte(str)); idx != nil {
		return 2, idx[0], fmt.Sprintf("Mögliche Posix-Escapes: '%s'", str[idx[0]:min(5, idx[1]-idx[0])])
	}
	return
}

func CheckOGDBBox(str string) (bool, error) {
	if !utf8.ValidString(str) {
		return false, fmt.Errorf("Zeichenfolge ist nicht durchgängig gültig als UTF8 kodiert")
	}
	if idx := regexpbboxWKT.FindIndex([]byte(str)); idx == nil {
		return false, fmt.Errorf("Keine gültige WKT-Angabe einer BoundingBox")
	}
	return true, nil
}

func (md *MetaData) Check() (message []ogdat.CheckMessage, err error) {
	const pflichtfeldfehlt = "Pflichtfeld nicht gesetzt"

	ogdset := ogdat.GetOGDSetForVersion(Version)
	if ogdset == nil {
		return nil, fmt.Errorf("Beschreibung für OGD Version %s ist nicht vorhanden, check kann nicht durchgeführt werden", Version)
	}

nextbeschreibung:
	for _, elm := range ogdset.Beschreibung {

		// check required fields for their presence. However, if the
		// cardinality on a required fiels is defined as 'N', it may be ok
		// that the field is not present, in which case we check
		// later explicitely and issue a warning
		if elm.IsRequired() && elm.Anzahl != "N" {
			ielements := reflect.TypeOf(md).Elem().NumField()
			for i := 0; i < ielements; i++ {
				f := reflect.TypeOf(md).Elem().Field(i)
				if ogdat.GetIDFromMetaDataStructField(f) == elm.ID && ogdat.MetaDataStructFieldIsNil(f) {
					message = append(message, ogdat.CheckMessage{Type: 3, OGDID: ogdat.GetIDFromMetaDataStructField(f), Text: pflichtfeldfehlt})
					break nextbeschreibung // required field is not present - nothing more to check
				}
			}
		}

		switch elm.OGD_Kurzname {
		case "metadata_identifier":
			if md.Extras.Metadata_Identifier.UUID == nil {
				message = append(message, ogdat.CheckMessage{
					Type:  3,
					OGDID: elm.ID,
					Text:  fmt.Sprintf("Feldwert vom Typ UUID erwartet, Wert ist aber keine UUID: '%s'", md.Extras.Metadata_Identifier.Raw)})
			}
		case "metadata_modified":
			if md.Extras.Metadata_Modified.Format != CustomTimeSpecifier2 {
				message = append(message, ogdat.CheckMessage{
					Type:  3,
					OGDID: elm.ID,
					Text:  fmt.Sprintf("Feldwert vom Typ ÖNORM ISO 8601 'YYYY-MM-DD' erwartet, Wert entspricht aber nicht diesem Typ: '%s'", md.Extras.Metadata_Modified.Raw)})
			}
		case "title":
			if err, idx, msg := CheckOGDTextStringForSaneCharacters(*md.Title); err > 0 {
				message = append(message, ogdat.CheckMessage{
					Type:  err,
					OGDID: elm.ID,
					Text:  fmt.Sprintf("Zeichenfolge enthält potentiell ungeeignete Zeichen ab Position %d: '%s'", idx, msg)})
			}
		case "description":
			if err, idx, msg := CheckOGDTextStringForSaneCharacters(*md.Description); err > 0 {
				message = append(message, ogdat.CheckMessage{
					Type:  err,
					OGDID: elm.ID,
					Text:  fmt.Sprintf("Zeichenfolge enthält potentiell ungeeignete Zeichen ab Position %d: '%s'", idx, msg)})
			}
		case "categorization":
			if cat := md.Extras.Categorization; cat == nil {
				message = append(message, ogdat.CheckMessage{
					Type:  2,
					OGDID: elm.ID,
					Text:  "Die Kategorisierung darf zwar mit Kardinalität 'N' optional auftreten, jedoch sollte zumindest eine Zuordnung getroffen werden"})

			} else {
				for _, element := range cat {
					if element.NumID == -1 {
						message = append(message, ogdat.CheckMessage{
							Type:  3,
							OGDID: elm.ID,
							Text:  fmt.Sprintf("Die Kategorie '%s' ist keine normierte OGD-Kategorie", element.ID)})
					}
				}
			}
		case "keywords":
			if keywords := md.Schlagworte; keywords == nil {
				message = append(message, ogdat.CheckMessage{
					Type:  2,
					OGDID: elm.ID,
					Text:  "Schlagworte dürfen zwar mit Kardinalität 'N' optional auftreten, die Angabe von Schlagworten wäre aber wünschenswert"})

			}
		case "maintainer":
			if err, idx, msg := CheckOGDTextStringForSaneCharacters(*md.Maintainer); err > 0 {
				message = append(message, ogdat.CheckMessage{
					Type:  err,
					OGDID: elm.ID,
					Text:  fmt.Sprintf("Zeichenfolge enthält potentiell ungeeignete Zeichen ab Position %d: '%s'", idx, msg)})
			}
		case "license":
			if err, idx, msg := CheckOGDTextStringForSaneCharacters(*md.License); err > 0 {
				message = append(message, ogdat.CheckMessage{
					Type:  err,
					OGDID: elm.ID,
					Text:  fmt.Sprintf("Zeichenfolge enthält potentiell ungeeignete Zeichen ab Position %d: '%s'", idx, msg)})
			}
		case "begin_datetime":
			if md.Extras.Begin_DateTime.Format != CustomTimeSpecifier1 {
				message = append(message, ogdat.CheckMessage{
					Type:  3,
					OGDID: elm.ID,
					Text:  fmt.Sprintf("Feldwert vom Typ ÖNORM ISO 8601 TM_Primitive 'YYYY-MM-DDThh:mm:ss' erwartet, Wert entspricht aber nicht diesem Typ: '%s'", md.Extras.Begin_DateTime.Raw)})
			}
			// ###################### OPTIONALE FELDER ######################
		case "schema_name":
			if schemaname := md.Extras.Schema_Name; schemaname != nil {
				if err, idx, msg := CheckOGDTextStringForSaneCharacters(*schemaname); err > 0 {
					message = append(message, ogdat.CheckMessage{
						Type:  err,
						OGDID: elm.ID,
						Text:  fmt.Sprintf("Zeichenfolge enthält potentiell ungeeignete Zeichen ab Position %d: '%s'", idx, msg)})
				}
				const ogdschemaspec = "OGD Austria Metadata 2.0"
				if *schemaname != ogdschemaspec {
					message = append(message, ogdat.CheckMessage{
						Type:  1,
						OGDID: elm.ID,
						Text:  fmt.Sprintf("Schemabezeichnung als '%s' erwartet, der Wert ist aber '%s'", ogdschemaspec, *schemaname)})
				}
			}
		case "schema_language":
			if lang := md.Extras.Schema_Language; lang != nil {
				const ogdschemalanguage = "ger"
				if *lang != ogdschemalanguage {
					message = append(message, ogdat.CheckMessage{
						Type:  3,
						OGDID: elm.ID,
						Text:  fmt.Sprintf("Schemasprache als '%s' erwartet, der Wert ist aber '%s'", ogdschemalanguage, *lang)})
				}
			}
		case "schema_characterset":
			if charset := md.Extras.Schema_Characterset; charset != nil {
				const ogdschemacharacterset = "utf8"
				if *charset != ogdschemacharacterset {
					message = append(message, ogdat.CheckMessage{
						Type:  3,
						OGDID: elm.ID,
						Text:  fmt.Sprintf("Characterset des Schemas als '%s' erwartet, der Wert ist aber '%s'", ogdschemacharacterset, *charset)})
				}
			}
		case "metadata_linkage":
			for _, element := range md.Extras.Metadata_Linkage {
				if element.URL == nil {
					message = append(message, ogdat.CheckMessage{
						Type:  3,
						OGDID: elm.ID,
						Text:  fmt.Sprintf("Gültigen Verweis (Link) erwartet, der Wert '%s' stellt keinen gültigen Link dar", element.Raw)})
				}
			}
		case "attribute_description":
			if desc := md.Extras.Attribute_Description; desc != nil {
				const ogddesclen = 20
				if i := len(*desc); i < ogddesclen {
					message = append(message, ogdat.CheckMessage{
						Type:  2,
						OGDID: elm.ID,
						Text:  fmt.Sprintf("Beschreibung enthält weniger als %d Zeichen", i)})

				}
				if err, idx, msg := CheckOGDTextStringForSaneCharacters(*desc); err > 0 {
					message = append(message, ogdat.CheckMessage{
						Type:  err,
						OGDID: elm.ID,
						Text:  fmt.Sprintf("Zeichenfolge enthält potentiell ungeeignete Zeichen ab Position %d: '%s'", idx, msg)})
				}
			}
		case "maintainer_link":
			if link := md.Extras.Maintainer_Link; link != nil {
				if link.URL == nil {
					message = append(message, ogdat.CheckMessage{
						Type:  3,
						OGDID: elm.ID,
						Text:  fmt.Sprintf("Gültigen Verweis (Link) erwartet, der Wert '%s' stellt keinen gültigen Link dar", link.Raw)})
				}
			}
		case "publisher":
			if publisher := md.Extras.Publisher; publisher != nil {
				if err, idx, msg := CheckOGDTextStringForSaneCharacters(*publisher); err > 0 {
					message = append(message, ogdat.CheckMessage{
						Type:  err,
						OGDID: elm.ID,
						Text:  fmt.Sprintf("Zeichenfolge enthält potentiell ungeeignete Zeichen ab Position %d: '%s'", idx, msg)})
				}
			}
		case "geographic_toponym":
			if toponym := md.Extras.Geographich_Toponym; toponym != nil {
				if err, idx, msg := CheckOGDTextStringForSaneCharacters(*toponym); err > 0 {
					message = append(message, ogdat.CheckMessage{
						Type:  err,
						OGDID: elm.ID,
						Text:  fmt.Sprintf("Zeichenfolge enthält potentiell ungeeignete Zeichen ab Position %d: '%s'", idx, msg)})
				}
			}
		case "geographic_bbox":
			if bbox := md.Extras.Geographic_BBox; bbox != nil {
				if ok, err := CheckOGDBBox(*bbox); !ok {
					message = append(message, ogdat.CheckMessage{
						Type:  3,
						OGDID: elm.ID,
						Text:  fmt.Sprintf("Zeichenfolge enthält keinen gültigen WKT für die örtliche Begrenzung (Boundingbox): '%s'", err)})
				}
			}
		}
	}
	return

	/*
		 		case "resource_url":
			  if keywords := md.Resources.Url; keywords == nil {
			  message = append(message, ogdat.CheckMessage{
						Type:  2,
						OGDID: elm.ID,
						Text:  "Schlagworte dürfen zwar mit Karidnalität 'N' optional auftreten, die Angabe von Schlagworten wäre aber wünschenswert"})
	*/
}

func init() {
	isolangfilemap = loadisolanguagefile(iso639file)
}
<|endoftext|>


<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package internal

import (
	"path"
	"time"

	"golang.org/x/mod/module"
	"golang.org/x/pkgsite/internal/licenses"
	"golang.org/x/pkgsite/internal/source"
	"golang.org/x/pkgsite/internal/stdlib"
	"golang.org/x/pkgsite/internal/version"
)

const (
	// LatestVersion signifies the latest available version in requests to the
	// proxy client.
	LatestVersion = "latest"

	// MasterVersion signifies the version at master.
	MasterVersion = "master"

	// UnknownModulePath signifies that the module path for a given package
	// path is ambiguous or not known. This is because requests to the
	// frontend can come in the form of <import-path>[@<version>], and it is
	// not clear which part of the import-path is the module path.
	UnknownModulePath = "unknownModulePath"
)

// ModuleInfo holds metadata associated with a module.
type ModuleInfo struct {
	ModulePath        string
	Version           string
	CommitTime        time.Time
	ReadmeFilePath    string
	ReadmeContents    string
	VersionType       version.Type
	IsRedistributable bool
	HasGoMod          bool // whether the module zip has a go.mod file
	SourceInfo        *source.Info
}

// VersionMap holds metadata associated with module queries for a version.
type VersionMap struct {
	ModulePath       string
	RequestedVersion string
	ResolvedVersion  string
	GoModPath        string
	Status           int
	Error            string
}

// SeriesPath returns the series path for the module.
//
// A series is a group of modules that share the same base path and are assumed
// to be major-version variants.
//
// The series path is the module path without the version. For most modules,
// this will be the module path for all module versions with major version 0 or
// 1. For gopkg.in modules, the series path does not correspond to any module
// version.
//
// Examples:
// The module paths "a/b" and "a/b/v2"  both have series path "a/b".
// The module paths "gopkg.in/yaml.v1" and "gopkg.in/yaml.v2" both have series path "gopkg.in/yaml".
func (v *ModuleInfo) SeriesPath() string {
	return SeriesPathForModule(v.ModulePath)
}

// SeriesPathForModule returns the series path for the provided modulePath.
func SeriesPathForModule(modulePath string) string {
	seriesPath, _, _ := module.SplitPathVersion(modulePath)
	return seriesPath
}

// V1Path returns the path for version 1 of the package whose path
// is modulePath + "/" + suffix. If modulePath is the standard
// library, then V1Path returns suffix.
func V1Path(modulePath, suffix string) string {
	if modulePath == stdlib.ModulePath {
		return suffix
	}
	return path.Join(SeriesPathForModule(modulePath), suffix)
}

// A Module is a specific, reproducible build of a module.
type Module struct {
	ModuleInfo
	Packages []*Package
	// Licenses holds all licenses within this module version, including those
	// that may be contained in nested subdirectories.
	Licenses []*licenses.License

	Directories []*DirectoryNew
}

// A Package is a group of one or more Go source files with the same package
// header. Packages are part of a module.
type Package struct {
	Path              string
	Name              string
	Synopsis          string
	IsRedistributable bool
	Licenses          []*licenses.Metadata // metadata of applicable version licenses
	Imports           []string
	DocumentationHTML string
	// The values of the GOOS and GOARCH environment variables used to parse the package.
	GOOS   string
	GOARCH string

	// V1Path is the package path of a package with major version 1 in a given series.
	V1Path string
}

// VersionedPackage is a Package along with its corresponding module
// information.
type VersionedPackage struct {
	Package
	ModuleInfo
}

// Directory represents a folder in a module version, and all of the packages
// inside that folder.
type Directory struct {
	ModuleInfo
	Path     string
	Packages []*Package
}

// VersionedDirectory is a DirectoryNew along with its corresponding module
// information.
type VersionedDirectory struct {
	DirectoryNew
	ModuleInfo
}

// DirectoryNew is a folder in a module version, and all of the packages
// inside that folder. It will replace Directory once everything has been migrated.
type DirectoryNew struct {
	Path              string
	V1Path            string
	IsRedistributable bool
	Licenses          []*licenses.Metadata // metadata of applicable version licenses
	Readme            *Readme
	Package           *PackageNew
}

// PackageNew is a group of one or more Go source files with the same package
// header. A PackageNew is part of a directory.
// It will replace Package once everything has been migrated.
type PackageNew struct {
	Name          string
	Path          string
	Documentation *Documentation
	Imports       []string
}

// VersionedPackageNew is a PackageNew along with associated module information.
type VersionedPackageNew struct {
	PackageNew
	ModuleInfo
}

// Documentation is the rendered documentation for a given package
// for a specific GOOS and GOARCH.
type Documentation struct {
	// The values of the GOOS and GOARCH environment variables used to parse the package.
	GOOS     string
	GOARCH   string
	Synopsis string
	HTML     string
}

// Readme is a README at a given directory.
type Readme struct {
	Filepath string
	Contents string
}

// IndexVersion holds the version information returned by the module index.
type IndexVersion struct {
	Path      string
	Version   string
	Timestamp time.Time
}

// ModuleVersionState holds a worker module version state.
type ModuleVersionState struct {
	ModulePath string
	Version    string

	// IndexTimestamp is the timestamp received from the Index for this version,
	// which should correspond to the time this version was committed to the
	// Index.
	IndexTimestamp time.Time
	// CreatedAt is the time this version was originally inserted into the
	// module version state table.
	CreatedAt time.Time

	// Status is the most recent HTTP status code received from the Fetch service
	// for this version, or nil if no request to the fetch service has been made.
	Status int
	// Error is the most recent HTTP response body received from the Fetch
	// service, for a response with an unsuccessful status code. It is used for
	// debugging only, and has no semantic significance.
	Error string
	// TryCount is the number of times a fetch of this version has been
	// attempted.
	TryCount int
	// LastProcessedAt is the last time this version was updated with a result
	// from the fetch service.
	LastProcessedAt *time.Time
	// NextProcessedAfter is the next time a fetch for this version should be
	// attempted.
	NextProcessedAfter time.Time

	// AppVersion is the value of the GAE_VERSION environment variable, which is
	// set by app engine. It is a timestamp in the format 20190709t112655 that
	// is close to, but not the same as, the deployment time. For example, the
	// deployment time for the above timestamp might be Jul 9, 2019, 11:29:59 AM.
	AppVersion string

	// GoModPath is the path declared in the go.mod file.
	GoModPath string

	// NumPackages it the number of packages that were processed as part of the
	// module (regardless of whether the processing was successful).
	NumPackages *int
}

// PackageVersionState holds a worker package version state. It is associated
// with a given module version state.
type PackageVersionState struct {
	PackagePath string
	ModulePath  string
	Version     string
	Status      int
	Error       string
}

// SearchResult represents a single search result from SearchDocuments.
type SearchResult struct {
	Name        string
	PackagePath string
	ModulePath  string
	Version     string
	Synopsis    string
	Licenses    []string

	CommitTime time.Time
	// Score is used to sort items in an array of SearchResult.
	Score float64

	// NumImportedBy is the number of packages that import Package.
	NumImportedBy uint64

	// NumResults is the total number of packages that were returned for this search.
	NumResults uint64
	// Approximate reports whether NumResults is an approximate count. NumResults
	// can be approximate if search scanned only a subset of documents, and
	// result count is estimated using the hyperloglog algorithm.
	Approximate bool
}

// A FieldSet is a bit set of struct fields. It is used to avoid reading large
// struct fields from the data store. FieldSet is also the type of the
// individual bit values. (Think of them as singleton sets.)
//
// NoFields (the zero value) is the empty set. AllFields is the set containing
// every field.
//
// FieldSet bits are unique across the entire project, because some types are
// concatenations (via embedding) of others. For example, a VersionedPackage
// contains the fields of both a ModuleInfo and a Package, so we can't use the
// same bits for both ModuleInfo's ReadmeContents field and Package's
// DocumentationHTML field.
type FieldSet int64

// MinimalFields is the empty FieldSet.
const MinimalFields FieldSet = 0

// AllFields is the FieldSet that contains all fields.
const AllFields FieldSet = -1

// StringFieldMissing is the value for string fields that are not present
// in a struct. We use it to distinguish a (possibly valid) empty string
// from a field that was never populated.
const StringFieldMissing = "!MISSING"

// FieldSet bits for fields that can be conditionally read from the data store.
const (
	WithReadmeContents FieldSet = 1 << iota
	WithDocumentationHTML
)
<commit_msg>internal: remove unused VersionedPackageNew<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package internal

import (
	"path"
	"time"

	"golang.org/x/mod/module"
	"golang.org/x/pkgsite/internal/licenses"
	"golang.org/x/pkgsite/internal/source"
	"golang.org/x/pkgsite/internal/stdlib"
	"golang.org/x/pkgsite/internal/version"
)

const (
	// LatestVersion signifies the latest available version in requests to the
	// proxy client.
	LatestVersion = "latest"

	// MasterVersion signifies the version at master.
	MasterVersion = "master"

	// UnknownModulePath signifies that the module path for a given package
	// path is ambiguous or not known. This is because requests to the
	// frontend can come in the form of <import-path>[@<version>], and it is
	// not clear which part of the import-path is the module path.
	UnknownModulePath = "unknownModulePath"
)

// ModuleInfo holds metadata associated with a module.
type ModuleInfo struct {
	ModulePath        string
	Version           string
	CommitTime        time.Time
	ReadmeFilePath    string
	ReadmeContents    string
	VersionType       version.Type
	IsRedistributable bool
	HasGoMod          bool // whether the module zip has a go.mod file
	SourceInfo        *source.Info
}

// VersionMap holds metadata associated with module queries for a version.
type VersionMap struct {
	ModulePath       string
	RequestedVersion string
	ResolvedVersion  string
	GoModPath        string
	Status           int
	Error            string
}

// SeriesPath returns the series path for the module.
//
// A series is a group of modules that share the same base path and are assumed
// to be major-version variants.
//
// The series path is the module path without the version. For most modules,
// this will be the module path for all module versions with major version 0 or
// 1. For gopkg.in modules, the series path does not correspond to any module
// version.
//
// Examples:
// The module paths "a/b" and "a/b/v2"  both have series path "a/b".
// The module paths "gopkg.in/yaml.v1" and "gopkg.in/yaml.v2" both have series path "gopkg.in/yaml".
func (v *ModuleInfo) SeriesPath() string {
	return SeriesPathForModule(v.ModulePath)
}

// SeriesPathForModule returns the series path for the provided modulePath.
func SeriesPathForModule(modulePath string) string {
	seriesPath, _, _ := module.SplitPathVersion(modulePath)
	return seriesPath
}

// V1Path returns the path for version 1 of the package whose path
// is modulePath + "/" + suffix. If modulePath is the standard
// library, then V1Path returns suffix.
func V1Path(modulePath, suffix string) string {
	if modulePath == stdlib.ModulePath {
		return suffix
	}
	return path.Join(SeriesPathForModule(modulePath), suffix)
}

// A Module is a specific, reproducible build of a module.
type Module struct {
	ModuleInfo
	Packages []*Package
	// Licenses holds all licenses within this module version, including those
	// that may be contained in nested subdirectories.
	Licenses []*licenses.License

	Directories []*DirectoryNew
}

// A Package is a group of one or more Go source files with the same package
// header. Packages are part of a module.
type Package struct {
	Path              string
	Name              string
	Synopsis          string
	IsRedistributable bool
	Licenses          []*licenses.Metadata // metadata of applicable version licenses
	Imports           []string
	DocumentationHTML string
	// The values of the GOOS and GOARCH environment variables used to parse the package.
	GOOS   string
	GOARCH string

	// V1Path is the package path of a package with major version 1 in a given series.
	V1Path string
}

// VersionedPackage is a Package along with its corresponding module
// information.
type VersionedPackage struct {
	Package
	ModuleInfo
}

// Directory represents a folder in a module version, and all of the packages
// inside that folder.
type Directory struct {
	ModuleInfo
	Path     string
	Packages []*Package
}

// VersionedDirectory is a DirectoryNew along with its corresponding module
// information.
type VersionedDirectory struct {
	DirectoryNew
	ModuleInfo
}

// DirectoryNew is a folder in a module version, and all of the packages
// inside that folder. It will replace Directory once everything has been migrated.
type DirectoryNew struct {
	Path              string
	V1Path            string
	IsRedistributable bool
	Licenses          []*licenses.Metadata // metadata of applicable version licenses
	Readme            *Readme
	Package           *PackageNew
}

// PackageNew is a group of one or more Go source files with the same package
// header. A PackageNew is part of a directory.
// It will replace Package once everything has been migrated.
type PackageNew struct {
	Name          string
	Path          string
	Documentation *Documentation
	Imports       []string
}

// Documentation is the rendered documentation for a given package
// for a specific GOOS and GOARCH.
type Documentation struct {
	// The values of the GOOS and GOARCH environment variables used to parse the package.
	GOOS     string
	GOARCH   string
	Synopsis string
	HTML     string
}

// Readme is a README at a given directory.
type Readme struct {
	Filepath string
	Contents string
}

// IndexVersion holds the version information returned by the module index.
type IndexVersion struct {
	Path      string
	Version   string
	Timestamp time.Time
}

// ModuleVersionState holds a worker module version state.
type ModuleVersionState struct {
	ModulePath string
	Version    string

	// IndexTimestamp is the timestamp received from the Index for this version,
	// which should correspond to the time this version was committed to the
	// Index.
	IndexTimestamp time.Time
	// CreatedAt is the time this version was originally inserted into the
	// module version state table.
	CreatedAt time.Time

	// Status is the most recent HTTP status code received from the Fetch service
	// for this version, or nil if no request to the fetch service has been made.
	Status int
	// Error is the most recent HTTP response body received from the Fetch
	// service, for a response with an unsuccessful status code. It is used for
	// debugging only, and has no semantic significance.
	Error string
	// TryCount is the number of times a fetch of this version has been
	// attempted.
	TryCount int
	// LastProcessedAt is the last time this version was updated with a result
	// from the fetch service.
	LastProcessedAt *time.Time
	// NextProcessedAfter is the next time a fetch for this version should be
	// attempted.
	NextProcessedAfter time.Time

	// AppVersion is the value of the GAE_VERSION environment variable, which is
	// set by app engine. It is a timestamp in the format 20190709t112655 that
	// is close to, but not the same as, the deployment time. For example, the
	// deployment time for the above timestamp might be Jul 9, 2019, 11:29:59 AM.
	AppVersion string

	// GoModPath is the path declared in the go.mod file.
	GoModPath string

	// NumPackages it the number of packages that were processed as part of the
	// module (regardless of whether the processing was successful).
	NumPackages *int
}

// PackageVersionState holds a worker package version state. It is associated
// with a given module version state.
type PackageVersionState struct {
	PackagePath string
	ModulePath  string
	Version     string
	Status      int
	Error       string
}

// SearchResult represents a single search result from SearchDocuments.
type SearchResult struct {
	Name        string
	PackagePath string
	ModulePath  string
	Version     string
	Synopsis    string
	Licenses    []string

	CommitTime time.Time
	// Score is used to sort items in an array of SearchResult.
	Score float64

	// NumImportedBy is the number of packages that import Package.
	NumImportedBy uint64

	// NumResults is the total number of packages that were returned for this search.
	NumResults uint64
	// Approximate reports whether NumResults is an approximate count. NumResults
	// can be approximate if search scanned only a subset of documents, and
	// result count is estimated using the hyperloglog algorithm.
	Approximate bool
}

// A FieldSet is a bit set of struct fields. It is used to avoid reading large
// struct fields from the data store. FieldSet is also the type of the
// individual bit values. (Think of them as singleton sets.)
//
// NoFields (the zero value) is the empty set. AllFields is the set containing
// every field.
//
// FieldSet bits are unique across the entire project, because some types are
// concatenations (via embedding) of others. For example, a VersionedPackage
// contains the fields of both a ModuleInfo and a Package, so we can't use the
// same bits for both ModuleInfo's ReadmeContents field and Package's
// DocumentationHTML field.
type FieldSet int64

// MinimalFields is the empty FieldSet.
const MinimalFields FieldSet = 0

// AllFields is the FieldSet that contains all fields.
const AllFields FieldSet = -1

// StringFieldMissing is the value for string fields that are not present
// in a struct. We use it to distinguish a (possibly valid) empty string
// from a field that was never populated.
const StringFieldMissing = "!MISSING"

// FieldSet bits for fields that can be conditionally read from the data store.
const (
	WithReadmeContents FieldSet = 1 << iota
	WithDocumentationHTML
)
<|endoftext|>


<commit_before>package keys

import "fmt"

// Kind expresses usage of the ambient internal key.
type Kind int

const (
	// Delete represents deletion of this key.
	Delete = 0
	// Value represents value setting of this key.
	Value   = 1
	maxKind = Value

	// Seek is maximum(Value, Delete), which is a valid Kind and
	// serves as termination point for keys with same sequence.
	Seek = maxKind
)

func (k Kind) String() string {
	switch k {
	case Delete:
		return "value deletion"
	case Value:
		return "value setting"
	}
	return fmt.Sprintf("unknown kind: %d", k)
}
<commit_msg>Fix document for keys.Kind Seek<commit_after>package keys

import "fmt"

// Kind expresses usage of the ambient internal key.
type Kind int

const (
	// Delete represents deletion of this key.
	Delete = 0
	// Value represents value setting of this key.
	Value   = 1
	maxKind = Value

	// Seek is maximum(Value, Delete), which is a valid Kind and
	// serves as start point for keys with same sequence.
	//
	// See InternalComparator.Compare for ordering among internal keys.
	Seek = maxKind
)

func (k Kind) String() string {
	switch k {
	case Delete:
		return "value deletion"
	case Value:
		return "value setting"
	}
	return fmt.Sprintf("unknown kind: %d", k)
}
<|endoftext|>


<commit_before>package util

import (
	"go/ast"
	"io/ioutil"
	"os"
	"strings"
	"text/template"

	"github.com/gedex/inflector"
	"github.com/serenize/snaker"
)

func GetFuncsMap() template.FuncMap {
	return template.FuncMap{
		"snakecase": snaker.CamelToSnake,
		"plural":    inflector.Pluralize,
		"unexport":  strings.ToLower,
		"exported":  ast.IsExported,
		"primitive": Primitive,
		"numerical": Numerical,
		"lower":     strings.ToLower,
		"upper":     strings.ToTitle,
		"zeroValue": ZeroValue,
	}
}

func Plural(s string) string {
	return inflector.Pluralize(s)
}

func Snakecase(s string) string {
	return snaker.CamelToSnake(s)
}

func GetPath(path string) string {
	return strings.TrimSuffix(path, string(os.PathSeparator))
}

func GetImportPath(path string) string {
	return GetPath(strings.Replace(path, os.Getenv("GOPATH")+string(os.PathSeparator)+"src"+string(os.PathSeparator), "", 1))
}

func GetFullPath(path string, subpath string) string {
	return path + string(os.PathSeparator) + strings.Replace(subpath, "/", string(os.PathSeparator), -1)
}

func Mkdir(name string) error {
	return os.MkdirAll(name, 0777)
}

func SaveFile(p string, data []byte) error {
	return ioutil.WriteFile(p, data, 0666)
}

func Primitive(a string) bool {
	if a == "uint" || a == "uint8" || a == "uint16" || a == "uint32" || a == "uint64" ||
		a == "int" || a == "int8" || a == "int16" || a == "int32" || a == "int64" ||
		a == "float32" || a == "float64" ||
		a == "complex32" || a == "complex64" ||
		a == "byte" || a == "[]byte" ||
		a == "time.Time" ||
		a == "rune" ||
		a == "bool" ||
		a == "string" {
		return true
	}
	return false
}

func ZeroValue(a string) string {
	if a == "uint" || a == "uint8" || a == "uint16" || a == "uint32" || a == "uint64" ||
		a == "int" || a == "int8" || a == "int16" || a == "int32" || a == "int64" ||
		a == "float32" || a == "float64" ||
		a == "complex32" || a == "complex64" {
		return "0"
	}
	if a == "byte" || a == "[]byte" || a == "*time.Time" {
		return "nil"
	}
	if a == "rune" {
		return "''"
	}
	if a == "bool" {
		return "f"
	}
	if a == "string" {
		return "\"\""
	}
	return ""
}

func Numerical(a string) bool {
	if a == "uint" || a == "uint8" || a == "uint16" || a == "uint32" || a == "uint64" ||
		a == "int" || a == "int8" || a == "int16" || a == "int32" || a == "int64" ||
		a == "time.Time" ||
		a == "float32" || a == "float64" ||
		a == "complex32" || a == "complex64" {
		return true
	}
	return false
}
<commit_msg>Changed templates funcs<commit_after>package util

import (
	"go/ast"
	"io/ioutil"
	"os"
	"strings"
	"text/template"

	"github.com/gedex/inflector"
	"github.com/serenize/snaker"
)

func GetFuncsMap() template.FuncMap {
	return template.FuncMap{
		"plural":    inflector.Pluralize,
		"unexport":  strings.ToLower,
		"exported":  ast.IsExported,
		"primitive": Primitive,
		"numerical": Numerical,
		"lower":     strings.ToLower,
		"upper":     Upper,
		"zeroValue": ZeroValue,
	}
}

func Plural(s string) string {
	return inflector.Pluralize(s)
}

func Upper(s string) string {
	return strings.ToUpper(string(s[0])) + s[1:]
}

func Snakecase(s string) string {
	return snaker.CamelToSnake(s)
}

func GetPath(path string) string {
	return strings.TrimSuffix(path, string(os.PathSeparator))
}

func GetImportPath(path string) string {
	return GetPath(strings.Replace(path, os.Getenv("GOPATH")+string(os.PathSeparator)+"src"+string(os.PathSeparator), "", 1))
}

func GetFullPath(path string, subpath string) string {
	return path + string(os.PathSeparator) + strings.Replace(subpath, "/", string(os.PathSeparator), -1)
}

func Mkdir(name string) error {
	return os.MkdirAll(name, 0777)
}

func SaveFile(p string, data []byte) error {
	return ioutil.WriteFile(p, data, 0666)
}

func Primitive(a string) bool {
	if a == "uint" || a == "uint8" || a == "uint16" || a == "uint32" || a == "uint64" ||
		a == "int" || a == "int8" || a == "int16" || a == "int32" || a == "int64" ||
		a == "float32" || a == "float64" ||
		a == "complex32" || a == "complex64" ||
		a == "byte" || a == "[]byte" ||
		a == "time.Time" ||
		a == "rune" ||
		a == "bool" ||
		a == "string" {
		return true
	}
	return false
}

func ZeroValue(a string) string {
	if a == "uint" || a == "uint8" || a == "uint16" || a == "uint32" || a == "uint64" ||
		a == "int" || a == "int8" || a == "int16" || a == "int32" || a == "int64" ||
		a == "float32" || a == "float64" ||
		a == "complex32" || a == "complex64" {
		return "0"
	}
	if a == "byte" || a == "[]byte" || a == "*time.Time" {
		return "nil"
	}
	if a == "rune" {
		return "''"
	}
	if a == "bool" {
		return "f"
	}
	if a == "string" {
		return "\"\""
	}
	return ""
}

func Numerical(a string) bool {
	if a == "uint" || a == "uint8" || a == "uint16" || a == "uint32" || a == "uint64" ||
		a == "int" || a == "int8" || a == "int16" || a == "int32" || a == "int64" ||
		a == "time.Time" ||
		a == "float32" || a == "float64" ||
		a == "complex32" || a == "complex64" {
		return true
	}
	return false
}
<|endoftext|>