<commit_before>package dynamodb

import (
	"fmt"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/gruntwork-io/terragrunt/aws_helper"
	"github.com/gruntwork-io/terragrunt/errors"
	"github.com/gruntwork-io/terragrunt/options"
	"github.com/gruntwork-io/terragrunt/util"
)

// DynamoDB only allows 10 table creates/deletes simultaneously. To ensure we don't hit this error, especially when
// running many automated tests in parallel, we use a counting semaphore
var tableCreateDeleteSemaphore = NewCountingSemaphore(10)

// Terraform requires the DynamoDB table to have a primary key with this name
const ATTR_LOCK_ID = "LockID"

// Default is to retry for up to 5 minutes
const MAX_RETRIES_WAITING_FOR_TABLE_TO_BE_ACTIVE = 30
const SLEEP_BETWEEN_TABLE_STATUS_CHECKS = 10 * time.Second

const DYNAMODB_PAY_PER_REQUEST_BILLING_MODE = "PAY_PER_REQUEST"

// Create an authenticated client for DynamoDB
func CreateDynamoDbClient(config *aws_helper.AwsSessionConfig, terragruntOptions *options.TerragruntOptions) (*dynamodb.DynamoDB, error) {
	session, err := aws_helper.CreateAwsSession(config, terragruntOptions)
	if err != nil {
		return nil, err
	}

	return dynamodb.New(session), nil
}

// Create the lock table in DynamoDB if it doesn't already exist
func CreateLockTableIfNecessary(tableName string, tags map[string]string, client *dynamodb.DynamoDB, terragruntOptions *options.TerragruntOptions) error {
	tableExists, err := LockTableExistsAndIsActive(tableName, client)
	if err != nil {
		return err
	}

	if !tableExists {
		terragruntOptions.Logger.Printf("Lock table %s does not exist in DynamoDB. Will need to create it just this first time.", tableName)
		return CreateLockTable(tableName, tags, client, terragruntOptions)
	}

	return nil
}

// Return true if the lock table exists in DynamoDB and is in "active" state
func LockTableExistsAndIsActive(tableName string, client *dynamodb.DynamoDB) (bool, error) {
	output, err := client.DescribeTable(&dynamodb.DescribeTableInput{TableName: aws.String(tableName)})
	if err != nil {
		if awsErr, isAwsErr := err.(awserr.Error); isAwsErr && awsErr.Code() == "ResourceNotFoundException" {
			return false, nil
		} else {
			return false, errors.WithStackTrace(err)
		}
	}

	return *output.Table.TableStatus == dynamodb.TableStatusActive, nil
}

// Return true if the lock table's SSEncryption is turned on
func LockTableCheckSSEncryptionIsOn(tableName string, client *dynamodb.DynamoDB) (bool, error) {
	output, err := client.DescribeTable(&dynamodb.DescribeTableInput{TableName: aws.String(tableName)})
	if err != nil {
		return false, errors.WithStackTrace(err)
	}

	return output.Table.SSEDescription != nil && aws.StringValue(output.Table.SSEDescription.Status) == dynamodb.SSEStatusEnabled, nil
}

// Create a lock table in DynamoDB and wait until it is in "active" state. If the table already exists, merely wait
// until it is in "active" state.
func CreateLockTable(tableName string, tags map[string]string, client *dynamodb.DynamoDB, terragruntOptions *options.TerragruntOptions) error {
	tableCreateDeleteSemaphore.Acquire()
	defer tableCreateDeleteSemaphore.Release()

	terragruntOptions.Logger.Printf("Creating table %s in DynamoDB", tableName)

	createTableOutput, err := client.CreateTable(&dynamodb.CreateTableInput{
		TableName:   aws.String(tableName),
		BillingMode: aws.String(DYNAMODB_PAY_PER_REQUEST_BILLING_MODE),
		AttributeDefinitions: []*dynamodb.AttributeDefinition{
			{
				AttributeName: aws.String(ATTR_LOCK_ID),
				AttributeType: aws.String(dynamodb.ScalarAttributeTypeS),
			},
		},
		KeySchema: []*dynamodb.KeySchemaElement{
			{
				AttributeName: aws.String(ATTR_LOCK_ID),
				KeyType:       aws.String(dynamodb.KeyTypeHash),
			},
		},
	})

	if err != nil {
		if isTableAlreadyBeingCreatedOrUpdatedError(err) {
			terragruntOptions.Logger.Printf("Looks like someone created table %s at the same time. Will wait for it to be in active state.", tableName)
		} else {
			return errors.WithStackTrace(err)
		}
	}

	err = waitForTableToBeActive(tableName, client, MAX_RETRIES_WAITING_FOR_TABLE_TO_BE_ACTIVE, SLEEP_BETWEEN_TABLE_STATUS_CHECKS, terragruntOptions)

	if err != nil {
		return err
	}

	if createTableOutput != nil && createTableOutput.TableDescription != nil && createTableOutput.TableDescription.TableArn != nil {
		// Do not tag in case somebody else had created the table

		err = tagTableIfTagsGiven(tags, createTableOutput.TableDescription.TableArn, client, terragruntOptions)

		if err != nil {
			return errors.WithStackTrace(err)
		}
	}

	return nil
}

func tagTableIfTagsGiven(tags map[string]string, tableArn *string, client *dynamodb.DynamoDB, terragruntOptions *options.TerragruntOptions) error {

	if tags == nil || len(tags) == 0 {
		terragruntOptions.Logger.Printf("No tags for lock table given.")
		return nil
	}

	// we were able to create the table successfully, now add tags
	terragruntOptions.Logger.Printf("Adding tags to lock table: %s", tags)

	var tagsConverted []*dynamodb.Tag

	for k, v := range tags {
		tagsConverted = append(tagsConverted, &dynamodb.Tag{Key: aws.String(k), Value: aws.String(v)})
	}

	var input = dynamodb.TagResourceInput{
		ResourceArn: tableArn,
		Tags:        tagsConverted}

	_, err := client.TagResource(&input)

	return err
}

// Delete the given table in DynamoDB
func DeleteTable(tableName string, client *dynamodb.DynamoDB) error {
	tableCreateDeleteSemaphore.Acquire()
	defer tableCreateDeleteSemaphore.Release()

	_, err := client.DeleteTable(&dynamodb.DeleteTableInput{TableName: aws.String(tableName)})
	return err
}

// Return true if the given error is the error message returned by AWS when the resource already exists and is being
// updated by someone else
func isTableAlreadyBeingCreatedOrUpdatedError(err error) bool {
	awsErr, isAwsErr := err.(awserr.Error)
	return isAwsErr && awsErr.Code() == "ResourceInUseException"
}

// Wait for the given DynamoDB table to be in the "active" state. If it's not in "active" state, sleep for the
// specified amount of time, and try again, up to a maximum of maxRetries retries.
func waitForTableToBeActive(tableName string, client *dynamodb.DynamoDB, maxRetries int, sleepBetweenRetries time.Duration, terragruntOptions *options.TerragruntOptions) error {
	return waitForTableToBeActiveWithRandomSleep(tableName, client, maxRetries, sleepBetweenRetries, sleepBetweenRetries, terragruntOptions)
}

// Waits for the given table as described above, but sleeps a random amount of time greater than sleepBetweenRetriesMin
// and less than sleepBetweenRetriesMax between tries. This is to avoid an AWS issue where all waiting requests fire at
// the same time, which continually triggered AWS's "subscriber limit exceeded" API error.
func waitForTableToBeActiveWithRandomSleep(tableName string, client *dynamodb.DynamoDB, maxRetries int, sleepBetweenRetriesMin time.Duration, sleepBetweenRetriesMax time.Duration, terragruntOptions *options.TerragruntOptions) error {
	for i := 0; i < maxRetries; i++ {
		tableReady, err := LockTableExistsAndIsActive(tableName, client)
		if err != nil {
			return err
		}

		if tableReady {
			terragruntOptions.Logger.Printf("Success! Table %s is now in active state.", tableName)
			return nil
		}

		sleepBetweenRetries := util.GetRandomTime(sleepBetweenRetriesMin, sleepBetweenRetriesMax)
		terragruntOptions.Logger.Printf("Table %s is not yet in active state. Will check again after %s.", tableName, sleepBetweenRetries)
		time.Sleep(sleepBetweenRetries)
	}

	return errors.WithStackTrace(TableActiveRetriesExceeded{TableName: tableName, Retries: maxRetries})
}

// Encrypt the TFState Lock table - If Necessary
func UpdateLockTableSetSSEncryptionOnIfNecessary(tableName string, client *dynamodb.DynamoDB, terragruntOptions *options.TerragruntOptions) error {
	tableSSEncrypted, err := LockTableCheckSSEncryptionIsOn(tableName, client)
	if err != nil {
		return errors.WithStackTrace(err)
	}

	if tableSSEncrypted {
		terragruntOptions.Logger.Printf("Table %s already has encryption enabled", tableName)
		return nil
	}

	tableCreateDeleteSemaphore.Acquire()
	defer tableCreateDeleteSemaphore.Release()

	terragruntOptions.Logger.Printf("Enabling server-side encryption on table %s in AWS DynamoDB", tableName)

	input := &dynamodb.UpdateTableInput{
		SSESpecification: &dynamodb.SSESpecification{
			Enabled: aws.Bool(true),
			SSEType: aws.String("KMS"),
		},
		TableName: aws.String(tableName),
	}

	if _, err := client.UpdateTable(input); err != nil {
		if isTableAlreadyBeingCreatedOrUpdatedError(err) {
			terragruntOptions.Logger.Printf("Looks like someone is already updating table %s at the same time. Will wait for that update to complete.", tableName)
		} else {
			return errors.WithStackTrace(err)
		}
	}

	if err := waitForEncryptionToBeEnabled(tableName, client, terragruntOptions); err != nil {
		return errors.WithStackTrace(err)
	}

	return waitForTableToBeActive(tableName, client, MAX_RETRIES_WAITING_FOR_TABLE_TO_BE_ACTIVE, SLEEP_BETWEEN_TABLE_STATUS_CHECKS, terragruntOptions)
}

// Wait until encryption is enabled for the given table
func waitForEncryptionToBeEnabled(tableName string, client *dynamodb.DynamoDB, terragruntOptions *options.TerragruntOptions) error {
	maxRetries := 15
	sleepBetweenRetries := 20 * time.Second

	terragruntOptions.Logger.Printf("Waiting for encryption to be enabled on table %s", tableName)

	for i := 0; i < maxRetries; i++ {
		tableSSEncrypted, err := LockTableCheckSSEncryptionIsOn(tableName, client)
		if err != nil {
			return errors.WithStackTrace(err)
		}

		if tableSSEncrypted {
			terragruntOptions.Logger.Printf("Encryption is now enabled for table %s!", tableName)
			return nil
		}

		terragruntOptions.Logger.Printf("Encryption is still not enabled for table %s. Will sleep for %v and try again.", tableName, sleepBetweenRetries)
		time.Sleep(sleepBetweenRetries)
	}

	return errors.WithStackTrace(TableEncryptedRetriesExceeded{TableName: tableName, Retries: maxRetries})
}

// Custom error types

type TableActiveRetriesExceeded struct {
	TableName string
	Retries   int
}

func (err TableActiveRetriesExceeded) Error() string {
	return fmt.Sprintf("Table %s is still not in active state after %d retries.", err.TableName, err.Retries)
}

type TableDoesNotExist struct {
	TableName  string
	Underlying error
}

func (err TableDoesNotExist) Error() string {
	return fmt.Sprintf("Table %s does not exist in DynamoDB! Original error from AWS: %v", err.TableName, err.Underlying)
}

type TableEncryptedRetriesExceeded struct {
	TableName string
	Retries   int
}

func (err TableEncryptedRetriesExceeded) Error() string {
	return fmt.Sprintf("Table %s still does not have encryption enabled after %d retries.", err.TableName, err.Retries)
}
<commit_msg>Updates CreateTable to instantiate `CreateTableInput` vars separately<commit_after>package dynamodb

import (
	"fmt"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/gruntwork-io/terragrunt/aws_helper"
	"github.com/gruntwork-io/terragrunt/errors"
	"github.com/gruntwork-io/terragrunt/options"
	"github.com/gruntwork-io/terragrunt/util"
)

// DynamoDB only allows 10 table creates/deletes simultaneously. To ensure we don't hit this error, especially when
// running many automated tests in parallel, we use a counting semaphore
var tableCreateDeleteSemaphore = NewCountingSemaphore(10)

// Terraform requires the DynamoDB table to have a primary key with this name
const ATTR_LOCK_ID = "LockID"

// Default is to retry for up to 5 minutes
const MAX_RETRIES_WAITING_FOR_TABLE_TO_BE_ACTIVE = 30
const SLEEP_BETWEEN_TABLE_STATUS_CHECKS = 10 * time.Second

const DYNAMODB_PAY_PER_REQUEST_BILLING_MODE = "PAY_PER_REQUEST"

// Create an authenticated client for DynamoDB
func CreateDynamoDbClient(config *aws_helper.AwsSessionConfig, terragruntOptions *options.TerragruntOptions) (*dynamodb.DynamoDB, error) {
	session, err := aws_helper.CreateAwsSession(config, terragruntOptions)
	if err != nil {
		return nil, err
	}

	return dynamodb.New(session), nil
}

// Create the lock table in DynamoDB if it doesn't already exist
func CreateLockTableIfNecessary(tableName string, tags map[string]string, client *dynamodb.DynamoDB, terragruntOptions *options.TerragruntOptions) error {
	tableExists, err := LockTableExistsAndIsActive(tableName, client)
	if err != nil {
		return err
	}

	if !tableExists {
		terragruntOptions.Logger.Printf("Lock table %s does not exist in DynamoDB. Will need to create it just this first time.", tableName)
		return CreateLockTable(tableName, tags, client, terragruntOptions)
	}

	return nil
}

// Return true if the lock table exists in DynamoDB and is in "active" state
func LockTableExistsAndIsActive(tableName string, client *dynamodb.DynamoDB) (bool, error) {
	output, err := client.DescribeTable(&dynamodb.DescribeTableInput{TableName: aws.String(tableName)})
	if err != nil {
		if awsErr, isAwsErr := err.(awserr.Error); isAwsErr && awsErr.Code() == "ResourceNotFoundException" {
			return false, nil
		} else {
			return false, errors.WithStackTrace(err)
		}
	}

	return *output.Table.TableStatus == dynamodb.TableStatusActive, nil
}

// Return true if the lock table's SSEncryption is turned on
func LockTableCheckSSEncryptionIsOn(tableName string, client *dynamodb.DynamoDB) (bool, error) {
	output, err := client.DescribeTable(&dynamodb.DescribeTableInput{TableName: aws.String(tableName)})
	if err != nil {
		return false, errors.WithStackTrace(err)
	}

	return output.Table.SSEDescription != nil && aws.StringValue(output.Table.SSEDescription.Status) == dynamodb.SSEStatusEnabled, nil
}

// Create a lock table in DynamoDB and wait until it is in "active" state. If the table already exists, merely wait
// until it is in "active" state.
func CreateLockTable(tableName string, tags map[string]string, client *dynamodb.DynamoDB, terragruntOptions *options.TerragruntOptions) error {
	tableCreateDeleteSemaphore.Acquire()
	defer tableCreateDeleteSemaphore.Release()

	terragruntOptions.Logger.Printf("Creating table %s in DynamoDB", tableName)

	attributeDefinitions := []*dynamodb.AttributeDefinition{
		{AttributeName: aws.String(ATTR_LOCK_ID), AttributeType: aws.String(dynamodb.ScalarAttributeTypeS)},
	}

	keySchema := []*dynamodb.KeySchemaElement{
		{AttributeName: aws.String(ATTR_LOCK_ID), KeyType: aws.String(dynamodb.KeyTypeHash)},
	}

	createTableOutput, err := client.CreateTable(&dynamodb.CreateTableInput{
		TableName:   aws.String(tableName),
		BillingMode: aws.String(DYNAMODB_PAY_PER_REQUEST_BILLING_MODE),
		AttributeDefinitions: attributeDefinitions,
		KeySchema: keySchema,
	})

	if err != nil {
		if isTableAlreadyBeingCreatedOrUpdatedError(err) {
			terragruntOptions.Logger.Printf("Looks like someone created table %s at the same time. Will wait for it to be in active state.", tableName)
		} else {
			return errors.WithStackTrace(err)
		}
	}

	err = waitForTableToBeActive(tableName, client, MAX_RETRIES_WAITING_FOR_TABLE_TO_BE_ACTIVE, SLEEP_BETWEEN_TABLE_STATUS_CHECKS, terragruntOptions)

	if err != nil {
		return err
	}

	if createTableOutput != nil && createTableOutput.TableDescription != nil && createTableOutput.TableDescription.TableArn != nil {
		// Do not tag in case somebody else had created the table

		err = tagTableIfTagsGiven(tags, createTableOutput.TableDescription.TableArn, client, terragruntOptions)

		if err != nil {
			return errors.WithStackTrace(err)
		}
	}

	return nil
}

func tagTableIfTagsGiven(tags map[string]string, tableArn *string, client *dynamodb.DynamoDB, terragruntOptions *options.TerragruntOptions) error {

	if tags == nil || len(tags) == 0 {
		terragruntOptions.Logger.Printf("No tags for lock table given.")
		return nil
	}

	// we were able to create the table successfully, now add tags
	terragruntOptions.Logger.Printf("Adding tags to lock table: %s", tags)

	var tagsConverted []*dynamodb.Tag

	for k, v := range tags {
		tagsConverted = append(tagsConverted, &dynamodb.Tag{Key: aws.String(k), Value: aws.String(v)})
	}

	var input = dynamodb.TagResourceInput{
		ResourceArn: tableArn,
		Tags:        tagsConverted}

	_, err := client.TagResource(&input)

	return err
}

// Delete the given table in DynamoDB
func DeleteTable(tableName string, client *dynamodb.DynamoDB) error {
	tableCreateDeleteSemaphore.Acquire()
	defer tableCreateDeleteSemaphore.Release()

	_, err := client.DeleteTable(&dynamodb.DeleteTableInput{TableName: aws.String(tableName)})
	return err
}

// Return true if the given error is the error message returned by AWS when the resource already exists and is being
// updated by someone else
func isTableAlreadyBeingCreatedOrUpdatedError(err error) bool {
	awsErr, isAwsErr := err.(awserr.Error)
	return isAwsErr && awsErr.Code() == "ResourceInUseException"
}

// Wait for the given DynamoDB table to be in the "active" state. If it's not in "active" state, sleep for the
// specified amount of time, and try again, up to a maximum of maxRetries retries.
func waitForTableToBeActive(tableName string, client *dynamodb.DynamoDB, maxRetries int, sleepBetweenRetries time.Duration, terragruntOptions *options.TerragruntOptions) error {
	return waitForTableToBeActiveWithRandomSleep(tableName, client, maxRetries, sleepBetweenRetries, sleepBetweenRetries, terragruntOptions)
}

// Waits for the given table as described above, but sleeps a random amount of time greater than sleepBetweenRetriesMin
// and less than sleepBetweenRetriesMax between tries. This is to avoid an AWS issue where all waiting requests fire at
// the same time, which continually triggered AWS's "subscriber limit exceeded" API error.
func waitForTableToBeActiveWithRandomSleep(tableName string, client *dynamodb.DynamoDB, maxRetries int, sleepBetweenRetriesMin time.Duration, sleepBetweenRetriesMax time.Duration, terragruntOptions *options.TerragruntOptions) error {
	for i := 0; i < maxRetries; i++ {
		tableReady, err := LockTableExistsAndIsActive(tableName, client)
		if err != nil {
			return err
		}

		if tableReady {
			terragruntOptions.Logger.Printf("Success! Table %s is now in active state.", tableName)
			return nil
		}

		sleepBetweenRetries := util.GetRandomTime(sleepBetweenRetriesMin, sleepBetweenRetriesMax)
		terragruntOptions.Logger.Printf("Table %s is not yet in active state. Will check again after %s.", tableName, sleepBetweenRetries)
		time.Sleep(sleepBetweenRetries)
	}

	return errors.WithStackTrace(TableActiveRetriesExceeded{TableName: tableName, Retries: maxRetries})
}

// Encrypt the TFState Lock table - If Necessary
func UpdateLockTableSetSSEncryptionOnIfNecessary(tableName string, client *dynamodb.DynamoDB, terragruntOptions *options.TerragruntOptions) error {
	tableSSEncrypted, err := LockTableCheckSSEncryptionIsOn(tableName, client)
	if err != nil {
		return errors.WithStackTrace(err)
	}

	if tableSSEncrypted {
		terragruntOptions.Logger.Printf("Table %s already has encryption enabled", tableName)
		return nil
	}

	tableCreateDeleteSemaphore.Acquire()
	defer tableCreateDeleteSemaphore.Release()

	terragruntOptions.Logger.Printf("Enabling server-side encryption on table %s in AWS DynamoDB", tableName)

	input := &dynamodb.UpdateTableInput{
		SSESpecification: &dynamodb.SSESpecification{
			Enabled: aws.Bool(true),
			SSEType: aws.String("KMS"),
		},
		TableName: aws.String(tableName),
	}

	if _, err := client.UpdateTable(input); err != nil {
		if isTableAlreadyBeingCreatedOrUpdatedError(err) {
			terragruntOptions.Logger.Printf("Looks like someone is already updating table %s at the same time. Will wait for that update to complete.", tableName)
		} else {
			return errors.WithStackTrace(err)
		}
	}

	if err := waitForEncryptionToBeEnabled(tableName, client, terragruntOptions); err != nil {
		return errors.WithStackTrace(err)
	}

	return waitForTableToBeActive(tableName, client, MAX_RETRIES_WAITING_FOR_TABLE_TO_BE_ACTIVE, SLEEP_BETWEEN_TABLE_STATUS_CHECKS, terragruntOptions)
}

// Wait until encryption is enabled for the given table
func waitForEncryptionToBeEnabled(tableName string, client *dynamodb.DynamoDB, terragruntOptions *options.TerragruntOptions) error {
	maxRetries := 15
	sleepBetweenRetries := 20 * time.Second

	terragruntOptions.Logger.Printf("Waiting for encryption to be enabled on table %s", tableName)

	for i := 0; i < maxRetries; i++ {
		tableSSEncrypted, err := LockTableCheckSSEncryptionIsOn(tableName, client)
		if err != nil {
			return errors.WithStackTrace(err)
		}

		if tableSSEncrypted {
			terragruntOptions.Logger.Printf("Encryption is now enabled for table %s!", tableName)
			return nil
		}

		terragruntOptions.Logger.Printf("Encryption is still not enabled for table %s. Will sleep for %v and try again.", tableName, sleepBetweenRetries)
		time.Sleep(sleepBetweenRetries)
	}

	return errors.WithStackTrace(TableEncryptedRetriesExceeded{TableName: tableName, Retries: maxRetries})
}

// Custom error types

type TableActiveRetriesExceeded struct {
	TableName string
	Retries   int
}

func (err TableActiveRetriesExceeded) Error() string {
	return fmt.Sprintf("Table %s is still not in active state after %d retries.", err.TableName, err.Retries)
}

type TableDoesNotExist struct {
	TableName  string
	Underlying error
}

func (err TableDoesNotExist) Error() string {
	return fmt.Sprintf("Table %s does not exist in DynamoDB! Original error from AWS: %v", err.TableName, err.Underlying)
}

type TableEncryptedRetriesExceeded struct {
	TableName string
	Retries   int
}

func (err TableEncryptedRetriesExceeded) Error() string {
	return fmt.Sprintf("Table %s still does not have encryption enabled after %d retries.", err.TableName, err.Retries)
}
<|endoftext|>


<commit_before>package indexer

import (
	"bytes"
	"encoding/binary"
	"errors"
	"github.com/couchbase/indexing/secondary/collatejson"
	"github.com/couchbase/indexing/secondary/common"
)

var (
	ErrSecKeyNil = errors.New("Secondary key array is empty")
)

// Special index keys
var (
	MinIndexKey = &NilIndexKey{cmp: -1, pcmp: -1}
	MaxIndexKey = &NilIndexKey{cmp: 1, pcmp: 1}
	NilJsonKey  = []byte("[]")
)

var (
	jsonEncoder *collatejson.Codec
	encBufPool  *common.BytesBufPool
)

const (
	maxIndexEntrySize = MAX_SEC_KEY_BUFFER_LEN + MAX_DOCID_LEN + 2
)

func init() {
	jsonEncoder = collatejson.NewCodec(16)
	encBufPool = common.NewByteBufferPool(maxIndexEntrySize)
}

// Generic index entry abstraction (primary or secondary)
// Represents a row in the index
type IndexEntry interface {
	ReadDocId([]byte) ([]byte, error)
	ReadSecKey([]byte) ([]byte, error)

	Bytes() []byte
	String() string
}

// Generic index key abstraction (primary or secondary)
// Represents a key supplied by the user for scan operation
type IndexKey interface {
	Compare(IndexEntry) int
	ComparePrefixFields(IndexEntry) int
	Bytes() []byte
	String() string
}

// Storage encoding for primary index entry
// Raw docid bytes are stored as the key
type primaryIndexEntry []byte

func NewPrimaryIndexEntry(docid []byte) (*primaryIndexEntry, error) {
	buf := append([]byte(nil), docid...)
	e := primaryIndexEntry(buf)
	return &e, nil
}

func BytesToPrimaryIndexEntry(b []byte) (*primaryIndexEntry, error) {
	e := primaryIndexEntry(b)
	return &e, nil
}

func (e *primaryIndexEntry) ReadDocId(buf []byte) ([]byte, error) {
	buf = append(buf, []byte(*e)...)
	return buf, nil
}

func (e *primaryIndexEntry) ReadSecKey(buf []byte) ([]byte, error) {
	return buf, nil
}

func (e *primaryIndexEntry) Bytes() []byte {
	return []byte(*e)
}

func (e *primaryIndexEntry) String() string {
	return string(*e)
}

// Storage encoding for secondary index entry
// Format:
// [collate_json_encoded_sec_key][raw_docid_bytes][len_of_docid_2_bytes]
type secondaryIndexEntry []byte

func NewSecondaryIndexEntry(key []byte, docid []byte) (*secondaryIndexEntry, error) {
	var err error
	var buf []byte

	if isNilJsonKey(key) {
		return nil, ErrSecKeyNil
	}

	poolBuf := encBufPool.Get()
	defer encBufPool.Put(poolBuf)
	if buf, err = jsonEncoder.Encode(key, (*poolBuf)[:0]); err != nil {
		return nil, err
	}

	buf = append(buf, docid...)
	buf = buf[:len(buf)+2]
	offset := len(buf) - 2
	binary.LittleEndian.PutUint16(buf[offset:offset+2], uint16(len(docid)))

	buf = append([]byte(nil), buf[:len(buf)]...)
	e := secondaryIndexEntry(buf)
	return &e, nil
}

func BytesToSecondaryIndexEntry(b []byte) (*secondaryIndexEntry, error) {
	e := secondaryIndexEntry(b)
	return &e, nil
}

func (e *secondaryIndexEntry) lenDocId() int {
	rbuf := []byte(*e)
	offset := len(rbuf) - 2
	l := binary.LittleEndian.Uint16(rbuf[offset : offset+2])
	return int(l)
}

func (e secondaryIndexEntry) ReadDocId(buf []byte) ([]byte, error) {
	doclen := e.lenDocId()
	offset := len(e) - doclen - 2
	buf = append(buf, e[offset:offset+doclen]...)

	return buf, nil
}

func (e secondaryIndexEntry) ReadSecKey(buf []byte) ([]byte, error) {
	var err error
	doclen := e.lenDocId()

	encoded := e[0 : len(e)-doclen-2]
	if buf, err = jsonEncoder.Decode(encoded, buf); err != nil {
		return nil, err
	}

	return buf, nil
}

func (e *secondaryIndexEntry) Bytes() []byte {
	return []byte(*e)
}

func (e *secondaryIndexEntry) String() string {
	buf := make([]byte, MAX_SEC_KEY_LEN*4)
	buf, _ = e.ReadSecKey(buf)
	buf = append(buf, ':')
	buf, _ = e.ReadDocId(buf)

	return string(buf)
}

type NilIndexKey struct {
	cmp  int
	pcmp int
}

func (k *NilIndexKey) Compare(entry IndexEntry) int {
	return k.cmp
}

func (k *NilIndexKey) ComparePrefixFields(entry IndexEntry) int {
	return k.pcmp
}

func (k *NilIndexKey) Bytes() []byte {
	return nil
}

func (k *NilIndexKey) String() string {
	return "nil"
}

type primaryKey []byte

func NewPrimaryKey(docid []byte) (IndexKey, error) {
	if len(docid) == 0 {
		return &NilIndexKey{}, nil
	}
	k := primaryKey(docid)
	return &k, nil
}

func (k *primaryKey) Compare(entry IndexEntry) int {
	return bytes.Compare(*k, entry.Bytes())
}

func (k *primaryKey) ComparePrefixFields(entry IndexEntry) int {
	kbytes := []byte(*k)
	klen := len(kbytes)

	if klen > len(entry.Bytes()) {
		return 1
	}

	return bytes.Compare(*k, entry.Bytes()[:klen])
}

func (k *primaryKey) Bytes() []byte {
	return *k
}

func (k *primaryKey) String() string {
	return string(*k)
}

type secondaryKey []byte

func NewSecondaryKey(key []byte) (IndexKey, error) {
	if isNilJsonKey(key) {
		return &NilIndexKey{}, nil
	}

	var err error
	buf := make([]byte, 0, MAX_SEC_KEY_LEN)
	if buf, err = jsonEncoder.Encode(key, buf); err != nil {
		return nil, err
	}

	buf = append([]byte(nil), buf[:len(buf)]...)

	k := secondaryKey(buf)
	return &k, nil
}

func (k *secondaryKey) Compare(entry IndexEntry) int {
	kbytes := []byte(*k)
	klen := len(kbytes)

	if klen > len(entry.Bytes()) {
		return 1
	}

	return bytes.Compare(kbytes[:klen], entry.Bytes()[:klen])
}

// A compound secondary index entry would be an array
// of indexed fields. A compound index on N fields
// also should be equivalent to other indexes with 1 to N
// prefix fields.
// When a user supplies a low or high key constraints,
// indexing systems should check the number of fields
// in the user supplied key and treat target index as
// an index created on that n prefix fields.
//
// Collatejson encoding puts a terminator character at the
// end of encoded bytes to represent termination of json array.
// Since we want to match partial secondary keys against fullset
// compound index entries, we would remove the last byte from the
// encoded user supplied secondary key to match prefixes.
func (k *secondaryKey) ComparePrefixFields(entry IndexEntry) int {
	kbytes := []byte(*k)
	klen := len(kbytes)
	if klen > len(entry.Bytes()) {
		return 1
	}
	return bytes.Compare(kbytes[:klen-1], entry.Bytes()[:klen-1])
}

func (k *secondaryKey) Bytes() []byte {
	return *k
}

func (k *secondaryKey) String() string {
	buf := make([]byte, 0, MAX_SEC_KEY_LEN)
	buf, _ = jsonEncoder.Decode(*k, buf)
	return string(buf)
}

func isNilJsonKey(k []byte) bool {
	return bytes.Equal(NilJsonKey, k) || len(k) == 0
}
<commit_msg>MB-15090 index_entry: Fix secondary key encode buf len<commit_after>package indexer

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"github.com/couchbase/indexing/secondary/collatejson"
	"github.com/couchbase/indexing/secondary/common"
)

var (
	ErrSecKeyNil     = errors.New("Secondary key array is empty")
	ErrSecKeyTooLong = errors.New(fmt.Sprintf("Secondary key is too long (> %d)", MAX_SEC_KEY_LEN))
)

// Special index keys
var (
	MinIndexKey = &NilIndexKey{cmp: -1, pcmp: -1}
	MaxIndexKey = &NilIndexKey{cmp: 1, pcmp: 1}
	NilJsonKey  = []byte("[]")
)

var (
	jsonEncoder *collatejson.Codec
	encBufPool  *common.BytesBufPool
)

const (
	maxIndexEntrySize = MAX_SEC_KEY_BUFFER_LEN + MAX_DOCID_LEN + 2
)

func init() {
	jsonEncoder = collatejson.NewCodec(16)
	encBufPool = common.NewByteBufferPool(maxIndexEntrySize)
}

// Generic index entry abstraction (primary or secondary)
// Represents a row in the index
type IndexEntry interface {
	ReadDocId([]byte) ([]byte, error)
	ReadSecKey([]byte) ([]byte, error)

	Bytes() []byte
	String() string
}

// Generic index key abstraction (primary or secondary)
// Represents a key supplied by the user for scan operation
type IndexKey interface {
	Compare(IndexEntry) int
	ComparePrefixFields(IndexEntry) int
	Bytes() []byte
	String() string
}

// Storage encoding for primary index entry
// Raw docid bytes are stored as the key
type primaryIndexEntry []byte

func NewPrimaryIndexEntry(docid []byte) (*primaryIndexEntry, error) {
	buf := append([]byte(nil), docid...)
	e := primaryIndexEntry(buf)
	return &e, nil
}

func BytesToPrimaryIndexEntry(b []byte) (*primaryIndexEntry, error) {
	e := primaryIndexEntry(b)
	return &e, nil
}

func (e *primaryIndexEntry) ReadDocId(buf []byte) ([]byte, error) {
	buf = append(buf, []byte(*e)...)
	return buf, nil
}

func (e *primaryIndexEntry) ReadSecKey(buf []byte) ([]byte, error) {
	return buf, nil
}

func (e *primaryIndexEntry) Bytes() []byte {
	return []byte(*e)
}

func (e *primaryIndexEntry) String() string {
	return string(*e)
}

// Storage encoding for secondary index entry
// Format:
// [collate_json_encoded_sec_key][raw_docid_bytes][len_of_docid_2_bytes]
type secondaryIndexEntry []byte

func NewSecondaryIndexEntry(key []byte, docid []byte) (*secondaryIndexEntry, error) {
	var err error
	var buf []byte

	if isNilJsonKey(key) {
		return nil, ErrSecKeyNil
	}

	poolBuf := encBufPool.Get()
	defer encBufPool.Put(poolBuf)
	if buf, err = jsonEncoder.Encode(key, (*poolBuf)[:0]); err != nil {
		return nil, err
	}

	buf = append(buf, docid...)
	buf = buf[:len(buf)+2]
	offset := len(buf) - 2
	binary.LittleEndian.PutUint16(buf[offset:offset+2], uint16(len(docid)))

	buf = append([]byte(nil), buf[:len(buf)]...)
	e := secondaryIndexEntry(buf)
	return &e, nil
}

func BytesToSecondaryIndexEntry(b []byte) (*secondaryIndexEntry, error) {
	e := secondaryIndexEntry(b)
	return &e, nil
}

func (e *secondaryIndexEntry) lenDocId() int {
	rbuf := []byte(*e)
	offset := len(rbuf) - 2
	l := binary.LittleEndian.Uint16(rbuf[offset : offset+2])
	return int(l)
}

func (e secondaryIndexEntry) ReadDocId(buf []byte) ([]byte, error) {
	doclen := e.lenDocId()
	offset := len(e) - doclen - 2
	buf = append(buf, e[offset:offset+doclen]...)

	return buf, nil
}

func (e secondaryIndexEntry) ReadSecKey(buf []byte) ([]byte, error) {
	var err error
	doclen := e.lenDocId()

	encoded := e[0 : len(e)-doclen-2]
	if buf, err = jsonEncoder.Decode(encoded, buf); err != nil {
		return nil, err
	}

	return buf, nil
}

func (e *secondaryIndexEntry) Bytes() []byte {
	return []byte(*e)
}

func (e *secondaryIndexEntry) String() string {
	buf := make([]byte, MAX_SEC_KEY_LEN*4)
	buf, _ = e.ReadSecKey(buf)
	buf = append(buf, ':')
	buf, _ = e.ReadDocId(buf)

	return string(buf)
}

type NilIndexKey struct {
	cmp  int
	pcmp int
}

func (k *NilIndexKey) Compare(entry IndexEntry) int {
	return k.cmp
}

func (k *NilIndexKey) ComparePrefixFields(entry IndexEntry) int {
	return k.pcmp
}

func (k *NilIndexKey) Bytes() []byte {
	return nil
}

func (k *NilIndexKey) String() string {
	return "nil"
}

type primaryKey []byte

func NewPrimaryKey(docid []byte) (IndexKey, error) {
	if len(docid) == 0 {
		return &NilIndexKey{}, nil
	}
	k := primaryKey(docid)
	return &k, nil
}

func (k *primaryKey) Compare(entry IndexEntry) int {
	return bytes.Compare(*k, entry.Bytes())
}

func (k *primaryKey) ComparePrefixFields(entry IndexEntry) int {
	kbytes := []byte(*k)
	klen := len(kbytes)

	if klen > len(entry.Bytes()) {
		return 1
	}

	return bytes.Compare(*k, entry.Bytes()[:klen])
}

func (k *primaryKey) Bytes() []byte {
	return *k
}

func (k *primaryKey) String() string {
	return string(*k)
}

type secondaryKey []byte

func NewSecondaryKey(key []byte) (IndexKey, error) {
	if len(key) > MAX_SEC_KEY_LEN {
		return nil, ErrSecKeyTooLong
	}

	if isNilJsonKey(key) {
		return &NilIndexKey{}, nil
	}

	var err error
	buf := make([]byte, 0, MAX_SEC_KEY_BUFFER_LEN)
	if buf, err = jsonEncoder.Encode(key, buf); err != nil {
		return nil, err
	}

	buf = append([]byte(nil), buf[:len(buf)]...)

	k := secondaryKey(buf)
	return &k, nil
}

func (k *secondaryKey) Compare(entry IndexEntry) int {
	kbytes := []byte(*k)
	klen := len(kbytes)

	if klen > len(entry.Bytes()) {
		return 1
	}

	return bytes.Compare(kbytes[:klen], entry.Bytes()[:klen])
}

// A compound secondary index entry would be an array
// of indexed fields. A compound index on N fields
// also should be equivalent to other indexes with 1 to N
// prefix fields.
// When a user supplies a low or high key constraints,
// indexing systems should check the number of fields
// in the user supplied key and treat target index as
// an index created on that n prefix fields.
//
// Collatejson encoding puts a terminator character at the
// end of encoded bytes to represent termination of json array.
// Since we want to match partial secondary keys against fullset
// compound index entries, we would remove the last byte from the
// encoded user supplied secondary key to match prefixes.
func (k *secondaryKey) ComparePrefixFields(entry IndexEntry) int {
	kbytes := []byte(*k)
	klen := len(kbytes)
	if klen > len(entry.Bytes()) {
		return 1
	}
	return bytes.Compare(kbytes[:klen-1], entry.Bytes()[:klen-1])
}

func (k *secondaryKey) Bytes() []byte {
	return *k
}

func (k *secondaryKey) String() string {
	buf := make([]byte, 0, MAX_SEC_KEY_LEN)
	buf, _ = jsonEncoder.Decode(*k, buf)
	return string(buf)
}

func isNilJsonKey(k []byte) bool {
	return bytes.Equal(NilJsonKey, k) || len(k) == 0
}
<|endoftext|>


<commit_before>// Copyright 2020 guylewin, guy@lewin.co.il
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package auth

import (
	"fmt"
	"time"

	"github.com/fatedier/frp/models/msg"
	"github.com/fatedier/frp/utils/util"

	"github.com/vaughan0/go-ini"
)

type tokenConfig struct {
	// Token specifies the authorization token used to create keys to be sent
	// to the server. The server must have a matching token for authorization
	// to succeed.  By default, this value is "".
	Token string `json:"token"`
}

func getDefaultTokenConf() tokenConfig {
	return tokenConfig{
		Token: "",
	}
}

func unmarshalTokenConfFromIni(conf ini.File) tokenConfig {
	var (
		tmpStr string
		ok     bool
	)

	cfg := getDefaultTokenConf()

	if tmpStr, ok = conf.Get("common", "token"); ok {
		cfg.Token = tmpStr
	}

	return cfg
}

type TokenAuthSetterVerifier struct {
	baseConfig

	token string
}

func NewTokenAuth(baseCfg baseConfig, cfg tokenConfig) *TokenAuthSetterVerifier {
	return &TokenAuthSetterVerifier{
		baseConfig: baseCfg,
		token:      cfg.Token,
	}
}

func (auth *TokenAuthSetterVerifier) SetLogin(loginMsg *msg.Login) (err error) {
	loginMsg.PrivilegeKey = util.GetAuthKey(auth.token, loginMsg.Timestamp)
	return nil
}

func (auth *TokenAuthSetterVerifier) SetPing(pingMsg *msg.Ping) error {
	if !auth.AuthenticateHeartBeats {
		return nil
	}

	pingMsg.Timestamp = time.Now().Unix()
	pingMsg.PrivilegeKey = util.GetAuthKey(auth.token, pingMsg.Timestamp)
	return nil
}

func (auth *TokenAuthSetterVerifier) SetNewWorkConn(newWorkConnMsg *msg.NewWorkConn) error {
	if !auth.AuthenticateHeartBeats {
		return nil
	}

	newWorkConnMsg.Timestamp = time.Now().Unix()
	newWorkConnMsg.PrivilegeKey = util.GetAuthKey(auth.token, newWorkConnMsg.Timestamp)
	return nil
}

func (auth *TokenAuthSetterVerifier) VerifyLogin(loginMsg *msg.Login) error {
	if util.GetAuthKey(auth.token, loginMsg.Timestamp) != loginMsg.PrivilegeKey {
		return fmt.Errorf("token in login doesn't match token from configuration")
	}
	return nil
}

func (auth *TokenAuthSetterVerifier) VerifyPing(pingMsg *msg.Ping) error {
	if !auth.AuthenticateHeartBeats {
		return nil
	}

	if util.GetAuthKey(auth.token, pingMsg.Timestamp) != pingMsg.PrivilegeKey {
		return fmt.Errorf("token in heartbeat doesn't match token from configuration")
	}
	return nil
}

func (auth *TokenAuthSetterVerifier) VerifyNewWorkConn(newWorkConnMsg *msg.NewWorkConn) error {
	if !auth.AuthenticateNewWorkConns {
		return nil
	}

	if util.GetAuthKey(auth.token, newWorkConnMsg.Timestamp) != newWorkConnMsg.PrivilegeKey {
		return fmt.Errorf("token in NewWorkConn doesn't match token from configuration")
	}
	return nil
}
<commit_msg>fix: auth token bug (#1762)<commit_after>// Copyright 2020 guylewin, guy@lewin.co.il
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package auth

import (
	"fmt"
	"time"

	"github.com/fatedier/frp/models/msg"
	"github.com/fatedier/frp/utils/util"

	"github.com/vaughan0/go-ini"
)

type tokenConfig struct {
	// Token specifies the authorization token used to create keys to be sent
	// to the server. The server must have a matching token for authorization
	// to succeed.  By default, this value is "".
	Token string `json:"token"`
}

func getDefaultTokenConf() tokenConfig {
	return tokenConfig{
		Token: "",
	}
}

func unmarshalTokenConfFromIni(conf ini.File) tokenConfig {
	var (
		tmpStr string
		ok     bool
	)

	cfg := getDefaultTokenConf()

	if tmpStr, ok = conf.Get("common", "token"); ok {
		cfg.Token = tmpStr
	}

	return cfg
}

type TokenAuthSetterVerifier struct {
	baseConfig

	token string
}

func NewTokenAuth(baseCfg baseConfig, cfg tokenConfig) *TokenAuthSetterVerifier {
	return &TokenAuthSetterVerifier{
		baseConfig: baseCfg,
		token:      cfg.Token,
	}
}

func (auth *TokenAuthSetterVerifier) SetLogin(loginMsg *msg.Login) (err error) {
	loginMsg.PrivilegeKey = util.GetAuthKey(auth.token, loginMsg.Timestamp)
	return nil
}

func (auth *TokenAuthSetterVerifier) SetPing(pingMsg *msg.Ping) error {
	if !auth.AuthenticateHeartBeats {
		return nil
	}

	pingMsg.Timestamp = time.Now().Unix()
	pingMsg.PrivilegeKey = util.GetAuthKey(auth.token, pingMsg.Timestamp)
	return nil
}

func (auth *TokenAuthSetterVerifier) SetNewWorkConn(newWorkConnMsg *msg.NewWorkConn) error {
	if !auth.AuthenticateNewWorkConns {
		return nil
	}

	newWorkConnMsg.Timestamp = time.Now().Unix()
	newWorkConnMsg.PrivilegeKey = util.GetAuthKey(auth.token, newWorkConnMsg.Timestamp)
	return nil
}

func (auth *TokenAuthSetterVerifier) VerifyLogin(loginMsg *msg.Login) error {
	if util.GetAuthKey(auth.token, loginMsg.Timestamp) != loginMsg.PrivilegeKey {
		return fmt.Errorf("token in login doesn't match token from configuration")
	}
	return nil
}

func (auth *TokenAuthSetterVerifier) VerifyPing(pingMsg *msg.Ping) error {
	if !auth.AuthenticateHeartBeats {
		return nil
	}

	if util.GetAuthKey(auth.token, pingMsg.Timestamp) != pingMsg.PrivilegeKey {
		return fmt.Errorf("token in heartbeat doesn't match token from configuration")
	}
	return nil
}

func (auth *TokenAuthSetterVerifier) VerifyNewWorkConn(newWorkConnMsg *msg.NewWorkConn) error {
	if !auth.AuthenticateNewWorkConns {
		return nil
	}

	if util.GetAuthKey(auth.token, newWorkConnMsg.Timestamp) != newWorkConnMsg.PrivilegeKey {
		return fmt.Errorf("token in NewWorkConn doesn't match token from configuration")
	}
	return nil
}
<|endoftext|>


<commit_before>package input

import (
	"testing"

	"github.com/ambientsound/pms/input"
)

// TestLexer tests the input.NextToken() function, checking that it correctly
// splits up input lines into Token structs.
func TestLexer(t *testing.T) {
	i := 0
	pos := 0
	str := "the  quick <brown>\t \tfoxee # adds a comment"
	token := input.Token{}

	checks := []input.Token{
		input.Token{Class: input.TokenIdentifier, Runes: []rune("the")},
		input.Token{Class: input.TokenIdentifier, Runes: []rune("quick")},
		input.Token{Class: input.TokenIdentifier, Runes: []rune("<brown>")},
		input.Token{Class: input.TokenIdentifier, Runes: []rune("foxee")},
		input.Token{Class: input.TokenComment, Runes: []rune("# adds a comment")},
		input.Token{Class: input.TokenEnd, Runes: nil},
	}

	for {

		if i == len(checks) {
			if token.Class == input.TokenEnd {
				break
			}
			t.Fatalf("Tokenizer generated too many tokens!")
		}

		check := checks[i]
		token, npos := input.NextToken(str[pos:])
		pos += npos

		t.Logf("Token %d: pos=%d, runes='%s', input='%s'", i, pos, string(token.Runes), str)

		if token.Class != check.Class {
			t.Fatalf("Token class for token %d is wrong; expected %d but got %d", i+1, check.Class, token.Class)
		}

		if string(check.Runes) != string(token.Runes) {
			t.Fatalf("String check against token %d failed; expected '%s' but got '%s'", i+1,
				string(check.Runes),
				string(token.Runes),
			)
		}

		i++
	}
}
<commit_msg>Rename input tests to package input_test<commit_after>package input_test

import (
	"testing"

	"github.com/ambientsound/pms/input"
)

// TestLexer tests the input.NextToken() function, checking that it correctly
// splits up input lines into Token structs.
func TestLexer(t *testing.T) {
	i := 0
	pos := 0
	str := "the  quick <brown>\t \tfoxee # adds a comment"
	token := input.Token{}

	checks := []input.Token{
		input.Token{Class: input.TokenIdentifier, Runes: []rune("the")},
		input.Token{Class: input.TokenIdentifier, Runes: []rune("quick")},
		input.Token{Class: input.TokenIdentifier, Runes: []rune("<brown>")},
		input.Token{Class: input.TokenIdentifier, Runes: []rune("foxee")},
		input.Token{Class: input.TokenComment, Runes: []rune("# adds a comment")},
		input.Token{Class: input.TokenEnd, Runes: nil},
	}

	for {

		if i == len(checks) {
			if token.Class == input.TokenEnd {
				break
			}
			t.Fatalf("Tokenizer generated too many tokens!")
		}

		check := checks[i]
		token, npos := input.NextToken(str[pos:])
		pos += npos

		t.Logf("Token %d: pos=%d, runes='%s', input='%s'", i, pos, string(token.Runes), str)

		if token.Class != check.Class {
			t.Fatalf("Token class for token %d is wrong; expected %d but got %d", i+1, check.Class, token.Class)
		}

		if string(check.Runes) != string(token.Runes) {
			t.Fatalf("String check against token %d failed; expected '%s' but got '%s'", i+1,
				string(check.Runes),
				string(token.Runes),
			)
		}

		i++
	}
}
<|endoftext|>


<commit_before>package engine

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("World", func() {
	Context("when created", func() {
		It("it able to provide lore", func() {
			seed := NewSeed("shroom")
			world := NewWorld([]Entity{})
		})
	})
})
<commit_msg>working on tests<commit_after>package engine

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("World", func() {
	Context("when created", func() {
		It("it able to provide lore", func() {
			seed := NewSeed("shroom", 0, 0, 0, nil)
			world := NewWorld([]Entity{seed})
			Expect(world.GetEntities()).NotTo(BeEmpty())
		})
	})
})
<|endoftext|>


<commit_before>package ogdatv21

import (
	"code.google.com/p/go-uuid/uuid"
	"encoding/json"
	"github.com/the42/ogdat"
	"net/url"
	"reflect"
	"strconv"
	"time"
)

const Version = "OGD Austria Metadata 2.1" // Version 2.1: 15.10.2012
const specfile = "ogdat_spec-2.1.csv"
const TimeSpecifier = "2006-01-02T15:04:05" // RFC 3339 = ISO 8601 ohne Zeitzone

/// BEGIN:check wheater this code may be factored out
const (
	Time2 = time.RFC3339Nano
	Time3 = time.RFC3339
	Time1 = TimeSpecifier
	TimeFormatUnknow
)

type Kategorie struct {
	NumID       int `json:"-"`
	ID          string
	PrettyName  string `json:"-"`
	RDFProperty string `json:"-"`
}

func (kat *Kategorie) String() string {
	return kat.PrettyName
}

var (
	Arbeit           = Kategorie{NumID: 1, ID: "arbeit", PrettyName: "Arbeit", RDFProperty: ""}
	Bevoelkerung     = Kategorie{NumID: 2, ID: "bevölkerung", PrettyName: "Bevölkerung", RDFProperty: ""}
	BildungForschung = Kategorie{NumID: 3, ID: "bildung-und-forschung", PrettyName: "Bildung und Forschung", RDFProperty: ""}
	FinanzRW         = Kategorie{NumID: 4, ID: "finanzen-und-rechnungswesen", PrettyName: "Finanzen und Rechnungswesen", RDFProperty: ""}
	GeographPlanung  = Kategorie{NumID: 5, ID: "geographie-und-planung", PrettyName: "Geographie und Planung", RDFProperty: ""}
	GesellSoziales   = Kategorie{NumID: 6, ID: "gesellschaft-und-soziales", PrettyName: "Gesellschaft und Soziales", RDFProperty: ""}
	Gesundheit       = Kategorie{NumID: 7, ID: "gesundheit", PrettyName: "Gesundheit", RDFProperty: ""}
	KunstKultur      = Kategorie{NumID: 8, ID: "kunst-und-kultur", PrettyName: "Kunst und Kultur", RDFProperty: ""}
	LandFW           = Kategorie{NumID: 9, ID: "land-und-forstwirtschaft", PrettyName: "Land und Forstwirtschaft", RDFProperty: ""}
	SportFZ          = Kategorie{NumID: 10, ID: "sport-und-freizeit", PrettyName: "Sport und Freizeit", RDFProperty: ""}
	Umwelt           = Kategorie{NumID: 11, ID: "umwelt", PrettyName: "Umwelt", RDFProperty: ""}
	VerkehrTechnik   = Kategorie{NumID: 12, ID: "verkehr-und-technik", PrettyName: "Verkehr und Technik", RDFProperty: ""}
	VerwaltPol       = Kategorie{NumID: 13, ID: "verwaltung-und-politik", PrettyName: "Verwaltung und Politik", RDFProperty: ""}
	WirtTourism      = Kategorie{NumID: 14, ID: "wirtschaft-und-tourismus", PrettyName: "Wirtschaft und Tourismus", RDFProperty: ""}
)

var categories = []Kategorie{
	Arbeit,
	Bevoelkerung,
	BildungForschung,
	FinanzRW,
	GeographPlanung,
	GesellSoziales,
	Gesundheit,
	KunstKultur,
	LandFW,
	SportFZ,
	Umwelt,
	VerkehrTechnik,
	VerwaltPol,
	WirtTourism,
}

var categorymap = make(map[string]Kategorie)

type Tags string
type ResourceSpecifier string

type Cycle struct {
	NumID                       int
	DomainCode                  string
	MD_MaintenanceFrequencyCode string
	Name_DE                     string
}

var (
	CycCont     = Cycle{1, "001", "continual", "kontinuierlich"}
	CycDaily    = Cycle{2, "002", "daily", "täglich"}
	CycWeekly   = Cycle{3, "003", "weekly", "wöchentlich"}
	CycFortNly  = Cycle{4, "004", "fortnightly", "14-tägig"}
	CycMonthly  = Cycle{5, "005", "monthly", "monatlich"}
	CycQuart    = Cycle{6, "006", "quarterly", "quartalsweise"}
	CycBiAnn    = Cycle{7, "007", "biannually", "halbjährlich"}
	CycAnnually = Cycle{8, "008", "annually", "jährlich"}
	CycNeeded   = Cycle{9, "009", "asNeeded", "nach Bedarf"}
	CycIrreg    = Cycle{10, "010", "irregular", "unregelmäßig"}
	CycNP       = Cycle{11, "011", "notPlanned", "nicht geplant"}
	CycUnknown  = Cycle{12, "012", "unknown", "unbekannt"}
)

var cycles = []Cycle{
	CycCont,
	CycDaily,
	CycWeekly,
	CycFortNly,
	CycMonthly,
	CycQuart,
	CycBiAnn,
	CycAnnually,
	CycNeeded,
	CycIrreg,
	CycNP,
	CycUnknown,
}

type Url struct {
	*url.URL
	Raw string
}

type Identifier struct {
	*uuid.UUID
	Raw string
}

func (id *Identifier) String() string {
	return id.Raw
}

type Time struct {
	time.Time
	Raw    string
	Format string
}

func (time *Time) String() string {
	return time.Raw
}

func (cyc *Cycle) String() string {
	return cyc.Name_DE
}

func cmpstrtocycle(raw string, cyc Cycle) bool {
	if raw == cyc.Name_DE || raw == cyc.DomainCode || raw == cyc.MD_MaintenanceFrequencyCode {
		return true
	}
	if len(raw) > 0 {
		if i, err := strconv.Atoi(raw); err == nil && i == cyc.NumID {
			return true
		}
	}
	return false
}

func (cyc *Cycle) UnmarshalJSON(data []byte) error {
	var raw string
	if err := json.Unmarshal(data, &raw); err != nil {
		return err
	}

	var found bool
	var idx int
	var matchcyc Cycle

	for idx, matchcyc = range cycles {
		if found := cmpstrtocycle(raw, matchcyc); found == true {
			break
		}
	}

	if found {
		*cyc = cycles[idx]
	} else {
		cyc.NumID = -1
		cyc.Name_DE = "**** NON cycle spec **** - " + raw
		cyc.MD_MaintenanceFrequencyCode = cyc.Name_DE
	}
	return nil
}

func (ogdtime *Time) UnmarshalJSON(data []byte) error {
	var raw string
	if err := json.Unmarshal(data, &raw); err != nil {
		return err
	}
	ogdtime.Raw = raw

	ogdtime.Format = Time1
	t, err := time.Parse(ogdtime.Format, raw)
	if err != nil {
		ogdtime.Format = Time2
		t, err = time.Parse(ogdtime.Format, raw)
		if err != nil {
			ogdtime.Format = Time3
			t, err = time.Parse(ogdtime.Format, raw)
			if err != nil {
				ogdtime.Format = TimeFormatUnknow
			}
		}
	}
	ogdtime.Time = t
	return nil
}

func (u *Url) UnmarshalJSON(data []byte) error {
	var raw string
	if err := json.Unmarshal(data, &raw); err != nil {
		return err
	}
	u.Raw = raw
	url, _ := url.Parse(raw) // an actuall error is not important. If url can not be parsed, result will be nil, which is fine here
	u.URL = url
	return nil
}

func (id *Identifier) UnmarshalJSON(data []byte) error {
	var raw string
	if err := json.Unmarshal(data, &raw); err != nil {
		return err
	}

	id.Raw = string(raw)
	if uuid := uuid.Parse(raw); uuid != nil {
		id.UUID = &uuid
	}
	return nil
}

func (kat *Kategorie) UnmarshalJSON(data []byte) error {
	var raw string
	if err := json.Unmarshal(data, &raw); err != nil {
		return err
	}

	corecat, found := categorymap[raw]
	if !found {
		kat.NumID = -1
		kat.ID = raw
		kat.PrettyName = "**** NON core category **** - " + kat.ID
	} else {
		*kat = corecat
	}
	return nil
}

/// END:check wheater this code may be factored out

type Extras struct {
	// Core
	Metadata_Identifier *Identifier `json:"metadata_identifier" ogdat:"ID:1"` // CKAN uses since API Version 2 a UUID V4, cf. https://github.com/okfn/ckan/blob/master/ckan/model/types.py
	Metadata_Modified   *Time       `json:"metadata_modified ogdat:"ID:5"`
	Categorization      []Kategorie `json:"categorization ogdat:"ID:10"`
	Begin_DateTime      *Time       `json:"begin_datetime" ogdat:"ID:24"`

	// Optional
	Schema_Name           *string `json:"schema_name" ogdat:"ID:2"`
	Schema_Language       *string `json:"schema_language" ogdat:"ID:3"`     // always "ger"
	Schema_Characterset   *string `json:"schema_characterset" ogdat:"ID:4"` // always "utf8", cf. https://www.ghrsst.org/files/download.php?m=documents&f=ISO%2019115%20.pdf
	MetaData_Linkage      []Url   `json:"metadata_linkage" ogdat:"ID:6"`
	Attribute_Description *string `json:"attribute_description" ogdat:"ID:12"`
	Maintainer_Link       *Url    `json:"maintainer_link" ogdat:"ID:13"`
	Publisher             *string `json:"publisher" ogdat:"ID:20"`
	Geographich_Toponym   *string `json:"geographic_toponym" ogdat:"ID:22"`

	/*  ON/EN/ISO 19115:2003: westBL (344) & eastBL (345) & southBL (346) & northBL (347)
	 * TODO: Specifiaction says a WKT of POLYGON should be used, which would make a
	 * POLYGON ((-180.00 -90.00, 180.00 90.00)) but Example states
	 * POLYGON (-180.00 -90.00, 180.00 90.00)
	 * UNDER CLARIFICATION
	 */
	Geographic_BBox  *string `json:"geographic_bbox" ogdat:"ID:23"`
	End_DateTime     *Time   `json:"end_datetime" ogdat:"ID:25"`
	Update_Frequency *Cycle  `json:"update_frequency" ogdat:"ID:26"`
	Lineage_Quality  *string `json:"lineage_quality" ogdat:"ID:27"`
	EnTitleDesc      *string `json:"en_title_and_desc" ogdat:"ID:28"`
}

type Resource struct {
	// Core
	URL    *Url              `json:"url" ogdat:"ID:14"`
	Format ResourceSpecifier `json:"format" ogdat:"ID:15"`

	// Optional
	Name         *string `json:"name" ogdat:"ID:16"`
	Created      *Time   `json:"created" ogdat:"ID:17"`
	LastModified *Time   `json:"last_modified" ogdat:"ID:18"`

	/*
	 * dcat:bytes a rdf:Property, owl:DatatypeProperty;
	 * rdfs:isDefinedBy <http://www.w3.org/ns/dcat>;
	 * rdfs:label "size in bytes";
	 * rdfs:comment "describe size of resource in bytes";
	 * rdfs:domain dcat:Distribution;
	 * rdfs:range xsd:integer .
	 */
	Size             *string `json:"size" ogdat:"ID:29"`
	License_Citation *string `json:"license_citation" ogdat:"ID:30"`
	Language         *string `json:"language" ogdat:"ID:31"`
	/* Here we have a problem in spec 2.1. which says "nach ISO\IEC 10646-1", which means utf-8, utf-16 and utf-32.
	 * We would certainly support more encodings, as eg.
	 * ISO 19115 / B.5.10 MD_CharacterSetCode<> or
	 * http://www.iana.org/assignments/character-sets/character-sets.xml
	 */
	Encoding *string `json:"characterset" ogdat:"ID:32"`
}

type MetaData struct {
	// Core
	Title       string `json:"title" ogdat:"ID:8"`
	Description string `json:"notes" ogdat:"ID:9"`
	Schlagworte []Tags `json:"tags" ogdat:"ID:11"`
	Maintainer  string `json:"maintainer" ogdat:"ID:19"`
	License     string `json:"license" ogdat:"ID:21"` // Sollte URI des Lizenzdokuments sein

	// nested structs
	Extras   `json:"extras"`
	Resource []Resource `json:"resources"`
}

func (md *MetaData) GetBeschreibungForFieldName(name string) *ogdat.Beschreibung {
	if f, ok := reflect.TypeOf(md).Elem().FieldByName(name); ok {
		if id := ogdat.GetIDFromMetaDataStructField(f); id > -1 {
			beschreibung, _ := ogdat.GetOGDSetForVersion(Version).GetBeschreibungForID(id)
			return beschreibung
		}
	}
	return nil
}

func init() {
	for idx, val := range categories {
		categorymap[val.ID] = categories[idx]
	}
	ogdat.RegisterFromCSVFile(Version, specfile)
}
<commit_msg>make all struct members of OGDAT pointers<commit_after>package ogdatv21

import (
	"code.google.com/p/go-uuid/uuid"
	"encoding/json"
	"github.com/the42/ogdat"
	"net/url"
	"reflect"
	"strconv"
	"time"
)

const Version = "OGD Austria Metadata 2.1" // Version 2.1: 15.10.2012
const specfile = "ogdat_spec-2.1.csv"
const TimeSpecifier = "2006-01-02T15:04:05" // RFC 3339 = ISO 8601 ohne Zeitzone

/// BEGIN:check wheater this code may be factored out
const (
	Time2 = time.RFC3339Nano
	Time3 = time.RFC3339
	Time1 = TimeSpecifier
	TimeFormatUnknow
)

type Kategorie struct {
	NumID       int `json:"-"`
	ID          string
	PrettyName  string `json:"-"`
	RDFProperty string `json:"-"`
}

func (kat *Kategorie) String() string {
	return kat.PrettyName
}

var (
	Arbeit           = Kategorie{NumID: 1, ID: "arbeit", PrettyName: "Arbeit", RDFProperty: ""}
	Bevoelkerung     = Kategorie{NumID: 2, ID: "bevölkerung", PrettyName: "Bevölkerung", RDFProperty: ""}
	BildungForschung = Kategorie{NumID: 3, ID: "bildung-und-forschung", PrettyName: "Bildung und Forschung", RDFProperty: ""}
	FinanzRW         = Kategorie{NumID: 4, ID: "finanzen-und-rechnungswesen", PrettyName: "Finanzen und Rechnungswesen", RDFProperty: ""}
	GeographPlanung  = Kategorie{NumID: 5, ID: "geographie-und-planung", PrettyName: "Geographie und Planung", RDFProperty: ""}
	GesellSoziales   = Kategorie{NumID: 6, ID: "gesellschaft-und-soziales", PrettyName: "Gesellschaft und Soziales", RDFProperty: ""}
	Gesundheit       = Kategorie{NumID: 7, ID: "gesundheit", PrettyName: "Gesundheit", RDFProperty: ""}
	KunstKultur      = Kategorie{NumID: 8, ID: "kunst-und-kultur", PrettyName: "Kunst und Kultur", RDFProperty: ""}
	LandFW           = Kategorie{NumID: 9, ID: "land-und-forstwirtschaft", PrettyName: "Land und Forstwirtschaft", RDFProperty: ""}
	SportFZ          = Kategorie{NumID: 10, ID: "sport-und-freizeit", PrettyName: "Sport und Freizeit", RDFProperty: ""}
	Umwelt           = Kategorie{NumID: 11, ID: "umwelt", PrettyName: "Umwelt", RDFProperty: ""}
	VerkehrTechnik   = Kategorie{NumID: 12, ID: "verkehr-und-technik", PrettyName: "Verkehr und Technik", RDFProperty: ""}
	VerwaltPol       = Kategorie{NumID: 13, ID: "verwaltung-und-politik", PrettyName: "Verwaltung und Politik", RDFProperty: ""}
	WirtTourism      = Kategorie{NumID: 14, ID: "wirtschaft-und-tourismus", PrettyName: "Wirtschaft und Tourismus", RDFProperty: ""}
)

var categories = []Kategorie{
	Arbeit,
	Bevoelkerung,
	BildungForschung,
	FinanzRW,
	GeographPlanung,
	GesellSoziales,
	Gesundheit,
	KunstKultur,
	LandFW,
	SportFZ,
	Umwelt,
	VerkehrTechnik,
	VerwaltPol,
	WirtTourism,
}

var categorymap = make(map[string]Kategorie)

type Tags string
type ResourceSpecifier string

type Cycle struct {
	NumID                       int
	DomainCode                  string
	MD_MaintenanceFrequencyCode string
	Name_DE                     string
}

var (
	CycCont     = Cycle{1, "001", "continual", "kontinuierlich"}
	CycDaily    = Cycle{2, "002", "daily", "täglich"}
	CycWeekly   = Cycle{3, "003", "weekly", "wöchentlich"}
	CycFortNly  = Cycle{4, "004", "fortnightly", "14-tägig"}
	CycMonthly  = Cycle{5, "005", "monthly", "monatlich"}
	CycQuart    = Cycle{6, "006", "quarterly", "quartalsweise"}
	CycBiAnn    = Cycle{7, "007", "biannually", "halbjährlich"}
	CycAnnually = Cycle{8, "008", "annually", "jährlich"}
	CycNeeded   = Cycle{9, "009", "asNeeded", "nach Bedarf"}
	CycIrreg    = Cycle{10, "010", "irregular", "unregelmäßig"}
	CycNP       = Cycle{11, "011", "notPlanned", "nicht geplant"}
	CycUnknown  = Cycle{12, "012", "unknown", "unbekannt"}
)

var cycles = []Cycle{
	CycCont,
	CycDaily,
	CycWeekly,
	CycFortNly,
	CycMonthly,
	CycQuart,
	CycBiAnn,
	CycAnnually,
	CycNeeded,
	CycIrreg,
	CycNP,
	CycUnknown,
}

type Url struct {
	*url.URL
	Raw string
}

type Identifier struct {
	*uuid.UUID
	Raw string
}

func (id *Identifier) String() string {
	return id.Raw
}

type Time struct {
	time.Time
	Raw    string
	Format string
}

func (time *Time) String() string {
	return time.Raw
}

func (cyc *Cycle) String() string {
	return cyc.Name_DE
}

func cmpstrtocycle(raw string, cyc Cycle) bool {
	if raw == cyc.Name_DE || raw == cyc.DomainCode || raw == cyc.MD_MaintenanceFrequencyCode {
		return true
	}
	if len(raw) > 0 {
		if i, err := strconv.Atoi(raw); err == nil && i == cyc.NumID {
			return true
		}
	}
	return false
}

func (cyc *Cycle) UnmarshalJSON(data []byte) error {
	var raw string
	if err := json.Unmarshal(data, &raw); err != nil {
		return err
	}

	var found bool
	var idx int
	var matchcyc Cycle

	for idx, matchcyc = range cycles {
		if found := cmpstrtocycle(raw, matchcyc); found == true {
			break
		}
	}

	if found {
		*cyc = cycles[idx]
	} else {
		cyc.NumID = -1
		cyc.Name_DE = "**** NON cycle spec **** - " + raw
		cyc.MD_MaintenanceFrequencyCode = cyc.Name_DE
	}
	return nil
}

func (ogdtime *Time) UnmarshalJSON(data []byte) error {
	var raw string
	if err := json.Unmarshal(data, &raw); err != nil {
		return err
	}
	ogdtime.Raw = raw

	ogdtime.Format = Time1
	t, err := time.Parse(ogdtime.Format, raw)
	if err != nil {
		ogdtime.Format = Time2
		t, err = time.Parse(ogdtime.Format, raw)
		if err != nil {
			ogdtime.Format = Time3
			t, err = time.Parse(ogdtime.Format, raw)
			if err != nil {
				ogdtime.Format = TimeFormatUnknow
			}
		}
	}
	ogdtime.Time = t
	return nil
}

func (u *Url) UnmarshalJSON(data []byte) error {
	var raw string
	if err := json.Unmarshal(data, &raw); err != nil {
		return err
	}
	u.Raw = raw
	url, _ := url.Parse(raw) // an actuall error is not important. If url can not be parsed, result will be nil, which is fine here
	u.URL = url
	return nil
}

func (id *Identifier) UnmarshalJSON(data []byte) error {
	var raw string
	if err := json.Unmarshal(data, &raw); err != nil {
		return err
	}

	id.Raw = string(raw)
	if uuid := uuid.Parse(raw); uuid != nil {
		id.UUID = &uuid
	}
	return nil
}

func (kat *Kategorie) UnmarshalJSON(data []byte) error {
	var raw string
	if err := json.Unmarshal(data, &raw); err != nil {
		return err
	}

	corecat, found := categorymap[raw]
	if !found {
		kat.NumID = -1
		kat.ID = raw
		kat.PrettyName = "**** NON core category **** - " + kat.ID
	} else {
		*kat = corecat
	}
	return nil
}

/// END:check wheater this code may be factored out

type Extras struct {
	// Core
	Metadata_Identifier *Identifier `json:"metadata_identifier" ogdat:"ID:1"` // CKAN uses since API Version 2 a UUID V4, cf. https://github.com/okfn/ckan/blob/master/ckan/model/types.py
	Metadata_Modified   *Time       `json:"metadata_modified ogdat:"ID:5"`
	Categorization      []Kategorie `json:"categorization ogdat:"ID:10"`
	Begin_DateTime      *Time       `json:"begin_datetime" ogdat:"ID:24"`

	// Optional
	Schema_Name           *string `json:"schema_name" ogdat:"ID:2"`
	Schema_Language       *string `json:"schema_language" ogdat:"ID:3"`     // always "ger"
	Schema_Characterset   *string `json:"schema_characterset" ogdat:"ID:4"` // always "utf8", cf. https://www.ghrsst.org/files/download.php?m=documents&f=ISO%2019115%20.pdf
	MetaData_Linkage      []Url   `json:"metadata_linkage" ogdat:"ID:6"`
	Attribute_Description *string `json:"attribute_description" ogdat:"ID:12"`
	Maintainer_Link       *Url    `json:"maintainer_link" ogdat:"ID:13"`
	Publisher             *string `json:"publisher" ogdat:"ID:20"`
	Geographich_Toponym   *string `json:"geographic_toponym" ogdat:"ID:22"`

	/*  ON/EN/ISO 19115:2003: westBL (344) & eastBL (345) & southBL (346) & northBL (347)
	 * TODO: Specifiaction says a WKT of POLYGON should be used, which would make a
	 * POLYGON ((-180.00 -90.00, 180.00 90.00)) but Example states
	 * POLYGON (-180.00 -90.00, 180.00 90.00)
	 * UNDER CLARIFICATION
	 */
	Geographic_BBox  *string `json:"geographic_bbox" ogdat:"ID:23"`
	End_DateTime     *Time   `json:"end_datetime" ogdat:"ID:25"`
	Update_Frequency *Cycle  `json:"update_frequency" ogdat:"ID:26"`
	Lineage_Quality  *string `json:"lineage_quality" ogdat:"ID:27"`
	EnTitleDesc      *string `json:"en_title_and_desc" ogdat:"ID:28"`
}

type Resource struct {
	// Core
	URL    *Url               `json:"url" ogdat:"ID:14"`
	Format *ResourceSpecifier `json:"format" ogdat:"ID:15"`

	// Optional
	Name         *string `json:"name" ogdat:"ID:16"`
	Created      *Time   `json:"created" ogdat:"ID:17"`
	LastModified *Time   `json:"last_modified" ogdat:"ID:18"`

	/*
	 * dcat:bytes a rdf:Property, owl:DatatypeProperty;
	 * rdfs:isDefinedBy <http://www.w3.org/ns/dcat>;
	 * rdfs:label "size in bytes";
	 * rdfs:comment "describe size of resource in bytes";
	 * rdfs:domain dcat:Distribution;
	 * rdfs:range xsd:integer .
	 */
	Size             *string `json:"size" ogdat:"ID:29"`
	License_Citation *string `json:"license_citation" ogdat:"ID:30"`
	Language         *string `json:"language" ogdat:"ID:31"`
	/* Here we have a problem in spec 2.1. which says "nach ISO\IEC 10646-1", which means utf-8, utf-16 and utf-32.
	 * We would certainly support more encodings, as eg.
	 * ISO 19115 / B.5.10 MD_CharacterSetCode<> or
	 * http://www.iana.org/assignments/character-sets/character-sets.xml
	 */
	Encoding *string `json:"characterset" ogdat:"ID:32"`
}

type MetaData struct {
	// Core
	Title       *string `json:"title" ogdat:"ID:8"`
	Description *string `json:"notes" ogdat:"ID:9"`
	Schlagworte []Tags  `json:"tags" ogdat:"ID:11"`
	Maintainer  *string `json:"maintainer" ogdat:"ID:19"`
	License     *string `json:"license" ogdat:"ID:21"` // Sollte URI des Lizenzdokuments sein

	// nested structs
	Extras   `json:"extras"`
	Resource []Resource `json:"resources"`
}

func (md *MetaData) GetBeschreibungForFieldName(name string) *ogdat.Beschreibung {
	if f, ok := reflect.TypeOf(md).Elem().FieldByName(name); ok {
		if id := ogdat.GetIDFromMetaDataStructField(f); id > -1 {
			beschreibung, _ := ogdat.GetOGDSetForVersion(Version).GetBeschreibungForID(id)
			return beschreibung
		}
	}
	return nil
}

func init() {
	for idx, val := range categories {
		categorymap[val.ID] = categories[idx]
	}
	ogdat.RegisterFromCSVFile(Version, specfile)
}
<|endoftext|>


<commit_before>package cmd

import (
	"bytes"
	"context"
	"os"
	"os/exec"
	"strings"

	"golang.org/x/sync/errgroup"

	"github.com/gobuffalo/buffalo/meta"
	"github.com/gobuffalo/envy"
	"github.com/markbates/deplist"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
)

var setupOptions = struct {
	verbose       bool
	updateGoDeps  bool
	dropDatabases bool
}{}

type setupCheck func(meta.App) error

var setupCmd = &cobra.Command{
	Use:   "setup",
	Short: "Setups a newly created, or recently checked out application.",
	Long: `Setup runs through checklist to make sure dependencies are setup correcly.

Dependencies (if used):
* Runs "dep ensure" to install required Go dependencies.

Asset Pipeline (if used):
* Runs "npm install" or "yarn install" to install asset dependencies.

Database (if used):
* Runs "buffalo db create -a" to create databases.
* Runs "buffalo db migrate" to run database migrations.
* Runs "buffalo task db:seed" to seed the database (if the task exists).

Tests:
* Runs "buffalo test" to confirm the application's tests are running properly.
`,
	RunE: func(cmd *cobra.Command, args []string) error {
		app := meta.New(".")
		for _, check := range []setupCheck{assetCheck, updateGoDepsCheck, databaseCheck, testCheck} {
			err := check(app)
			if err != nil {
				return errors.WithStack(err)
			}
		}
		return nil
	},
}

func updateGoDepsCheck(app meta.App) error {
	if app.WithModules {
		c := exec.Command(envy.Get("GO_BIN", "go"), "get")
		return run(c)
	}
	if app.WithDep {
		// use github.com/golang/dep
		args := []string{"ensure"}
		if setupOptions.verbose {
			args = append(args, "-v")
		}
		if setupOptions.updateGoDeps {
			args = append(args, "--update")
		}
		err := run(exec.Command("dep", args...))
		if err != nil {
			return errors.WithStack(err)
		}
		return nil
	}

	// go old school with the installation
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	wg, _ := errgroup.WithContext(ctx)
	deps, err := deplist.List()
	if err != nil {
		return errors.WithStack(err)
	}

	deps["github.com/gobuffalo/suite"] = "github.com/gobuffalo/suite"

	for dep := range deps {
		args := []string{"get"}
		if setupOptions.verbose {
			args = append(args, "-v")
		}
		if setupOptions.updateGoDeps {
			args = append(args, "-u")
		}
		args = append(args, dep)
		c := exec.Command(envy.Get("GO_BIN", "go"), args...)
		f := func() error {
			return run(c)
		}
		wg.Go(f)
	}
	err = wg.Wait()
	if err != nil {
		return errors.Errorf("We encountered the following error trying to install and update the dependencies for this application:\n%s", err)
	}
	return nil
}

func testCheck(meta.App) error {
	err := run(exec.Command("buffalo", "test"))
	if err != nil {
		return errors.Errorf("We encountered the following error when trying to run your applications tests:\n%s", err)
	}
	return nil
}

func databaseCheck(app meta.App) error {
	if !app.WithPop {
		return nil
	}
	for _, check := range []setupCheck{dbCreateCheck, dbMigrateCheck, dbSeedCheck} {
		err := check(app)
		if err != nil {
			return err
		}
	}
	return nil
}

func dbCreateCheck(meta.App) error {
	if setupOptions.dropDatabases {
		err := run(exec.Command("buffalo", "pop", "drop", "-a"))
		if err != nil {
			return errors.Errorf("We encountered an error when trying to drop your application's databases. Please check to make sure that your database server is running and that the username and passwords found in the database.yml are properly configured and set up on your database server.\n %s", err)
		}
	}
	err := run(exec.Command("buffalo", "pop", "create", "-a"))
	if err != nil {
		return errors.Errorf("We encountered an error when trying to create your application's databases. Please check to make sure that your database server is running and that the username and passwords found in the database.yml are properly configured and set up on your database server.\n %s", err)
	}
	return nil
}

func dbMigrateCheck(meta.App) error {
	err := run(exec.Command("buffalo", "pop", "migrate"))
	if err != nil {
		return errors.Errorf("We encountered the following error when trying to migrate your database:\n%s", err)
	}
	return nil
}

func dbSeedCheck(meta.App) error {
	cmd := exec.Command("buffalo", "t", "list")
	out, err := cmd.Output()
	if err != nil {
		// no tasks configured, so return
		return nil
	}
	if bytes.Contains(out, []byte("db:seed")) {
		err := run(exec.Command("buffalo", "task", "db:seed"))
		if err != nil {
			return errors.Errorf("We encountered the following error when trying to seed your database:\n%s", err)
		}
	}
	return nil
}

func assetCheck(app meta.App) error {
	if !app.WithWebpack {
		return nil
	}
	if app.WithYarn {
		return yarnCheck(app)
	}
	return npmCheck(app)
}

func npmCheck(app meta.App) error {
	err := nodeCheck(app)
	if err != nil {
		return errors.WithStack(err)
	}
	err = run(exec.Command("npm", "install", "--no-progress"))
	if err != nil {
		return errors.Errorf("We encountered the following error when trying to install your asset dependencies using npm:\n%s", err)
	}
	return nil
}

func yarnCheck(app meta.App) error {
	if err := nodeCheck(app); err != nil {
		return errors.WithStack(err)
	}
	if _, err := exec.LookPath("yarnpkg"); err != nil {
		err := run(exec.Command("npm", "install", "-g", "yarn"))
		if err != nil {
			return errors.Errorf("This application require yarn, and we could not find it installed on your system. We tried to install it for you, but ran into the following error:\n%s", err)
		}
	}
	if err := run(exec.Command("yarnpkg", "install", "--no-progress")); err != nil {
		return errors.Errorf("We encountered the following error when trying to install your asset dependencies using yarn:\n%s", err)
	}
	return nil
}

func nodeCheck(meta.App) error {
	if _, err := exec.LookPath("node"); err != nil {
		return errors.New("this application requires node, and we could not find it installed on your system please install node and try again")
	}
	if _, err := exec.LookPath("npm"); err != nil {
		return errors.New("this application requires npm, and we could not find it installed on your system please install npm and try again")
	}
	return nil
}

func run(cmd *exec.Cmd) error {
	logrus.Infof("--> %s\n", strings.Join(cmd.Args, " "))
	cmd.Stdin = os.Stdin
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	return cmd.Run()
}

func init() {
	setupCmd.Flags().BoolVarP(&setupOptions.verbose, "verbose", "v", false, "run with verbose output")
	setupCmd.Flags().BoolVarP(&setupOptions.updateGoDeps, "update", "u", false, "run go get -u against the application's Go dependencies")
	setupCmd.Flags().BoolVarP(&setupOptions.dropDatabases, "drop", "d", false, "drop existing databases")

	decorate("setup", setupCmd)
	RootCmd.AddCommand(setupCmd)
}
<commit_msg>install dep if missing during buffalo setup for dep app (#1371)<commit_after>package cmd

import (
	"bytes"
	"context"
	"os"
	"os/exec"
	"strings"

	"golang.org/x/sync/errgroup"

	"github.com/gobuffalo/buffalo/meta"
	"github.com/gobuffalo/envy"
	"github.com/markbates/deplist"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
)

var setupOptions = struct {
	verbose       bool
	updateGoDeps  bool
	dropDatabases bool
}{}

type setupCheck func(meta.App) error

var setupCmd = &cobra.Command{
	Use:   "setup",
	Short: "Setups a newly created, or recently checked out application.",
	Long: `Setup runs through checklist to make sure dependencies are setup correcly.

Dependencies (if used):
* Runs "dep ensure" to install required Go dependencies.

Asset Pipeline (if used):
* Runs "npm install" or "yarn install" to install asset dependencies.

Database (if used):
* Runs "buffalo db create -a" to create databases.
* Runs "buffalo db migrate" to run database migrations.
* Runs "buffalo task db:seed" to seed the database (if the task exists).

Tests:
* Runs "buffalo test" to confirm the application's tests are running properly.
`,
	RunE: func(cmd *cobra.Command, args []string) error {
		app := meta.New(".")
		for _, check := range []setupCheck{assetCheck, updateGoDepsCheck, databaseCheck, testCheck} {
			err := check(app)
			if err != nil {
				return errors.WithStack(err)
			}
		}
		return nil
	},
}

func updateGoDepsCheck(app meta.App) error {
	if app.WithModules {
		c := exec.Command(envy.Get("GO_BIN", "go"), "get")
		return run(c)
	}
	if app.WithDep {
		if _, err := exec.LookPath("dep"); err != nil {
			if err := run(exec.Command(envy.Get("GO_BIN", "go"), "get", "github.com/golang/dep/cmd/dep")); err != nil {
				return errors.WithStack(err)
			}
		}
		args := []string{"ensure"}
		if setupOptions.verbose {
			args = append(args, "-v")
		}
		if setupOptions.updateGoDeps {
			args = append(args, "--update")
		}
		err := run(exec.Command("dep", args...))
		if err != nil {
			return errors.WithStack(err)
		}
		return nil
	}

	// go old school with the installation
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	wg, _ := errgroup.WithContext(ctx)
	deps, err := deplist.List()
	if err != nil {
		return errors.WithStack(err)
	}

	deps["github.com/gobuffalo/suite"] = "github.com/gobuffalo/suite"

	for dep := range deps {
		args := []string{"get"}
		if setupOptions.verbose {
			args = append(args, "-v")
		}
		if setupOptions.updateGoDeps {
			args = append(args, "-u")
		}
		args = append(args, dep)
		c := exec.Command(envy.Get("GO_BIN", "go"), args...)
		f := func() error {
			return run(c)
		}
		wg.Go(f)
	}
	err = wg.Wait()
	if err != nil {
		return errors.Errorf("We encountered the following error trying to install and update the dependencies for this application:\n%s", err)
	}
	return nil
}

func testCheck(meta.App) error {
	err := run(exec.Command("buffalo", "test"))
	if err != nil {
		return errors.Errorf("We encountered the following error when trying to run your applications tests:\n%s", err)
	}
	return nil
}

func databaseCheck(app meta.App) error {
	if !app.WithPop {
		return nil
	}
	for _, check := range []setupCheck{dbCreateCheck, dbMigrateCheck, dbSeedCheck} {
		err := check(app)
		if err != nil {
			return err
		}
	}
	return nil
}

func dbCreateCheck(meta.App) error {
	if setupOptions.dropDatabases {
		err := run(exec.Command("buffalo", "pop", "drop", "-a"))
		if err != nil {
			return errors.Errorf("We encountered an error when trying to drop your application's databases. Please check to make sure that your database server is running and that the username and passwords found in the database.yml are properly configured and set up on your database server.\n %s", err)
		}
	}
	err := run(exec.Command("buffalo", "pop", "create", "-a"))
	if err != nil {
		return errors.Errorf("We encountered an error when trying to create your application's databases. Please check to make sure that your database server is running and that the username and passwords found in the database.yml are properly configured and set up on your database server.\n %s", err)
	}
	return nil
}

func dbMigrateCheck(meta.App) error {
	err := run(exec.Command("buffalo", "pop", "migrate"))
	if err != nil {
		return errors.Errorf("We encountered the following error when trying to migrate your database:\n%s", err)
	}
	return nil
}

func dbSeedCheck(meta.App) error {
	cmd := exec.Command("buffalo", "t", "list")
	out, err := cmd.Output()
	if err != nil {
		// no tasks configured, so return
		return nil
	}
	if bytes.Contains(out, []byte("db:seed")) {
		err := run(exec.Command("buffalo", "task", "db:seed"))
		if err != nil {
			return errors.Errorf("We encountered the following error when trying to seed your database:\n%s", err)
		}
	}
	return nil
}

func assetCheck(app meta.App) error {
	if !app.WithWebpack {
		return nil
	}
	if app.WithYarn {
		return yarnCheck(app)
	}
	return npmCheck(app)
}

func npmCheck(app meta.App) error {
	err := nodeCheck(app)
	if err != nil {
		return errors.WithStack(err)
	}
	err = run(exec.Command("npm", "install", "--no-progress"))
	if err != nil {
		return errors.Errorf("We encountered the following error when trying to install your asset dependencies using npm:\n%s", err)
	}
	return nil
}

func yarnCheck(app meta.App) error {
	if err := nodeCheck(app); err != nil {
		return errors.WithStack(err)
	}
	if _, err := exec.LookPath("yarnpkg"); err != nil {
		err := run(exec.Command("npm", "install", "-g", "yarn"))
		if err != nil {
			return errors.Errorf("This application require yarn, and we could not find it installed on your system. We tried to install it for you, but ran into the following error:\n%s", err)
		}
	}
	if err := run(exec.Command("yarnpkg", "install", "--no-progress")); err != nil {
		return errors.Errorf("We encountered the following error when trying to install your asset dependencies using yarn:\n%s", err)
	}
	return nil
}

func nodeCheck(meta.App) error {
	if _, err := exec.LookPath("node"); err != nil {
		return errors.New("this application requires node, and we could not find it installed on your system please install node and try again")
	}
	if _, err := exec.LookPath("npm"); err != nil {
		return errors.New("this application requires npm, and we could not find it installed on your system please install npm and try again")
	}
	return nil
}

func run(cmd *exec.Cmd) error {
	logrus.Infof("--> %s\n", strings.Join(cmd.Args, " "))
	cmd.Stdin = os.Stdin
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	return cmd.Run()
}

func init() {
	setupCmd.Flags().BoolVarP(&setupOptions.verbose, "verbose", "v", false, "run with verbose output")
	setupCmd.Flags().BoolVarP(&setupOptions.updateGoDeps, "update", "u", false, "run go get -u against the application's Go dependencies")
	setupCmd.Flags().BoolVarP(&setupOptions.dropDatabases, "drop", "d", false, "drop existing databases")

	decorate("setup", setupCmd)
	RootCmd.AddCommand(setupCmd)
}
<|endoftext|>


<commit_before>package apidAnalytics

import (
	"bufio"
	"compress/gzip"
	"crypto/rand"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"time"
)

const fileExtension = ".txt.gz"

// Channel where analytics records are buffered before being dumped to a
// file as write to file should not performed in the Http Thread
var internalBuffer chan axRecords
// channel to indicate that internalBuffer channel is closed
var doneInternalBufferChan chan bool

// Channel where close bucket event is published i.e. when a bucket
// is ready to be closed based on collection interval
var closeBucketEvent chan bucket
// channel to indicate that closeBucketEvent channel is closed
var doneClosebucketChan chan bool

// Map from timestampt to bucket
var bucketMap map[int64]bucket

// RW lock for bucketMap  since the cache can be
// read while its being written to and vice versa
var bucketMaplock = sync.RWMutex{}

type bucket struct {
	keyTS   int64
	DirName string
	// We need file handle and writer to close the file
	FileWriter fileWriter
}

// This struct will store open file handle and writer to close the file
type fileWriter struct {
	file *os.File
	gw   *gzip.Writer
	bw   *bufio.Writer
}

func initBufferingManager() {
	internalBuffer = make(chan axRecords,
		config.GetInt(analyticsBufferChannelSize))
	closeBucketEvent = make(chan bucket)
	doneInternalBufferChan = make(chan bool)
	doneClosebucketChan = make(chan bool)

	bucketMaplock.Lock()
	bucketMap = make(map[int64]bucket)
	bucketMaplock.Unlock()

	// Keep polling the internal buffer for new messages
	go func() {
		for {
			records, more := <-internalBuffer
			if more {
				err := save(records)
				if err != nil {
					log.Errorf("Could not save %d messages to file"+
						" due to: %v", len(records.Records), err)
				}
			} else {
				// indicates a close signal was sent on the channel
				log.Debugf("Closing channel internal buffer")
				doneInternalBufferChan <- true
				return
			}

		}
	}()

	// Keep polling the closeEvent channel to see if bucket is ready to be closed
	go func() {
		for {
			bucket, more := <-closeBucketEvent
			if more {
				log.Debugf("Close Event received for bucket: %s",
					bucket.DirName)

				// close open file
				closeGzipFile(bucket.FileWriter)

				dirToBeClosed := filepath.Join(localAnalyticsTempDir, bucket.DirName)
				stagingPath := filepath.Join(localAnalyticsStagingDir, bucket.DirName)
				// close files in tmp folder and move directory to
				// staging to indicate its ready for upload
				err := os.Rename(dirToBeClosed, stagingPath)
				if err != nil {
					log.Errorf("Cannot move directory '%s' from"+
						" tmp to staging folder due to '%s", bucket.DirName, err)
				} else {
					// Remove bucket from bucket map once its closed successfully
					bucketMaplock.Lock()
					delete(bucketMap, bucket.keyTS)
					bucketMaplock.Unlock()
				}
			} else {
				// indicates a close signal was sent on the channel
				log.Debugf("Closing channel close bucketevent")
				doneClosebucketChan <- true
				return
			}
		}
	}()
}

// Save records to correct file based on what timestamp data is being collected for
func save(records axRecords) error {
	bucket, err := getBucketForTimestamp(time.Now(), records.Tenant)
	if err != nil {
		return err
	}
	writeGzipFile(bucket.FileWriter, records.Records)
	return nil
}

func getBucketForTimestamp(now time.Time, tenant tenant) (bucket, error) {
	// first based on current timestamp and collection interval,
	// determine the timestamp of the bucket
	ts := now.Unix() / int64(config.GetInt(analyticsCollectionInterval)) * int64(config.GetInt(analyticsCollectionInterval))

	bucketMaplock.RLock()
	b, exists := bucketMap[ts]
	bucketMaplock.RUnlock()

	if exists {
		return b, nil
	} else {
		timestamp := time.Unix(ts, 0).Format(timestampLayout)

		// endtimestamp of bucket = starttimestamp + collectionInterval
		endTime := time.Unix(ts+int64(config.GetInt(analyticsCollectionInterval)), 0)
		endtimestamp := endTime.Format(timestampLayout)

		dirName := tenant.Org + "~" + tenant.Env + "~" + timestamp
		newPath := filepath.Join(localAnalyticsTempDir, dirName)
		// create dir
		err := os.Mkdir(newPath, os.ModePerm)
		if err != nil {
			return bucket{}, fmt.Errorf("Cannot create directory "+
				"'%s' to buffer messages '%v'", dirName, err)
		}

		// create file for writing
		// Format: <4DigitRandomHex>_<TSStart>.<TSEnd>_<APIDINSTANCEUUID>_writer_0.txt.gz
		fileName := getRandomHex() + "_" + timestamp + "." +
			endtimestamp + "_" +
			config.GetString("apigeesync_apid_instance_id") +
			"_writer_0" + fileExtension
		completeFilePath := filepath.Join(newPath, fileName)
		fw, err := createGzipFile(completeFilePath)
		if err != nil {
			return bucket{}, err
		}

		newBucket := bucket{keyTS: ts, DirName: dirName, FileWriter: fw}

		bucketMaplock.Lock()
		bucketMap[ts] = newBucket
		bucketMaplock.Unlock()

		//Send event to close directory after endTime + 5
		// seconds to make sure all buffers are flushed to file
		timer := time.After(endTime.Sub(time.Now()) + time.Second*5)
		go func() {
			<-timer
			closeBucketEvent <- newBucket
		}()
		return newBucket, nil
	}
}

// 4 digit Hex is prefixed to each filename to improve
// how s3 partitions the files being uploaded
func getRandomHex() string {
	buff := make([]byte, 2)
	rand.Read(buff)
	return fmt.Sprintf("%x", buff)
}

func createGzipFile(s string) (fileWriter, error) {
	file, err := os.OpenFile(s, os.O_WRONLY|os.O_CREATE, os.ModePerm)
	if err != nil {
		return fileWriter{},
			fmt.Errorf("Cannot create file '%s' "+
				"to buffer messages '%v'", s, err)
	}
	gw := gzip.NewWriter(file)
	bw := bufio.NewWriter(gw)
	return fileWriter{file, gw, bw}, nil
}

func writeGzipFile(fw fileWriter, records []interface{}) {
	// write each record as a new line to the bufferedWriter
	for _, eachRecord := range records {
		s, _ := json.Marshal(eachRecord)
		_, err := (fw.bw).WriteString(string(s))
		if err != nil {
			log.Errorf("Write to file failed '%v'", err)
		}
		(fw.bw).WriteString("\n")
	}
	// Flush entire batch of records to file vs each message
	fw.bw.Flush()
	fw.gw.Flush()
}

func closeGzipFile(fw fileWriter) {
	fw.bw.Flush()
	fw.gw.Close()
	fw.file.Close()
}
<commit_msg>[UAP-854] Cleaned up syntax for reading on a channel until close is received<commit_after>package apidAnalytics

import (
	"bufio"
	"compress/gzip"
	"crypto/rand"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"time"
)

const fileExtension = ".txt.gz"

// Channel where analytics records are buffered before being dumped to a
// file as write to file should not performed in the Http Thread
var internalBuffer chan axRecords
// channel to indicate that internalBuffer channel is closed
var doneInternalBufferChan chan bool

// Channel where close bucket event is published i.e. when a bucket
// is ready to be closed based on collection interval
var closeBucketEvent chan bucket
// channel to indicate that closeBucketEvent channel is closed
var doneClosebucketChan chan bool

// Map from timestampt to bucket
var bucketMap map[int64]bucket

// RW lock for bucketMap  since the cache can be
// read while its being written to and vice versa
var bucketMaplock = sync.RWMutex{}

type bucket struct {
	keyTS   int64
	DirName string
	// We need file handle and writer to close the file
	FileWriter fileWriter
}

// This struct will store open file handle and writer to close the file
type fileWriter struct {
	file *os.File
	gw   *gzip.Writer
	bw   *bufio.Writer
}

func initBufferingManager() {
	internalBuffer = make(chan axRecords,
		config.GetInt(analyticsBufferChannelSize))
	closeBucketEvent = make(chan bucket)
	doneInternalBufferChan = make(chan bool)
	doneClosebucketChan = make(chan bool)

	bucketMaplock.Lock()
	bucketMap = make(map[int64]bucket)
	bucketMaplock.Unlock()

	// Keep polling the internal buffer for new messages
	go func() {
		for records := range internalBuffer {
			err := save(records)
			if err != nil {
				log.Errorf("Could not save %d messages to file"+
					" due to: %v", len(records.Records), err)
			}
		}
		// indicates a close signal was sent on the channel
		log.Debugf("Closing channel internal buffer")
		doneInternalBufferChan <- true
	}()

	// Keep polling the closeEvent channel to see if bucket is ready to be closed
	go func() {
		for bucket := range closeBucketEvent {
			log.Debugf("Close Event received for bucket: %s",
				bucket.DirName)

			// close open file
			closeGzipFile(bucket.FileWriter)

			dirToBeClosed := filepath.Join(localAnalyticsTempDir, bucket.DirName)
			stagingPath := filepath.Join(localAnalyticsStagingDir, bucket.DirName)
			// close files in tmp folder and move directory to
			// staging to indicate its ready for upload
			err := os.Rename(dirToBeClosed, stagingPath)
			if err != nil {
				log.Errorf("Cannot move directory '%s' from" +
					" tmp to staging folder due to '%s", bucket.DirName, err)
			} else {
				// Remove bucket from bucket map once its closed successfully
				bucketMaplock.Lock()
				delete(bucketMap, bucket.keyTS)
				bucketMaplock.Unlock()
			}
		}
		// indicates a close signal was sent on the channel
		log.Debugf("Closing channel close bucketevent")
		doneClosebucketChan <- true
	}()
}

// Save records to correct file based on what timestamp data is being collected for
func save(records axRecords) error {
	bucket, err := getBucketForTimestamp(time.Now(), records.Tenant)
	if err != nil {
		return err
	}
	writeGzipFile(bucket.FileWriter, records.Records)
	return nil
}

func getBucketForTimestamp(now time.Time, tenant tenant) (bucket, error) {
	// first based on current timestamp and collection interval,
	// determine the timestamp of the bucket
	ts := now.Unix() / int64(config.GetInt(analyticsCollectionInterval)) * int64(config.GetInt(analyticsCollectionInterval))

	bucketMaplock.RLock()
	b, exists := bucketMap[ts]
	bucketMaplock.RUnlock()

	if exists {
		return b, nil
	} else {
		timestamp := time.Unix(ts, 0).Format(timestampLayout)

		// endtimestamp of bucket = starttimestamp + collectionInterval
		endTime := time.Unix(ts+int64(config.GetInt(analyticsCollectionInterval)), 0)
		endtimestamp := endTime.Format(timestampLayout)

		dirName := tenant.Org + "~" + tenant.Env + "~" + timestamp
		newPath := filepath.Join(localAnalyticsTempDir, dirName)
		// create dir
		err := os.Mkdir(newPath, os.ModePerm)
		if err != nil {
			return bucket{}, fmt.Errorf("Cannot create directory "+
				"'%s' to buffer messages '%v'", dirName, err)
		}

		// create file for writing
		// Format: <4DigitRandomHex>_<TSStart>.<TSEnd>_<APIDINSTANCEUUID>_writer_0.txt.gz
		fileName := getRandomHex() + "_" + timestamp + "." +
			endtimestamp + "_" +
			config.GetString("apigeesync_apid_instance_id") +
			"_writer_0" + fileExtension
		completeFilePath := filepath.Join(newPath, fileName)
		fw, err := createGzipFile(completeFilePath)
		if err != nil {
			return bucket{}, err
		}

		newBucket := bucket{keyTS: ts, DirName: dirName, FileWriter: fw}

		bucketMaplock.Lock()
		bucketMap[ts] = newBucket
		bucketMaplock.Unlock()

		//Send event to close directory after endTime + 5
		// seconds to make sure all buffers are flushed to file
		timer := time.After(endTime.Sub(time.Now()) + time.Second*5)
		go func() {
			<-timer
			closeBucketEvent <- newBucket
		}()
		return newBucket, nil
	}
}

// 4 digit Hex is prefixed to each filename to improve
// how s3 partitions the files being uploaded
func getRandomHex() string {
	buff := make([]byte, 2)
	rand.Read(buff)
	return fmt.Sprintf("%x", buff)
}

func createGzipFile(s string) (fileWriter, error) {
	file, err := os.OpenFile(s, os.O_WRONLY|os.O_CREATE, os.ModePerm)
	if err != nil {
		return fileWriter{},
			fmt.Errorf("Cannot create file '%s' "+
				"to buffer messages '%v'", s, err)
	}
	gw := gzip.NewWriter(file)
	bw := bufio.NewWriter(gw)
	return fileWriter{file, gw, bw}, nil
}

func writeGzipFile(fw fileWriter, records []interface{}) {
	// write each record as a new line to the bufferedWriter
	for _, eachRecord := range records {
		s, _ := json.Marshal(eachRecord)
		_, err := (fw.bw).WriteString(string(s))
		if err != nil {
			log.Errorf("Write to file failed '%v'", err)
		}
		(fw.bw).WriteString("\n")
	}
	// Flush entire batch of records to file vs each message
	fw.bw.Flush()
	fw.gw.Flush()
}

func closeGzipFile(fw fileWriter) {
	fw.bw.Flush()
	fw.gw.Close()
	fw.file.Close()
}
<|endoftext|>