<commit_before>// Package provide GOlang API for http://aws.amazon.com/kinesis/
package kinesis

import (
  "encoding/base64"
  "encoding/json"
  "fmt"
  "net/http"
  "strings"
  "time"
)

const (
  ACTION_KEY = "Action"
)

var (
  timeNow = time.Now
)

type Region struct {
  Name string
}

var USEast = Region{"us-east-1"}
var USWest2 = Region{"us-west-2"}
var EUWest = Region{"eu-west-1"}

// Structure for kinesis client
type Kinesis struct {
  client  *Client
  Region  string
  Version string
}

// Interface implemented by Kinesis
type KinesisClient interface {
  CreateStream(StreamName string, ShardCount int) error
  DeleteStream(StreamName string) error
  MergeShards(args *RequestArgs) error
  SplitShard(args *RequestArgs) error
  ListStreams(args *RequestArgs) (resp *ListStreamsResp, err error)
  DescribeStream(args *RequestArgs) (resp *DescribeStreamResp, err error)
  GetShardIterator(args *RequestArgs) (resp *GetShardIteratorResp, err error)
  GetRecords(args *RequestArgs) (resp *GetRecordsResp, err error)
  PutRecord(args *RequestArgs) (resp *PutRecordResp, err error)
}

// Initialize new client for AWS Kinesis
func New(access_key string, secret_key string, region Region) *Kinesis {
  keys := &Auth{
    AccessKey: access_key,
    SecretKey: secret_key,
  }
  return &Kinesis{client: NewClient(keys), Version: "20131202", Region: GetRegion(region)}
}

// Create params object for request
func makeParams(action string) map[string]string {
  params := make(map[string]string)
  params[ACTION_KEY] = action
  return params
}

// RequestArgs store params for request
type RequestArgs struct {
  params map[string]interface{}
}

// NewFilter creates a new Filter.
func NewArgs() *RequestArgs {
  return &RequestArgs{make(map[string]interface{})}
}

// Add appends a filtering parameter with the given name and value(s).
func (f *RequestArgs) Add(name string, value interface{}) {
  f.params[name] = value
}

func (f *RequestArgs) AddData(value []byte) {
  enc := base64.StdEncoding
  buf := make([]byte, enc.EncodedLen(len(value)))
  enc.Encode(buf, value)
  f.params["Data"] = buf
}

// Error represent error from Kinesis API
type Error struct {
  // HTTP status code (200, 403, ...)
  StatusCode int
  // error code ("UnsupportedOperation", ...)
  Code string
  // The human-oriented error message
  Message   string
  RequestId string
}

// Return error message from error object
func (err *Error) Error() string {
  if err.Code == "" {
    return err.Message
  }
  return fmt.Sprintf("%s (%s)", err.Message, err.Code)
}

type jsonErrors struct {
  Message string
}

func buildError(r *http.Response) error {
  errors := jsonErrors{}
  json.NewDecoder(r.Body).Decode(&errors)

  var err Error
  err.Message = errors.Message
  err.StatusCode = r.StatusCode
  if err.Message == "" {
    err.Message = r.Status
  }
  return &err
}

// Query by AWS API
func (kinesis *Kinesis) query(params map[string]string, data interface{}, resp interface{}) error {
  jsonData, err := json.Marshal(data)
  if err != nil {
    return err
  }

  // request
  request, err := http.NewRequest("POST", fmt.Sprintf("https://kinesis.%s.amazonaws.com", kinesis.Region), strings.NewReader(string(jsonData)))
  if err != nil {
    return err
  }
  // headers
  request.Header.Set("Content-Type", "application/x-amz-json-1.1")
  request.Header.Set("X-Amz-Target", fmt.Sprintf("Kinesis_%s.%s", kinesis.Version, params[ACTION_KEY]))
  request.Header.Set("User-Agent", "Golang Kinesis")
  // response
  response, err := kinesis.client.Do(request)
  if err != nil {
    return err
  }
  defer response.Body.Close()

  if response.StatusCode != 200 {
    return buildError(response)
  }

  if resp == nil {
    return nil
  }
  return json.NewDecoder(response.Body).Decode(resp)
}

// CreateStream adds a new Amazon Kinesis stream to your AWS account
// StreamName is a name of stream, ShardCount is number of shards
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_CreateStream.html
func (kinesis *Kinesis) CreateStream(StreamName string, ShardCount int) error {
  params := makeParams("CreateStream")
  requestParams := struct {
    StreamName string
    ShardCount int
  }{
    StreamName,
    ShardCount,
  }
  err := kinesis.query(params, requestParams, nil)
  if err != nil {
    return err
  }
  return nil
}

// DeleteStream deletes a stream and all of its shards and data from your AWS account
// StreamName is a name of stream
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_DeleteStream.html
func (kinesis *Kinesis) DeleteStream(StreamName string) error {
  params := makeParams("DeleteStream")
  requestParams := struct {
    StreamName string
  }{
    StreamName,
  }
  err := kinesis.query(params, requestParams, nil)
  if err != nil {
    return err
  }
  return nil
}

// MergeShards merges two adjacent shards in a stream and combines them into a single shard to reduce the stream's capacity to ingest and transport data
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_MergeShards.html
func (kinesis *Kinesis) MergeShards(args *RequestArgs) error {
  params := makeParams("MergeShards")
  err := kinesis.query(params, args.params, nil)
  if err != nil {
    return err
  }
  return nil
}

// SplitShard splits a shard into two new shards in the stream, to increase the stream's capacity to ingest and transport data
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_SplitShard.html
func (kinesis *Kinesis) SplitShard(args *RequestArgs) error {
  params := makeParams("SplitShard")
  err := kinesis.query(params, args.params, nil)
  if err != nil {
    return err
  }
  return nil
}

// ListStreamsResp stores the information that provides by ListStreams API call
type ListStreamsResp struct {
  HasMoreStreams bool
  StreamNames    []string
}

// ListStreams returns an array of the names of all the streams that are associated with the AWS account making the ListStreams request
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_ListStreams.html
func (kinesis *Kinesis) ListStreams(args *RequestArgs) (resp *ListStreamsResp, err error) {
  params := makeParams("ListStreams")
  resp = &ListStreamsResp{}
  err = kinesis.query(params, args.params, resp)
  if err != nil {
    return nil, err
  }
  return
}

// DescribeStreamShards stores the information about list of shards inside DescribeStreamResp
type DescribeStreamShards struct {
  AdjacentParentShardId string
  HashKeyRange          struct {
    EndingHashKey   string
    StartingHashKey string
  }
  ParentShardId       string
  SequenceNumberRange struct {
    EndingSequenceNumber   string
    StartingSequenceNumber string
  }
  ShardId string
}

// DescribeStreamResp stores the information that provides by DescribeStream API call
type DescribeStreamResp struct {
  StreamDescription struct {
    HasMoreShards bool
    Shards        []DescribeStreamShards
    StreamARN     string
    StreamName    string
    StreamStatus  string
  }
}

// DescribeStream returns the following information about the stream: the current status of the stream,
// the stream Amazon Resource Name (ARN), and an array of shard objects that comprise the stream.
// For each shard object there is information about the hash key and sequence number ranges that
// the shard spans, and the IDs of any earlier shards that played in a role in a MergeShards or
// SplitShard operation that created the shard
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_DescribeStream.html
func (kinesis *Kinesis) DescribeStream(args *RequestArgs) (resp *DescribeStreamResp, err error) {
  params := makeParams("DescribeStream")
  resp = &DescribeStreamResp{}
  err = kinesis.query(params, args.params, resp)
  if err != nil {
    return nil, err
  }
  return
}

// GetShardIteratorResp stores the information that provides by GetShardIterator API call
type GetShardIteratorResp struct {
  ShardIterator string
}

// GetShardIterator returns a shard iterator
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetShardIterator.html
func (kinesis *Kinesis) GetShardIterator(args *RequestArgs) (resp *GetShardIteratorResp, err error) {
  params := makeParams("GetShardIterator")
  resp = &GetShardIteratorResp{}
  err = kinesis.query(params, args.params, resp)
  if err != nil {
    return nil, err
  }
  return
}

// GetNextRecordsRecords stores the information that provides by GetNextRecordsResp
type GetRecordsRecords struct {
  Data           []byte
  PartitionKey   string
  SequenceNumber string
}

func (r GetRecordsRecords) GetData() ([]byte, error) {
  enc := base64.StdEncoding
  dbuf := make([]byte, enc.DecodedLen(len(r.Data)))
  n, err := enc.Decode(dbuf, r.Data)
  return dbuf[:n], err
}

// GetNextRecordsResp stores the information that provides by GetNextRecords API call
type GetRecordsResp struct {
  NextShardIterator string
  Records           []GetRecordsRecords
}

// GetRecords returns one or more data records from a shard
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetRecords.html
func (kinesis *Kinesis) GetRecords(args *RequestArgs) (resp *GetRecordsResp, err error) {
  params := makeParams("GetRecords")
  resp = &GetRecordsResp{}
  err = kinesis.query(params, args.params, resp)
  if err != nil {
    return nil, err
  }
  return
}

// PutRecordResp stores the information that provides by PutRecord API call
type PutRecordResp struct {
  SequenceNumber string
  ShardId        string
}

// PutRecord puts a data record into an Amazon Kinesis stream from a producer
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecord.html
func (kinesis *Kinesis) PutRecord(args *RequestArgs) (resp *PutRecordResp, err error) {
  params := makeParams("PutRecord")
  resp = &PutRecordResp{}
  err = kinesis.query(params, args.params, resp)
  if err != nil {
    return nil, err
  }
  return
}
<commit_msg>Converts the base64 encoded data from []byte to string.<commit_after>// Package provide GOlang API for http://aws.amazon.com/kinesis/
package kinesis

import (
  "encoding/base64"
  "encoding/json"
  "fmt"
  "net/http"
  "strings"
  "time"
)

const (
  ACTION_KEY = "Action"
)

var (
  timeNow = time.Now
)

type Region struct {
  Name string
}

var USEast = Region{"us-east-1"}
var USWest2 = Region{"us-west-2"}
var EUWest = Region{"eu-west-1"}

// Structure for kinesis client
type Kinesis struct {
  client  *Client
  Region  string
  Version string
}

// Interface implemented by Kinesis
type KinesisClient interface {
  CreateStream(StreamName string, ShardCount int) error
  DeleteStream(StreamName string) error
  MergeShards(args *RequestArgs) error
  SplitShard(args *RequestArgs) error
  ListStreams(args *RequestArgs) (resp *ListStreamsResp, err error)
  DescribeStream(args *RequestArgs) (resp *DescribeStreamResp, err error)
  GetShardIterator(args *RequestArgs) (resp *GetShardIteratorResp, err error)
  GetRecords(args *RequestArgs) (resp *GetRecordsResp, err error)
  PutRecord(args *RequestArgs) (resp *PutRecordResp, err error)
}

// Initialize new client for AWS Kinesis
func New(access_key string, secret_key string, region Region) *Kinesis {
  keys := &Auth{
    AccessKey: access_key,
    SecretKey: secret_key,
  }
  return &Kinesis{client: NewClient(keys), Version: "20131202", Region: GetRegion(region)}
}

// Create params object for request
func makeParams(action string) map[string]string {
  params := make(map[string]string)
  params[ACTION_KEY] = action
  return params
}

// RequestArgs store params for request
type RequestArgs struct {
  params map[string]interface{}
}

// NewFilter creates a new Filter.
func NewArgs() *RequestArgs {
  return &RequestArgs{make(map[string]interface{})}
}

// Add appends a filtering parameter with the given name and value(s).
func (f *RequestArgs) Add(name string, value interface{}) {
  f.params[name] = value
}

func (f *RequestArgs) AddData(value []byte) {
  enc := base64.StdEncoding
  buf := make([]byte, enc.EncodedLen(len(value)))
  enc.Encode(buf, value)
  f.params["Data"] = string(buf)
}

// Error represent error from Kinesis API
type Error struct {
  // HTTP status code (200, 403, ...)
  StatusCode int
  // error code ("UnsupportedOperation", ...)
  Code string
  // The human-oriented error message
  Message   string
  RequestId string
}

// Return error message from error object
func (err *Error) Error() string {
  if err.Code == "" {
    return err.Message
  }
  return fmt.Sprintf("%s (%s)", err.Message, err.Code)
}

type jsonErrors struct {
  Message string
}

func buildError(r *http.Response) error {
  errors := jsonErrors{}
  json.NewDecoder(r.Body).Decode(&errors)

  var err Error
  err.Message = errors.Message
  err.StatusCode = r.StatusCode
  if err.Message == "" {
    err.Message = r.Status
  }
  return &err
}

// Query by AWS API
func (kinesis *Kinesis) query(params map[string]string, data interface{}, resp interface{}) error {
  jsonData, err := json.Marshal(data)
  if err != nil {
    return err
  }

  // request
  request, err := http.NewRequest("POST", fmt.Sprintf("https://kinesis.%s.amazonaws.com", kinesis.Region), strings.NewReader(string(jsonData)))
  if err != nil {
    return err
  }
  // headers
  request.Header.Set("Content-Type", "application/x-amz-json-1.1")
  request.Header.Set("X-Amz-Target", fmt.Sprintf("Kinesis_%s.%s", kinesis.Version, params[ACTION_KEY]))
  request.Header.Set("User-Agent", "Golang Kinesis")
  // response
  response, err := kinesis.client.Do(request)
  if err != nil {
    return err
  }
  defer response.Body.Close()

  if response.StatusCode != 200 {
    return buildError(response)
  }

  if resp == nil {
    return nil
  }
  return json.NewDecoder(response.Body).Decode(resp)
}

// CreateStream adds a new Amazon Kinesis stream to your AWS account
// StreamName is a name of stream, ShardCount is number of shards
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_CreateStream.html
func (kinesis *Kinesis) CreateStream(StreamName string, ShardCount int) error {
  params := makeParams("CreateStream")
  requestParams := struct {
    StreamName string
    ShardCount int
  }{
    StreamName,
    ShardCount,
  }
  err := kinesis.query(params, requestParams, nil)
  if err != nil {
    return err
  }
  return nil
}

// DeleteStream deletes a stream and all of its shards and data from your AWS account
// StreamName is a name of stream
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_DeleteStream.html
func (kinesis *Kinesis) DeleteStream(StreamName string) error {
  params := makeParams("DeleteStream")
  requestParams := struct {
    StreamName string
  }{
    StreamName,
  }
  err := kinesis.query(params, requestParams, nil)
  if err != nil {
    return err
  }
  return nil
}

// MergeShards merges two adjacent shards in a stream and combines them into a single shard to reduce the stream's capacity to ingest and transport data
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_MergeShards.html
func (kinesis *Kinesis) MergeShards(args *RequestArgs) error {
  params := makeParams("MergeShards")
  err := kinesis.query(params, args.params, nil)
  if err != nil {
    return err
  }
  return nil
}

// SplitShard splits a shard into two new shards in the stream, to increase the stream's capacity to ingest and transport data
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_SplitShard.html
func (kinesis *Kinesis) SplitShard(args *RequestArgs) error {
  params := makeParams("SplitShard")
  err := kinesis.query(params, args.params, nil)
  if err != nil {
    return err
  }
  return nil
}

// ListStreamsResp stores the information that provides by ListStreams API call
type ListStreamsResp struct {
  HasMoreStreams bool
  StreamNames    []string
}

// ListStreams returns an array of the names of all the streams that are associated with the AWS account making the ListStreams request
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_ListStreams.html
func (kinesis *Kinesis) ListStreams(args *RequestArgs) (resp *ListStreamsResp, err error) {
  params := makeParams("ListStreams")
  resp = &ListStreamsResp{}
  err = kinesis.query(params, args.params, resp)
  if err != nil {
    return nil, err
  }
  return
}

// DescribeStreamShards stores the information about list of shards inside DescribeStreamResp
type DescribeStreamShards struct {
  AdjacentParentShardId string
  HashKeyRange          struct {
    EndingHashKey   string
    StartingHashKey string
  }
  ParentShardId       string
  SequenceNumberRange struct {
    EndingSequenceNumber   string
    StartingSequenceNumber string
  }
  ShardId string
}

// DescribeStreamResp stores the information that provides by DescribeStream API call
type DescribeStreamResp struct {
  StreamDescription struct {
    HasMoreShards bool
    Shards        []DescribeStreamShards
    StreamARN     string
    StreamName    string
    StreamStatus  string
  }
}

// DescribeStream returns the following information about the stream: the current status of the stream,
// the stream Amazon Resource Name (ARN), and an array of shard objects that comprise the stream.
// For each shard object there is information about the hash key and sequence number ranges that
// the shard spans, and the IDs of any earlier shards that played in a role in a MergeShards or
// SplitShard operation that created the shard
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_DescribeStream.html
func (kinesis *Kinesis) DescribeStream(args *RequestArgs) (resp *DescribeStreamResp, err error) {
  params := makeParams("DescribeStream")
  resp = &DescribeStreamResp{}
  err = kinesis.query(params, args.params, resp)
  if err != nil {
    return nil, err
  }
  return
}

// GetShardIteratorResp stores the information that provides by GetShardIterator API call
type GetShardIteratorResp struct {
  ShardIterator string
}

// GetShardIterator returns a shard iterator
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetShardIterator.html
func (kinesis *Kinesis) GetShardIterator(args *RequestArgs) (resp *GetShardIteratorResp, err error) {
  params := makeParams("GetShardIterator")
  resp = &GetShardIteratorResp{}
  err = kinesis.query(params, args.params, resp)
  if err != nil {
    return nil, err
  }
  return
}

// GetNextRecordsRecords stores the information that provides by GetNextRecordsResp
type GetRecordsRecords struct {
  Data           []byte
  PartitionKey   string
  SequenceNumber string
}

func (r GetRecordsRecords) GetData() ([]byte, error) {
  enc := base64.StdEncoding
  dbuf := make([]byte, enc.DecodedLen(len(r.Data)))
  n, err := enc.Decode(dbuf, r.Data)
  return dbuf[:n], err
}

// GetNextRecordsResp stores the information that provides by GetNextRecords API call
type GetRecordsResp struct {
  NextShardIterator string
  Records           []GetRecordsRecords
}

// GetRecords returns one or more data records from a shard
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetRecords.html
func (kinesis *Kinesis) GetRecords(args *RequestArgs) (resp *GetRecordsResp, err error) {
  params := makeParams("GetRecords")
  resp = &GetRecordsResp{}
  err = kinesis.query(params, args.params, resp)
  if err != nil {
    return nil, err
  }
  return
}

// PutRecordResp stores the information that provides by PutRecord API call
type PutRecordResp struct {
  SequenceNumber string
  ShardId        string
}

// PutRecord puts a data record into an Amazon Kinesis stream from a producer
// more info http://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecord.html
func (kinesis *Kinesis) PutRecord(args *RequestArgs) (resp *PutRecordResp, err error) {
  params := makeParams("PutRecord")
  resp = &PutRecordResp{}
  err = kinesis.query(params, args.params, resp)
  if err != nil {
    return nil, err
  }
  return
}
<|endoftext|>


<commit_before>// Copyright 2017, TCN Inc.
// All rights reserved.

// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:

//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//     * Neither the name of TCN Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.

// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package generator_test

import (
	"compress/gzip"

	"github.com/golang/protobuf/proto"
	"github.com/golang/protobuf/protoc-gen-go/descriptor"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	_ "github.com/tcncloud/protoc-gen-persist/examples"
	"github.com/tcncloud/protoc-gen-persist/generator"

	"bytes"
	"io/ioutil"
	"testing"
)

func TestGenerator(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Generator Suite")
}

var _ = Describe("IsServicePersistEnabled", func() {
	Describe("for a service that implement custom extension persist.ql", func() {
		It("should return true", func() {
			var descr descriptor.FileDescriptorProto
			bReader, err := gzip.NewReader(bytes.NewReader(proto.FileDescriptor("examples/example1.proto")))
			defer bReader.Close()
			if err != nil {
				Fail("Fatal error loading file descriptor")
			}

			buf, err := ioutil.ReadAll(bReader)
			if err != nil {
				Fail("Can' decompress file descriptor data")
			}

			err = proto.Unmarshal(buf, &descr)
			Expect(err).To(BeNil())

			Expect(generator.IsServicePersistEnabled(descr.Service[0])).To(Equal(true))
		})

	})
})
<commit_msg>reorganized tests<commit_after>// Copyright 2017, TCN Inc.
// All rights reserved.

// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:

//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//     * Neither the name of TCN Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.

// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package generator_test

import (
	"compress/gzip"

	"github.com/golang/protobuf/proto"
	"github.com/golang/protobuf/protoc-gen-go/descriptor"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	_ "github.com/tcncloud/protoc-gen-persist/examples"
	"github.com/tcncloud/protoc-gen-persist/generator"

	"bytes"
	"io/ioutil"
	"testing"
)

func TestGenerator(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Generator Suite")
}

var descr descriptor.FileDescriptorProto

var _ = BeforeSuite(func() {
	bReader, err := gzip.NewReader(bytes.NewReader(proto.FileDescriptor("examples/example1.proto")))
	defer bReader.Close()
	if err != nil {
		Fail("Fatal error loading file descriptor")
	}

	buf, err := ioutil.ReadAll(bReader)
	if err != nil {
		Fail("Can' decompress file descriptor data")
	}

	err = proto.Unmarshal(buf, &descr)
	Expect(err).To(BeNil())

})

var _ = Describe("IsServicePersistEnabled", func() {
	Describe("for a service that implement custom extension persist.ql", func() {
		It("should return true", func() {

			Expect(generator.IsServicePersistEnabled(descr.Service[0])).To(Equal(true))
		})

	})
})
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"github.com/spf13/cobra"

	"github.com/NebulousLabs/Sia/modules"
)

var (
	hostCmd = &cobra.Command{
		Use:   "host",
		Short: "Perform host actions",
		Long:  "View or modify host settings. Modifying host settings also announces the host to the network.",
		Run:   wrap(hoststatuscmd),
	}

	hostConfigCmd = &cobra.Command{
		Use:   "config [setting] [value]",
		Short: "Modify host settings",
		Long: `Modify host settings.
Available settings:
	totalstorage
	minfilesize
	maxfilesize
	minduration
	maxduration
	windowsize
	price
	collateral`,
		Run: wrap(hostconfigcmd),
	}

	hostAnnounceCmd = &cobra.Command{
		Use:   "announce",
		Short: "Announce yourself as a host",
		Long:  "Announce yourself as a host on the network.",
		Run:   wrap(hostannouncecmd)}

	hostStatusCmd = &cobra.Command{
		Use:   "status",
		Short: "View host settings",
		Long:  "View host settings, including available storage, price, and more.",
		Run:   wrap(hoststatuscmd),
	}
)

func hostconfigcmd(param, value string) {
	err := post("/host/configure", param+"="+value)
	if err != nil {
		fmt.Println("Could not update host settings:", err)
		return
	}
	fmt.Println("Host settings updated.")
}

func hostannouncecmd() {
	err := post("/host/announce", "")
	if err != nil {
		fmt.Println("Could not announce host:", err)
		return
	}
	fmt.Println("Host announcement submitted to network.")
}

func hoststatuscmd() {
	info := new(modules.HostInfo)
	err := getAPI("/host/status", info)
	if err != nil {
		fmt.Println("Could not fetch host settings:", err)
		return
	}
	fmt.Printf(`Host settings:
Storage:      %v (%v used)
Price:        %v coins
Collateral:   %v
Max Filesize: %v
Max Duration: %v
Contracts:    %v
`, filesizeUnits(info.TotalStorage), filesizeUnits(info.TotalStorage-info.StorageRemaining), info.Price, info.Collateral, info.MaxFilesize, info.MaxDuration, info.NumContracts)
}
<commit_msg>clarify host settings<commit_after>package main

import (
	"fmt"

	"github.com/spf13/cobra"

	"github.com/NebulousLabs/Sia/modules"
)

var (
	hostCmd = &cobra.Command{
		Use:   "host",
		Short: "Perform host actions",
		Long:  "View or modify host settings. Modifying host settings also announces the host to the network.",
		Run:   wrap(hoststatuscmd),
	}

	hostConfigCmd = &cobra.Command{
		Use:   "config [setting] [value]",
		Short: "Modify host settings",
		Long: `Modify host settings.
Available settings:
	totalstorage
	minfilesize
	maxfilesize
	minduration
	maxduration
	windowsize
	price (in hastings)
	collateral`,
		Run: wrap(hostconfigcmd),
	}

	hostAnnounceCmd = &cobra.Command{
		Use:   "announce",
		Short: "Announce yourself as a host",
		Long:  "Announce yourself as a host on the network.",
		Run:   wrap(hostannouncecmd)}

	hostStatusCmd = &cobra.Command{
		Use:   "status",
		Short: "View host settings",
		Long:  "View host settings, including available storage, price, and more.",
		Run:   wrap(hoststatuscmd),
	}
)

func hostconfigcmd(param, value string) {
	err := post("/host/configure", param+"="+value)
	if err != nil {
		fmt.Println("Could not update host settings:", err)
		return
	}
	fmt.Println("Host settings updated.")
}

func hostannouncecmd() {
	err := post("/host/announce", "")
	if err != nil {
		fmt.Println("Could not announce host:", err)
		return
	}
	fmt.Println("Host announcement submitted to network.")
}

func hoststatuscmd() {
	info := new(modules.HostInfo)
	err := getAPI("/host/status", info)
	if err != nil {
		fmt.Println("Could not fetch host settings:", err)
		return
	}
	fmt.Printf(`Host settings:
Storage:      %v (%v used)
Price:        %v coins
Collateral:   %v
Max Filesize: %v
Max Duration: %v
Contracts:    %v
`, filesizeUnits(info.TotalStorage), filesizeUnits(info.TotalStorage-info.StorageRemaining), info.Price, info.Collateral, info.MaxFilesize, info.MaxDuration, info.NumContracts)
}
<|endoftext|>


<commit_before>package image

import (
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/filter"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/triggers"
	"time"
)

type Annotation struct {
	Object *hash.Hash // These are mutually exclusive.
	URL    string
}

type DirectoryMetadata struct {
	OwnerGroup string
}

type Directory struct {
	Name     string
	Metadata DirectoryMetadata
}

type Image struct {
	CreatedBy    string // Username. Set by imageserver. Empty: unauthenticated.
	Filter       *filter.Filter
	FileSystem   *filesystem.FileSystem
	Triggers     *triggers.Triggers
	ReleaseNotes *Annotation
	BuildLog     *Annotation
	ExpiresAt    time.Time
}

// Verify will perform some self-consistency checks on the image. If a problem
// is found, an error is returned.
func (image *Image) Verify() error {
	return image.verify()
}

// VerifyRequiredPaths will verify if required paths are present in the image.
// The table of required paths is given by requiredPaths. If the image is a
// sparse image (has no filter), then this check is skipped. If a problem is
// found, an error is returned.
// This function will create cache data associated with the image, consuming
// more memory.
func (image *Image) VerifyRequiredPaths(requiredPaths map[string]rune) error {
	return image.verifyRequiredPaths(requiredPaths)
}

func SortDirectories(directories []Directory) {
	sortDirectories(directories)
}
<commit_msg>Add lib/image.Image.CreatedOn field.<commit_after>package image

import (
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/filter"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/triggers"
	"time"
)

type Annotation struct {
	Object *hash.Hash // These are mutually exclusive.
	URL    string
}

type DirectoryMetadata struct {
	OwnerGroup string
}

type Directory struct {
	Name     string
	Metadata DirectoryMetadata
}

type Image struct {
	CreatedBy    string // Username. Set by imageserver. Empty: unauthenticated.
	Filter       *filter.Filter
	FileSystem   *filesystem.FileSystem
	Triggers     *triggers.Triggers
	ReleaseNotes *Annotation
	BuildLog     *Annotation
	CreatedOn    time.Time
	ExpiresAt    time.Time
}

// Verify will perform some self-consistency checks on the image. If a problem
// is found, an error is returned.
func (image *Image) Verify() error {
	return image.verify()
}

// VerifyRequiredPaths will verify if required paths are present in the image.
// The table of required paths is given by requiredPaths. If the image is a
// sparse image (has no filter), then this check is skipped. If a problem is
// found, an error is returned.
// This function will create cache data associated with the image, consuming
// more memory.
func (image *Image) VerifyRequiredPaths(requiredPaths map[string]rune) error {
	return image.verifyRequiredPaths(requiredPaths)
}

func SortDirectories(directories []Directory) {
	sortDirectories(directories)
}
<|endoftext|>


<commit_before>package lib

import (
	"encoding/json"
	"time"

	"github.com/bitly/go-simplejson"
	"github.com/rcrowley/go-metrics"
	"github.com/streadway/amqp"
	"github.com/travis-ci/worker/lib/backend"
	"github.com/travis-ci/worker/lib/context"
	gocontext "golang.org/x/net/context"
)

// A JobQueue allows getting Jobs out of an AMQP queue.
type JobQueue struct {
	conn  *amqp.Connection
	queue string
}

type amqpJob struct {
	conn            *amqp.Connection
	delivery        amqp.Delivery
	payload         JobPayload
	rawPayload      *simplejson.Json
	startAttributes backend.StartAttributes
}

type amqpPayloadStartAttrs struct {
	Config backend.StartAttributes `json:"config"`
}

func (j amqpJob) Payload() JobPayload {
	return j.payload
}

func (j amqpJob) RawPayload() *simplejson.Json {
	return j.rawPayload
}

func (j amqpJob) StartAttributes() backend.StartAttributes {
	return j.startAttributes
}

func (j amqpJob) Error(ctx gocontext.Context, errMessage string) error {
	log, err := j.LogWriter(ctx)
	if err != nil {
		return err
	}

	_, err = log.WriteAndClose([]byte(errMessage))
	if err != nil {
		return err
	}

	return j.Finish(FinishStateErrored)
}

func (j amqpJob) Requeue() error {
	metrics.GetOrRegisterMeter("worker.job.requeue", metrics.DefaultRegistry).Mark(1)

	amqpChan, err := j.conn.Channel()
	if err != nil {
		return err
	}
	defer amqpChan.Close()

	body := map[string]interface{}{
		"id":    j.Payload().Job.ID,
		"state": "reset",
	}

	bodyBytes, err := json.Marshal(body)
	if err != nil {
		return err
	}

	_, err = amqpChan.QueueDeclare("reporting.jobs.builds", true, false, false, false, nil)
	if err != nil {
		return err
	}

	amqpChan.Publish("", "reporting.jobs.builds", false, false, amqp.Publishing{
		ContentType:  "application/json",
		DeliveryMode: amqp.Persistent,
		Timestamp:    time.Now(),
		Type:         "job:test:reset",
		Body:         bodyBytes,
	})

	j.delivery.Ack(false)

	return nil
}

func (j amqpJob) Received() error {
	amqpChan, err := j.conn.Channel()
	if err != nil {
		return err
	}
	defer amqpChan.Close()

	body := map[string]interface{}{
		"id":          j.Payload().Job.ID,
		"state":       "received",
		"received_at": time.Now().Format(time.RFC3339),
	}

	bodyBytes, err := json.Marshal(body)
	if err != nil {
		return err
	}

	_, err = amqpChan.QueueDeclare("reporting.jobs.builds", true, false, false, false, nil)
	if err != nil {
		return err
	}

	return amqpChan.Publish("", "reporting.jobs.builds", false, false, amqp.Publishing{
		ContentType:  "application/json",
		DeliveryMode: amqp.Persistent,
		Timestamp:    time.Now(),
		Type:         "job:test:receive",
		Body:         bodyBytes,
	})
}

func (j amqpJob) Started() error {
	amqpChan, err := j.conn.Channel()
	if err != nil {
		return err
	}
	defer amqpChan.Close()

	body := map[string]interface{}{
		"id":         j.Payload().Job.ID,
		"state":      "started",
		"started_at": time.Now().Format(time.RFC3339),
	}

	bodyBytes, err := json.Marshal(body)
	if err != nil {
		return err
	}

	_, err = amqpChan.QueueDeclare("reporting.jobs.builds", true, false, false, false, nil)
	if err != nil {
		return err
	}

	amqpChan.Publish("", "reporting.jobs.builds", false, false, amqp.Publishing{
		ContentType:  "application/json",
		DeliveryMode: amqp.Persistent,
		Timestamp:    time.Now(),
		Type:         "job:test:start",
		Body:         bodyBytes,
	})

	return nil
}

func (j amqpJob) Finish(state FinishState) error {
	amqpChan, err := j.conn.Channel()
	if err != nil {
		return err
	}
	defer amqpChan.Close()

	body := map[string]interface{}{
		"id":          j.Payload().Job.ID,
		"state":       state,
		"finished_at": time.Now().Format(time.RFC3339),
	}

	bodyBytes, err := json.Marshal(body)
	if err != nil {
		return err
	}

	_, err = amqpChan.QueueDeclare("reporting.jobs.builds", true, false, false, false, nil)
	if err != nil {
		return err
	}

	amqpChan.Publish("", "reporting.jobs.builds", false, false, amqp.Publishing{
		ContentType:  "application/json",
		DeliveryMode: amqp.Persistent,
		Timestamp:    time.Now(),
		Type:         "job:test:finish",
		Body:         bodyBytes,
	})

	j.delivery.Ack(false)

	return nil
}

func (j amqpJob) LogWriter(ctx gocontext.Context) (LogWriter, error) {
	return NewLogWriter(ctx, j.conn, j.payload.Job.ID)
}

// NewJobQueue creates a JobQueue backed by the given AMQP connections and
// connects to the AMQP queue with the given name. The queue will be declared
// in AMQP when this function is called, so an error could be raised if the
// queue already exists, but with different attributes than we expect.
func NewJobQueue(conn *amqp.Connection, queue string) (*JobQueue, error) {
	channel, err := conn.Channel()
	if err != nil {
		return nil, err
	}

	_, err = channel.QueueDeclare(queue, true, false, false, false, nil)
	if err != nil {
		return nil, err
	}

	err = channel.Close()
	if err != nil {
		return nil, err
	}

	return &JobQueue{
		conn:  conn,
		queue: queue,
	}, nil
}

// Jobs creates a new consumer on the queue, and returns three channels. The
// first channel gets sent every BuildJob that we receive from AMQP. The
// stopChan is a channel that can be closed in order to stop the consumer.
func (q *JobQueue) Jobs(ctx gocontext.Context) (outChan <-chan Job, err error) {
	channel, err := q.conn.Channel()
	if err != nil {
		return
	}

	err = channel.Qos(1, 0, false)
	if err != nil {
		return
	}

	deliveries, err := channel.Consume(q.queue, "build-job-consumer", false, false, false, false, nil)
	if err != nil {
		return
	}

	buildJobChan := make(chan Job)
	outChan = buildJobChan

	go func() {
		for delivery := range deliveries {
			var buildJob amqpJob
			var startAttrs amqpPayloadStartAttrs

			err := json.Unmarshal(delivery.Body, &buildJob.payload)
			if err != nil {
				context.LoggerFromContext(ctx).WithField("err", err).Error("payload JSON parse error")
				// This should be a regular Ack, but for now we'll requeue it until a Ruby worker picks it up
				delivery.Reject(true)
				continue
			}

			err = json.Unmarshal(delivery.Body, &startAttrs)
			if err != nil {
				context.LoggerFromContext(ctx).WithField("err", err).Error("start attributes JSON parse error")
				// This should be a regular Ack, but for now we'll requeue it until a Ruby worker picks it up
				delivery.Reject(true)
				continue
			}

			buildJob.rawPayload, err = simplejson.NewJson(delivery.Body)
			if err != nil {
				context.LoggerFromContext(ctx).WithField("err", err).Error("raw payload JSON parse error")
				// This should be a regular Ack, but for now we'll requeue it until a Ruby worker picks it up
				delivery.Reject(true)
				continue
			}

			buildJob.startAttributes = startAttrs.Config
			buildJob.conn = q.conn
			buildJob.delivery = delivery

			buildJobChan <- buildJob
		}

		channel.Close()
	}()

	return
}
<commit_msg>job-queue: ack if we can't parse JSON<commit_after>package lib

import (
	"encoding/json"
	"time"

	"github.com/bitly/go-simplejson"
	"github.com/rcrowley/go-metrics"
	"github.com/streadway/amqp"
	"github.com/travis-ci/worker/lib/backend"
	"github.com/travis-ci/worker/lib/context"
	gocontext "golang.org/x/net/context"
)

// A JobQueue allows getting Jobs out of an AMQP queue.
type JobQueue struct {
	conn  *amqp.Connection
	queue string
}

type amqpJob struct {
	conn            *amqp.Connection
	delivery        amqp.Delivery
	payload         JobPayload
	rawPayload      *simplejson.Json
	startAttributes backend.StartAttributes
}

type amqpPayloadStartAttrs struct {
	Config backend.StartAttributes `json:"config"`
}

func (j amqpJob) Payload() JobPayload {
	return j.payload
}

func (j amqpJob) RawPayload() *simplejson.Json {
	return j.rawPayload
}

func (j amqpJob) StartAttributes() backend.StartAttributes {
	return j.startAttributes
}

func (j amqpJob) Error(ctx gocontext.Context, errMessage string) error {
	log, err := j.LogWriter(ctx)
	if err != nil {
		return err
	}

	_, err = log.WriteAndClose([]byte(errMessage))
	if err != nil {
		return err
	}

	return j.Finish(FinishStateErrored)
}

func (j amqpJob) Requeue() error {
	metrics.GetOrRegisterMeter("worker.job.requeue", metrics.DefaultRegistry).Mark(1)

	amqpChan, err := j.conn.Channel()
	if err != nil {
		return err
	}
	defer amqpChan.Close()

	body := map[string]interface{}{
		"id":    j.Payload().Job.ID,
		"state": "reset",
	}

	bodyBytes, err := json.Marshal(body)
	if err != nil {
		return err
	}

	_, err = amqpChan.QueueDeclare("reporting.jobs.builds", true, false, false, false, nil)
	if err != nil {
		return err
	}

	amqpChan.Publish("", "reporting.jobs.builds", false, false, amqp.Publishing{
		ContentType:  "application/json",
		DeliveryMode: amqp.Persistent,
		Timestamp:    time.Now(),
		Type:         "job:test:reset",
		Body:         bodyBytes,
	})

	j.delivery.Ack(false)

	return nil
}

func (j amqpJob) Received() error {
	amqpChan, err := j.conn.Channel()
	if err != nil {
		return err
	}
	defer amqpChan.Close()

	body := map[string]interface{}{
		"id":          j.Payload().Job.ID,
		"state":       "received",
		"received_at": time.Now().Format(time.RFC3339),
	}

	bodyBytes, err := json.Marshal(body)
	if err != nil {
		return err
	}

	_, err = amqpChan.QueueDeclare("reporting.jobs.builds", true, false, false, false, nil)
	if err != nil {
		return err
	}

	return amqpChan.Publish("", "reporting.jobs.builds", false, false, amqp.Publishing{
		ContentType:  "application/json",
		DeliveryMode: amqp.Persistent,
		Timestamp:    time.Now(),
		Type:         "job:test:receive",
		Body:         bodyBytes,
	})
}

func (j amqpJob) Started() error {
	amqpChan, err := j.conn.Channel()
	if err != nil {
		return err
	}
	defer amqpChan.Close()

	body := map[string]interface{}{
		"id":         j.Payload().Job.ID,
		"state":      "started",
		"started_at": time.Now().Format(time.RFC3339),
	}

	bodyBytes, err := json.Marshal(body)
	if err != nil {
		return err
	}

	_, err = amqpChan.QueueDeclare("reporting.jobs.builds", true, false, false, false, nil)
	if err != nil {
		return err
	}

	amqpChan.Publish("", "reporting.jobs.builds", false, false, amqp.Publishing{
		ContentType:  "application/json",
		DeliveryMode: amqp.Persistent,
		Timestamp:    time.Now(),
		Type:         "job:test:start",
		Body:         bodyBytes,
	})

	return nil
}

func (j amqpJob) Finish(state FinishState) error {
	amqpChan, err := j.conn.Channel()
	if err != nil {
		return err
	}
	defer amqpChan.Close()

	body := map[string]interface{}{
		"id":          j.Payload().Job.ID,
		"state":       state,
		"finished_at": time.Now().Format(time.RFC3339),
	}

	bodyBytes, err := json.Marshal(body)
	if err != nil {
		return err
	}

	_, err = amqpChan.QueueDeclare("reporting.jobs.builds", true, false, false, false, nil)
	if err != nil {
		return err
	}

	amqpChan.Publish("", "reporting.jobs.builds", false, false, amqp.Publishing{
		ContentType:  "application/json",
		DeliveryMode: amqp.Persistent,
		Timestamp:    time.Now(),
		Type:         "job:test:finish",
		Body:         bodyBytes,
	})

	j.delivery.Ack(false)

	return nil
}

func (j amqpJob) LogWriter(ctx gocontext.Context) (LogWriter, error) {
	return NewLogWriter(ctx, j.conn, j.payload.Job.ID)
}

// NewJobQueue creates a JobQueue backed by the given AMQP connections and
// connects to the AMQP queue with the given name. The queue will be declared
// in AMQP when this function is called, so an error could be raised if the
// queue already exists, but with different attributes than we expect.
func NewJobQueue(conn *amqp.Connection, queue string) (*JobQueue, error) {
	channel, err := conn.Channel()
	if err != nil {
		return nil, err
	}

	_, err = channel.QueueDeclare(queue, true, false, false, false, nil)
	if err != nil {
		return nil, err
	}

	err = channel.Close()
	if err != nil {
		return nil, err
	}

	return &JobQueue{
		conn:  conn,
		queue: queue,
	}, nil
}

// Jobs creates a new consumer on the queue, and returns three channels. The
// first channel gets sent every BuildJob that we receive from AMQP. The
// stopChan is a channel that can be closed in order to stop the consumer.
func (q *JobQueue) Jobs(ctx gocontext.Context) (outChan <-chan Job, err error) {
	channel, err := q.conn.Channel()
	if err != nil {
		return
	}

	err = channel.Qos(1, 0, false)
	if err != nil {
		return
	}

	deliveries, err := channel.Consume(q.queue, "build-job-consumer", false, false, false, false, nil)
	if err != nil {
		return
	}

	buildJobChan := make(chan Job)
	outChan = buildJobChan

	go func() {
		for delivery := range deliveries {
			var buildJob amqpJob
			var startAttrs amqpPayloadStartAttrs

			err := json.Unmarshal(delivery.Body, &buildJob.payload)
			if err != nil {
				context.LoggerFromContext(ctx).WithField("err", err).Error("payload JSON parse error")
				delivery.Ack(false)
				continue
			}

			err = json.Unmarshal(delivery.Body, &startAttrs)
			if err != nil {
				context.LoggerFromContext(ctx).WithField("err", err).Error("start attributes JSON parse error")
				delivery.Ack(false)
				continue
			}

			buildJob.rawPayload, err = simplejson.NewJson(delivery.Body)
			if err != nil {
				context.LoggerFromContext(ctx).WithField("err", err).Error("raw payload JSON parse error")
				delivery.Ack(false)
				continue
			}

			buildJob.startAttributes = startAttrs.Config
			buildJob.conn = q.conn
			buildJob.delivery = delivery

			buildJobChan <- buildJob
		}

		channel.Close()
	}()

	return
}
<|endoftext|>


<commit_before>package main

import (
	"container/heap"
	"fmt"
	"github.com/jessevdk/go-flags"
	"github.com/tj/go-dropbox"
	"golang.org/x/text/unicode/norm"
	"os"
	"path/filepath"
	"strings"
)

/* TODO
- Clean up output formatting
- Parallelize local/remote file listing
- Add progress printing - maybe collect progress from remote/local listing through channels
- Test for more case issues - already handling when root folder is lowercased
  by Dropbox, but maybe other path components could be as well?
*/

// File stores the result of either Dropbox API or local file listing
type File struct {
	Path        string
	ContentHash string
}

// FileHeap is a list of Files sorted by path
type FileHeap []*File

func (h FileHeap) Len() int           { return len(h) }
func (h FileHeap) Less(i, j int) bool { return h[i].Path < h[j].Path }
func (h FileHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }

// Push a File onto the heap
func (h *FileHeap) Push(x interface{}) {
	// Push and Pop use pointer receivers because they modify the slice's length,
	// not just its contents.
	*h = append(*h, x.(*File))
}

// Pop a File off the heap
func (h *FileHeap) Pop() interface{} {
	old := *h
	n := len(old)
	x := old[n-1]
	*h = old[0 : n-1]
	return x
}

// PopOrNil pops a File off the heap or returns nil if there's nothing left
func (h *FileHeap) PopOrNil() *File {
	if h.Len() > 0 {
		return heap.Pop(h).(*File)
	}
	return nil
}

// ManifestComparison records the relative paths that differ between remote and
// local versions of a directory
type ManifestComparison struct {
	OnlyRemote      []string
	OnlyLocal       []string
	ContentMismatch []string
	Matches         int
	Misses          int
}

func main() {
	token := os.Getenv("DROPBOX_ACCESS_TOKEN")
	if token == "" {
		fmt.Fprintln(os.Stderr, "Missing Dropbox OAuth token! Please set the DROPBOX_ACCESS_TOKEN environment variable.")
		os.Exit(1)
	}

	var opts struct {
		Verbose          bool   `short:"v" long:"verbose" description:"Show verbose debug information"`
		RemoteRoot       string `short:"r" long:"remote" description:"Directory in Dropbox to verify" default:"/"`
		LocalRoot        string `short:"l" long:"local" description:"Local directory to compare to Dropbox contents" default:"."`
		CheckContentHash bool   `long:"check" description:"Check content hash of local files"`
	}

	_, err := flags.Parse(&opts)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}

	// Dropbox API uses empty string for root, but for figuring out relative
	// paths of the returned entries it's easier to use "/". Conversion is
	// handled before the API call.
	if opts.RemoteRoot == "" {
		opts.RemoteRoot = "/"
	}
	if opts.RemoteRoot[0] != '/' {
		opts.RemoteRoot = "/" + opts.RemoteRoot
	}

	localRoot, _ := filepath.Abs(opts.LocalRoot)

	dbxClient := dropbox.New(dropbox.NewConfig(token))

	fmt.Printf("Comparing Dropbox directory \"%v\" to local directory \"%v\"\n", opts.RemoteRoot, localRoot)
	if opts.CheckContentHash {
		fmt.Println("Checking content hashes.")
	}
	fmt.Println("")

	dropboxManifest, err := getDropboxManifest(dbxClient, opts.RemoteRoot)
	if err != nil {
		panic(err)
	}

	localManifest, err := getLocalManifest(localRoot, opts.CheckContentHash)
	if err != nil {
		panic(err)
	}

	manifestComparison := compareManifests(dropboxManifest, localManifest)

	printFileList(manifestComparison.OnlyRemote, "Files only in remote")
	printFileList(manifestComparison.OnlyLocal, "Files only in local")
	printFileList(manifestComparison.ContentMismatch, "Files whose contents don't match")
	total := manifestComparison.Matches + manifestComparison.Misses
	fmt.Println("SUMMARY:")
	fmt.Printf("Files matched: %d/%d\n", manifestComparison.Matches, total)
	fmt.Printf("Files not matched: %d/%d\n", manifestComparison.Misses, total)
}

func getDropboxManifest(dbxClient *dropbox.Client, rootPath string) (manifest *FileHeap, err error) {
	manifest = &FileHeap{}
	heap.Init(manifest)
	cursor := ""
	keepGoing := true

	for keepGoing {
		var resp *dropbox.ListFolderOutput
		if cursor != "" {
			arg := &dropbox.ListFolderContinueInput{Cursor: cursor}
			resp, err = dbxClient.Files.ListFolderContinue(arg)
		} else {
			apiPath := rootPath
			if apiPath == "/" {
				apiPath = ""
			}
			arg := &dropbox.ListFolderInput{
				Path:             apiPath,
				Recursive:        true,
				IncludeMediaInfo: false,
				IncludeDeleted:   false,
			}
			resp, err = dbxClient.Files.ListFolder(arg)
		}
		if err != nil {
			return
		}
		for _, entry := range resp.Entries {
			if entry.Tag == "file" {

				var relPath string
				relPath, err = normalizePath(rootPath, entry.PathDisplay)
				if err != nil {
					return
				}
				heap.Push(manifest, &File{
					Path:        relPath,
					ContentHash: entry.ContentHash,
				})
			}
		}

		cursor = resp.Cursor
		keepGoing = resp.HasMore

		// DEBUG info
		fmt.Printf("%d dropbox files listed          \r", manifest.Len())
	}

	return
}

func getLocalManifest(localRoot string, contentHash bool) (manifest *FileHeap, err error) {
	manifest = &FileHeap{}
	heap.Init(manifest)

	err = filepath.Walk(localRoot, func(entryPath string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if info.Mode().IsRegular() {
			relPath, err := normalizePath(localRoot, entryPath)
			if err != nil {
				return err
			}

			hash := ""
			if contentHash {
				hash, err = dropbox.FileContentHash(entryPath)
				if err != nil {
					return err
				}
			}

			heap.Push(manifest, &File{
				Path:        relPath,
				ContentHash: hash,
			})

			// DEBUG info
			fmt.Printf("%d local files listed          \r", manifest.Len())
		}

		return nil
	})

	return
}

func normalizePath(root string, entryPath string) (string, error) {
	relPath, err := filepath.Rel(root, entryPath)
	if err != nil {
		return "", err
	}
	if relPath[0:3] == "../" {
		// try lowercase root instead
		relPath, err = filepath.Rel(strings.ToLower(root), entryPath)
		if err != nil {
			return "", err
		}
	}

	// Normalize Unicode combining characters
	relPath = norm.NFC.String(relPath)
	return relPath, nil
}

func compareManifests(remoteManifest, localManifest *FileHeap) *ManifestComparison {
	// 1. Pop a path off both remote and local manifests.
	// 2. While remote & local are both not nil:
	//    Compare remote & local:
	//    a. If local is nil or local > remote, this file is only in remote. Record and pop remote again.
	//    b. If remote is nil or local < remote, this file is only in local. Record and pop local again.
	//    c. If local == remote, check for content mismatch. Record if necessary and pop both again.
	comparison := &ManifestComparison{}
	local := localManifest.PopOrNil()
	remote := remoteManifest.PopOrNil()
	for local != nil || remote != nil {
		if local == nil || local.Path > remote.Path {
			comparison.OnlyRemote = append(comparison.OnlyRemote, remote.Path)
			comparison.Misses++
			remote = remoteManifest.PopOrNil()
		} else if remote == nil || local.Path < remote.Path {
			comparison.OnlyLocal = append(comparison.OnlyLocal, local.Path)
			comparison.Misses++
			local = localManifest.PopOrNil()
		} else if remote.Path == local.Path {
			if !compareFileContents(remote, local) {
				comparison.ContentMismatch = append(comparison.ContentMismatch, local.Path)
				comparison.Misses++
			} else {
				comparison.Matches++
			}
			local = localManifest.PopOrNil()
			remote = remoteManifest.PopOrNil()
		} else {
			// This should be unreachable...
			panic(fmt.Sprintf("Unexpected condition! Local: %v, remote: %v", local, remote))
		}
	}
	return comparison
}

func compareFileContents(remote, local *File) bool {
	if remote.ContentHash == "" || local.ContentHash == "" {
		// Missing content hash for one of the files, possibly intentionally,
		// so can't compare. Assume that presence of both is enough to
		// validate.
		return true
	}
	return remote.ContentHash == local.ContentHash
}

func printFileList(files []string, description string) {
	fmt.Printf("%s: %d\n\n", description, len(files))
	for _, path := range files {
		fmt.Println(path)
	}
	if len(files) > 0 {
		fmt.Print("\n\n")
	}
}
<commit_msg>Skip .DS_Store files<commit_after>package main

import (
	"container/heap"
	"fmt"
	"github.com/jessevdk/go-flags"
	"github.com/tj/go-dropbox"
	"golang.org/x/text/unicode/norm"
	"os"
	"path/filepath"
	"strings"
)

/* TODO
- Clean up output formatting
- Parallelize local/remote file listing
- Add progress printing - maybe collect progress from remote/local listing through channels
- Test for more case issues - already handling when root folder is lowercased
  by Dropbox, but maybe other path components could be as well?
- Ignore more file names in skipLocalFile - see https://www.dropbox.com/help/syncing-uploads/files-not-syncing
*/

// File stores the result of either Dropbox API or local file listing
type File struct {
	Path        string
	ContentHash string
}

// FileHeap is a list of Files sorted by path
type FileHeap []*File

func (h FileHeap) Len() int           { return len(h) }
func (h FileHeap) Less(i, j int) bool { return h[i].Path < h[j].Path }
func (h FileHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }

// Push a File onto the heap
func (h *FileHeap) Push(x interface{}) {
	// Push and Pop use pointer receivers because they modify the slice's length,
	// not just its contents.
	*h = append(*h, x.(*File))
}

// Pop a File off the heap
func (h *FileHeap) Pop() interface{} {
	old := *h
	n := len(old)
	x := old[n-1]
	*h = old[0 : n-1]
	return x
}

// PopOrNil pops a File off the heap or returns nil if there's nothing left
func (h *FileHeap) PopOrNil() *File {
	if h.Len() > 0 {
		return heap.Pop(h).(*File)
	}
	return nil
}

// ManifestComparison records the relative paths that differ between remote and
// local versions of a directory
type ManifestComparison struct {
	OnlyRemote      []string
	OnlyLocal       []string
	ContentMismatch []string
	Matches         int
	Misses          int
}

func main() {
	token := os.Getenv("DROPBOX_ACCESS_TOKEN")
	if token == "" {
		fmt.Fprintln(os.Stderr, "Missing Dropbox OAuth token! Please set the DROPBOX_ACCESS_TOKEN environment variable.")
		os.Exit(1)
	}

	var opts struct {
		Verbose          bool   `short:"v" long:"verbose" description:"Show verbose debug information"`
		RemoteRoot       string `short:"r" long:"remote" description:"Directory in Dropbox to verify" default:"/"`
		LocalRoot        string `short:"l" long:"local" description:"Local directory to compare to Dropbox contents" default:"."`
		CheckContentHash bool   `long:"check" description:"Check content hash of local files"`
	}

	_, err := flags.Parse(&opts)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}

	// Dropbox API uses empty string for root, but for figuring out relative
	// paths of the returned entries it's easier to use "/". Conversion is
	// handled before the API call.
	if opts.RemoteRoot == "" {
		opts.RemoteRoot = "/"
	}
	if opts.RemoteRoot[0] != '/' {
		opts.RemoteRoot = "/" + opts.RemoteRoot
	}

	localRoot, _ := filepath.Abs(opts.LocalRoot)

	dbxClient := dropbox.New(dropbox.NewConfig(token))

	fmt.Printf("Comparing Dropbox directory \"%v\" to local directory \"%v\"\n", opts.RemoteRoot, localRoot)
	if opts.CheckContentHash {
		fmt.Println("Checking content hashes.")
	}
	fmt.Println("")

	dropboxManifest, err := getDropboxManifest(dbxClient, opts.RemoteRoot)
	if err != nil {
		panic(err)
	}

	localManifest, err := getLocalManifest(localRoot, opts.CheckContentHash)
	if err != nil {
		panic(err)
	}

	manifestComparison := compareManifests(dropboxManifest, localManifest)

	printFileList(manifestComparison.OnlyRemote, "Files only in remote")
	printFileList(manifestComparison.OnlyLocal, "Files only in local")
	printFileList(manifestComparison.ContentMismatch, "Files whose contents don't match")
	total := manifestComparison.Matches + manifestComparison.Misses
	fmt.Println("SUMMARY:")
	fmt.Printf("Files matched: %d/%d\n", manifestComparison.Matches, total)
	fmt.Printf("Files not matched: %d/%d\n", manifestComparison.Misses, total)
}

func getDropboxManifest(dbxClient *dropbox.Client, rootPath string) (manifest *FileHeap, err error) {
	manifest = &FileHeap{}
	heap.Init(manifest)
	cursor := ""
	keepGoing := true

	for keepGoing {
		var resp *dropbox.ListFolderOutput
		if cursor != "" {
			arg := &dropbox.ListFolderContinueInput{Cursor: cursor}
			resp, err = dbxClient.Files.ListFolderContinue(arg)
		} else {
			apiPath := rootPath
			if apiPath == "/" {
				apiPath = ""
			}
			arg := &dropbox.ListFolderInput{
				Path:             apiPath,
				Recursive:        true,
				IncludeMediaInfo: false,
				IncludeDeleted:   false,
			}
			resp, err = dbxClient.Files.ListFolder(arg)
		}
		if err != nil {
			return
		}
		for _, entry := range resp.Entries {
			if entry.Tag == "file" {

				var relPath string
				relPath, err = normalizePath(rootPath, entry.PathDisplay)
				if err != nil {
					return
				}
				heap.Push(manifest, &File{
					Path:        relPath,
					ContentHash: entry.ContentHash,
				})
			}
		}

		cursor = resp.Cursor
		keepGoing = resp.HasMore

		// DEBUG info
		fmt.Printf("%d dropbox files listed          \r", manifest.Len())
	}

	return
}

func getLocalManifest(localRoot string, contentHash bool) (manifest *FileHeap, err error) {
	manifest = &FileHeap{}
	heap.Init(manifest)

	err = filepath.Walk(localRoot, func(entryPath string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if info.Mode().IsRegular() && !skipLocalFile(entryPath) {
			relPath, err := normalizePath(localRoot, entryPath)
			if err != nil {
				return err
			}

			hash := ""
			if contentHash {
				hash, err = dropbox.FileContentHash(entryPath)
				if err != nil {
					return err
				}
			}

			heap.Push(manifest, &File{
				Path:        relPath,
				ContentHash: hash,
			})

			// DEBUG info
			fmt.Printf("%d local files listed          \r", manifest.Len())
		}

		return nil
	})

	return
}

func normalizePath(root string, entryPath string) (string, error) {
	relPath, err := filepath.Rel(root, entryPath)
	if err != nil {
		return "", err
	}
	if relPath[0:3] == "../" {
		// try lowercase root instead
		relPath, err = filepath.Rel(strings.ToLower(root), entryPath)
		if err != nil {
			return "", err
		}
	}

	// Normalize Unicode combining characters
	relPath = norm.NFC.String(relPath)
	return relPath, nil
}

func skipLocalFile(path string) bool {
	// TODO handle more ignored file names
	if filepath.Base(path) == ".DS_Store" {
		return true
	}
	return false
}

func compareManifests(remoteManifest, localManifest *FileHeap) *ManifestComparison {
	// 1. Pop a path off both remote and local manifests.
	// 2. While remote & local are both not nil:
	//    Compare remote & local:
	//    a. If local is nil or local > remote, this file is only in remote. Record and pop remote again.
	//    b. If remote is nil or local < remote, this file is only in local. Record and pop local again.
	//    c. If local == remote, check for content mismatch. Record if necessary and pop both again.
	comparison := &ManifestComparison{}
	local := localManifest.PopOrNil()
	remote := remoteManifest.PopOrNil()
	for local != nil || remote != nil {
		if local == nil || local.Path > remote.Path {
			comparison.OnlyRemote = append(comparison.OnlyRemote, remote.Path)
			comparison.Misses++
			remote = remoteManifest.PopOrNil()
		} else if remote == nil || local.Path < remote.Path {
			comparison.OnlyLocal = append(comparison.OnlyLocal, local.Path)
			comparison.Misses++
			local = localManifest.PopOrNil()
		} else if remote.Path == local.Path {
			if !compareFileContents(remote, local) {
				comparison.ContentMismatch = append(comparison.ContentMismatch, local.Path)
				comparison.Misses++
			} else {
				comparison.Matches++
			}
			local = localManifest.PopOrNil()
			remote = remoteManifest.PopOrNil()
		} else {
			// This should be unreachable...
			panic(fmt.Sprintf("Unexpected condition! Local: %v, remote: %v", local, remote))
		}
	}
	return comparison
}

func compareFileContents(remote, local *File) bool {
	if remote.ContentHash == "" || local.ContentHash == "" {
		// Missing content hash for one of the files, possibly intentionally,
		// so can't compare. Assume that presence of both is enough to
		// validate.
		return true
	}
	return remote.ContentHash == local.ContentHash
}

func printFileList(files []string, description string) {
	fmt.Printf("%s: %d\n\n", description, len(files))
	for _, path := range files {
		fmt.Println(path)
	}
	if len(files) > 0 {
		fmt.Print("\n\n")
	}
}
<|endoftext|>


<commit_before>package cattp

import(
	"testing"
	"time"
	"os"
	"log"
)

func init() {
	Log = log.New(os.Stdout,LOGPREFIX,log.Flags())
}

func TestSetup(t *testing.T) {
	RetransmitTimeout = 100 * time.Millisecond
	//TODO: START a tracer for the entire session
}

func TestReceiveWindowExceed(t *testing.T) {
	p := NewDataACK(0,1,9000,11,100,10,[]byte{})
	rw := NewReceiveWindow(0,10)
	_,err := rw.Receive(p)
	if err == nil {// Should be error here
		t.Errorf("Window should be exceeded here.")
	}
}

func TestReceiveWindowOutOfOrder(t *testing.T) {
	p := NewDataACK(0,1,9000,8,100,10,[]byte{})
	rw := NewReceiveWindow(5,10)
	ret,err := rw.Receive(p)
	if len(ret) > 0 || err != nil {
		t.Errorf("OutOf Order insert 1 failed: %d,%s",len(ret),err)
	}

	p = NewDataACK(0,1,9000,7,100,10,[]byte{})
	ret,err = rw.Receive(p)
	if len(ret) > 0 || err != nil {
		t.Errorf("OutOf Order insert 2 failed: %d,%s",len(ret),err)
	}

	p = NewDataACK(0,1,9000,6,100,10,[]byte{})
	ret,err = rw.Receive(p)
	if len(ret) != 3  || err != nil || ret [0].SeqNo() != 6{
		t.Errorf("OutOf Order insert 3 failed: &d,%d,%s",ret[0].SeqNo(),len(ret),err)
	}

	p = NewDataACK(0,1,9000,9,100,10,[]byte{})
	ret,err = rw.Receive(p)
	if len(ret) != 1  || err != nil || ret[0].SeqNo() != 9{
		t.Errorf("OutOf Order insert 4 failed: %d,%d,%s",ret[0].SeqNo(),len(ret),err)
	}
}

func TestBlindConnect(t *testing.T) {
	testdata := []byte("TESTDATA")
	var receivedData []byte

	srv,err := Listen("localhost:8770",9000,EchoHandler)
	if err != nil {
		t.Errorf("Could ot create listen server: %s",err)
	}

	time.Sleep(200 * time.Millisecond)

	c,err  := ConnectWait("localhost:8770",1,9000,[]byte{},func(c *Connection,ps []*Header,data []byte) { receivedData = data })
	if err != nil {
		t.Errorf("Connection failed: %s",err)
	}
	if c.state != OPEN {
		t.Errorf("Connection is not in state OPEN: %s",c.StateS())
	}

	c.Write(testdata)

	time.Sleep(200 * time.Millisecond)

	if string(receivedData) != string(testdata) {
		t.Errorf("Data not properly transmitted.")
	}

	err = srv.Close()
	if err != nil {
		t.Errorf("Server Close failed: %s",err)
	}
	time.Sleep(200 * time.Millisecond)

	err = c.Close()
	if err != nil {
		t.Errorf("Client Close failed: %s",err)
	}

	srv.Wait()
	if srv.countClients() != 0 {
		t.Errorf("Client connection should have been closed. % open",srv.countClients())
	}
}

func TestNULPDU(t *testing.T) {
	srv,err := Listen("localhost:8770",9000,EchoHandler)
	if err != nil {
		t.Errorf("Could ot create listen server: %s",err)
	}

	c,err := ConnectWait("localhost:8770",1,9000,[]byte{},func(c *Connection,ps []*Header,data []byte) { })
	if err != nil {
		t.Errorf("Could not connect to server: %s",err)
	}

	p := NewNUL(0,c.LocalPort(), c.RemotePort(),c.SND_NXT_SEQ_NB(), c.RCV_CUR_SEQ_NB(), c.receiveWindow.WindowSize())
	c.Send(p)

	//Wait for ACK
	time.Sleep(200 * time.Millisecond)

	//Check if ack was received.
	if c.sendWindow.last != p.SeqNo() {
		t.Errorf("Did not receive ACK for NUL PDU.")
	}

	c.Close()
	srv.CloseWait()
}

func TestSYNRetransmit(t *testing.T) {
	c,err := ConnectWait("localhost:8770",1,9000,[]byte{},LogHandler)
	if err == nil {
		t.Errorf("Connect should have failed.")
	}

	if c.sendWindow.rtcount < RetransmitCount {
		t.Errorf("SYN has not been retransmitted.")
	}
}

func TestDATACKRetransmit(t *testing.T) {
	srv,err := Listen("localhost:8770",9000,EchoHandler)
	if err != nil {
		t.Errorf("Could ot create listen server: %s",err)
	}

	c,err:= ConnectWait("localhost:8770",1,9000,[]byte{},LogHandler)

	srv.Kill(false)
	c.Write([]byte{'a','b'})

	c.WaitForClose()

	if c.sendWindow.rtcount < RetransmitCount {
		t.Errorf("ACK has not been retransmitted.")
	}
}

func TestNULEAK(t *testing.T) {
	srv,err := Listen("localhost:8770",9000,EchoHandler)
	var rec []byte
	if err != nil {
		t.Errorf("Could ot create listen server: %s",err)
	}

	c,err:= ConnectWait("localhost:8770",1,9000,[]byte{},func(c *Connection, h []*Header, data []byte) { rec = data })

	time.Sleep(200 * time.Millisecond)

	x := c.SND_NXT_SEQ_NB()

	p := NewNUL(0,c.LocalPort(), c.RemotePort(),x+1, c.RCV_CUR_SEQ_NB(), c.receiveWindow.WindowSize())
	c.Send(p)

	time.Sleep(200 * time.Millisecond)

	p = NewNUL(0,c.LocalPort(), c.RemotePort(),x, c.RCV_CUR_SEQ_NB(), c.receiveWindow.WindowSize())
	c.Send(p)

	c.Write([]byte{'a','b'})

	time.Sleep(200 * time.Millisecond)

	if rec[0] != 'a' {
		t.Errorf("Data has not been properly transmitted.")
	}

	c.Close()
	srv.CloseWait()
}

func TestEAKsAndWindowSize(t *testing.T) {
	l := 34
	srv,err := Listen("localhost:8770",9000,func(c *Connection, h []*Header, data []byte) { }, SocketParameters{RECEIVEWINDOW_SIZE: uint16(l+1)})

	if err != nil {
		t.Errorf("Could ot create listen server: %s",err)
	}

	c,err:= ConnectWait("localhost:8770",1,9000,[]byte{},func(c *Connection, h []*Header, data []byte) { },SocketParameters{RECEIVEWINDOW_SIZE: uint16(l+1)})

	//Simulate lost packet to receive eaks.
	//c.sendWindow.last++
	c.sendWindow.newest++

	//Send five packets and expect eaks
	for i:=0;i<l;i++ {
		c.Write([]byte{'1'})
		time.Sleep(50 * time.Millisecond)
	}

	oos := int(c.sendWindow.newest - c.sendWindow.last - 1)

	if oos != l {
		t.Errorf("Invalid count of EAKs received: %d/%d",oos,l)
	}

	c.Close()
	srv.CloseWait()
}

// Test identification field.
func TestIdentification(t *testing.T) {
	var client string
	var server string

	srv,err := Listen("localhost:8770",9000,func(c *Connection, h []*Header, data []byte) {
		client = string(c.Identification())
		c.Write(data)
	}, SocketParameters{IDENTIFICATION: []byte("SERVER")})
	if err != nil {
		t.Errorf("Could ot create listen server: %s",err)
	}

	c,err:= ConnectWait("localhost:8770",1,9000,[]byte("CLIENT"),func(con *Connection, h []*Header, data []byte) {
		server = string(con.Identification())
	})

	c.Write([]byte("Hello World!"))

	time.Sleep(100 * time.Millisecond)

	if client != "CLIENT" {
		t.Errorf("Client identification incorrect: %s",client)
	}

	if server != "SERVER" {
		t.Errorf("Server identification incorrect: %s", server)
	}
	c.Close()
	srv.CloseWait()
}

// Test long identification field.
func TestLongIdentification(t *testing.T) {
	sid := ""
	for i:=0;i < 50; i++ {
		sid = sid + "0123456789"
	}

	srv,err := Listen("localhost:8770",9000,func(c *Connection, h []*Header, data []byte) {
		c.Write(data)
	}, SocketParameters{IDENTIFICATION: []byte(sid)})
	if err != nil {
		t.Errorf("Could not create listen server: %s",err)
	}

	c,err:= ConnectWait("localhost:8770",1,9000,[]byte{},func(c *Connection, h []*Header, data []byte) {})
	if err != nil {
		t.Errorf("Could not connect to server: %s",err)
	}
	c.Write([]byte("Hello World!"))
	time.Sleep(100 * time.Millisecond)

	if string(c.Identification()) != sid[:MaxIdenficationLen] {
		t.Errorf("Identification incorrect: %d/%d",len(c.Identification()),len(sid))
	}

	c.Close()
	srv.CloseWait()
}

func TestICCID(t *testing.T) {
	var riccid string
	srv,err := Listen("localhost:8770",9000,func(c *Connection, h []*Header, data []byte) {
		riccid = string(Nibble(c.Identification()))
		c.Write(data)
	})
	if err != nil {
		t.Errorf("Could not connect to server: %s",err)
	}

	iccid := "8934FFFFFFFFFFFFFFFF"

	c,err:= ConnectWait("localhost:8770",1,9000,Nibble([]byte(iccid)),func(c *Connection, h []*Header, data []byte) {})
	c.Write([]byte{'0'})
	time.Sleep(100 * time.Millisecond)

	if riccid != iccid {
		t.Errorf("ICCID incorrect: %s/%s",riccid,iccid)
	}

	c.Close()
	srv.CloseWait()
}
func TestLargePacket(t *testing.T) {
	mes := ""
	for i:=0; i < 200;i++ {
		mes = mes + "0123456789"
	}

	ret := ""

	srv,err := Listen("localhost:8770",9000,func(c *Connection, h []*Header, data []byte) {
		c.Write(data)
	})

	if err != nil {
		t.Errorf("Could not connect to server: %s",err)
	}

	c,err:= ConnectWait("localhost:8770",1,9000,[]byte{},func(c *Connection, h []*Header, data []byte) {
		ret = ret + string(data)
	})

	c.Write([]byte(mes))
	time.Sleep(100 * time.Millisecond)

	if mes != ret {
		t.Errorf("Sending big packet of len %d failed. %d/%d",len(mes),len(ret),len(mes))
	}

	c.Close()
	srv.CloseWait()
}

// As Long as no client is connected, server socket should be closeable immediately.
func TestOpenAndClose(t *testing.T) {

	srv,err := Listen("localhost:8770",9000,func(c *Connection, h []*Header, data []byte) { })
	if err != nil {
		t.Errorf("Could ot create listen server: %s",err)
	}
	srv.CloseWait()
}

func TestWrongPort(t *testing.T) {
	srv,err := Listen("localhost:8770",9000,EchoHandler)
	if err != nil {
		t.Errorf("Could ot create listen server: %s",err)
	}

	if _,err = ConnectWait("localhost:8770",1,9001,[]byte{},LogHandler); err == nil {
		t.Errorf("Connection should not have been established.")
	}

	if c,err := ConnectWait("localhost:8770",1,9000,[]byte{},LogHandler); err != nil {
		t.Errorf("Connection should have been established.")
	} else {
		c.Close()
	}

	srv.CloseWait()
}

func TestMultipleServerPorts(t *testing.T) {
	var buf0,buf1 []byte
	srv,err := Listen("localhost:8770",9000,func (c *Connection,ps []*Header, data []byte) {
		c.Write([]byte("9000"))
	})
	if err != nil {
		t.Errorf("Could ot create listen server: %s",err)
	}
	srv.SetListener(9001, func (c *Connection, ps []*Header, data []byte) {
		c.Write([]byte("9001"))
	})

	c1,err := ConnectWait("localhost:8770",1,9001,[]byte{}, func (co *Connection, ps[]*Header, data []byte) {
		buf1 = data
		co.Close()
	})
	if err != nil{
		t.Errorf("Faild to connect: %s",err)
	}

	c0,err := ConnectWait("localhost:8770",1,9000,[]byte{}, func (co *Connection, ps[]*Header, data []byte) {
		buf0 = data
		co.Close()
	});
	if err != nil {
		t.Errorf("Failed to connect: %s",err)
	}

	c0.Write([]byte("X"))
	c1.Write([]byte("X"))

	time.Sleep(100 * time.Millisecond)

	if string(buf0) != "9000" {
		t.Errorf("Failed to connect to correct port: %s/%s",string(buf0),"9000")
	}

	if string(buf1) != "9001" {
		t.Errorf("Failed to connect to correct port: %s/%s",string(buf0),"9001")
	}

	srv.CloseWait()
}
<commit_msg>Fixed iccid hex encoding.<commit_after>package cattp

import(
	"testing"
	"time"
	"os"
	"log"
	"encoding/hex"
)

func init() {
	Log = log.New(os.Stdout,LOGPREFIX,log.Flags())
}

func TestSetup(t *testing.T) {
	RetransmitTimeout = 100 * time.Millisecond
	//TODO: START a tracer for the entire session
}

func TestReceiveWindowExceed(t *testing.T) {
	p := NewDataACK(0,1,9000,11,100,10,[]byte{})
	rw := NewReceiveWindow(0,10)
	_,err := rw.Receive(p)
	if err == nil {// Should be error here
		t.Errorf("Window should be exceeded here.")
	}
}

func TestReceiveWindowOutOfOrder(t *testing.T) {
	p := NewDataACK(0,1,9000,8,100,10,[]byte{})
	rw := NewReceiveWindow(5,10)
	ret,err := rw.Receive(p)
	if len(ret) > 0 || err != nil {
		t.Errorf("OutOf Order insert 1 failed: %d,%s",len(ret),err)
	}

	p = NewDataACK(0,1,9000,7,100,10,[]byte{})
	ret,err = rw.Receive(p)
	if len(ret) > 0 || err != nil {
		t.Errorf("OutOf Order insert 2 failed: %d,%s",len(ret),err)
	}

	p = NewDataACK(0,1,9000,6,100,10,[]byte{})
	ret,err = rw.Receive(p)
	if len(ret) != 3  || err != nil || ret [0].SeqNo() != 6{
		t.Errorf("OutOf Order insert 3 failed: &d,%d,%s",ret[0].SeqNo(),len(ret),err)
	}

	p = NewDataACK(0,1,9000,9,100,10,[]byte{})
	ret,err = rw.Receive(p)
	if len(ret) != 1  || err != nil || ret[0].SeqNo() != 9{
		t.Errorf("OutOf Order insert 4 failed: %d,%d,%s",ret[0].SeqNo(),len(ret),err)
	}
}

func TestBlindConnect(t *testing.T) {
	testdata := []byte("TESTDATA")
	var receivedData []byte

	srv,err := Listen("localhost:8770",9000,EchoHandler)
	if err != nil {
		t.Errorf("Could ot create listen server: %s",err)
	}

	time.Sleep(200 * time.Millisecond)

	c,err  := ConnectWait("localhost:8770",1,9000,[]byte{},func(c *Connection,ps []*Header,data []byte) { receivedData = data })
	if err != nil {
		t.Errorf("Connection failed: %s",err)
	}
	if c.state != OPEN {
		t.Errorf("Connection is not in state OPEN: %s",c.StateS())
	}

	c.Write(testdata)

	time.Sleep(200 * time.Millisecond)

	if string(receivedData) != string(testdata) {
		t.Errorf("Data not properly transmitted.")
	}

	err = srv.Close()
	if err != nil {
		t.Errorf("Server Close failed: %s",err)
	}
	time.Sleep(200 * time.Millisecond)

	err = c.Close()
	if err != nil {
		t.Errorf("Client Close failed: %s",err)
	}

	srv.Wait()
	if srv.countClients() != 0 {
		t.Errorf("Client connection should have been closed. % open",srv.countClients())
	}
}

func TestNULPDU(t *testing.T) {
	srv,err := Listen("localhost:8770",9000,EchoHandler)
	if err != nil {
		t.Errorf("Could ot create listen server: %s",err)
	}

	c,err := ConnectWait("localhost:8770",1,9000,[]byte{},func(c *Connection,ps []*Header,data []byte) { })
	if err != nil {
		t.Errorf("Could not connect to server: %s",err)
	}

	p := NewNUL(0,c.LocalPort(), c.RemotePort(),c.SND_NXT_SEQ_NB(), c.RCV_CUR_SEQ_NB(), c.receiveWindow.WindowSize())
	c.Send(p)

	//Wait for ACK
	time.Sleep(200 * time.Millisecond)

	//Check if ack was received.
	if c.sendWindow.last != p.SeqNo() {
		t.Errorf("Did not receive ACK for NUL PDU.")
	}

	c.Close()
	srv.CloseWait()
}

func TestSYNRetransmit(t *testing.T) {
	c,err := ConnectWait("localhost:8770",1,9000,[]byte{},LogHandler)
	if err == nil {
		t.Errorf("Connect should have failed.")
	}

	if c.sendWindow.rtcount < RetransmitCount {
		t.Errorf("SYN has not been retransmitted.")
	}
}

func TestDATACKRetransmit(t *testing.T) {
	srv,err := Listen("localhost:8770",9000,EchoHandler)
	if err != nil {
		t.Errorf("Could ot create listen server: %s",err)
	}

	c,err:= ConnectWait("localhost:8770",1,9000,[]byte{},LogHandler)

	srv.Kill(false)
	c.Write([]byte{'a','b'})

	c.WaitForClose()

	if c.sendWindow.rtcount < RetransmitCount {
		t.Errorf("ACK has not been retransmitted.")
	}
}

func TestNULEAK(t *testing.T) {
	srv,err := Listen("localhost:8770",9000,EchoHandler)
	var rec []byte
	if err != nil {
		t.Errorf("Could ot create listen server: %s",err)
	}

	c,err:= ConnectWait("localhost:8770",1,9000,[]byte{},func(c *Connection, h []*Header, data []byte) { rec = data })

	time.Sleep(200 * time.Millisecond)

	x := c.SND_NXT_SEQ_NB()

	p := NewNUL(0,c.LocalPort(), c.RemotePort(),x+1, c.RCV_CUR_SEQ_NB(), c.receiveWindow.WindowSize())
	c.Send(p)

	time.Sleep(200 * time.Millisecond)

	p = NewNUL(0,c.LocalPort(), c.RemotePort(),x, c.RCV_CUR_SEQ_NB(), c.receiveWindow.WindowSize())
	c.Send(p)

	c.Write([]byte{'a','b'})

	time.Sleep(200 * time.Millisecond)

	if rec[0] != 'a' {
		t.Errorf("Data has not been properly transmitted.")
	}

	c.Close()
	srv.CloseWait()
}

func TestEAKsAndWindowSize(t *testing.T) {
	l := 34
	srv,err := Listen("localhost:8770",9000,func(c *Connection, h []*Header, data []byte) { }, SocketParameters{RECEIVEWINDOW_SIZE: uint16(l+1)})

	if err != nil {
		t.Errorf("Could ot create listen server: %s",err)
	}

	c,err:= ConnectWait("localhost:8770",1,9000,[]byte{},func(c *Connection, h []*Header, data []byte) { },SocketParameters{RECEIVEWINDOW_SIZE: uint16(l+1)})

	//Simulate lost packet to receive eaks.
	//c.sendWindow.last++
	c.sendWindow.newest++

	//Send five packets and expect eaks
	for i:=0;i<l;i++ {
		c.Write([]byte{'1'})
		time.Sleep(50 * time.Millisecond)
	}

	oos := int(c.sendWindow.newest - c.sendWindow.last - 1)

	if oos != l {
		t.Errorf("Invalid count of EAKs received: %d/%d",oos,l)
	}

	c.Close()
	srv.CloseWait()
}

// Test identification field.
func TestIdentification(t *testing.T) {
	var client string
	var server string

	srv,err := Listen("localhost:8770",9000,func(c *Connection, h []*Header, data []byte) {
		client = string(c.Identification())
		c.Write(data)
	}, SocketParameters{IDENTIFICATION: []byte("SERVER")})
	if err != nil {
		t.Errorf("Could ot create listen server: %s",err)
	}

	c,err:= ConnectWait("localhost:8770",1,9000,[]byte("CLIENT"),func(con *Connection, h []*Header, data []byte) {
		server = string(con.Identification())
	})

	c.Write([]byte("Hello World!"))

	time.Sleep(100 * time.Millisecond)

	if client != "CLIENT" {
		t.Errorf("Client identification incorrect: %s",client)
	}

	if server != "SERVER" {
		t.Errorf("Server identification incorrect: %s", server)
	}
	c.Close()
	srv.CloseWait()
}

// Test long identification field.
func TestLongIdentification(t *testing.T) {
	sid := ""
	for i:=0;i < 50; i++ {
		sid = sid + "0123456789"
	}

	srv,err := Listen("localhost:8770",9000,func(c *Connection, h []*Header, data []byte) {
		c.Write(data)
	}, SocketParameters{IDENTIFICATION: []byte(sid)})
	if err != nil {
		t.Errorf("Could not create listen server: %s",err)
	}

	c,err:= ConnectWait("localhost:8770",1,9000,[]byte{},func(c *Connection, h []*Header, data []byte) {})
	if err != nil {
		t.Errorf("Could not connect to server: %s",err)
	}
	c.Write([]byte("Hello World!"))
	time.Sleep(100 * time.Millisecond)

	if string(c.Identification()) != sid[:MaxIdenficationLen] {
		t.Errorf("Identification incorrect: %d/%d",len(c.Identification()),len(sid))
	}

	c.Close()
	srv.CloseWait()
}

func TestICCID(t *testing.T) {
	var riccid string
	srv,err := Listen("localhost:8770",9000,func(c *Connection, h []*Header, data []byte) {
		riccid = string(Nibble(c.Identification()))
		c.Write(data)
	})
	if err != nil {
		t.Errorf("Could not connect to server: %s",err)
	}

	iccid,_ := hex.DecodeString("8934FFFFFFFFFFFFFFFF")

	c,err:= ConnectWait("localhost:8770",1,9000,Nibble([]byte(iccid)),func(c *Connection, h []*Header, data []byte) {})
	c.Write([]byte("ICCID Test"))
	time.Sleep(100 * time.Millisecond)

	if riccid != string(iccid) {
		t.Errorf("ICCID incorrect: %s/%s",riccid,iccid)
	}

	c.Close()
	srv.CloseWait()
}
func TestLargePacket(t *testing.T) {
	mes := ""
	for i:=0; i < 200;i++ {
		mes = mes + "0123456789"
	}

	ret := ""

	srv,err := Listen("localhost:8770",9000,func(c *Connection, h []*Header, data []byte) {
		c.Write(data)
	})

	if err != nil {
		t.Errorf("Could not connect to server: %s",err)
	}

	c,err:= ConnectWait("localhost:8770",1,9000,[]byte{},func(c *Connection, h []*Header, data []byte) {
		ret = ret + string(data)
	})

	c.Write([]byte(mes))
	time.Sleep(100 * time.Millisecond)

	if mes != ret {
		t.Errorf("Sending big packet of len %d failed. %d/%d",len(mes),len(ret),len(mes))
	}

	c.Close()
	srv.CloseWait()
}

// As Long as no client is connected, server socket should be closeable immediately.
func TestOpenAndClose(t *testing.T) {

	srv,err := Listen("localhost:8770",9000,func(c *Connection, h []*Header, data []byte) { })
	if err != nil {
		t.Errorf("Could ot create listen server: %s",err)
	}
	srv.CloseWait()
}

func TestWrongPort(t *testing.T) {
	srv,err := Listen("localhost:8770",9000,EchoHandler)
	if err != nil {
		t.Errorf("Could ot create listen server: %s",err)
	}

	if _,err = ConnectWait("localhost:8770",1,9001,[]byte{},LogHandler); err == nil {
		t.Errorf("Connection should not have been established.")
	}

	if c,err := ConnectWait("localhost:8770",1,9000,[]byte{},LogHandler); err != nil {
		t.Errorf("Connection should have been established.")
	} else {
		c.Close()
	}

	srv.CloseWait()
}

func TestMultipleServerPorts(t *testing.T) {
	var buf0,buf1 []byte
	srv,err := Listen("localhost:8770",9000,func (c *Connection,ps []*Header, data []byte) {
		c.Write([]byte("9000"))
	})
	if err != nil {
		t.Errorf("Could ot create listen server: %s",err)
	}
	srv.SetListener(9001, func (c *Connection, ps []*Header, data []byte) {
		c.Write([]byte("9001"))
	})

	c1,err := ConnectWait("localhost:8770",1,9001,[]byte{}, func (co *Connection, ps[]*Header, data []byte) {
		buf1 = data
		co.Close()
	})
	if err != nil{
		t.Errorf("Faild to connect: %s",err)
	}

	c0,err := ConnectWait("localhost:8770",1,9000,[]byte{}, func (co *Connection, ps[]*Header, data []byte) {
		buf0 = data
		co.Close()
	});
	if err != nil {
		t.Errorf("Failed to connect: %s",err)
	}

	c0.Write([]byte("X"))
	c1.Write([]byte("X"))

	time.Sleep(100 * time.Millisecond)

	if string(buf0) != "9000" {
		t.Errorf("Failed to connect to correct port: %s/%s",string(buf0),"9000")
	}

	if string(buf1) != "9001" {
		t.Errorf("Failed to connect to correct port: %s/%s",string(buf0),"9001")
	}

	srv.CloseWait()
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Andy Leap, Google
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal in the Software without restriction, including without limitation the
// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
// sell copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
// IN THE SOFTWARE.

// This file includes backwards compatibility support for microformats v1.

package microformats

import (
	"net/url"
	"path"
	"strings"
)

var (
	backcompatRootMap = map[string]string{
		"adr":               "h-adr",
		"geo":               "h-geo",
		"hentry":            "h-entry",
		"hfeed":             "h-feed",
		"hnews":             "h-news",
		"hproduct":          "h-product",
		"hrecipe":           "h-recipe",
		"hresume":           "h-resume",
		"hreview":           "h-review",
		"hreview-aggregate": "h-review-aggregate",
		"vcard":             "h-card",
		"vevent":            "h-event",
	}

	backcompatPropertyMap = map[string]string{
		"author":            "p-author",
		"description":       "p-description",
		"job-title":         "p-job-title",
		"organization-name": "p-organization-name",
		"organization-unit": "p-organization-unit",
		"published":         "dt-published",
		"summary":           "p-summary",
		"title":             "p-title",
		"worst":             "p-worst",
	}

	backcompatPropertyOverrideMap = map[string]map[string]string{
		"h-adr": map[string]string{
			"country-name":     "p-country-name",
			"extended-address": "p-extended-address",
			"locality":         "p-locality",
			"post-office-box":  "p-post-office-box",
			"postal-code":      "p-postal-code",
			"region":           "p-region",
			"street-address":   "p-street-address",
		},
		"h-card": map[string]string{
			"additional-name":  "p-additional-name",
			"adr":              "p-adr",
			"agent":            "p-agent",
			"bday":             "dt-bday",
			"category":         "p-category",
			"class":            "p-class",
			"email":            "u-email",
			"family-name":      "p-family-name",
			"fn":               "p-name",
			"geo":              "p-geo",
			"given-name":       "p-given-name",
			"honorific-prefix": "p-honorific-prefix",
			"honorific-suffix": "p-honorific-suffix",
			"key":              "u-key",
			"label":            "p-label",
			"logo":             "u-logo",
			"mailer":           "p-mailer",
			"nickname":         "p-nickname",
			"note":             "p-note",
			"org":              "p-org",
			"photo":            "u-photo",
			"rev":              "dt-rev",
			"role":             "p-role",
			"sort-string":      "p-sort-string",
			"sound":            "u-sound",
			"tel":              "p-tel",
			"title":            "p-job-title",
			"tz":               "p-tz",
			"uid":              "u-uid",
			"url":              "u-url",
		},
		"h-entry": map[string]string{
			"author":        "p-author",
			"category":      "p-category",
			"entry-content": "e-content",
			"entry-summary": "p-summary",
			"entry-title":   "p-name",
			"published":     "dt-published",
			"summary":       "p-summary",
			"updated":       "dt-updated",
		},
		"h-event": map[string]string{
			"attendee":    "p-attendee",
			"category":    "p-category",
			"description": "p-description",
			"dtend":       "dt-end",
			"dtstart":     "dt-start",
			"duration":    "dt-duration",
			"geo":         "p-geo",
			"location":    "p-location",
			"summary":     "p-name",
			"url":         "u-url",
		},
		"h-feed": map[string]string{
			"author": "p-author",
			"entry":  "p-entry",
			"photo":  "u-photo",
			"url":    "u-url",
		},
		"h-geo": map[string]string{
			"latitude":  "p-latitude",
			"longitude": "p-longitude",
		},
		"h-item": map[string]string{
			"fn":    "p-name",
			"photo": "u-photo",
			"url":   "u-url",
		},
		"h-news": map[string]string{
			"dateline":   "p-dateline",
			"entry":      "p-entry",
			"geo":        "p-geo",
			"source-org": "p-source-org",
		},
		"h-product": map[string]string{
			"brand":       "p-brand",
			"category":    "p-category",
			"description": "p-description",
			"fn":          "p-name",
			"identifier":  "u-identifier",
			"listing":     "p-listing",
			"photo":       "u-photo",
			"price":       "p-price",
			"review":      "p-review",
			"url":         "u-url",
		},
		"h-recipe": map[string]string{
			"author":       "p-author",
			"category":     "p-category",
			"duration":     "dt-duration",
			"fn":           "p-name",
			"ingredient":   "p-ingredient",
			"instructions": "e-instructions",
			"nutrition":    "p-nutrition",
			"photo":        "u-photo",
			"summary":      "p-summary",
			"yield":        "p-yield",
		},
		"h-resume": map[string]string{
			"affiliation":  "p-affiliation",
			"contact":      "p-contact",
			"education":    "p-education",
			"experience":   "p-experience",
			"publications": "p-publications",
			"skill":        "p-skill",
			"summary":      "p-summary",
		},
		"h-review": map[string]string{
			"best":        "p-best",
			"description": "e-content",
			"dtreviewed":  "dt-reviewed",
			"item":        "p-item",
			"rating":      "p-rating",
			"reviewer":    "p-author",
			"summary":     "p-name",
			"worst":       "p-worst",
		},
		"h-review-aggregate": map[string]string{
			"average": "p-average",
			"best":    "p-best",
			"count":   "p-count",
			"item":    "p-item",
			"rating":  "p-rating",
			"summary": "p-name",
			"votes":   "p-votes",
			"worst":   "p-worst",
		},
	}

	backcompatRelMap = map[string]map[string]string{
		"h-card": map[string]string{
			"tag": "u-category",
		},
		"h-entry": map[string]string{
			"bookmark": "u-url",
			"tag":      "u-category",
		},
		"h-feed": map[string]string{
			"tag": "u-category",
		},
		"h-news": map[string]string{
			"principles": "u-principles",
		},
		"h-recipe": map[string]string{
			"tag": "u-category",
		},
		"h-review": map[string]string{
			"bookmark": "u-url",
			"tag":      "u-category",
		},
	}
)

func backcompatRootClasses(classes []string, parent *Microformat) []string {
	var rootclasses []string
	var itemClass bool
	for _, class := range classes {
		if c, ok := backcompatRootMap[class]; ok {
			rootclasses = append(rootclasses, c)
		}
		if class == "item" {
			itemClass = true
		}
	}

	// handle implied h-item microformat inside of h-review
	if len(rootclasses) == 0 && parent != nil && itemClass {
		for _, t := range parent.Type {
			if t == "h-review" || t == "h-review-aggregate" {
				rootclasses = append(rootclasses, "h-item")
			}
		}
	}

	return rootclasses
}

func backcompatPropertyClasses(classes []string, rels []string, context []string) []string {
	var classmap = make(map[string]string)
	for _, class := range classes {
		for _, ctx := range context {
			if c, ok := backcompatPropertyOverrideMap[ctx][class]; ok {
				parts := strings.SplitN(c, "-", 2)
				classmap[parts[1]] = c
			}
		}
	}
	for _, rel := range rels {
		for _, ctx := range context {
			if c, ok := backcompatRelMap[ctx][rel]; ok {
				parts := strings.SplitN(c, "-", 2)
				classmap[parts[1]] = c
			}
		}
	}

	var propertyclasses []string
	for _, c := range classmap {
		propertyclasses = append(propertyclasses, c)
	}
	return propertyclasses
}

// strip provided URL to its last path segment to serve as a category value.
func backcompatURLCategory(s string) string {
	if s == "" {
		return s
	}
	if p, err := url.Parse(s); err == nil {
		return path.Base(p.Path)
	}
	return s
}
<commit_msg>remve unused backcompat property map<commit_after>// Copyright (c) 2015 Andy Leap, Google
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal in the Software without restriction, including without limitation the
// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
// sell copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
// IN THE SOFTWARE.

// This file includes backwards compatibility support for microformats v1.

package microformats

import (
	"net/url"
	"path"
	"strings"
)

var (
	backcompatRootMap = map[string]string{
		"adr":               "h-adr",
		"geo":               "h-geo",
		"hentry":            "h-entry",
		"hfeed":             "h-feed",
		"hnews":             "h-news",
		"hproduct":          "h-product",
		"hrecipe":           "h-recipe",
		"hresume":           "h-resume",
		"hreview":           "h-review",
		"hreview-aggregate": "h-review-aggregate",
		"vcard":             "h-card",
		"vevent":            "h-event",
	}

	backcompatPropertyMap = map[string]map[string]string{
		"h-adr": map[string]string{
			"country-name":     "p-country-name",
			"extended-address": "p-extended-address",
			"locality":         "p-locality",
			"post-office-box":  "p-post-office-box",
			"postal-code":      "p-postal-code",
			"region":           "p-region",
			"street-address":   "p-street-address",
		},
		"h-card": map[string]string{
			"additional-name":  "p-additional-name",
			"adr":              "p-adr",
			"agent":            "p-agent",
			"bday":             "dt-bday",
			"category":         "p-category",
			"class":            "p-class",
			"email":            "u-email",
			"family-name":      "p-family-name",
			"fn":               "p-name",
			"geo":              "p-geo",
			"given-name":       "p-given-name",
			"honorific-prefix": "p-honorific-prefix",
			"honorific-suffix": "p-honorific-suffix",
			"key":              "u-key",
			"label":            "p-label",
			"logo":             "u-logo",
			"mailer":           "p-mailer",
			"nickname":         "p-nickname",
			"note":             "p-note",
			"org":              "p-org",
			"photo":            "u-photo",
			"rev":              "dt-rev",
			"role":             "p-role",
			"sort-string":      "p-sort-string",
			"sound":            "u-sound",
			"tel":              "p-tel",
			"title":            "p-job-title",
			"tz":               "p-tz",
			"uid":              "u-uid",
			"url":              "u-url",
		},
		"h-entry": map[string]string{
			"author":        "p-author",
			"category":      "p-category",
			"entry-content": "e-content",
			"entry-summary": "p-summary",
			"entry-title":   "p-name",
			"published":     "dt-published",
			"summary":       "p-summary",
			"updated":       "dt-updated",
		},
		"h-event": map[string]string{
			"attendee":    "p-attendee",
			"category":    "p-category",
			"description": "p-description",
			"dtend":       "dt-end",
			"dtstart":     "dt-start",
			"duration":    "dt-duration",
			"geo":         "p-geo",
			"location":    "p-location",
			"summary":     "p-name",
			"url":         "u-url",
		},
		"h-feed": map[string]string{
			"author": "p-author",
			"entry":  "p-entry",
			"photo":  "u-photo",
			"url":    "u-url",
		},
		"h-geo": map[string]string{
			"latitude":  "p-latitude",
			"longitude": "p-longitude",
		},
		"h-item": map[string]string{
			"fn":    "p-name",
			"photo": "u-photo",
			"url":   "u-url",
		},
		"h-news": map[string]string{
			"dateline":   "p-dateline",
			"entry":      "p-entry",
			"geo":        "p-geo",
			"source-org": "p-source-org",
		},
		"h-product": map[string]string{
			"brand":       "p-brand",
			"category":    "p-category",
			"description": "p-description",
			"fn":          "p-name",
			"identifier":  "u-identifier",
			"listing":     "p-listing",
			"photo":       "u-photo",
			"price":       "p-price",
			"review":      "p-review",
			"url":         "u-url",
		},
		"h-recipe": map[string]string{
			"author":       "p-author",
			"category":     "p-category",
			"duration":     "dt-duration",
			"fn":           "p-name",
			"ingredient":   "p-ingredient",
			"instructions": "e-instructions",
			"nutrition":    "p-nutrition",
			"photo":        "u-photo",
			"summary":      "p-summary",
			"yield":        "p-yield",
		},
		"h-resume": map[string]string{
			"affiliation":  "p-affiliation",
			"contact":      "p-contact",
			"education":    "p-education",
			"experience":   "p-experience",
			"publications": "p-publications",
			"skill":        "p-skill",
			"summary":      "p-summary",
		},
		"h-review": map[string]string{
			"best":        "p-best",
			"description": "e-content",
			"dtreviewed":  "dt-reviewed",
			"item":        "p-item",
			"rating":      "p-rating",
			"reviewer":    "p-author",
			"summary":     "p-name",
			"worst":       "p-worst",
		},
		"h-review-aggregate": map[string]string{
			"average": "p-average",
			"best":    "p-best",
			"count":   "p-count",
			"item":    "p-item",
			"rating":  "p-rating",
			"summary": "p-name",
			"votes":   "p-votes",
			"worst":   "p-worst",
		},
	}

	backcompatRelMap = map[string]map[string]string{
		"h-card": map[string]string{
			"tag": "u-category",
		},
		"h-entry": map[string]string{
			"bookmark": "u-url",
			"tag":      "u-category",
		},
		"h-feed": map[string]string{
			"tag": "u-category",
		},
		"h-news": map[string]string{
			"principles": "u-principles",
		},
		"h-recipe": map[string]string{
			"tag": "u-category",
		},
		"h-review": map[string]string{
			"bookmark": "u-url",
			"tag":      "u-category",
		},
	}
)

func backcompatRootClasses(classes []string, parent *Microformat) []string {
	var rootclasses []string
	var itemClass bool
	for _, class := range classes {
		if c, ok := backcompatRootMap[class]; ok {
			rootclasses = append(rootclasses, c)
		}
		if class == "item" {
			itemClass = true
		}
	}

	// handle implied h-item microformat inside of h-review
	if len(rootclasses) == 0 && parent != nil && itemClass {
		for _, t := range parent.Type {
			if t == "h-review" || t == "h-review-aggregate" {
				rootclasses = append(rootclasses, "h-item")
			}
		}
	}

	return rootclasses
}

func backcompatPropertyClasses(classes []string, rels []string, context []string) []string {
	var classmap = make(map[string]string)
	for _, class := range classes {
		for _, ctx := range context {
			if c, ok := backcompatPropertyMap[ctx][class]; ok {
				parts := strings.SplitN(c, "-", 2)
				classmap[parts[1]] = c
			}
		}
	}
	for _, rel := range rels {
		for _, ctx := range context {
			if c, ok := backcompatRelMap[ctx][rel]; ok {
				parts := strings.SplitN(c, "-", 2)
				classmap[parts[1]] = c
			}
		}
	}

	var propertyclasses []string
	for _, c := range classmap {
		propertyclasses = append(propertyclasses, c)
	}
	return propertyclasses
}

// strip provided URL to its last path segment to serve as a category value.
func backcompatURLCategory(s string) string {
	if s == "" {
		return s
	}
	if p, err := url.Parse(s); err == nil {
		return path.Base(p.Path)
	}
	return s
}
<|endoftext|>