<commit_before>package safemap_test

import (
	"fmt"
	"testing"

	. "github.com/aleasoluciones/goaleasoluciones/safemap"

	"github.com/stretchr/testify/assert"
)

func TestInitialLenIsZero(t *testing.T) {
	t.Parallel()
	assert.Equal(t, NewSafeMap().Len(), 0)
}

func TestKeyNotFound(t *testing.T) {
	t.Parallel()
	result, found := NewSafeMap().Find("unknown_key")
	assert.Nil(t, result)
	assert.False(t, found)
}

func TestNewSafeMapKeyInsertion(t *testing.T) {
	t.Parallel()
	sm := NewSafeMap()
	sm.Insert("key", "value")
	assert.Equal(t, sm.Len(), 1)
	result, found := sm.Find("key")
	assert.Equal(t, result, "value")
	assert.True(t, found)
}

func TestDeleteAKey(t *testing.T) {
	t.Parallel()
	sm := NewSafeMap()
	sm.Insert("key", "value")
	sm.Delete("key")
	result, found := sm.Find("key")
	assert.Nil(t, result)
	assert.False(t, found)
}

func TestDeleteANonExistingKey(t *testing.T) {
	t.Parallel()
	sm := NewSafeMap()
	sm.Delete("key")
	result, found := sm.Find("key")
	assert.Nil(t, result)
	assert.False(t, found)
}

func TestUpdateAnExistingKey(t *testing.T) {
	t.Parallel()
	sm := NewSafeMap()
	sm.Insert("key", "value")
	sm.Update("key", func(value Value, found bool) Value {
		return value.(string) + "_updated"
	})
	result, found := sm.Find("key")
	assert.Equal(t, result, "value_updated")
	assert.True(t, found)
}

func TestUpdateANonExistingKey(t *testing.T) {
	t.Parallel()
	sm := NewSafeMap()
	sm.Update("key", func(value Value, found bool) Value {
		return fmt.Sprintf("%s_updated", value)
	})
	result, found := sm.Find("key")
	assert.Equal(t, result, "%!s(<nil>)_updated")
	assert.True(t, found)
}

func TestClose(t *testing.T) {
	t.Parallel()
	sm := NewSafeMap()
	sm.Close()
	assert.Panics(t, func() {
		sm.Insert("key", "value")
	})
}

func TestKeys(t *testing.T) {
	t.Parallel()
	sm := NewSafeMap()
	sm.Insert("key", "value")
	sm.Insert("key1", "value1")

	result := sm.Keys()

	assert.Len(t, result, 2)
	assert.Contains(t, result, "key")
	assert.Contains(t, result, "key1")
}
<commit_msg>Update safemap_test.go<commit_after>// Copyright 2014 Alea Soluciones SLL. All rights reserved.  Use of this
// source code is governed by a MIT-style license that can be found in the
// LICENSE file.

package safemap_test

import (
	"fmt"
	"testing"

	. "github.com/aleasoluciones/goaleasoluciones/safemap"

	"github.com/stretchr/testify/assert"
)

func TestInitialLenIsZero(t *testing.T) {
	t.Parallel()
	assert.Equal(t, NewSafeMap().Len(), 0)
}

func TestKeyNotFound(t *testing.T) {
	t.Parallel()
	result, found := NewSafeMap().Find("unknown_key")
	assert.Nil(t, result)
	assert.False(t, found)
}

func TestNewSafeMapKeyInsertion(t *testing.T) {
	t.Parallel()
	sm := NewSafeMap()
	sm.Insert("key", "value")
	assert.Equal(t, sm.Len(), 1)
	result, found := sm.Find("key")
	assert.Equal(t, result, "value")
	assert.True(t, found)
}

func TestDeleteAKey(t *testing.T) {
	t.Parallel()
	sm := NewSafeMap()
	sm.Insert("key", "value")
	sm.Delete("key")
	result, found := sm.Find("key")
	assert.Nil(t, result)
	assert.False(t, found)
}

func TestDeleteANonExistingKey(t *testing.T) {
	t.Parallel()
	sm := NewSafeMap()
	sm.Delete("key")
	result, found := sm.Find("key")
	assert.Nil(t, result)
	assert.False(t, found)
}

func TestUpdateAnExistingKey(t *testing.T) {
	t.Parallel()
	sm := NewSafeMap()
	sm.Insert("key", "value")
	sm.Update("key", func(value Value, found bool) Value {
		return value.(string) + "_updated"
	})
	result, found := sm.Find("key")
	assert.Equal(t, result, "value_updated")
	assert.True(t, found)
}

func TestUpdateANonExistingKey(t *testing.T) {
	t.Parallel()
	sm := NewSafeMap()
	sm.Update("key", func(value Value, found bool) Value {
		return fmt.Sprintf("%s_updated", value)
	})
	result, found := sm.Find("key")
	assert.Equal(t, result, "%!s(<nil>)_updated")
	assert.True(t, found)
}

func TestClose(t *testing.T) {
	t.Parallel()
	sm := NewSafeMap()
	sm.Close()
	assert.Panics(t, func() {
		sm.Insert("key", "value")
	})
}

func TestKeys(t *testing.T) {
	t.Parallel()
	sm := NewSafeMap()
	sm.Insert("key", "value")
	sm.Insert("key1", "value1")

	result := sm.Keys()

	assert.Len(t, result, 2)
	assert.Contains(t, result, "key")
	assert.Contains(t, result, "key1")
}
<|endoftext|>


<commit_before>package page_writer

import (
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/chrislusf/seaweedfs/weed/util/mem"
	"os"
)

var (
	_ = PageChunk(&SwapFileChunk{})
)

type ActualChunkIndex int

type SwapFile struct {
	dir                     string
	file                    *os.File
	logicToActualChunkIndex map[LogicChunkIndex]ActualChunkIndex
	chunkSize               int64
	freeActualChunkList     []ActualChunkIndex
}

type SwapFileChunk struct {
	swapfile         *SwapFile
	usage            *ChunkWrittenIntervalList
	logicChunkIndex  LogicChunkIndex
	actualChunkIndex ActualChunkIndex
}

func NewSwapFile(dir string, chunkSize int64) *SwapFile {
	return &SwapFile{
		dir:                     dir,
		file:                    nil,
		logicToActualChunkIndex: make(map[LogicChunkIndex]ActualChunkIndex),
		chunkSize:               chunkSize,
	}
}
func (sf *SwapFile) FreeResource() {
	if sf.file != nil {
		sf.file.Close()
		os.Remove(sf.file.Name())
	}
}

func (sf *SwapFile) NewTempFileChunk(logicChunkIndex LogicChunkIndex) (tc *SwapFileChunk) {
	if sf.file == nil {
		var err error
		sf.file, err = os.CreateTemp(sf.dir, "")
		if err != nil {
			glog.Errorf("create swap file: %v", err)
			return nil
		}
	}
	actualChunkIndex, found := sf.logicToActualChunkIndex[logicChunkIndex]
	if !found {
		if len(sf.freeActualChunkList) > 0 {
			actualChunkIndex = sf.freeActualChunkList[0]
			sf.freeActualChunkList = sf.freeActualChunkList[1:]
		} else {
			actualChunkIndex = ActualChunkIndex(len(sf.logicToActualChunkIndex))
		}
		sf.logicToActualChunkIndex[logicChunkIndex] = actualChunkIndex
	}

	return &SwapFileChunk{
		swapfile:         sf,
		usage:            newChunkWrittenIntervalList(),
		logicChunkIndex:  logicChunkIndex,
		actualChunkIndex: actualChunkIndex,
	}
}

func (sc *SwapFileChunk) FreeResource() {
	sc.swapfile.freeActualChunkList = append(sc.swapfile.freeActualChunkList, sc.actualChunkIndex)
	delete(sc.swapfile.logicToActualChunkIndex, sc.logicChunkIndex)
}

func (sc *SwapFileChunk) WriteDataAt(src []byte, offset int64) (n int) {
	innerOffset := offset % sc.swapfile.chunkSize
	var err error
	n, err = sc.swapfile.file.WriteAt(src, int64(sc.actualChunkIndex)*sc.swapfile.chunkSize+innerOffset)
	if err == nil {
		sc.usage.MarkWritten(innerOffset, innerOffset+int64(n))
	} else {
		glog.Errorf("failed to write swap file %s: %v", sc.swapfile.file.Name(), err)
	}
	return
}

func (sc *SwapFileChunk) ReadDataAt(p []byte, off int64) (maxStop int64) {
	chunkStartOffset := int64(sc.logicChunkIndex) * sc.swapfile.chunkSize
	for t := sc.usage.head.next; t != sc.usage.tail; t = t.next {
		logicStart := max(off, chunkStartOffset+t.StartOffset)
		logicStop := min(off+int64(len(p)), chunkStartOffset+t.stopOffset)
		if logicStart < logicStop {
			actualStart := logicStart - chunkStartOffset + int64(sc.actualChunkIndex)*sc.swapfile.chunkSize
			if _, err := sc.swapfile.file.ReadAt(p[logicStart-off:logicStop-off], actualStart); err != nil {
				glog.Errorf("failed to reading swap file %s: %v", sc.swapfile.file.Name(), err)
				break
			}
			maxStop = max(maxStop, logicStop)
		}
	}
	return
}

func (sc *SwapFileChunk) IsComplete() bool {
	return sc.usage.IsComplete(sc.swapfile.chunkSize)
}

func (sc *SwapFileChunk) WrittenSize() int64 {
	return sc.usage.WrittenSize()
}

func (sc *SwapFileChunk) SaveContent(saveFn SaveToStorageFunc) {
	if saveFn == nil {
		return
	}
	for t := sc.usage.head.next; t != sc.usage.tail; t = t.next {
		data := mem.Allocate(int(t.Size()))
		sc.swapfile.file.ReadAt(data, t.StartOffset+int64(sc.actualChunkIndex)*sc.swapfile.chunkSize)
		reader := util.NewBytesReader(data)
		saveFn(reader, int64(sc.logicChunkIndex)*sc.swapfile.chunkSize+t.StartOffset, t.Size(), func() {
		})
		mem.Free(data)
	}
	sc.usage = newChunkWrittenIntervalList()
}
<commit_msg>avoid concurrent access to map object<commit_after>package page_writer

import (
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/chrislusf/seaweedfs/weed/util/mem"
	"os"
	"sync"
)

var (
	_ = PageChunk(&SwapFileChunk{})
)

type ActualChunkIndex int

type SwapFile struct {
	dir                         string
	file                        *os.File
	logicToActualChunkIndex     map[LogicChunkIndex]ActualChunkIndex
	logicToActualChunkIndexLock sync.Mutex
	chunkSize                   int64
	freeActualChunkList         []ActualChunkIndex
}

type SwapFileChunk struct {
	swapfile         *SwapFile
	usage            *ChunkWrittenIntervalList
	logicChunkIndex  LogicChunkIndex
	actualChunkIndex ActualChunkIndex
}

func NewSwapFile(dir string, chunkSize int64) *SwapFile {
	return &SwapFile{
		dir:                     dir,
		file:                    nil,
		logicToActualChunkIndex: make(map[LogicChunkIndex]ActualChunkIndex),
		chunkSize:               chunkSize,
	}
}
func (sf *SwapFile) FreeResource() {
	if sf.file != nil {
		sf.file.Close()
		os.Remove(sf.file.Name())
	}
}

func (sf *SwapFile) NewTempFileChunk(logicChunkIndex LogicChunkIndex) (tc *SwapFileChunk) {
	if sf.file == nil {
		var err error
		sf.file, err = os.CreateTemp(sf.dir, "")
		if err != nil {
			glog.Errorf("create swap file: %v", err)
			return nil
		}
	}
	sf.logicToActualChunkIndexLock.Lock()
	defer sf.logicToActualChunkIndexLock.Unlock()
	actualChunkIndex, found := sf.logicToActualChunkIndex[logicChunkIndex]
	if !found {
		if len(sf.freeActualChunkList) > 0 {
			actualChunkIndex = sf.freeActualChunkList[0]
			sf.freeActualChunkList = sf.freeActualChunkList[1:]
		} else {
			actualChunkIndex = ActualChunkIndex(len(sf.logicToActualChunkIndex))
		}
		sf.logicToActualChunkIndex[logicChunkIndex] = actualChunkIndex
	}

	return &SwapFileChunk{
		swapfile:         sf,
		usage:            newChunkWrittenIntervalList(),
		logicChunkIndex:  logicChunkIndex,
		actualChunkIndex: actualChunkIndex,
	}
}

func (sc *SwapFileChunk) FreeResource() {
	sc.swapfile.logicToActualChunkIndexLock.Lock()
	defer sc.swapfile.logicToActualChunkIndexLock.Unlock()

	sc.swapfile.freeActualChunkList = append(sc.swapfile.freeActualChunkList, sc.actualChunkIndex)
	delete(sc.swapfile.logicToActualChunkIndex, sc.logicChunkIndex)
}

func (sc *SwapFileChunk) WriteDataAt(src []byte, offset int64) (n int) {
	innerOffset := offset % sc.swapfile.chunkSize
	var err error
	n, err = sc.swapfile.file.WriteAt(src, int64(sc.actualChunkIndex)*sc.swapfile.chunkSize+innerOffset)
	if err == nil {
		sc.usage.MarkWritten(innerOffset, innerOffset+int64(n))
	} else {
		glog.Errorf("failed to write swap file %s: %v", sc.swapfile.file.Name(), err)
	}
	return
}

func (sc *SwapFileChunk) ReadDataAt(p []byte, off int64) (maxStop int64) {
	chunkStartOffset := int64(sc.logicChunkIndex) * sc.swapfile.chunkSize
	for t := sc.usage.head.next; t != sc.usage.tail; t = t.next {
		logicStart := max(off, chunkStartOffset+t.StartOffset)
		logicStop := min(off+int64(len(p)), chunkStartOffset+t.stopOffset)
		if logicStart < logicStop {
			actualStart := logicStart - chunkStartOffset + int64(sc.actualChunkIndex)*sc.swapfile.chunkSize
			if _, err := sc.swapfile.file.ReadAt(p[logicStart-off:logicStop-off], actualStart); err != nil {
				glog.Errorf("failed to reading swap file %s: %v", sc.swapfile.file.Name(), err)
				break
			}
			maxStop = max(maxStop, logicStop)
		}
	}
	return
}

func (sc *SwapFileChunk) IsComplete() bool {
	return sc.usage.IsComplete(sc.swapfile.chunkSize)
}

func (sc *SwapFileChunk) WrittenSize() int64 {
	return sc.usage.WrittenSize()
}

func (sc *SwapFileChunk) SaveContent(saveFn SaveToStorageFunc) {
	if saveFn == nil {
		return
	}
	for t := sc.usage.head.next; t != sc.usage.tail; t = t.next {
		data := mem.Allocate(int(t.Size()))
		sc.swapfile.file.ReadAt(data, t.StartOffset+int64(sc.actualChunkIndex)*sc.swapfile.chunkSize)
		reader := util.NewBytesReader(data)
		saveFn(reader, int64(sc.logicChunkIndex)*sc.swapfile.chunkSize+t.StartOffset, t.Size(), func() {
		})
		mem.Free(data)
	}
	sc.usage = newChunkWrittenIntervalList()
}
<|endoftext|>


<commit_before>package s3_backend

import (
	"fmt"
	"io"
	"os"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3iface"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
	"github.com/chrislusf/seaweedfs/weed/storage/backend"
	"github.com/google/uuid"
)

func init() {
	backend.BackendStorageFactories["s3"] = &S3BackendFactory{}
}

type S3BackendFactory struct {
}

func (factory *S3BackendFactory) StorageType() backend.StorageType {
	return backend.StorageType("s3")
}
func (factory *S3BackendFactory) BuildStorage(configuration backend.StringProperties, id string) (backend.BackendStorage, error) {
	return newS3BackendStorage(configuration, id)
}

type S3BackendStorage struct {
	id                    string
	aws_access_key_id     string
	aws_secret_access_key string
	region                string
	bucket                string
	conn                  s3iface.S3API
}

func newS3BackendStorage(configuration backend.StringProperties, id string) (s *S3BackendStorage, err error) {
	s = &S3BackendStorage{}
	s.id = id
	s.aws_access_key_id = configuration.GetString("aws_access_key_id")
	s.aws_secret_access_key = configuration.GetString("aws_secret_access_key")
	s.region = configuration.GetString("region")
	s.bucket = configuration.GetString("bucket")
	s.conn, err = createSession(s.aws_access_key_id, s.aws_secret_access_key, s.region)

	glog.V(0).Infof("created backend storage s3.%s for region %s bucket %s", s.id, s.region, s.bucket)
	return
}

func (s *S3BackendStorage) ToProperties() map[string]string {
	m := make(map[string]string)
	m["aws_access_key_id"] = s.aws_access_key_id
	m["aws_secret_access_key"] = s.aws_secret_access_key
	m["region"] = s.region
	m["bucket"] = s.bucket
	return m
}

func (s *S3BackendStorage) NewStorageFile(key string, tierInfo *volume_server_pb.VolumeTierInfo) backend.BackendStorageFile {
	if strings.HasPrefix(key, "/") {
		key = key[1:]
	}

	f := &S3BackendStorageFile{
		backendStorage: s,
		key:            key,
		tierInfo:       tierInfo,
	}

	return f
}

func (s *S3BackendStorage) CopyFile(f *os.File, fn func(progressed int64, percentage float32) error) (key string, size int64, err error) {
	randomUuid, _ := uuid.NewRandom()
	key = randomUuid.String()

	glog.V(1).Infof("copying dat file of", f.Name(), "to remote s3", s.id, "as", key)

	size, err = uploadToS3(s.conn, f.Name(), s.bucket, key, fn)

	return
}

type S3BackendStorageFile struct {
	backendStorage *S3BackendStorage
	key            string
	tierInfo       *volume_server_pb.VolumeTierInfo
}

func (s3backendStorageFile S3BackendStorageFile) ReadAt(p []byte, off int64) (n int, err error) {

	bytesRange := fmt.Sprintf("bytes=%d-%d", off, off+int64(len(p))-1)

	// glog.V(0).Infof("read %s %s", s3backendStorageFile.key, bytesRange)

	getObjectOutput, getObjectErr := s3backendStorageFile.backendStorage.conn.GetObject(&s3.GetObjectInput{
		Bucket: &s3backendStorageFile.backendStorage.bucket,
		Key:    &s3backendStorageFile.key,
		Range:  &bytesRange,
	})

	if getObjectErr != nil {
		return 0, fmt.Errorf("bucket %s GetObject %s: %v", s3backendStorageFile.backendStorage.bucket, s3backendStorageFile.key, getObjectErr)
	}
	defer getObjectOutput.Body.Close()

	glog.V(4).Infof("read %s %s", s3backendStorageFile.key, bytesRange)
	glog.V(4).Infof("content range: %s, contentLength: %d", *getObjectOutput.ContentRange, *getObjectOutput.ContentLength)

	for {
		if n, err = getObjectOutput.Body.Read(p); err == nil && n < len(p) {
			p = p[n:]
		} else {
			break
		}
	}

	if err == io.EOF {
		err = nil
	}

	return
}

func (s3backendStorageFile S3BackendStorageFile) WriteAt(p []byte, off int64) (n int, err error) {
	panic("implement me")
}

func (s3backendStorageFile S3BackendStorageFile) Truncate(off int64) error {
	panic("implement me")
}

func (s3backendStorageFile S3BackendStorageFile) Close() error {
	return nil
}

func (s3backendStorageFile S3BackendStorageFile) GetStat() (datSize int64, modTime time.Time, err error) {

	files := s3backendStorageFile.tierInfo.GetFiles()

	if len(files)==0 {
		err = fmt.Errorf("remote file info not found")
		return
	}

	datSize = int64(files[0].FileSize)
	modTime = time.Unix(int64(files[0].ModifiedTime),0)

	return
}

func (s3backendStorageFile S3BackendStorageFile) String() string {
	return s3backendStorageFile.key
}

func (s3backendStorageFile *S3BackendStorageFile) GetName() string {
	return "s3"
}

func (s3backendStorageFile S3BackendStorageFile) Instantiate(src *os.File) error {
	panic("implement me")
}
<commit_msg>fix test<commit_after>package s3_backend

import (
	"fmt"
	"io"
	"os"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3iface"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
	"github.com/chrislusf/seaweedfs/weed/storage/backend"
	"github.com/google/uuid"
)

func init() {
	backend.BackendStorageFactories["s3"] = &S3BackendFactory{}
}

type S3BackendFactory struct {
}

func (factory *S3BackendFactory) StorageType() backend.StorageType {
	return backend.StorageType("s3")
}
func (factory *S3BackendFactory) BuildStorage(configuration backend.StringProperties, id string) (backend.BackendStorage, error) {
	return newS3BackendStorage(configuration, id)
}

type S3BackendStorage struct {
	id                    string
	aws_access_key_id     string
	aws_secret_access_key string
	region                string
	bucket                string
	conn                  s3iface.S3API
}

func newS3BackendStorage(configuration backend.StringProperties, id string) (s *S3BackendStorage, err error) {
	s = &S3BackendStorage{}
	s.id = id
	s.aws_access_key_id = configuration.GetString("aws_access_key_id")
	s.aws_secret_access_key = configuration.GetString("aws_secret_access_key")
	s.region = configuration.GetString("region")
	s.bucket = configuration.GetString("bucket")
	s.conn, err = createSession(s.aws_access_key_id, s.aws_secret_access_key, s.region)

	glog.V(0).Infof("created backend storage s3.%s for region %s bucket %s", s.id, s.region, s.bucket)
	return
}

func (s *S3BackendStorage) ToProperties() map[string]string {
	m := make(map[string]string)
	m["aws_access_key_id"] = s.aws_access_key_id
	m["aws_secret_access_key"] = s.aws_secret_access_key
	m["region"] = s.region
	m["bucket"] = s.bucket
	return m
}

func (s *S3BackendStorage) NewStorageFile(key string, tierInfo *volume_server_pb.VolumeTierInfo) backend.BackendStorageFile {
	if strings.HasPrefix(key, "/") {
		key = key[1:]
	}

	f := &S3BackendStorageFile{
		backendStorage: s,
		key:            key,
		tierInfo:       tierInfo,
	}

	return f
}

func (s *S3BackendStorage) CopyFile(f *os.File, fn func(progressed int64, percentage float32) error) (key string, size int64, err error) {
	randomUuid, _ := uuid.NewRandom()
	key = randomUuid.String()

	glog.V(1).Infof("copying dat file of %s to remote s3.%s as %s", f.Name(), s.id, key)

	size, err = uploadToS3(s.conn, f.Name(), s.bucket, key, fn)

	return
}

type S3BackendStorageFile struct {
	backendStorage *S3BackendStorage
	key            string
	tierInfo       *volume_server_pb.VolumeTierInfo
}

func (s3backendStorageFile S3BackendStorageFile) ReadAt(p []byte, off int64) (n int, err error) {

	bytesRange := fmt.Sprintf("bytes=%d-%d", off, off+int64(len(p))-1)

	// glog.V(0).Infof("read %s %s", s3backendStorageFile.key, bytesRange)

	getObjectOutput, getObjectErr := s3backendStorageFile.backendStorage.conn.GetObject(&s3.GetObjectInput{
		Bucket: &s3backendStorageFile.backendStorage.bucket,
		Key:    &s3backendStorageFile.key,
		Range:  &bytesRange,
	})

	if getObjectErr != nil {
		return 0, fmt.Errorf("bucket %s GetObject %s: %v", s3backendStorageFile.backendStorage.bucket, s3backendStorageFile.key, getObjectErr)
	}
	defer getObjectOutput.Body.Close()

	glog.V(4).Infof("read %s %s", s3backendStorageFile.key, bytesRange)
	glog.V(4).Infof("content range: %s, contentLength: %d", *getObjectOutput.ContentRange, *getObjectOutput.ContentLength)

	for {
		if n, err = getObjectOutput.Body.Read(p); err == nil && n < len(p) {
			p = p[n:]
		} else {
			break
		}
	}

	if err == io.EOF {
		err = nil
	}

	return
}

func (s3backendStorageFile S3BackendStorageFile) WriteAt(p []byte, off int64) (n int, err error) {
	panic("implement me")
}

func (s3backendStorageFile S3BackendStorageFile) Truncate(off int64) error {
	panic("implement me")
}

func (s3backendStorageFile S3BackendStorageFile) Close() error {
	return nil
}

func (s3backendStorageFile S3BackendStorageFile) GetStat() (datSize int64, modTime time.Time, err error) {

	files := s3backendStorageFile.tierInfo.GetFiles()

	if len(files)==0 {
		err = fmt.Errorf("remote file info not found")
		return
	}

	datSize = int64(files[0].FileSize)
	modTime = time.Unix(int64(files[0].ModifiedTime),0)

	return
}

func (s3backendStorageFile S3BackendStorageFile) String() string {
	return s3backendStorageFile.key
}

func (s3backendStorageFile *S3BackendStorageFile) GetName() string {
	return "s3"
}

func (s3backendStorageFile S3BackendStorageFile) Instantiate(src *os.File) error {
	panic("implement me")
}
<|endoftext|>


<commit_before>package cli

import (
    "fmt"
    "strconv"
)

type Parser interface {
    Match([]string) ([]string, bool)
    Capture([]string) ([]string, map[string]interface{})
}

type CompleteParser struct {
    parsers []Parser
}

func (self CompleteParser) Match(values []string) ([]string, bool) {
    remainingValues := values

    for _, parser := range self.parsers {
        var ok bool
        remainingValues, ok = parser.Match(remainingValues)
        if !ok {
            return remainingValues, false
        }
    }

    return remainingValues, len(remainingValues) == 0
}

func (self CompleteParser) Capture(values []string) ([]string, map[string]interface{}) {
    remainingValues := values
    data := map[string]interface{}{}

    for _, parser := range self.parsers {
        var captured map[string]interface{}
        remainingValues, captured = parser.Capture(remainingValues)
        for key, value := range captured {
            data[key] = value
        }
    }

    return remainingValues, data
}

func (self CompleteParser) String() string {
    return fmt.Sprintf("CompleteParser %v", self.parsers)
}

type EqualParser struct {
    value string
}

func (self EqualParser) Match(values []string) ([]string, bool) {
    if len(values) == 0 {
        return values, false
    }

    if self.value == values[0] {
        return values[1:], true
    }

    return values, false
}

func (self EqualParser) Capture(values []string) ([]string, map[string]interface{}) {
    remainingValues, _ := self.Match(values)
    return remainingValues, nil
}

func (self EqualParser) String() string {
    return fmt.Sprintf("EqualParser '%s'", self.value)
}

type CaptureGroupParser struct {
    value string
}

func (self CaptureGroupParser) Match(values []string) ([]string, bool) {
    if len(values) == 0 {
        return values, false
    }

    return values[1:], true
}

func (self CaptureGroupParser) key() string {
    return self.value[1:len(self.value) - 1]
}

func (self CaptureGroupParser) Capture(values []string) ([]string, map[string]interface{}) {
    if remainingValues, ok := self.Match(values); ok {
        return remainingValues, map[string]interface{}{self.key(): values[0]}
    }

    return values, nil
}

func (self CaptureGroupParser) String() string {
    return fmt.Sprintf("CaptureGroupParser '%s'", self.value)
}

type BoolFlagParser struct {
    pattern string
    key string
    omitValue bool
    defaultValue bool
}

func (self BoolFlagParser) Match(values []string) ([]string, bool) {
    if self.omitValue {
        if len(values) == 0 {
            return values, false
        }

        if self.pattern == values[0] {
            return values[1:], true
        }

        return values, false
    } else {
        if len(values) < 2 {
            return values, false
        }

        if self.pattern != values[0] {
            return values, false
        }

        // Check that value is a valid boolean
        if _, err := strconv.ParseBool(values[1]); err != nil {
            return values, false
        }

        return values[2:], true
    }
}

func (self BoolFlagParser) Capture(values []string) ([]string, map[string]interface{}) {
    remainingValues, ok := self.Match(values)
    if !ok && !self.omitValue {
        return remainingValues, map[string]interface{}{self.key: self.defaultValue}
    }
    return remainingValues, map[string]interface{}{self.key: ok}
}

func (self BoolFlagParser) String() string {
    return fmt.Sprintf("BoolFlagParser '%s'", self.pattern)
}

type StringFlagParser struct {
    pattern string
    key string
    defaultValue string
}

func (self StringFlagParser) Match(values []string) ([]string, bool) {
    if len(values) < 2 {
        return values, false
    }

    if self.pattern != values[0] {
        return values, false
    }

    return values[2:], true
}

func (self StringFlagParser) Capture(values []string) ([]string, map[string]interface{}) {
    remainingValues, ok := self.Match(values)
    if ok {
        return remainingValues, map[string]interface{}{self.key: values[1]}
    }

    return values, map[string]interface{}{self.key: self.defaultValue}
}

func (self StringFlagParser) String() string {
    return fmt.Sprintf("StringFlagParser '%s'", self.pattern)
}

type IntFlagParser struct {
    pattern string
    key string
    defaultValue int64
}

func (self IntFlagParser) Match(values []string) ([]string, bool) {
    if len(values) < 2 {
        return values, false
    }

    if self.pattern != values[0] {
        return values, false
    }

    // Check that value is a valid integer
    if _, err := strconv.ParseInt(values[1], 10, 64); err != nil {
        return values, false
    }

    return values[2:], true
}

func (self IntFlagParser) Capture(values []string) ([]string, map[string]interface{}) {
    remainingValues, ok := self.Match(values)
    if ok {
        n, _ := strconv.ParseInt(values[1], 10, 64)
        return remainingValues, map[string]interface{}{self.key: n}
    }

    return values, map[string]interface{}{self.key: self.defaultValue}
}

func (self IntFlagParser) String() string {
    return fmt.Sprintf("IntFlagParser '%s'", self.pattern)
}

type StringSliceFlagParser struct {
    pattern string
    key string
    defaultValue []string
}

func (self StringSliceFlagParser) Match(values []string) ([]string, bool) {
    if len(values) < 2 {
        return values, false
    }

    var remainingValues []string

    for i := 0; i < len(values); i++ {
        if values[i] == self.pattern && i + 1 < len(values) {
            i++
            continue
        }
        remainingValues = append(remainingValues, values[i])
    }

    return remainingValues, len(values) != len(remainingValues)
}

func (self StringSliceFlagParser) Capture(values []string) ([]string, map[string]interface{}) {
    remainingValues, ok := self.Match(values)
    if !ok {
        return values, map[string]interface{}{self.key: self.defaultValue}
    }

    var captured []string

    for i := 0; i < len(values); i++ {
        if values[i] == self.pattern && i + 1 < len(values) {
            captured = append(captured, values[i + 1])
        }
    }

    return remainingValues, map[string]interface{}{self.key: captured}
}

func (self StringSliceFlagParser) String() string {
    return fmt.Sprintf("StringSliceFlagParser '%s'", self.pattern)
}

type FlagParser struct {
    parsers []Parser
}

func (self FlagParser) Match(values []string) ([]string, bool) {
    remainingValues := values
    var oneOrMoreMatches bool

    for _, parser := range self.parsers {
        var ok bool
        remainingValues, ok = parser.Match(remainingValues)
        if ok {
            oneOrMoreMatches = true
        }
    }

    // Recurse while we have one or more matches
    if oneOrMoreMatches {
        return self.Match(remainingValues)
    }

    return remainingValues, true
}

func (self FlagParser) Capture(values []string) ([]string, map[string]interface{}) {
    data := map[string]interface{}{}
    remainingValues := values

    for _, parser := range self.parsers {
        var captured map[string]interface{}
        remainingValues, captured = parser.Capture(remainingValues)
        for key, value := range captured {
            // Skip value if it already exists and new value is an empty string
            if _, exists := data[key]; exists && value == "" {
                continue
            }

            data[key] = value
        }
    }
    return remainingValues, data
}

func (self FlagParser) String() string {
    return fmt.Sprintf("FlagParser %v", self.parsers)
}

type ShortCircuitParser struct {
    parsers []Parser
}

func (self ShortCircuitParser) Match(values []string) ([]string, bool) {
    remainingValues := values

    for _, parser := range self.parsers {
        var ok bool
        remainingValues, ok = parser.Match(remainingValues)
        if ok {
            return remainingValues, true
        }
    }

    return remainingValues, false
}

func (self ShortCircuitParser) Capture(values []string) ([]string, map[string]interface{}) {
    if len(self.parsers) == 0 {
        return values, nil
    }

    for _, parser := range self.parsers {
        if _, ok := parser.Match(values); ok {
            return parser.Capture(values)
        }
    }

    // No parsers matched at this point,
    // just return the capture value of the first one
    return self.parsers[0].Capture(values)
}

func (self ShortCircuitParser) String() string {
    return fmt.Sprintf("ShortCircuitParser %v", self.parsers)
}
<commit_msg>Improve parsers<commit_after>package cli

import (
    "fmt"
    "strconv"
)

type Parser interface {
    Match([]string) ([]string, bool)
    Capture([]string) ([]string, map[string]interface{})
}

type EqualParser struct {
    value string
}

func (self EqualParser) Match(values []string) ([]string, bool) {
    if len(values) == 0 {
        return values, false
    }

    if self.value == values[0] {
        return values[1:], true
    }

    return values, false
}

func (self EqualParser) Capture(values []string) ([]string, map[string]interface{}) {
    remainingValues, _ := self.Match(values)
    return remainingValues, nil
}

func (self EqualParser) String() string {
    return fmt.Sprintf("EqualParser '%s'", self.value)
}

type CaptureGroupParser struct {
    value string
}

func (self CaptureGroupParser) Match(values []string) ([]string, bool) {
    if len(values) == 0 {
        return values, false
    }

    return values[1:], true
}

func (self CaptureGroupParser) key() string {
    return self.value[1:len(self.value) - 1]
}

func (self CaptureGroupParser) Capture(values []string) ([]string, map[string]interface{}) {
    if remainingValues, ok := self.Match(values); ok {
        return remainingValues, map[string]interface{}{self.key(): values[0]}
    }

    return values, nil
}

func (self CaptureGroupParser) String() string {
    return fmt.Sprintf("CaptureGroupParser '%s'", self.value)
}

type BoolFlagParser struct {
    pattern string
    key string
    omitValue bool
    defaultValue bool
}

func (self BoolFlagParser) Match(values []string) ([]string, bool) {
    if self.omitValue {
        return flagKeyMatch(self.pattern, values, 0)
    }

    remaining, value, ok := flagKeyValueMatch(self.pattern, values, 0)
    if !ok {
        return remaining, false
    }

    // Check that value is a valid boolean
    if _, err := strconv.ParseBool(value); err != nil {
        return remaining, false
    }

    return remaining, true
}

func (self BoolFlagParser) Capture(values []string) ([]string, map[string]interface{}) {
    if self.omitValue {
        remaining, ok := flagKeyMatch(self.pattern, values, 0)
        return remaining, map[string]interface{}{self.key: ok}
    }

    remaining, value, ok := flagKeyValueMatch(self.pattern, values, 0)
    if !ok {
        return remaining, map[string]interface{}{self.key: self.defaultValue}
    }

    b, _ := strconv.ParseBool(value)
    return remaining, map[string]interface{}{self.key: b}
}

func (self BoolFlagParser) String() string {
    return fmt.Sprintf("BoolFlagParser '%s'", self.pattern)
}

type StringFlagParser struct {
    pattern string
    key string
    defaultValue string
}

func (self StringFlagParser) Match(values []string) ([]string, bool) {
    remaining, _, ok := flagKeyValueMatch(self.pattern, values, 0)
    return remaining, ok
}

func (self StringFlagParser) Capture(values []string) ([]string, map[string]interface{}) {
    remaining, value, ok := flagKeyValueMatch(self.pattern, values, 0)
    if !ok {
        return remaining, map[string]interface{}{self.key: self.defaultValue}
    }

    return remaining, map[string]interface{}{self.key: value}
}

func (self StringFlagParser) String() string {
    return fmt.Sprintf("StringFlagParser '%s'", self.pattern)
}

type IntFlagParser struct {
    pattern string
    key string
    defaultValue int64
}

func (self IntFlagParser) Match(values []string) ([]string, bool) {
    remaining, value, ok := flagKeyValueMatch(self.pattern, values, 0)
    if !ok {
        return remaining, false
    }

    // Check that value is a valid integer
    if _, err := strconv.ParseInt(value, 10, 64); err != nil {
        return remaining, false
    }

    return remaining, true
}

func (self IntFlagParser) Capture(values []string) ([]string, map[string]interface{}) {
    remaining, value, ok := flagKeyValueMatch(self.pattern, values, 0)
    if !ok {
        return remaining, map[string]interface{}{self.key: self.defaultValue}
    }

    n, _ := strconv.ParseInt(value, 10, 64)
    return remaining, map[string]interface{}{self.key: n}
}

func (self IntFlagParser) String() string {
    return fmt.Sprintf("IntFlagParser '%s'", self.pattern)
}

type StringSliceFlagParser struct {
    pattern string
    key string
    defaultValue []string
}

func (self StringSliceFlagParser) Match(values []string) ([]string, bool) {
    if len(values) < 2 {
        return values, false
    }

    var remainingValues []string

    for i := 0; i < len(values); i++ {
        if values[i] == self.pattern && i + 1 < len(values) {
            i++
            continue
        }
        remainingValues = append(remainingValues, values[i])
    }

    return remainingValues, len(values) != len(remainingValues)
}

func (self StringSliceFlagParser) Capture(values []string) ([]string, map[string]interface{}) {
    remainingValues, ok := self.Match(values)
    if !ok {
        return values, map[string]interface{}{self.key: self.defaultValue}
    }

    var captured []string

    for i := 0; i < len(values); i++ {
        if values[i] == self.pattern && i + 1 < len(values) {
            captured = append(captured, values[i + 1])
        }
    }

    return remainingValues, map[string]interface{}{self.key: captured}
}

func (self StringSliceFlagParser) String() string {
    return fmt.Sprintf("StringSliceFlagParser '%s'", self.pattern)
}

type FlagParser struct {
    parsers []Parser
}

func (self FlagParser) Match(values []string) ([]string, bool) {
    remainingValues := values

    for _, parser := range self.parsers {
        remainingValues, _ = parser.Match(remainingValues)
    }
    return remainingValues, true
}

func (self FlagParser) Capture(values []string) ([]string, map[string]interface{}) {
    captured := map[string]interface{}{}
    remainingValues := values

    for _, parser := range self.parsers {
        var data map[string]interface{}
        remainingValues, data = parser.Capture(remainingValues)
        for key, value := range data {
            captured[key] = value
        }
    }

    return remainingValues, captured
}

func (self FlagParser) String() string {
    return fmt.Sprintf("FlagParser %v", self.parsers)
}

type ShortCircuitParser struct {
    parsers []Parser
}

func (self ShortCircuitParser) Match(values []string) ([]string, bool) {
    remainingValues := values

    for _, parser := range self.parsers {
        var ok bool
        remainingValues, ok = parser.Match(remainingValues)
        if ok {
            return remainingValues, true
        }
    }

    return remainingValues, false
}

func (self ShortCircuitParser) Capture(values []string) ([]string, map[string]interface{}) {
    if len(self.parsers) == 0 {
        return values, nil
    }

    for _, parser := range self.parsers {
        if _, ok := parser.Match(values); ok {
            return parser.Capture(values)
        }
    }

    // No parsers matched at this point,
    // just return the capture value of the first one
    return self.parsers[0].Capture(values)
}

func (self ShortCircuitParser) String() string {
    return fmt.Sprintf("ShortCircuitParser %v", self.parsers)
}

type CompleteParser struct {
    parsers []Parser
}

func (self CompleteParser) Match(values []string) ([]string, bool) {
    remainingValues := values

    for _, parser := range self.parsers {
        var ok bool
        remainingValues, ok = parser.Match(remainingValues)
        if !ok {
            return remainingValues, false
        }
    }

    return remainingValues, len(remainingValues) == 0
}

func (self CompleteParser) Capture(values []string) ([]string, map[string]interface{}) {
    remainingValues := values
    data := map[string]interface{}{}

    for _, parser := range self.parsers {
        var captured map[string]interface{}
        remainingValues, captured = parser.Capture(remainingValues)
        for key, value := range captured {
            data[key] = value
        }
    }

    return remainingValues, data
}

func (self CompleteParser) String() string {
    return fmt.Sprintf("CompleteParser %v", self.parsers)
}

func flagKeyValueMatch(key string, values []string, index int) ([]string, string, bool) {
    if index > len(values) - 2 {
        return values, "", false
    }

    if values[index] == key {
        value := values[index + 1]
        remaining := append(values[:index], values[index + 2:]...)
        return remaining, value, true
    }

    return flagKeyValueMatch(key, values, index + 1)
}

func flagKeyMatch(key string, values []string, index int) ([]string, bool) {
    if index > len(values) - 1 {
        return values, false
    }

    if values[index] == key {
        remaining := append(values[:index], values[index + 1:]...)
        return remaining, true
    }

    return flagKeyMatch(key, values, index + 1)
}
<|endoftext|>


<commit_before>package empire

import (
	"archive/tar"
	"bytes"
	"fmt"
	"io"
	"os"
	"path"
	"strings"

	"github.com/fsouza/go-dockerclient"
	"gopkg.in/yaml.v2"
)

var (
	// Procfile is the name of the Procfile file.
	Procfile = "Procfile"
)

// Extractor represents an object that can extract the process types from an
// image.
type Extractor interface {
	// Extract takes a repo in the form `remind101/r101-api`, and an image
	// id, and extracts the process types from the image.
	Extract(Image, *docker.AuthConfigurations) (CommandMap, error)
}

// NewExtractor returns a new Extractor instance.
func NewExtractor(socket, certPath string) (Extractor, error) {
	if socket == "" {
		return newExtractor(), nil
	}

	c, err := newDockerClient(socket, certPath)
	if err != nil {
		return nil, err
	}

	return &ProcfileExtractor{
		Client: c,
	}, nil
}

// extractor is a fake implementation of the Extractor interface.
type extractor struct{}

func newExtractor() *extractor {
	return &extractor{}
}

// Extract implements Extractor Extract.
func (e *extractor) Extract(image Image, auth *docker.AuthConfigurations) (CommandMap, error) {
	pm := make(CommandMap)

	// Just return some fake processes.
	pm[ProcessType("web")] = Command("./bin/web")

	return pm, nil
}

// ProcfileExtractor is an implementation of the Extractor interface that can
// pull a docker image and extract it's Procfile into a process.CommandMap.
type ProcfileExtractor struct {
	// Client is the docker client to use to pull the container image.
	Client interface {
		PullImage(docker.PullImageOptions, docker.AuthConfiguration) error
		InspectImage(string) (*docker.Image, error)
		InspectContainer(string) (*docker.Container, error)
		CreateContainer(docker.CreateContainerOptions) (*docker.Container, error)
		RemoveContainer(docker.RemoveContainerOptions) error
		CopyFromContainer(docker.CopyFromContainerOptions) error
	}

	// AuthConfiguration contains the docker AuthConfiguration.
	docker.AuthConfiguration
}

// Extract implements Extractor Extract.
func (e *ProcfileExtractor) Extract(image Image, auth *docker.AuthConfigurations) (CommandMap, error) {
	pm := make(CommandMap)

	if err := e.pullImage(image, auth); err != nil {
		return pm, err
	}

	c, err := e.createContainer(image)
	if err != nil {
		return pm, err
	}

	defer e.removeContainer(c.ID)

	procfile, err := e.procfile(c.ID)
	if err != nil {
		return pm, err
	}

	b, err := e.copyFile(c.ID, procfile)
	if err != nil {
		return pm, &ProcfileError{Err: err}
	}

	return ParseProcfile(b)
}

// procfile returns the path to the Procfile. If the container has a WORKDIR
// set, then this will return a path to the Procfile within that directory.
func (e *ProcfileExtractor) procfile(id string) (string, error) {
	p := ""

	c, err := e.Client.InspectContainer(id)
	if err != nil {
		return "", err
	}

	if c.Config != nil {
		p = c.Config.WorkingDir
	}

	return path.Join(p, Procfile), nil
}

// pullImage can pull a docker image from a repo, by it's imageID.
//
// Because docker does not support pulling an image by ID, we're assuming that
// the docker image has been tagged with it's own ID beforehand.
func (e *ProcfileExtractor) pullImage(i Image, auth *docker.AuthConfigurations) error {
	var a docker.AuthConfiguration

	if auth != nil {
		registry := strings.Split(string(i.Repo), "/")[0]
		if c, ok := auth.Configs[registry]; ok {
			a = c
		}
	}

	return e.Client.PullImage(docker.PullImageOptions{
		Repository:   string(i.Repo),
		Tag:          i.ID,
		OutputStream: os.Stdout,
	}, a)
}

// createContainer creates a new docker container for the given docker image.
func (e *ProcfileExtractor) createContainer(i Image) (*docker.Container, error) {
	return e.Client.CreateContainer(docker.CreateContainerOptions{
		Config: &docker.Config{
			Image: i.String(),
		},
	})
}

// removeContainer removes a container by it's ID.
func (e *ProcfileExtractor) removeContainer(containerID string) error {
	return e.Client.RemoveContainer(docker.RemoveContainerOptions{
		ID: containerID,
	})
}

// copyFile copies a file from a container.
func (e *ProcfileExtractor) copyFile(containerID, path string) ([]byte, error) {
	var buf bytes.Buffer
	if err := e.Client.CopyFromContainer(docker.CopyFromContainerOptions{
		Container:    containerID,
		Resource:     path,
		OutputStream: &buf,
	}); err != nil {
		return nil, err
	}

	// Open the tar archive for reading.
	r := bytes.NewReader(buf.Bytes())

	return firstFile(tar.NewReader(r))
}

// dockerClient is a fake docker client that can be used with the
// ProcfileExtractor.
type dockerClient struct {
	procfile string
}

func (c *dockerClient) PullImage(options docker.PullImageOptions, auth docker.AuthConfiguration) error {
	return nil
}

func (c *dockerClient) CreateContainer(options docker.CreateContainerOptions) (*docker.Container, error) {
	return &docker.Container{}, nil
}

func (c *dockerClient) RemoveContainer(options docker.RemoveContainerOptions) error {
	return nil
}

func (c *dockerClient) CopyFromContainer(options docker.CopyFromContainerOptions) error {
	pf := c.procfile
	tw := tar.NewWriter(options.OutputStream)
	if err := tw.WriteHeader(&tar.Header{
		Name: "/Procfile",
		Size: int64(len(pf)),
	}); err != nil {
		return err
	}

	if _, err := tw.Write([]byte(pf)); err != nil {
		return err
	}

	return nil
}

// Example instance: Procfile doesn't exist
type ProcfileError struct {
	Err error
}

func (e *ProcfileError) Error() string {
	return fmt.Sprintf("Procfile not found: %s", e.Err)
}

// ParseProcfile takes a byte slice representing a YAML Procfile and parses it
// into a processes.CommandMap.
func ParseProcfile(b []byte) (CommandMap, error) {
	pm := make(CommandMap)

	if err := yaml.Unmarshal(b, &pm); err != nil {
		return pm, err
	}

	return pm, nil
}

// firstFile extracts the first file from a tar archive.
func firstFile(tr *tar.Reader) ([]byte, error) {
	if _, err := tr.Next(); err != nil {
		return nil, err
	}

	var buf bytes.Buffer
	if _, err := io.Copy(&buf, tr); err != nil {
		return nil, err
	}

	return buf.Bytes(), nil
}

func newDockerClient(socket, certPath string) (*docker.Client, error) {
	if certPath != "" {
		cert := certPath + "/cert.pem"
		key := certPath + "/key.pem"
		ca := ""
		return docker.NewTLSClient(socket, cert, key, ca)
	}

	return docker.NewClient(socket)
}
<commit_msg>Remove unused func<commit_after>package empire

import (
	"archive/tar"
	"bytes"
	"fmt"
	"io"
	"os"
	"path"
	"strings"

	"github.com/fsouza/go-dockerclient"
	"gopkg.in/yaml.v2"
)

var (
	// Procfile is the name of the Procfile file.
	Procfile = "Procfile"
)

// Extractor represents an object that can extract the process types from an
// image.
type Extractor interface {
	// Extract takes a repo in the form `remind101/r101-api`, and an image
	// id, and extracts the process types from the image.
	Extract(Image, *docker.AuthConfigurations) (CommandMap, error)
}

// NewExtractor returns a new Extractor instance.
func NewExtractor(socket, certPath string) (Extractor, error) {
	if socket == "" {
		return newExtractor(), nil
	}

	c, err := newDockerClient(socket, certPath)
	if err != nil {
		return nil, err
	}

	return &ProcfileExtractor{
		Client: c,
	}, nil
}

// extractor is a fake implementation of the Extractor interface.
type extractor struct{}

func newExtractor() *extractor {
	return &extractor{}
}

// Extract implements Extractor Extract.
func (e *extractor) Extract(image Image, auth *docker.AuthConfigurations) (CommandMap, error) {
	pm := make(CommandMap)

	// Just return some fake processes.
	pm[ProcessType("web")] = Command("./bin/web")

	return pm, nil
}

// ProcfileExtractor is an implementation of the Extractor interface that can
// pull a docker image and extract it's Procfile into a process.CommandMap.
type ProcfileExtractor struct {
	// Client is the docker client to use to pull the container image.
	Client interface {
		PullImage(docker.PullImageOptions, docker.AuthConfiguration) error
		InspectContainer(string) (*docker.Container, error)
		CreateContainer(docker.CreateContainerOptions) (*docker.Container, error)
		RemoveContainer(docker.RemoveContainerOptions) error
		CopyFromContainer(docker.CopyFromContainerOptions) error
	}

	// AuthConfiguration contains the docker AuthConfiguration.
	docker.AuthConfiguration
}

// Extract implements Extractor Extract.
func (e *ProcfileExtractor) Extract(image Image, auth *docker.AuthConfigurations) (CommandMap, error) {
	pm := make(CommandMap)

	if err := e.pullImage(image, auth); err != nil {
		return pm, err
	}

	c, err := e.createContainer(image)
	if err != nil {
		return pm, err
	}

	defer e.removeContainer(c.ID)

	procfile, err := e.procfile(c.ID)
	if err != nil {
		return pm, err
	}

	b, err := e.copyFile(c.ID, procfile)
	if err != nil {
		return pm, &ProcfileError{Err: err}
	}

	return ParseProcfile(b)
}

// procfile returns the path to the Procfile. If the container has a WORKDIR
// set, then this will return a path to the Procfile within that directory.
func (e *ProcfileExtractor) procfile(id string) (string, error) {
	p := ""

	c, err := e.Client.InspectContainer(id)
	if err != nil {
		return "", err
	}

	if c.Config != nil {
		p = c.Config.WorkingDir
	}

	return path.Join(p, Procfile), nil
}

// pullImage can pull a docker image from a repo, by it's imageID.
//
// Because docker does not support pulling an image by ID, we're assuming that
// the docker image has been tagged with it's own ID beforehand.
func (e *ProcfileExtractor) pullImage(i Image, auth *docker.AuthConfigurations) error {
	var a docker.AuthConfiguration

	if auth != nil {
		registry := strings.Split(string(i.Repo), "/")[0]
		if c, ok := auth.Configs[registry]; ok {
			a = c
		}
	}

	return e.Client.PullImage(docker.PullImageOptions{
		Repository:   string(i.Repo),
		Tag:          i.ID,
		OutputStream: os.Stdout,
	}, a)
}

// createContainer creates a new docker container for the given docker image.
func (e *ProcfileExtractor) createContainer(i Image) (*docker.Container, error) {
	return e.Client.CreateContainer(docker.CreateContainerOptions{
		Config: &docker.Config{
			Image: i.String(),
		},
	})
}

// removeContainer removes a container by it's ID.
func (e *ProcfileExtractor) removeContainer(containerID string) error {
	return e.Client.RemoveContainer(docker.RemoveContainerOptions{
		ID: containerID,
	})
}

// copyFile copies a file from a container.
func (e *ProcfileExtractor) copyFile(containerID, path string) ([]byte, error) {
	var buf bytes.Buffer
	if err := e.Client.CopyFromContainer(docker.CopyFromContainerOptions{
		Container:    containerID,
		Resource:     path,
		OutputStream: &buf,
	}); err != nil {
		return nil, err
	}

	// Open the tar archive for reading.
	r := bytes.NewReader(buf.Bytes())

	return firstFile(tar.NewReader(r))
}

// dockerClient is a fake docker client that can be used with the
// ProcfileExtractor.
type dockerClient struct {
	procfile string
}

func (c *dockerClient) PullImage(options docker.PullImageOptions, auth docker.AuthConfiguration) error {
	return nil
}

func (c *dockerClient) CreateContainer(options docker.CreateContainerOptions) (*docker.Container, error) {
	return &docker.Container{}, nil
}

func (c *dockerClient) RemoveContainer(options docker.RemoveContainerOptions) error {
	return nil
}

func (c *dockerClient) CopyFromContainer(options docker.CopyFromContainerOptions) error {
	pf := c.procfile
	tw := tar.NewWriter(options.OutputStream)
	if err := tw.WriteHeader(&tar.Header{
		Name: "/Procfile",
		Size: int64(len(pf)),
	}); err != nil {
		return err
	}

	if _, err := tw.Write([]byte(pf)); err != nil {
		return err
	}

	return nil
}

// Example instance: Procfile doesn't exist
type ProcfileError struct {
	Err error
}

func (e *ProcfileError) Error() string {
	return fmt.Sprintf("Procfile not found: %s", e.Err)
}

// ParseProcfile takes a byte slice representing a YAML Procfile and parses it
// into a processes.CommandMap.
func ParseProcfile(b []byte) (CommandMap, error) {
	pm := make(CommandMap)

	if err := yaml.Unmarshal(b, &pm); err != nil {
		return pm, err
	}

	return pm, nil
}

// firstFile extracts the first file from a tar archive.
func firstFile(tr *tar.Reader) ([]byte, error) {
	if _, err := tr.Next(); err != nil {
		return nil, err
	}

	var buf bytes.Buffer
	if _, err := io.Copy(&buf, tr); err != nil {
		return nil, err
	}

	return buf.Bytes(), nil
}

func newDockerClient(socket, certPath string) (*docker.Client, error) {
	if certPath != "" {
		cert := certPath + "/cert.pem"
		key := certPath + "/key.pem"
		ca := ""
		return docker.NewTLSClient(socket, cert, key, ca)
	}

	return docker.NewClient(socket)
}
<|endoftext|>


<commit_before>// Package hex implements Well Known Binary encoding and decoding of strings.
package hex

import (
	"encoding/binary"
	"encoding/hex"

	"github.com/twpayne/go-geom"
	"github.com/twpayne/go-geom/encoding/wkb"
)

func Encode(g geom.T, byteOrder binary.ByteOrder) (string, error) {
	wkb, err := wkb.Marshal(g, byteOrder)
	if err != nil {
		return "", err
	}
	return hex.EncodeToString(wkb), nil
}

func Decode(s string) (geom.T, error) {
	data, err := hex.DecodeString(s)
	if err != nil {
		return nil, err
	}
	return wkb.Unmarshal(data)
}
<commit_msg>Add some Hex documentation<commit_after>// Package hex implements Well Known Binary encoding and decoding of strings.
package hex

import (
	"encoding/binary"
	"encoding/hex"

	"github.com/twpayne/go-geom"
	"github.com/twpayne/go-geom/encoding/wkb"
)

// Encode encodes an arbitrary geometry to a string.
func Encode(g geom.T, byteOrder binary.ByteOrder) (string, error) {
	wkb, err := wkb.Marshal(g, byteOrder)
	if err != nil {
		return "", err
	}
	return hex.EncodeToString(wkb), nil
}

// Decode decodes an arbitrary geometry from a string.
func Decode(s string) (geom.T, error) {
	data, err := hex.DecodeString(s)
	if err != nil {
		return nil, err
	}
	return wkb.Unmarshal(data)
}
<|endoftext|>


<commit_before>package handlers

import (
	mwr "github.com/gilcrest/go-API-template/pkg/api/server/middleware"
	"github.com/gilcrest/go-API-template/pkg/env"
	"github.com/gorilla/mux"
)

// Dispatch is a way of organizing routing to handlers (versioning as well)
func Dispatch(env *env.Env, rtr *mux.Router) *mux.Router {

	// setup new instance of APIAudit
	audit := new(mwr.APIAudit)

	// match only POST requests on /api/appUser/create
	// This is the original (v1) version for the API and the response for this
	// will never change with versioning in order to maintain a stable contract
	rtr.Handle("/appUser", mwr.Adapt(Handler{env, CreateUserHandler}, mwr.LogRequest(env, audit))).
		Methods("POST").
		Headers("Content-Type", "application/json")

	// match only POST requests on /api/v1/appUser/create
	rtr.Handle("/v1/appUser", mwr.Adapt(Handler{env, CreateUserHandler}, mwr.LogRequest(env, audit))).
		Methods("POST").
		Headers("Content-Type", "application/json")

	return rtr
}
<commit_msg>Moved timer into adapt list<commit_after>package handlers

import (
	mwr "github.com/gilcrest/go-API-template/pkg/api/server/middleware"
	"github.com/gilcrest/go-API-template/pkg/env"
	"github.com/gorilla/mux"
)

// Dispatch is a way of organizing routing to handlers (versioning as well)
func Dispatch(env *env.Env, rtr *mux.Router) *mux.Router {

	// setup new instance of APIAudit
	audit := new(mwr.APIAudit)

	// match only POST requests on /api/appUser/create
	// This is the original (v1) version for the API and the response for this
	// will never change with versioning in order to maintain a stable contract
	rtr.Handle("/appUser", mwr.Adapt(Handler{env, CreateUserHandler}, mwr.LogRequest(env, audit))).
		Methods("POST").
		Headers("Content-Type", "application/json")

		// match only POST requests on /api/v1/appUser/create
	rtr.Handle("/v1/appUser", mwr.Adapt(Handler{env, CreateUserHandler}, mwr.LogRequest(env, audit), mwr.Timer(audit))).
		Methods("POST").
		Headers("Content-Type", "application/json")

	return rtr
}
<|endoftext|>


<commit_before>package handlers

import (
	mwr "github.com/gilcrest/go-API-template/pkg/api/server/middleware"
	"github.com/gilcrest/go-API-template/pkg/env"
	"github.com/gorilla/mux"
)

// Dispatch is a way of organizing routing to handlers (versioning as well)
func Dispatch(env *env.Env, rtr *mux.Router) *mux.Router {

	// setup new instance of APIAudit
	audit := new(mwr.APIAudit)

	// match only POST requests on /api/appUser/create
	// This is the original (v1) version for the API and the response for this
	// will never change with versioning in order to maintain a stable contract
	rtr.Handle("/appUser", mwr.Adapt(Handler{env, CreateUserHandler}, mwr.LogRequest(env, audit))).
		Methods("POST").
		Headers("Content-Type", "application/json")

		// match only POST requests on /api/v1/appUser/create
	rtr.Handle("/v1/appUser", mwr.Adapt(Handler{env, CreateUserHandler}, mwr.LogRequest(env, audit), mwr.Timer(audit))).
		Methods("POST").
		Headers("Content-Type", "application/json")

	return rtr
}
<commit_msg>make handler calls consistent<commit_after>package handlers

import (
	mwr "github.com/gilcrest/go-API-template/pkg/api/server/middleware"
	"github.com/gilcrest/go-API-template/pkg/env"
	"github.com/gorilla/mux"
)

// Dispatch is a way of organizing routing to handlers (versioning as well)
func Dispatch(env *env.Env, rtr *mux.Router) *mux.Router {

	// initialize new instance of APIAudit
	audit := new(mwr.APIAudit)

	// match only POST requests on /api/appUser/create
	// This is the original (v1) version for the API and the response for this
	// will never change with versioning in order to maintain a stable contract
	rtr.Handle("/appUser", mwr.Adapt(Handler{env, CreateUserHandler}, mwr.LogRequest(env, audit), mwr.LogResponse(env, audit))).
		Methods("POST").
		Headers("Content-Type", "application/json")

	// match only POST requests on /api/v1/appUser/create
	rtr.Handle("/v1/appUser", mwr.Adapt(Handler{env, CreateUserHandler}, mwr.LogRequest(env, audit), mwr.LogResponse(env, audit))).
		Methods("POST").
		Headers("Content-Type", "application/json")

	return rtr
}
<|endoftext|>


<commit_before>package resolver

import (
	"sync"
	"time"

	"github.com/google/go-cmp/cmp"
	"github.com/patrickmn/go-cache"

	"nimona.io/internal/rand"
	"nimona.io/pkg/context"
	"nimona.io/pkg/crypto"
	"nimona.io/pkg/errors"
	"nimona.io/pkg/hyperspace"
	"nimona.io/pkg/hyperspace/peerstore"
	"nimona.io/pkg/localpeer"
	"nimona.io/pkg/log"
	"nimona.io/pkg/network"
	"nimona.io/pkg/object"
	"nimona.io/pkg/peer"
)

var (
	hyperspaceAnnouncementType   = new(hyperspace.Announcement).Type()
	hyperspaceLookupResponseType = new(hyperspace.LookupResponse).Type()

	peerCacheTTL = 1 * time.Minute
)

const (
	ErrNoPeersToAsk = errors.Error("no peers to ask")
)

//go:generate mockgen -destination=../resolvermock/resolvermock_generated.go -package=resolvermock -source=resolver.go

type (
	Resolver interface {
		Lookup(
			ctx context.Context,
			opts ...LookupOption,
		) ([]*peer.ConnectionInfo, error)
		LookupPeer(
			ctx context.Context,
			publicKey crypto.PublicKey,
		) (*peer.ConnectionInfo, error)
	}
	resolver struct {
		context                        context.Context
		network                        network.Network
		localpeer                      localpeer.LocalPeer
		peerCache                      *peerstore.PeerCache
		localPeerAnnouncementCache     *hyperspace.Announcement
		localPeerAnnouncementCacheLock sync.RWMutex
		bootstrapPeers                 []*peer.ConnectionInfo
		blocklist                      *cache.Cache
	}
	// Option for customizing a new resolver
	Option func(*resolver)
)

// New returns a new resolver
func New(
	ctx context.Context,
	netw network.Network,
	opts ...Option,
) Resolver {
	r := &resolver{
		context: ctx,
		network: netw,
		peerCache: peerstore.NewPeerCache(
			time.Minute,
			"nimona_hyperspace_resolver",
		),
		localPeerAnnouncementCacheLock: sync.RWMutex{},
		bootstrapPeers:                 []*peer.ConnectionInfo{},
		blocklist:                      cache.New(time.Second*5, time.Second*60),
	}

	for _, opt := range opts {
		opt(r)
	}

	r.localpeer = r.network.LocalPeer()

	// we are listening for all incoming object types in order to learn about
	// new peers that are talking to us so we can announce ourselves to them
	go network.HandleEnvelopeSubscription(
		r.network.Subscribe(),
		r.handleObject,
	)

	// register self to network
	netw.RegisterResolver(r)

	for _, p := range r.bootstrapPeers {
		r.peerCache.Put(&hyperspace.Announcement{
			ConnectionInfo: p,
		}, 0)
	}

	go func() {
		r.announceSelf()
		announceOnUpdate, cf := r.localpeer.ListenForUpdates()
		defer cf()
		announceTicker := time.NewTicker(30 * time.Second)
		for {
			select {
			case <-announceTicker.C:
				r.announceSelf()
			case <-announceOnUpdate:
				r.announceSelf()
			}
		}
	}()

	return r
}

func (r *resolver) LookupPeer(
	ctx context.Context,
	publicKey crypto.PublicKey,
) (*peer.ConnectionInfo, error) {
	ps, err := r.Lookup(ctx, LookupByPeerKey(publicKey))
	if err != nil {
		return nil, err
	}
	if len(ps) == 0 {
		return nil, nil
	}
	return ps[0], nil
}

// Lookup finds and returns peer infos from a fingerprint
// TODO consider returning peers synchronously
func (r *resolver) Lookup(
	ctx context.Context,
	opts ...LookupOption,
) ([]*peer.ConnectionInfo, error) {
	if len(r.bootstrapPeers) == 0 {
		return nil, errors.New("no peers to ask")
	}

	logger := log.FromContext(ctx).With(
		log.String("method", "resolver.Lookup"),
	)
	logger.Debug("looking up")

	opt := ParseLookupOptions(opts...)
	bl := hyperspace.New(opt.Lookups...)

	// send content requests to recipients
	req := &hyperspace.LookupRequest{
		Metadata: object.Metadata{
			Owner: r.localpeer.GetPrimaryPeerKey().PublicKey(),
		},
		Nonce:       rand.String(12),
		QueryVector: bl,
	}
	reqObject := req.ToObject()

	// listen for lookup responses
	resSub := r.network.Subscribe(
		network.FilterByObjectType(hyperspaceLookupResponseType),
		func(e *network.Envelope) bool {
			v := e.Payload.Data["nonce"]
			rn, ok := v.(object.String)
			return ok && string(rn) == req.Nonce
		},
	)

	go func() {
		for _, bp := range r.bootstrapPeers {
			err := r.network.Send(
				ctx,
				reqObject,
				bp.PublicKey,
				network.SendWithConnectionInfo(bp),
			)
			if err != nil {
				logger.Debug("could send request to peer", log.Error(err))
				continue
			}
			logger.Debug(
				"asked peer",
				log.String("peer", bp.PublicKey.String()),
			)
		}
	}()

	// create channel to keep peers we find
	peers := []*peer.ConnectionInfo{}
	done := make(chan struct{})

	go func() {
		for {
			e, err := resSub.Next()
			if err != nil {
				break
			}
			r := &hyperspace.LookupResponse{}
			if err := r.FromObject(e.Payload); err != nil {
				continue
			}
			// TODO verify peer?
			for _, ann := range r.Announcements {
				peers = append(peers, ann.ConnectionInfo)
			}
			close(done)
			break
		}
	}()

	select {
	case <-ctx.Done():
		return nil, ctx.Err()
	case <-done:
		return peers, nil
	}
}

func (r *resolver) handleObject(
	e *network.Envelope,
) error {
	// attempt to recover correlation id from request id
	ctx := r.context

	// handle payload
	o := e.Payload
	if o.Type == hyperspaceAnnouncementType {
		v := &hyperspace.Announcement{}
		if err := v.FromObject(o); err != nil {
			return err
		}
		r.handleAnnouncement(ctx, v)
	}
	return nil
}

func (r *resolver) handleAnnouncement(
	ctx context.Context,
	p *hyperspace.Announcement,
) {
	logger := log.FromContext(ctx).With(
		log.String("method", "resolver.handleAnnouncement"),
		log.String("peer.publicKey", p.ConnectionInfo.PublicKey.String()),
		log.Strings("peer.addresses", p.ConnectionInfo.Addresses),
	)
	logger.Debug("adding peer to cache")
	r.peerCache.Put(p, peerCacheTTL)
}

func (r *resolver) announceSelf() {
	ctx := context.New(
		context.WithParent(r.context),
	)
	logger := log.FromContext(ctx).With(
		log.String("method", "resolver.announceSelf"),
	)
	n := 0
	for _, p := range r.bootstrapPeers {
		if err := r.network.Send(
			context.New(
				context.WithParent(ctx),
				context.WithTimeout(time.Second*3),
			),
			r.getLocalPeerAnnouncement().ToObject(),
			p.PublicKey,
			network.SendWithConnectionInfo(p),
		); err != nil {
			logger.Error(
				"error announcing self to bootstrap",
				log.String("peer", p.PublicKey.String()),
				log.Error(err),
			)
			continue
		}
		n++
	}
	if n == 0 {
		logger.Error(
			"failed to announce self to any bootstrap peers",
		)
		return
	}
	logger.Info(
		"announced self to bootstrap peers",
		log.Int("bootstrapPeers", n),
	)
}

func (r *resolver) getLocalPeerAnnouncement() *hyperspace.Announcement {
	r.localPeerAnnouncementCacheLock.RLock()
	lastAnnouncement := r.localPeerAnnouncementCache
	r.localPeerAnnouncementCacheLock.RUnlock()

	peerKey := r.localpeer.GetPrimaryPeerKey().PublicKey()
	certificates := r.localpeer.GetCertificates()
	cids := r.localpeer.GetCIDs()
	contentTypes := r.localpeer.GetContentTypes()
	addresses := r.localpeer.GetAddresses()
	relays := r.localpeer.GetRelays()

	// gather up peer key, certificates, content ids and types
	hs := contentTypes
	hs = append(hs, peerKey.String())
	for _, c := range cids {
		hs = append(hs, c.String())
	}
	for _, c := range certificates {
		if !c.Metadata.Signature.IsEmpty() {
			hs = append(hs, c.Metadata.Signature.Signer.String())
		}
	}
	vec := hyperspace.New(hs...)

	if lastAnnouncement != nil &&
		cmp.Equal(lastAnnouncement.ConnectionInfo.Addresses, addresses) &&
		cmp.Equal(lastAnnouncement.PeerVector, vec) {
		return lastAnnouncement
	}

	localPeerAnnouncementCache := &hyperspace.Announcement{
		Metadata: object.Metadata{
			Owner: peerKey,
		},
		Version: time.Now().Unix(),
		ConnectionInfo: &peer.ConnectionInfo{
			PublicKey: peerKey,
			Addresses: addresses,
			Relays:    relays,
		},
		PeerVector:       vec,
		PeerCapabilities: contentTypes,
	}

	r.localPeerAnnouncementCacheLock.Lock()
	r.localPeerAnnouncementCache = localPeerAnnouncementCache
	r.localPeerAnnouncementCacheLock.Unlock()

	return localPeerAnnouncementCache
}
<commit_msg>chore(hyperspace/resolver): continue after handling error<commit_after>package resolver

import (
	"sync"
	"time"

	"github.com/google/go-cmp/cmp"
	"github.com/patrickmn/go-cache"

	"nimona.io/internal/rand"
	"nimona.io/pkg/context"
	"nimona.io/pkg/crypto"
	"nimona.io/pkg/errors"
	"nimona.io/pkg/hyperspace"
	"nimona.io/pkg/hyperspace/peerstore"
	"nimona.io/pkg/localpeer"
	"nimona.io/pkg/log"
	"nimona.io/pkg/network"
	"nimona.io/pkg/object"
	"nimona.io/pkg/peer"
)

var (
	hyperspaceAnnouncementType   = new(hyperspace.Announcement).Type()
	hyperspaceLookupResponseType = new(hyperspace.LookupResponse).Type()

	peerCacheTTL = 1 * time.Minute
)

const (
	ErrNoPeersToAsk = errors.Error("no peers to ask")
)

//go:generate mockgen -destination=../resolvermock/resolvermock_generated.go -package=resolvermock -source=resolver.go

type (
	Resolver interface {
		Lookup(
			ctx context.Context,
			opts ...LookupOption,
		) ([]*peer.ConnectionInfo, error)
		LookupPeer(
			ctx context.Context,
			publicKey crypto.PublicKey,
		) (*peer.ConnectionInfo, error)
	}
	resolver struct {
		context                        context.Context
		network                        network.Network
		localpeer                      localpeer.LocalPeer
		peerCache                      *peerstore.PeerCache
		localPeerAnnouncementCache     *hyperspace.Announcement
		localPeerAnnouncementCacheLock sync.RWMutex
		bootstrapPeers                 []*peer.ConnectionInfo
		blocklist                      *cache.Cache
	}
	// Option for customizing a new resolver
	Option func(*resolver)
)

// New returns a new resolver
func New(
	ctx context.Context,
	netw network.Network,
	opts ...Option,
) Resolver {
	r := &resolver{
		context: ctx,
		network: netw,
		peerCache: peerstore.NewPeerCache(
			time.Minute,
			"nimona_hyperspace_resolver",
		),
		localPeerAnnouncementCacheLock: sync.RWMutex{},
		bootstrapPeers:                 []*peer.ConnectionInfo{},
		blocklist:                      cache.New(time.Second*5, time.Second*60),
	}

	for _, opt := range opts {
		opt(r)
	}

	r.localpeer = r.network.LocalPeer()

	// we are listening for all incoming object types in order to learn about
	// new peers that are talking to us so we can announce ourselves to them
	go network.HandleEnvelopeSubscription(
		r.network.Subscribe(),
		func(e *network.Envelope) error {
			go r.handleObject(e)
			return nil
		},
	)

	// register self to network
	netw.RegisterResolver(r)

	for _, p := range r.bootstrapPeers {
		r.peerCache.Put(&hyperspace.Announcement{
			ConnectionInfo: p,
		}, 0)
	}

	go func() {
		r.announceSelf()
		announceOnUpdate, cf := r.localpeer.ListenForUpdates()
		defer cf()
		announceTicker := time.NewTicker(30 * time.Second)
		for {
			select {
			case <-announceTicker.C:
				r.announceSelf()
			case <-announceOnUpdate:
				r.announceSelf()
			}
		}
	}()

	return r
}

func (r *resolver) LookupPeer(
	ctx context.Context,
	publicKey crypto.PublicKey,
) (*peer.ConnectionInfo, error) {
	ps, err := r.Lookup(ctx, LookupByPeerKey(publicKey))
	if err != nil {
		return nil, err
	}
	if len(ps) == 0 {
		return nil, nil
	}
	return ps[0], nil
}

// Lookup finds and returns peer infos from a fingerprint
// TODO consider returning peers synchronously
func (r *resolver) Lookup(
	ctx context.Context,
	opts ...LookupOption,
) ([]*peer.ConnectionInfo, error) {
	if len(r.bootstrapPeers) == 0 {
		return nil, errors.New("no peers to ask")
	}

	logger := log.FromContext(ctx).With(
		log.String("method", "resolver.Lookup"),
	)
	logger.Debug("looking up")

	opt := ParseLookupOptions(opts...)
	bl := hyperspace.New(opt.Lookups...)

	// send content requests to recipients
	req := &hyperspace.LookupRequest{
		Metadata: object.Metadata{
			Owner: r.localpeer.GetPrimaryPeerKey().PublicKey(),
		},
		Nonce:       rand.String(12),
		QueryVector: bl,
	}
	reqObject := req.ToObject()

	// listen for lookup responses
	resSub := r.network.Subscribe(
		network.FilterByObjectType(hyperspaceLookupResponseType),
		func(e *network.Envelope) bool {
			v := e.Payload.Data["nonce"]
			rn, ok := v.(object.String)
			return ok && string(rn) == req.Nonce
		},
	)

	go func() {
		for _, bp := range r.bootstrapPeers {
			err := r.network.Send(
				ctx,
				reqObject,
				bp.PublicKey,
				network.SendWithConnectionInfo(bp),
			)
			if err != nil {
				logger.Debug("could send request to peer", log.Error(err))
				continue
			}
			logger.Debug(
				"asked peer",
				log.String("peer", bp.PublicKey.String()),
			)
		}
	}()

	// create channel to keep peers we find
	peers := []*peer.ConnectionInfo{}
	done := make(chan struct{})

	go func() {
		for {
			e, err := resSub.Next()
			if err != nil {
				break
			}
			r := &hyperspace.LookupResponse{}
			if err := r.FromObject(e.Payload); err != nil {
				continue
			}
			// TODO verify peer?
			for _, ann := range r.Announcements {
				peers = append(peers, ann.ConnectionInfo)
			}
			close(done)
			break
		}
	}()

	select {
	case <-ctx.Done():
		return nil, ctx.Err()
	case <-done:
		return peers, nil
	}
}

func (r *resolver) handleObject(
	e *network.Envelope,
) {
	// attempt to recover correlation id from request id
	ctx := r.context

	logger := log.FromContext(ctx).With(
		log.String("method", "resolver.handleObject"),
		log.String("env.Sender", e.Sender.String()),
	)

	// handle payload
	o := e.Payload
	if o.Type == hyperspaceAnnouncementType {
		v := &hyperspace.Announcement{}
		if err := v.FromObject(o); err != nil {
			logger.Warn(
				"error handling announcement",
				log.Error(err),
			)
			return
		}
		r.handleAnnouncement(ctx, v)
	}
}

func (r *resolver) handleAnnouncement(
	ctx context.Context,
	p *hyperspace.Announcement,
) {
	logger := log.FromContext(ctx).With(
		log.String("method", "resolver.handleAnnouncement"),
		log.String("peer.publicKey", p.ConnectionInfo.PublicKey.String()),
		log.Strings("peer.addresses", p.ConnectionInfo.Addresses),
	)
	logger.Debug("adding peer to cache")
	r.peerCache.Put(p, peerCacheTTL)
}

func (r *resolver) announceSelf() {
	ctx := context.New(
		context.WithParent(r.context),
	)
	logger := log.FromContext(ctx).With(
		log.String("method", "resolver.announceSelf"),
	)
	n := 0
	for _, p := range r.bootstrapPeers {
		if err := r.network.Send(
			context.New(
				context.WithParent(ctx),
				context.WithTimeout(time.Second*3),
			),
			r.getLocalPeerAnnouncement().ToObject(),
			p.PublicKey,
			network.SendWithConnectionInfo(p),
		); err != nil {
			logger.Error(
				"error announcing self to bootstrap",
				log.String("peer", p.PublicKey.String()),
				log.Error(err),
			)
			continue
		}
		n++
	}
	if n == 0 {
		logger.Error(
			"failed to announce self to any bootstrap peers",
		)
		return
	}
	logger.Info(
		"announced self to bootstrap peers",
		log.Int("bootstrapPeers", n),
	)
}

func (r *resolver) getLocalPeerAnnouncement() *hyperspace.Announcement {
	r.localPeerAnnouncementCacheLock.RLock()
	lastAnnouncement := r.localPeerAnnouncementCache
	r.localPeerAnnouncementCacheLock.RUnlock()

	peerKey := r.localpeer.GetPrimaryPeerKey().PublicKey()
	certificates := r.localpeer.GetCertificates()
	cids := r.localpeer.GetCIDs()
	contentTypes := r.localpeer.GetContentTypes()
	addresses := r.localpeer.GetAddresses()
	relays := r.localpeer.GetRelays()

	// gather up peer key, certificates, content ids and types
	hs := contentTypes
	hs = append(hs, peerKey.String())
	for _, c := range cids {
		hs = append(hs, c.String())
	}
	for _, c := range certificates {
		if !c.Metadata.Signature.IsEmpty() {
			hs = append(hs, c.Metadata.Signature.Signer.String())
		}
	}
	vec := hyperspace.New(hs...)

	if lastAnnouncement != nil &&
		cmp.Equal(lastAnnouncement.ConnectionInfo.Addresses, addresses) &&
		cmp.Equal(lastAnnouncement.PeerVector, vec) {
		return lastAnnouncement
	}

	localPeerAnnouncementCache := &hyperspace.Announcement{
		Metadata: object.Metadata{
			Owner: peerKey,
		},
		Version: time.Now().Unix(),
		ConnectionInfo: &peer.ConnectionInfo{
			PublicKey: peerKey,
			Addresses: addresses,
			Relays:    relays,
		},
		PeerVector:       vec,
		PeerCapabilities: contentTypes,
	}

	r.localPeerAnnouncementCacheLock.Lock()
	r.localPeerAnnouncementCache = localPeerAnnouncementCache
	r.localPeerAnnouncementCacheLock.Unlock()

	return localPeerAnnouncementCache
}
<|endoftext|>


<commit_before>package parse

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"reflect"
	"strconv"
	"strings"
	"testing"
	"time"

	"golang.org/x/text/unicode/norm"
)

type checkNode struct {
	t          *testing.T
	parsedNode interface{}
	expectNode interface{}
	testPath   string
	pNodeName  string
	pFieldName string
	pFieldVal  interface{}
	pFieldType reflect.Type
	eFieldName string
	eFieldVal  interface{}
	eFieldType reflect.Type
}

func (c *checkNode) error(args ...interface{}) {
	c.t.Error(args...)
}

func (c *checkNode) errorf(format string, args ...interface{}) {
	c.t.Errorf(format, args...)
}

func (c *checkNode) checkField(parsedValue interface{}, expectedValue interface{}) {
	if parsedValue != expectedValue {
		c.dError()
	}
}

func (c *checkNode) dError() {
	var got, exp string

	switch c.pFieldVal.(type) {
	case NodeType:
		got = c.pFieldVal.(NodeType).String()
		exp = c.eFieldVal.(string)
	case StartPosition:
		got = c.pFieldVal.(StartPosition).String()
		exp = strconv.Itoa(int(c.eFieldVal.(float64)))
	case Line:
		got = c.pFieldVal.(Line).String()
		exp = strconv.Itoa(int(c.eFieldVal.(float64)))
	case systemMessageLevel:
		pNum := int(c.pFieldVal.(systemMessageLevel))
		pNumStr := " (" + strconv.Itoa(pNum) + ")"
		got = c.pFieldVal.(systemMessageLevel).String() + pNumStr
		smsLvl := int(systemMessageLevelFromString(c.eFieldVal.(string)))
		eNumStr := " (" + strconv.Itoa(smsLvl) + ")"
		exp = c.eFieldVal.(string) + eNumStr
	case string:
		got = c.pFieldVal.(string)
		exp = c.eFieldVal.(string)
	case int:
		got = strconv.Itoa(c.pFieldVal.(int))
		exp = strconv.Itoa(int(c.eFieldVal.(float64)))
	case rune:
		got = string(c.pFieldVal.(rune))
		exp = string(c.eFieldVal.(rune))
	}
	lPos := strconv.FormatFloat(c.expectNode.(map[string]interface{})["line"].(float64), 'f', -1, 64)
	sPos := reflect.Indirect(reflect.ValueOf(c.parsedNode)).FieldByName("StartPosition").Int()
	txt := reflect.Indirect(reflect.ValueOf(c.parsedNode)).FieldByName("Text").String()
	c.t.Errorf("[node  text:%q line:%s startpos:%d] Got %s = %q -- Expect %s = %q", txt, lPos, sPos, c.pFieldName, got, c.eFieldName, exp)
}

func (c *checkNode) updateState(eKey string, eVal interface{}, pVal reflect.Value, eNode interface{}, pNode interface{}) bool {
	c.expectNode = eNode
	c.parsedNode = pNode

	// Expected parser metadata
	c.eFieldName = eKey
	c.eFieldVal = eVal
	c.eFieldType = reflect.TypeOf(c.eFieldVal)

	// Actual parsed metadata
	c.pNodeName = pVal.Type().Name()
	c.pFieldName = strings.ToUpper(string(c.eFieldName[0]))
	c.pFieldName += c.eFieldName[1:]

	if !pVal.FieldByName(c.pFieldName).IsValid() {
		return false
	}

	c.pFieldVal = pVal.FieldByName(c.pFieldName).Interface()
	c.pFieldType = pVal.FieldByName(c.pFieldName).Type()

	// Overline adornment nodes can be null
	if c.eFieldName == "overLine" && c.eFieldVal == nil {
		return false
	} else if c.eFieldVal == nil {
		c.dError()
		return false
	}

	return true
}

func (c *checkNode) checkType(fieldNum int, parseFieldName string,
	parseFields reflect.Value, expectedFields map[string]interface{}) bool {

	if parseFieldName == "" {
		tlog(fmt.Sprintf("Check struct tags! parseFieldName = %s", parseFieldName))
		os.Exit(1)
	}

	pVal := parseFields.Field(fieldNum).Interface()

	check := func(cond bool) bool { return cond }

	switch parseFieldName {
	case "indentLength":
		// Some title nodes aren't indented.
		return check(pVal == 0)
	case "startPosition":
		// Most nodes begin at position one in the line, therefore we can ignore them if it hasn't been
		// specified in the expected nodes.
		return check(pVal.(StartPosition) == 0 || pVal.(StartPosition) == 1)
	case "line":
		// zero, then we ignore it.  systemMessage literal block nodes have no line position.
		return check(pVal.(Line).LineNumber() == 0)
	case "overLine":
		// Some sections don't have overlines
		return check(expectedFields[parseFieldName] == nil && pVal.(*AdornmentNode) == nil)
	case "nodeList":
		// Some Nodes don't have child nodes.
		return check(expectedFields[parseFieldName] == nil && pVal.(NodeList) == nil)
	case "text":
		// Some Nodes don't have text.
		return check(expectedFields[parseFieldName] == nil && pVal.(string) == "")
	case "length":
		return check(expectedFields[parseFieldName] == nil && pVal == 0)
	}
	return false
}

// checkMatchingFields compares the expected node output retrieved from the nodes.json file and the actual parser NodeList
// output. Returns an error if a mismatch is found.
func (c *checkNode) checkMatchingFields(eNodes interface{}, pNode Node) error {
	if eNodes == nil || pNode == nil {
		panic("arguments must not be nil!")
	}

	// If the value is missing in eNodes and nil in pNode than we can exclude it.
	eFields := eNodes.(map[string]interface{})
	pNodeVal := reflect.Indirect(reflect.ValueOf(pNode))

	// Check expected node to parsed node
	for eName := range eFields {
		var sfName string
		sfName = strings.ToUpper(eName[0:1]) + eName[1:]
		if _, in := pNodeVal.Type().FieldByName(sfName); !in {
			return fmt.Errorf("Parse Node missing field %q:\nParseNode:\n%sExpectNode:\n%s",
				sfName, spd.Sdump(pNode), spd.Sdump(eFields))
		}
	}

	// Compare pNode against eNodes
	for i := 0; i < pNodeVal.NumField(); i++ {
		pName := pNodeVal.Type().Field(i).Tag.Get("json")
		if c.checkType(i, pName, pNodeVal, eFields) {
			continue
		}

		eNode := eNodes.(map[string]interface{})
		if eNode[pName] == nil {
			pVal := pNodeVal.Field(i).Interface()
			return fmt.Errorf("NodeType: %q Missing field %q -- Parser got: %q == %v -- ExpectNode:\n%s\n",
				eNode["type"], pName, pName, pVal, spd.Sdump(eNode))
		}
	}
	return nil
}

func (c *checkNode) checkFieldByType() error {
	switch c.eFieldName {
	case "text":
		c.checkField(c.pFieldVal.(string), norm.NFC.String(c.eFieldVal.(string)))
	case "type":
		c.checkField(c.pFieldVal.(NodeType).String(), c.eFieldVal)
	case "messageType":
		c.checkField(c.pFieldVal.(parserMessage).String(), c.eFieldVal)
	case "level", "length", "indentLength":
		c.checkField(float64(c.pFieldVal.(int)), c.eFieldVal)
	case "line":
		c.checkField(float64(c.pFieldVal.(Line)), c.eFieldVal)
	case "startPosition":
		c.checkField(float64(c.pFieldVal.(StartPosition)), c.eFieldVal)
	case "indent", "overLine", "title", "underLine":
		return c.checkFields(c.eFieldVal, c.pFieldVal.(Node))
	case "term", "definition":
		return c.checkFields(c.eFieldVal, c.pFieldVal.(Node))
	case "nodeList":
		len1 := len(c.eFieldVal.([]interface{}))
		len2 := len(c.pFieldVal.(NodeList))
		if len1 != len2 {
			return fmt.Errorf("Expected NodeList values (len=%d) and parsed NodeList values (len=%d) "+
				"do not match!", len1, len2)
		}
		for num, node := range c.eFieldVal.([]interface{}) {
			// Store and reset the parser value, otherwise a panic will occur on the next iteration
			pFieldVal := c.pFieldVal
			if cerr := c.checkFields(node, c.pFieldVal.(NodeList)[num]); cerr != nil {
				return cerr
			}
			c.pFieldVal = pFieldVal
		}
	case "rune":
		c.checkField(string(c.pFieldVal.(rune)), c.eFieldVal)
	case "severity":
		c.checkField(c.pFieldVal.(systemMessageLevel).String(), c.eFieldVal)
	case "bullet":
		c.checkField(c.pFieldVal.(string), c.eFieldVal.(string))
	case "enumType":
		c.checkField(c.pFieldVal.(EnumListType).String(), c.eFieldVal)
	case "affix":
		c.checkField(c.pFieldVal.(EnumAffixType).String(), c.eFieldVal)
	default:
		c.t.Errorf("Type %q case is not implemented in checkFields!", c.eFieldName)
	}
	return nil
}

// checkFields is a recursive function that compares the expected node output to the parser output comparing the two objects
// field by field. eNodes is unmarshaled json input and pNode is the parser node to check.
func (c *checkNode) checkFields(eNodes interface{}, pNode Node) error {
	if eNodes == nil || pNode == nil {
		panic("arguments cannot be nil!")
	}
	if err := c.checkMatchingFields(eNodes, pNode); err != nil {
		c.t.Error(err)
	}
	for eKey, eVal := range eNodes.(map[string]interface{}) {
		pVal := reflect.Indirect(reflect.ValueOf(pNode))
		if !c.updateState(eKey, eVal, pVal, eNodes, pNode) {
			continue
		}
		if cerr := c.checkFieldByType(); cerr != nil {
			return cerr
		}
	}
	return nil
}

// checkParseNodes compares the expected parser output (*_nodes.json) against the actual parser output node by node.
func checkParseNodes(t *testing.T, eTree []interface{}, pNodes *NodeList, testPath string) {

	state := &checkNode{t: t, testPath: testPath}

	failTest := func(err error) {
		// Give all other output time to print
		time.Sleep(time.Second / 2)
		tlog(fmt.Sprintf("\nFAIL: %s\n", err.Error()))
		tlog("-----------------------------------------------------------------------------")
		tlog("Parse Nodes")
		tlog("-----------------------------------------------------------------------------")
		pnj, err := json.MarshalIndent(pNodes, "", "    ")
		if err != nil {
			tlog(fmt.Sprintf("ERROR: Could not marshal json! Error=%q", err.Error()))
			t.Fail()
		}
		tlog(string(pnj))
		// tlog(spd.Sdump(pNodes))
		tlog("-----------------------------------------------------------------------------")
		tlog("Expected Nodes")
		tlog("-----------------------------------------------------------------------------")
		enj, err := json.MarshalIndent(eTree, "", " ")
		if err != nil {
			tlog(fmt.Sprintf("ERROR: Could not marshal json! Error=%q", err.Error()))
			t.Fail()
		}
		tlog(string(enj))
		// tlog(spd.Sdump(eTree))
		t.FailNow()
	}

	if len(*pNodes) != len(eTree) {
		failTest(errors.New("The number of parsed nodes does not match expected nodes!"))
	}

	for eNum, eNode := range eTree {
		if cerr := state.checkFields(eNode, (*pNodes)[eNum]); cerr != nil {
			failTest(cerr)
		}
	}

	return
}
<commit_msg>Split up checkNode.checkFields(), round 2<commit_after>package parse

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"reflect"
	"strconv"
	"strings"
	"testing"
	"time"

	"golang.org/x/text/unicode/norm"
)

type checkNode struct {
	t          *testing.T
	parsedNode interface{}
	expectNode interface{}
	testPath   string
	pNodeName  string
	pFieldName string
	pFieldVal  interface{}
	pFieldType reflect.Type
	eFieldName string
	eFieldVal  interface{}
	eFieldType reflect.Type
}

func (c *checkNode) error(args ...interface{}) {
	c.t.Error(args...)
}

func (c *checkNode) errorf(format string, args ...interface{}) {
	c.t.Errorf(format, args...)
}

func (c *checkNode) checkField(parsedValue interface{}, expectedValue interface{}) {
	if parsedValue != expectedValue {
		c.dError()
	}
}

func (c *checkNode) dError() {
	var got, exp string

	switch c.pFieldVal.(type) {
	case NodeType:
		got = c.pFieldVal.(NodeType).String()
		exp = c.eFieldVal.(string)
	case StartPosition:
		got = c.pFieldVal.(StartPosition).String()
		exp = strconv.Itoa(int(c.eFieldVal.(float64)))
	case Line:
		got = c.pFieldVal.(Line).String()
		exp = strconv.Itoa(int(c.eFieldVal.(float64)))
	case systemMessageLevel:
		pNum := int(c.pFieldVal.(systemMessageLevel))
		pNumStr := " (" + strconv.Itoa(pNum) + ")"
		got = c.pFieldVal.(systemMessageLevel).String() + pNumStr
		smsLvl := int(systemMessageLevelFromString(c.eFieldVal.(string)))
		eNumStr := " (" + strconv.Itoa(smsLvl) + ")"
		exp = c.eFieldVal.(string) + eNumStr
	case string:
		got = c.pFieldVal.(string)
		exp = c.eFieldVal.(string)
	case int:
		got = strconv.Itoa(c.pFieldVal.(int))
		exp = strconv.Itoa(int(c.eFieldVal.(float64)))
	case rune:
		got = string(c.pFieldVal.(rune))
		exp = string(c.eFieldVal.(rune))
	}
	lPos := strconv.FormatFloat(c.expectNode.(map[string]interface{})["line"].(float64), 'f', -1, 64)
	sPos := reflect.Indirect(reflect.ValueOf(c.parsedNode)).FieldByName("StartPosition").Int()
	txt := reflect.Indirect(reflect.ValueOf(c.parsedNode)).FieldByName("Text").String()
	c.t.Errorf("[node  text:%q line:%s startpos:%d] Got %s = %q -- Expect %s = %q", txt, lPos, sPos, c.pFieldName, got, c.eFieldName, exp)
}

func (c *checkNode) updateState(eKey string, eVal interface{}, pVal reflect.Value, eNode interface{}, pNode interface{}) bool {
	c.expectNode = eNode
	c.parsedNode = pNode

	// Expected parser metadata
	c.eFieldName = eKey
	c.eFieldVal = eVal
	c.eFieldType = reflect.TypeOf(c.eFieldVal)

	// Actual parsed metadata
	c.pNodeName = pVal.Type().Name()
	c.pFieldName = strings.ToUpper(string(c.eFieldName[0]))
	c.pFieldName += c.eFieldName[1:]

	if !pVal.FieldByName(c.pFieldName).IsValid() {
		return false
	}

	c.pFieldVal = pVal.FieldByName(c.pFieldName).Interface()
	c.pFieldType = pVal.FieldByName(c.pFieldName).Type()

	// Overline adornment nodes can be null
	if c.eFieldName == "overLine" && c.eFieldVal == nil {
		return false
	} else if c.eFieldVal == nil {
		c.dError()
		return false
	}

	return true
}

func (c *checkNode) checkType(fieldNum int, parseFieldName string,
	parseFields reflect.Value, expectedFields map[string]interface{}) bool {

	if parseFieldName == "" {
		tlog(fmt.Sprintf("Check struct tags! parseFieldName = %s", parseFieldName))
		os.Exit(1)
	}

	pVal := parseFields.Field(fieldNum).Interface()

	check := func(cond bool) bool { return cond }

	switch parseFieldName {
	case "indentLength":
		// Some title nodes aren't indented.
		return check(pVal == 0)
	case "startPosition":
		// Most nodes begin at position one in the line, therefore we can ignore them if it hasn't been
		// specified in the expected nodes.
		return check(pVal.(StartPosition) == 0 || pVal.(StartPosition) == 1)
	case "line":
		// zero, then we ignore it.  systemMessage literal block nodes have no line position.
		return check(pVal.(Line).LineNumber() == 0)
	case "overLine":
		// Some sections don't have overlines
		return check(expectedFields[parseFieldName] == nil && pVal.(*AdornmentNode) == nil)
	case "nodeList":
		// Some Nodes don't have child nodes.
		return check(expectedFields[parseFieldName] == nil && pVal.(NodeList) == nil)
	case "text":
		// Some Nodes don't have text.
		return check(expectedFields[parseFieldName] == nil && pVal.(string) == "")
	case "length":
		return check(expectedFields[parseFieldName] == nil && pVal == 0)
	}
	return false
}

// checkMatchingFields compares the expected node output retrieved from the nodes.json file and the actual parser NodeList
// output. Returns an error if a mismatch is found.
func (c *checkNode) checkMatchingFields(eNodes interface{}, pNode Node) error {
	if eNodes == nil || pNode == nil {
		panic("arguments must not be nil!")
	}

	// If the value is missing in eNodes and nil in pNode than we can exclude it.
	eFields := eNodes.(map[string]interface{})
	pNodeVal := reflect.Indirect(reflect.ValueOf(pNode))

	// Check expected node to parsed node
	for eName := range eFields {
		var sfName string
		sfName = strings.ToUpper(eName[0:1]) + eName[1:]
		if _, in := pNodeVal.Type().FieldByName(sfName); !in {
			return fmt.Errorf("Parse Node missing field %q:\nParseNode:\n%sExpectNode:\n%s",
				sfName, spd.Sdump(pNode), spd.Sdump(eFields))
		}
	}

	// Compare pNode against eNodes
	for i := 0; i < pNodeVal.NumField(); i++ {
		pName := pNodeVal.Type().Field(i).Tag.Get("json")
		if c.checkType(i, pName, pNodeVal, eFields) {
			continue
		}

		eNode := eNodes.(map[string]interface{})
		if eNode[pName] == nil {
			pVal := pNodeVal.Field(i).Interface()
			return fmt.Errorf("NodeType: %q Missing field %q -- Parser got: %q == %v -- ExpectNode:\n%s\n",
				eNode["type"], pName, pName, pVal, spd.Sdump(eNode))
		}
	}
	return nil
}

func (c *checkNode) checkFieldNodeList() error {
	len1 := len(c.eFieldVal.([]interface{}))
	len2 := len(c.pFieldVal.(NodeList))
	if len1 != len2 {
		return fmt.Errorf("Expected NodeList values (len=%d) and parsed NodeList values (len=%d) do not match!",
			len1, len2)
	}
	for num, node := range c.eFieldVal.([]interface{}) {
		// Store and reset the parser value, otherwise a panic will occur on the next iteration
		pFieldVal := c.pFieldVal
		if cerr := c.checkFields(node, c.pFieldVal.(NodeList)[num]); cerr != nil {
			return cerr
		}
		c.pFieldVal = pFieldVal
	}
	return nil
}

func (c *checkNode) checkFieldByType() error {
	switch c.eFieldName {
	case "text":
		c.checkField(c.pFieldVal.(string), norm.NFC.String(c.eFieldVal.(string)))
	case "type":
		c.checkField(c.pFieldVal.(NodeType).String(), c.eFieldVal)
	case "messageType":
		c.checkField(c.pFieldVal.(parserMessage).String(), c.eFieldVal)
	case "level", "length", "indentLength":
		c.checkField(float64(c.pFieldVal.(int)), c.eFieldVal)
	case "line":
		c.checkField(float64(c.pFieldVal.(Line)), c.eFieldVal)
	case "startPosition":
		c.checkField(float64(c.pFieldVal.(StartPosition)), c.eFieldVal)
	case "indent", "overLine", "title", "underLine":
		return c.checkFields(c.eFieldVal, c.pFieldVal.(Node))
	case "term", "definition":
		return c.checkFields(c.eFieldVal, c.pFieldVal.(Node))
	case "nodeList":
		return c.checkFieldNodeList()
	case "rune":
		c.checkField(string(c.pFieldVal.(rune)), c.eFieldVal)
	case "severity":
		c.checkField(c.pFieldVal.(systemMessageLevel).String(), c.eFieldVal)
	case "bullet":
		c.checkField(c.pFieldVal.(string), c.eFieldVal.(string))
	case "enumType":
		c.checkField(c.pFieldVal.(EnumListType).String(), c.eFieldVal)
	case "affix":
		c.checkField(c.pFieldVal.(EnumAffixType).String(), c.eFieldVal)
	default:
		c.t.Errorf("Type %q case is not implemented in checkFields!", c.eFieldName)
	}
	return nil
}

// checkFields is a recursive function that compares the expected node output to the parser output comparing the two objects
// field by field. eNodes is unmarshaled json input and pNode is the parser node to check.
func (c *checkNode) checkFields(eNodes interface{}, pNode Node) error {
	if eNodes == nil || pNode == nil {
		panic("arguments cannot be nil!")
	}
	if err := c.checkMatchingFields(eNodes, pNode); err != nil {
		c.t.Error(err)
	}
	for eKey, eVal := range eNodes.(map[string]interface{}) {
		pVal := reflect.Indirect(reflect.ValueOf(pNode))
		if !c.updateState(eKey, eVal, pVal, eNodes, pNode) {
			continue
		}
		if cerr := c.checkFieldByType(); cerr != nil {
			return cerr
		}
	}
	return nil
}

// checkParseNodes compares the expected parser output (*_nodes.json) against the actual parser output node by node.
func checkParseNodes(t *testing.T, eTree []interface{}, pNodes *NodeList, testPath string) {

	state := &checkNode{t: t, testPath: testPath}

	failTest := func(err error) {
		// Give all other output time to print
		time.Sleep(time.Second / 2)
		tlog(fmt.Sprintf("\nFAIL: %s\n", err.Error()))
		tlog("-----------------------------------------------------------------------------")
		tlog("Parse Nodes")
		tlog("-----------------------------------------------------------------------------")
		pnj, err := json.MarshalIndent(pNodes, "", "    ")
		if err != nil {
			tlog(fmt.Sprintf("ERROR: Could not marshal json! Error=%q", err.Error()))
			t.Fail()
		}
		tlog(string(pnj))
		// tlog(spd.Sdump(pNodes))
		tlog("-----------------------------------------------------------------------------")
		tlog("Expected Nodes")
		tlog("-----------------------------------------------------------------------------")
		enj, err := json.MarshalIndent(eTree, "", " ")
		if err != nil {
			tlog(fmt.Sprintf("ERROR: Could not marshal json! Error=%q", err.Error()))
			t.Fail()
		}
		tlog(string(enj))
		// tlog(spd.Sdump(eTree))
		t.FailNow()
	}

	if len(*pNodes) != len(eTree) {
		failTest(errors.New("The number of parsed nodes does not match expected nodes!"))
	}

	for eNum, eNode := range eTree {
		if cerr := state.checkFields(eNode, (*pNodes)[eNum]); cerr != nil {
			failTest(cerr)
		}
	}

	return
}
<|endoftext|>