<commit_before>// Copyright (c) 2014 The SurgeMQ Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

import (
	"bufio"
	"fmt"
	"io"
	"sync/atomic"
	"runtime"
//"github.com/surgemq/message"
	"encoding/binary"
	"strconv"
)

var (
	bufcnt int64
	DefaultBufferSize int64

	DeviceInBufferSize int64
	DeviceOutBufferSize int64

	MasterInBufferSize int64
	MasterOutBufferSize int64
)

const (
	smallReadBlockSize = 512
	defaultReadBlockSize = 8192
	defaultWriteBlockSize = 8192
)

/**
2016.03.03 修改
bingbuffer结构体
 */
type buffer struct {
	readIndex  int64        //读序号
	writeIndex int64        //写序号
	ringBuffer []*ByteArray //环形buffer指针数组
	bufferSize int64        //初始化环形buffer指针数组大小
	mask       int64        //掩码：bufferSize-1
	done       int64        //是否完成
}

type ByteArray struct {
	bArray []byte
}

func (this *buffer)ReadCommit(index int64) {
	this.ringBuffer[index] = nil
}

/**
2016.03.03 添加
初始化ringbuffer
参数bufferSize：初始化环形buffer指针数组大小
 */
func newBuffer(size int64) (*buffer, error) {
	if size < 0 {
		return nil, bufio.ErrNegativeCount
	}
	if size == 0 {
		size = DefaultBufferSize
	}
	if !powerOfTwo64(size) {
		fmt.Printf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
		return nil, fmt.Errorf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
	}

	return &buffer{
		readIndex: int64(0), //读序号
		writeIndex: int64(0), //写序号
		ringBuffer: make([]*ByteArray, size), //环形buffer指针数组
		bufferSize: size, //初始化环形buffer指针数组大小
		mask:size - 1,
	}, nil
}

/**
2016.03.03 添加
获取当前读序号
 */
func (this *buffer)GetCurrentReadIndex() (int64) {
	return atomic.LoadInt64(&this.readIndex)
}
/**
2016.03.03 添加
获取当前写序号
 */
func (this *buffer)GetCurrentWriteIndex() (int64) {
	return atomic.LoadInt64(&this.writeIndex)
}

/**
2016.03.03 添加
读取ringbuffer指定的buffer指针，返回该指针并清空ringbuffer该位置存在的指针内容，以及将读序号加1
 */
func (this *buffer)ReadBuffer() (p []byte, index int64, ok bool) {
	ok = true
	p = nil

	readIndex := atomic.LoadInt64(&this.readIndex)
	writeIndex := atomic.LoadInt64(&this.writeIndex)
	switch  {
	case readIndex >= writeIndex:
		ok = false
	case writeIndex - readIndex > this.bufferSize:
		ok = false
	default:
		//index := buffer.readIndex % buffer.bufferSize
		index := readIndex & this.mask

		p_ := this.ringBuffer[index]
		//this.ringBuffer[index] = nil
		atomic.AddInt64(&this.readIndex, 1)
		p = p_.bArray

		if p == nil {
			ok = false
		}
	}
	return p, index, ok
}

/**
2016.03.03 添加
写入ringbuffer指针，以及将写序号加1
 */
func (this *buffer)WriteBuffer(in *[]byte) (ok bool) {
	ok = true

	readIndex := atomic.LoadInt64(&this.readIndex)
	writeIndex := atomic.LoadInt64(&this.writeIndex)
	switch  {
	case writeIndex - readIndex < 0:
		ok = false
	default:
		//index := buffer.writeIndex % buffer.bufferSize
		index := writeIndex & this.mask
		if this.ringBuffer[index] == nil {
			this.ringBuffer[index] = &ByteArray{bArray:in}
			atomic.AddInt64(&this.writeIndex, 1)
		}else {
			ok = false
		}
	}
	return ok
}

/**
2016.03.03 修改
关闭缓存
 */
func (this *buffer) Close() error {
	atomic.StoreInt64(&this.done, 1)
	return nil
}
/*

/**
2016.03.03 修改
向ringbuffer中写数据（从connection的中向ringbuffer中写）--生产者
*/
func (this *buffer) ReadFrom(r io.Reader) (int64, error) {
	defer this.Close()
	for {
		total := int64(0)
		if this.isDone() {
			return total, io.EOF
		}
		b := make([]byte, 5)
		n, err := r.Read(b[0:1])

		if n > 0 {
			total += int64(n)
			if err != nil {
				return total, err
			}
		}

		/**************************/
		cnt := 1

		// Let's read enough bytes to get the message header (msg type, remaining length)
		for {
			// If we have read 5 bytes and still not done, then there's a problem.
			if cnt > 4 {
				return 0, fmt.Errorf("sendrecv/peekMessageSize: 4th byte of remaining length has continuation bit set")
			}

			// Peek cnt bytes from the input buffer.
			_, err := r.Read(b[cnt:cnt + 1])
			if err != nil {
				return 0, err
			}

			// If we got enough bytes, then check the last byte to see if the continuation
			// bit is set. If so, increment cnt and continue peeking
			if b[cnt] >= 0x80 {
				cnt++
			} else {
				break
			}
		}

		// Get the remaining length of the message
		remlen, _ := binary.Uvarint(b[1:])

		// Total message length is remlen + 1 (msg type) + m (remlen bytes)
		len := int64(len(b))
		remlen_ := int64(remlen)
		total = remlen_ + int64(len)

		//mtype := message.MessageType(b[0] >> 4)
		/****************/
		//var msg message.Message
		//
		//msg, err = mtype.New()
		//if err != nil {
		//	return 0, err
		//}
		b_ := make([]byte, remlen_)
		_, err = r.Read(b_[0:])
		if err != nil {
			return 0, err
		}

		b = append(b, b_...)
		//n, err = msg.Decode(b)
		//if err != nil {
		//	return 0, err
		//}

		/*************************/

		for !this.WriteBuffer(&b) {
			runtime.Gosched()
		}

		return total, nil
	}
}

/**
2016.03.03 修改
 */
func (this *buffer) WriteTo(w io.Writer) (int64, error) {
	defer this.Close()
	total := int64(0)
	for {
		if this.isDone() {
			return total, io.EOF
		}
		p, index, ok := this.ReadBuffer()
		if !ok {
			runtime.Gosched()
		}
		defer this.ReadCommit(index)
		Log.Debugc(func() string {
			return fmt.Sprintf("p=" + strconv.FormatBool(p == nil))
		})
		Log.Debugc(func() string {
			return fmt.Sprintf("WriteTo函数》》读取*p：" + string(*p))
		})

		Log.Debugc(func() string {
			return fmt.Sprintf(" WriteTo(w io.Writer)(7)")
		})
		//
		//Log.Errorc(func() string {
		//	return fmt.Sprintf("msg::" + msg.Name())
		//})
		//
		//p := make([]byte, msg.Len())
		//_, err := msg.Encode(p)
		//if err != nil {
		//	Log.Errorc(func() string {
		//		return fmt.Sprintf("msg.Encode(p)")
		//	})
		//	return total, io.EOF
		//}
		// There's some data, let's process it first
		if len(*p) > 0 {
			n, err := w.Write(*p)
			total += int64(n)
			Log.Debugc(func() string {
				return fmt.Sprintf("Wrote %d bytes, totaling %d bytes", n, total)
			})

			if err != nil {
				Log.Errorc(func() string {
					return fmt.Sprintf("w.Write(p) error")
				})
				return total, err
			}
		}

		return total, nil
	}
}

/**
2016.03.03 修改
*/
func (this *buffer) isDone() bool {
	if atomic.LoadInt64(&this.done) == 1 {
		return true
	}

	return false
}

func powerOfTwo64(n int64) bool {
	return n != 0 && (n & (n - 1)) == 0
}

func roundUpPowerOfTwo64(n int64) int64 {
	n--
	n |= n >> 1
	n |= n >> 2
	n |= n >> 4
	n |= n >> 8
	n |= n >> 16
	n |= n >> 32
	n++

	return n
}
<commit_msg>添加测试日志<commit_after>// Copyright (c) 2014 The SurgeMQ Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

import (
	"bufio"
	"fmt"
	"io"
	"sync/atomic"
	"runtime"
//"github.com/surgemq/message"
	"encoding/binary"
	"strconv"
)

var (
	bufcnt int64
	DefaultBufferSize int64

	DeviceInBufferSize int64
	DeviceOutBufferSize int64

	MasterInBufferSize int64
	MasterOutBufferSize int64
)

const (
	smallReadBlockSize = 512
	defaultReadBlockSize = 8192
	defaultWriteBlockSize = 8192
)

/**
2016.03.03 修改
bingbuffer结构体
 */
type buffer struct {
	readIndex  int64        //读序号
	writeIndex int64        //写序号
	ringBuffer []*ByteArray //环形buffer指针数组
	bufferSize int64        //初始化环形buffer指针数组大小
	mask       int64        //掩码：bufferSize-1
	done       int64        //是否完成
}

type ByteArray struct {
	bArray []byte
}

func (this *buffer)ReadCommit(index int64) {
	this.ringBuffer[index] = nil
}

/**
2016.03.03 添加
初始化ringbuffer
参数bufferSize：初始化环形buffer指针数组大小
 */
func newBuffer(size int64) (*buffer, error) {
	if size < 0 {
		return nil, bufio.ErrNegativeCount
	}
	if size == 0 {
		size = DefaultBufferSize
	}
	if !powerOfTwo64(size) {
		fmt.Printf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
		return nil, fmt.Errorf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
	}

	return &buffer{
		readIndex: int64(0), //读序号
		writeIndex: int64(0), //写序号
		ringBuffer: make([]*ByteArray, size), //环形buffer指针数组
		bufferSize: size, //初始化环形buffer指针数组大小
		mask:size - 1,
	}, nil
}

/**
2016.03.03 添加
获取当前读序号
 */
func (this *buffer)GetCurrentReadIndex() (int64) {
	return atomic.LoadInt64(&this.readIndex)
}
/**
2016.03.03 添加
获取当前写序号
 */
func (this *buffer)GetCurrentWriteIndex() (int64) {
	return atomic.LoadInt64(&this.writeIndex)
}

/**
2016.03.03 添加
读取ringbuffer指定的buffer指针，返回该指针并清空ringbuffer该位置存在的指针内容，以及将读序号加1
 */
func (this *buffer)ReadBuffer() (p []byte, index int64, ok bool) {
	ok = true
	p = nil

	readIndex := atomic.LoadInt64(&this.readIndex)
	writeIndex := atomic.LoadInt64(&this.writeIndex)
	switch  {
	case readIndex >= writeIndex:
		ok = false
	case writeIndex - readIndex > this.bufferSize:
		ok = false
	default:
		//index := buffer.readIndex % buffer.bufferSize
		index := readIndex & this.mask

		p_ := this.ringBuffer[index]
		//this.ringBuffer[index] = nil
		atomic.AddInt64(&this.readIndex, 1)
		p = p_.bArray

		if p == nil {
			ok = false
		}
	}
	return p, index, ok
}

/**
2016.03.03 添加
写入ringbuffer指针，以及将写序号加1
 */
func (this *buffer)WriteBuffer(in *[]byte) (ok bool) {
	ok = true

	readIndex := atomic.LoadInt64(&this.readIndex)
	writeIndex := atomic.LoadInt64(&this.writeIndex)
	switch  {
	case writeIndex - readIndex < 0:
		ok = false
	default:
		//index := buffer.writeIndex % buffer.bufferSize
		index := writeIndex & this.mask
		if this.ringBuffer[index] == nil {
			this.ringBuffer[index] = &ByteArray{bArray:*in}
			atomic.AddInt64(&this.writeIndex, 1)
		}else {
			ok = false
		}
	}
	return ok
}

/**
2016.03.03 修改
关闭缓存
 */
func (this *buffer) Close() error {
	atomic.StoreInt64(&this.done, 1)
	return nil
}
/*

/**
2016.03.03 修改
向ringbuffer中写数据（从connection的中向ringbuffer中写）--生产者
*/
func (this *buffer) ReadFrom(r io.Reader) (int64, error) {
	defer this.Close()
	for {
		total := int64(0)
		if this.isDone() {
			return total, io.EOF
		}
		b := make([]byte, 5)
		n, err := r.Read(b[0:1])

		if n > 0 {
			total += int64(n)
			if err != nil {
				return total, err
			}
		}

		/**************************/
		cnt := 1

		// Let's read enough bytes to get the message header (msg type, remaining length)
		for {
			// If we have read 5 bytes and still not done, then there's a problem.
			if cnt > 4 {
				return 0, fmt.Errorf("sendrecv/peekMessageSize: 4th byte of remaining length has continuation bit set")
			}

			// Peek cnt bytes from the input buffer.
			_, err := r.Read(b[cnt:cnt + 1])
			if err != nil {
				return 0, err
			}

			// If we got enough bytes, then check the last byte to see if the continuation
			// bit is set. If so, increment cnt and continue peeking
			if b[cnt] >= 0x80 {
				cnt++
			} else {
				break
			}
		}

		// Get the remaining length of the message
		remlen, _ := binary.Uvarint(b[1:])

		// Total message length is remlen + 1 (msg type) + m (remlen bytes)
		len := int64(len(b))
		remlen_ := int64(remlen)
		total = remlen_ + int64(len)

		//mtype := message.MessageType(b[0] >> 4)
		/****************/
		//var msg message.Message
		//
		//msg, err = mtype.New()
		//if err != nil {
		//	return 0, err
		//}
		b_ := make([]byte, remlen_)
		_, err = r.Read(b_[0:])
		if err != nil {
			return 0, err
		}

		b = append(b, b_...)
		//n, err = msg.Decode(b)
		//if err != nil {
		//	return 0, err
		//}

		/*************************/

		for !this.WriteBuffer(&b) {
			runtime.Gosched()
		}

		return total, nil
	}
}

/**
2016.03.03 修改
 */
func (this *buffer) WriteTo(w io.Writer) (int64, error) {
	defer this.Close()
	total := int64(0)
	for {
		if this.isDone() {
			return total, io.EOF
		}
		p, index, ok := this.ReadBuffer()
		if !ok {
			runtime.Gosched()
		}
		defer this.ReadCommit(index)
		Log.Debugc(func() string {
			return fmt.Sprintf("p=" + strconv.FormatBool(p == nil))
		})
		Log.Debugc(func() string {
			return fmt.Sprintf("WriteTo函数》》读取*p：" + string(*p))
		})

		Log.Debugc(func() string {
			return fmt.Sprintf(" WriteTo(w io.Writer)(7)")
		})
		//
		//Log.Errorc(func() string {
		//	return fmt.Sprintf("msg::" + msg.Name())
		//})
		//
		//p := make([]byte, msg.Len())
		//_, err := msg.Encode(p)
		//if err != nil {
		//	Log.Errorc(func() string {
		//		return fmt.Sprintf("msg.Encode(p)")
		//	})
		//	return total, io.EOF
		//}
		// There's some data, let's process it first
		if len(*p) > 0 {
			n, err := w.Write(*p)
			total += int64(n)
			Log.Debugc(func() string {
				return fmt.Sprintf("Wrote %d bytes, totaling %d bytes", n, total)
			})

			if err != nil {
				Log.Errorc(func() string {
					return fmt.Sprintf("w.Write(p) error")
				})
				return total, err
			}
		}

		return total, nil
	}
}

/**
2016.03.03 修改
*/
func (this *buffer) isDone() bool {
	if atomic.LoadInt64(&this.done) == 1 {
		return true
	}

	return false
}

func powerOfTwo64(n int64) bool {
	return n != 0 && (n & (n - 1)) == 0
}

func roundUpPowerOfTwo64(n int64) int64 {
	n--
	n |= n >> 1
	n |= n >> 2
	n |= n >> 4
	n |= n >> 8
	n |= n >> 16
	n |= n >> 32
	n++

	return n
}
<|endoftext|>


<commit_before>package html

import (
	"errors"
	"net/http"

	"github.com/chrisolsen/ae/auth"
)

type page map[string]interface{}

// NewPage creates a new page
func NewPage() page {
	return page(make(map[string]interface{}))
}

// NewPageWithCSRFToken create a page with an initialized CSRF token
func NewPageWithCSRFToken(r *http.Request) page {
	p := NewPage()
	token := auth.NewCSRFToken(r)
	p["CSRFToken"] = token
	return p
}

// SetError sets any error that needs to be shown
func (p page) SetError(err interface{}) {
	switch err.(type) {
	case string:
		p["Error"] = errors.New(err.(string))
	case error:
		p["Error"] = err.(error)
	default:
		p["Error"] = nil
	}
}

// SetUser sets the current user
func (p page) SetUser(user interface{}) {
	p["CurrentUser"] = user
}

// Set sets the key and value
func (p page) Set(key string, val interface{}) {
	p[key] = val
}
<commit_msg>add helper to create pagination indexes<commit_after>package html

import (
	"errors"
	"math"
	"net/http"

	"github.com/chrisolsen/ae/auth"
)

type page map[string]interface{}

// NewPage creates a new page
func NewPage() page {
	return page(make(map[string]interface{}))
}

// NewPageWithCSRFToken create a page with an initialized CSRF token
func NewPageWithCSRFToken(r *http.Request) page {
	p := NewPage()
	token := auth.NewCSRFToken(r)
	p["CSRFToken"] = token
	return p
}

// SetError sets any error that needs to be shown
func (p page) SetError(err interface{}) {
	switch err.(type) {
	case string:
		p["Error"] = errors.New(err.(string))
	case error:
		p["Error"] = err.(error)
	default:
		p["Error"] = nil
	}
}

// SetUser sets the current user
func (p page) SetUser(user interface{}) {
	p["CurrentUser"] = user
}

// Set sets the key and value
func (p page) Set(key string, val interface{}) {
	p[key] = val
}

// SetPageOffsets sets an array within the page to allow it to be iterated through
// in the template to create pagination links.
//  // .go file
// 	p := html.NewPage()
// 	items := []string {"foo", "bar", "bits", ...}
// 	p.SetPageOffsets(len(items), 10)
//
//  // template
//  {{range $index, $offset := .Offsets}}
//      {{add $index 1}}
//  {{end}}
func (p page) SetPageOffsets(itemCount, pageSize int) {
	offsetCount := int(math.Ceil(float64(itemCount) / float64(pageSize)))
	offsets := make([]int, offsetCount)
	for i := 0; i < offsetCount; i++ {
		offsets[i] = i * 10
	}
	p.Set("Offsets", offsets)
}
<|endoftext|>


<commit_before>package tika

import (
	"context"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"time"

	"go.opentelemetry.io/otel/api/trace"
	"go.opentelemetry.io/otel/codes"
	"go.opentelemetry.io/otel/label"

	"github.com/ipfs-search/ipfs-search/extractor"
	"github.com/ipfs-search/ipfs-search/instr"
	"github.com/ipfs-search/ipfs-search/protocol"
	t "github.com/ipfs-search/ipfs-search/types"
)

// Extractor extracts metadata using the ipfs-tika server.
type Extractor struct {
	config   *Config
	client   *http.Client
	protocol protocol.Protocol

	*instr.Instrumentation
}

func (e *Extractor) get(ctx context.Context, url string) (resp *http.Response, err error) {
	// Temporarily disabled due to bug - the connection needs to be open until the response body has been read!
	// ctx, cancel := context.WithTimeout(ctx, e.config.RequestTimeout)
	// defer cancel()

	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
	if err != nil {
		// Errors here are programming errors.
		panic(fmt.Sprintf("creating request: %s", err))
	}

	return e.client.Do(req)
}

// retryingGet is an infinitely retrying GET on intermittent errors (e.g. server goes)
// TODO: Replace by proper circuit breakers.
func (e *Extractor) retryingGet(ctx context.Context, url string) (resp *http.Response, err error) {
	retries := 0

	for {
		log.Printf("Fetching metadata from '%s'", url)

		resp, err := e.get(ctx, url)

		if err == nil {
			// Success, we're done here.
			return resp, nil
		}

		if !shouldRetry(err) {
			// TODO: shouldRetry is probably a sensible update to go, which might simplify
			// shouldRetry - but better to have tracing infra in place before we go there.
			//
			// Any returned error will be of type *url.Error. The url.Error value's Timeout
			// method will report true if request timed out or was canceled.
			// Ref: https://golang.org/pkg/net/http/#Client.Do
			// Fatal error, don't retry
			return nil, err
		}

		retries++

		log.Printf("Retrying (%d) in %s", retries, e.config.RetryWait)
		time.Sleep(e.config.RetryWait)
	}
}

func (e *Extractor) getExtractURL(r *t.AnnotatedResource) string {
	// TODO: This should be TIKAURL?url=GATEWAYURL (or something similar)
	return e.protocol.GatewayURL(r)
}

// Extract metadata from a (potentially) referenced resource, updating
// Metadata or returning an error.
func (e *Extractor) Extract(ctx context.Context, r *t.AnnotatedResource, m interface{}) error {
	ctx, span := e.Tracer.Start(ctx, "extractor.tika.Extract",
		trace.WithAttributes(label.String("cid", r.ID)),
	)
	defer span.End()

	resp, err := e.retryingGet(ctx, e.getExtractURL(r))

	if err != nil {
		span.RecordError(ctx, err, trace.WithErrorStatus(codes.Error))
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		err := fmt.Errorf("unexpected status '%s' from ipfs-tika", resp.Status)
		span.RecordError(ctx, err, trace.WithErrorStatus(codes.Error))
		return err
	}

	// Parse resulting JSON
	if err := json.NewDecoder(resp.Body).Decode(m); err != nil {
		span.RecordError(ctx, err, trace.WithErrorStatus(codes.Error))
		return err
	}

	// TODO
	// Check for IPFS links in urls extracted from resource
	/*
	   for raw_url := range metadata.urls {
	       url, err := URL.Parse(raw_url)

	       if err != nil {
	           return err
	       }

	       if strings.HasPrefix(url.Path, "/ipfs/") {
	           // Found IPFS link!
	           args := crawlerArgs{
	               Hash:       link.Hash,
	               Name:       link.Name,
	               Size:       link.Size,
	               ParentHash: hash,
	           }

	       }
	   }
	*/

	return nil
}

// New returns a new Tika extractor.
func New(config *Config, client *http.Client, protocol protocol.Protocol, instr *instr.Instrumentation) extractor.Extractor {
	return &Extractor{
		config,
		client,
		protocol,
		instr,
	}
}

// Compile-time assurance that implementation satisfies interface.
var _ extractor.Extractor = &Extractor{}
<commit_msg>Workaround in Tika.<commit_after>package tika

import (
	"context"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"time"

	"go.opentelemetry.io/otel/api/trace"
	"go.opentelemetry.io/otel/codes"
	"go.opentelemetry.io/otel/label"

	"github.com/ipfs-search/ipfs-search/extractor"
	"github.com/ipfs-search/ipfs-search/instr"
	"github.com/ipfs-search/ipfs-search/protocol"
	t "github.com/ipfs-search/ipfs-search/types"
)

// Extractor extracts metadata using the ipfs-tika server.
type Extractor struct {
	config   *Config
	client   *http.Client
	protocol protocol.Protocol

	*instr.Instrumentation
}

func (e *Extractor) get(ctx context.Context, url string) (resp *http.Response, err error) {
	// Temporarily disabled due to bug - the connection needs to be open until the response body has been read!
	// ctx, cancel := context.WithTimeout(ctx, e.config.RequestTimeout)
	// defer cancel()

	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
	if err != nil {
		// Errors here are programming errors.
		panic(fmt.Sprintf("creating request: %s", err))
	}

	return e.client.Do(req)
}

// retryingGet is an infinitely retrying GET on intermittent errors (e.g. server goes)
// TODO: Replace by proper circuit breakers.
func (e *Extractor) retryingGet(ctx context.Context, url string) (resp *http.Response, err error) {
	retries := 0

	for {
		log.Printf("Fetching metadata from '%s'", url)

		resp, err := e.get(ctx, url)

		if err == nil {
			// Success, we're done here.
			return resp, nil
		}

		if !shouldRetry(err) {
			// TODO: shouldRetry is probably a sensible update to go, which might simplify
			// shouldRetry - but better to have tracing infra in place before we go there.
			//
			// Any returned error will be of type *url.Error. The url.Error value's Timeout
			// method will report true if request timed out or was canceled.
			// Ref: https://golang.org/pkg/net/http/#Client.Do
			// Fatal error, don't retry
			return nil, err
		}

		retries++

		log.Printf("Retrying (%d) in %s", retries, e.config.RetryWait)
		time.Sleep(e.config.RetryWait)
	}
}

func (e *Extractor) getExtractURL(r *t.AnnotatedResource) string {
	// TODO: This should be TIKAURL?url=GATEWAYURL (or something similar)
	gwURL := e.protocol.GatewayURL(r)
	u, _ := url.Parse(gwURL)
	return e.config.TikaServerURL + u.Path
}

// Extract metadata from a (potentially) referenced resource, updating
// Metadata or returning an error.
func (e *Extractor) Extract(ctx context.Context, r *t.AnnotatedResource, m interface{}) error {
	ctx, span := e.Tracer.Start(ctx, "extractor.tika.Extract",
		trace.WithAttributes(label.String("cid", r.ID)),
	)
	defer span.End()

	resp, err := e.retryingGet(ctx, e.getExtractURL(r))

	if err != nil {
		span.RecordError(ctx, err, trace.WithErrorStatus(codes.Error))
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		err := fmt.Errorf("unexpected status '%s' from ipfs-tika", resp.Status)
		span.RecordError(ctx, err, trace.WithErrorStatus(codes.Error))
		return err
	}

	// Parse resulting JSON
	if err := json.NewDecoder(resp.Body).Decode(m); err != nil {
		span.RecordError(ctx, err, trace.WithErrorStatus(codes.Error))
		return err
	}

	// TODO
	// Check for IPFS links in urls extracted from resource
	/*
	   for raw_url := range metadata.urls {
	       url, err := URL.Parse(raw_url)

	       if err != nil {
	           return err
	       }

	       if strings.HasPrefix(url.Path, "/ipfs/") {
	           // Found IPFS link!
	           args := crawlerArgs{
	               Hash:       link.Hash,
	               Name:       link.Name,
	               Size:       link.Size,
	               ParentHash: hash,
	           }

	       }
	   }
	*/

	return nil
}

// New returns a new Tika extractor.
func New(config *Config, client *http.Client, protocol protocol.Protocol, instr *instr.Instrumentation) extractor.Extractor {
	return &Extractor{
		config,
		client,
		protocol,
		instr,
	}
}

// Compile-time assurance that implementation satisfies interface.
var _ extractor.Extractor = &Extractor{}
<|endoftext|>


<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package saltpack

import (
	"crypto/hmac"
)

// RawBoxKey is the raw byte-representation of what a box key should
// look like --- a static 32-byte buffer
type RawBoxKey [32]byte

// SymmetricKey is a template for a symmetric key, a 32-byte static
// buffer.  Used for both NaCl SecretBox.
type SymmetricKey [32]byte

type KIDExtractor interface {
	// ToKID outputs the "key ID" that corresponds to this BoxPublicKey.
	// You can do whatever you'd like here, but probably it makes sense just
	// to output the public key as is.
	ToKID() []byte
}

// BoxPublicKey is an generic interface to NaCl's public key Box function.
type BoxPublicKey interface {
	KIDExtractor

	// ToRawBoxKeyPointer returns this public key as a *[32]byte,
	// for use with nacl.box.Seal
	ToRawBoxKeyPointer() *RawBoxKey

	// CreateEmphemeralKey creates an ephemeral key of the same type,
	// but totally random.
	CreateEphemeralKey() (BoxSecretKey, error)

	// HideIdentity returns true if we should hide the identity of this
	// key in our output message format.
	HideIdentity() bool
}

// BoxPrecomputedSharedKey results from a Precomputation below.
type BoxPrecomputedSharedKey interface {
	Unbox(nonce *Nonce, msg []byte) ([]byte, error)
	Box(nonce *Nonce, msg []byte) ([]byte, error)
}

// BoxSecretKey is the secret key corresponding to a BoxPublicKey
type BoxSecretKey interface {

	// Box boxes up data, sent from this secret key, and to the receiver
	// specified.
	Box(receiver BoxPublicKey, nonce *Nonce, msg []byte) ([]byte, error)

	// Unobx opens up the box, using this secret key as the receiver key
	// abd the give public key as the sender key.
	Unbox(sender BoxPublicKey, nonce *Nonce, msg []byte) ([]byte, error)

	// GetPublicKey gets the public key associated with this secret key.
	GetPublicKey() BoxPublicKey

	// Precompute computes a DH with the given key
	Precompute(sender BoxPublicKey) BoxPrecomputedSharedKey
}

// SigningSecretKey is a secret NaCl key that can sign messages.
type SigningSecretKey interface {
	// Sign signs message with this secret key.
	Sign(message []byte) ([]byte, error)

	// PublicKey gets the public key associated with this secret key.
	PublicKey() SigningPublicKey
}

// SigningPublicKey is a public NaCl key that can verify
// signatures.
type SigningPublicKey interface {
	KIDExtractor

	// Verify verifies that signature is a valid signature of message for
	// this public key.
	Verify(message []byte, signature []byte) error
}

// Keyring is an interface used with decryption; it is called to
// recover public or private keys during the decryption process.
// Calls can block on network action.
type Keyring interface {
	// LookupBoxSecretKey looks in the Keyring for the secret key corresponding
	// to one of the given Key IDs.  Returns the index and the key on success,
	// or -1 and nil on failure.
	LookupBoxSecretKey(kids [][]byte) (int, BoxSecretKey)

	// LookupBoxPublicKey returns a public key given the specified key ID.
	// For most cases, the key ID will be the key itself.
	LookupBoxPublicKey(kid []byte) BoxPublicKey

	// GetAllSecretKeys returns all keys, needed if we want to support
	// "hidden" receivers via trial and error
	GetAllSecretKeys() []BoxSecretKey

	// ImportEphemeralKey imports the ephemeral key into
	// BoxPublicKey format. This key has never been seen before, so
	// will be ephemeral.
	ImportEphemeralKey(kid []byte) BoxPublicKey
}

// SigKeyring is an interface used during verification to find
// the public key for the signer of a message.
type SigKeyring interface {
	// LookupSigningPublicKey returns a public signing key for the specified key ID.
	LookupSigningPublicKey(kid []byte) SigningPublicKey
}

// SecretKeyEqual returns true if the two secret keys are equal.
func SecretKeyEqual(sk1, sk2 BoxSecretKey) bool {
	return PublicKeyEqual(sk1.GetPublicKey(), sk2.GetPublicKey())
}

// PublicKeyEqual returns true if the two public keys are equal.
func PublicKeyEqual(pk1, pk2 BoxPublicKey) bool {
	return KIDEqual(pk1, pk2)
}

// KIDEqual return true if the KIDs for two keys are equal.
func KIDEqual(k1, k2 KIDExtractor) bool {
	return hmac.Equal(k1.ToKID(), k2.ToKID())
}
<commit_msg>Add comments on KIDExtractor<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package saltpack

import (
	"crypto/hmac"
)

// RawBoxKey is the raw byte-representation of what a box key should
// look like --- a static 32-byte buffer
type RawBoxKey [32]byte

// SymmetricKey is a template for a symmetric key, a 32-byte static
// buffer.  Used for both NaCl SecretBox.
type SymmetricKey [32]byte

// KIDExtractor key types can output a key ID corresponding to the
// key.
type KIDExtractor interface {
	// ToKID outputs the "key ID" that corresponds to this key.
	// You can do whatever you'd like here, but probably it makes sense just
	// to output the public key as is.
	ToKID() []byte
}

// BoxPublicKey is an generic interface to NaCl's public key Box function.
type BoxPublicKey interface {
	KIDExtractor

	// ToRawBoxKeyPointer returns this public key as a *[32]byte,
	// for use with nacl.box.Seal
	ToRawBoxKeyPointer() *RawBoxKey

	// CreateEmphemeralKey creates an ephemeral key of the same type,
	// but totally random.
	CreateEphemeralKey() (BoxSecretKey, error)

	// HideIdentity returns true if we should hide the identity of this
	// key in our output message format.
	HideIdentity() bool
}

// BoxPrecomputedSharedKey results from a Precomputation below.
type BoxPrecomputedSharedKey interface {
	Unbox(nonce *Nonce, msg []byte) ([]byte, error)
	Box(nonce *Nonce, msg []byte) ([]byte, error)
}

// BoxSecretKey is the secret key corresponding to a BoxPublicKey
type BoxSecretKey interface {

	// Box boxes up data, sent from this secret key, and to the receiver
	// specified.
	Box(receiver BoxPublicKey, nonce *Nonce, msg []byte) ([]byte, error)

	// Unobx opens up the box, using this secret key as the receiver key
	// abd the give public key as the sender key.
	Unbox(sender BoxPublicKey, nonce *Nonce, msg []byte) ([]byte, error)

	// GetPublicKey gets the public key associated with this secret key.
	GetPublicKey() BoxPublicKey

	// Precompute computes a DH with the given key
	Precompute(sender BoxPublicKey) BoxPrecomputedSharedKey
}

// SigningSecretKey is a secret NaCl key that can sign messages.
type SigningSecretKey interface {
	// Sign signs message with this secret key.
	Sign(message []byte) ([]byte, error)

	// PublicKey gets the public key associated with this secret key.
	PublicKey() SigningPublicKey
}

// SigningPublicKey is a public NaCl key that can verify
// signatures.
type SigningPublicKey interface {
	KIDExtractor

	// Verify verifies that signature is a valid signature of message for
	// this public key.
	Verify(message []byte, signature []byte) error
}

// Keyring is an interface used with decryption; it is called to
// recover public or private keys during the decryption process.
// Calls can block on network action.
type Keyring interface {
	// LookupBoxSecretKey looks in the Keyring for the secret key corresponding
	// to one of the given Key IDs.  Returns the index and the key on success,
	// or -1 and nil on failure.
	LookupBoxSecretKey(kids [][]byte) (int, BoxSecretKey)

	// LookupBoxPublicKey returns a public key given the specified key ID.
	// For most cases, the key ID will be the key itself.
	LookupBoxPublicKey(kid []byte) BoxPublicKey

	// GetAllSecretKeys returns all keys, needed if we want to support
	// "hidden" receivers via trial and error
	GetAllSecretKeys() []BoxSecretKey

	// ImportEphemeralKey imports the ephemeral key into
	// BoxPublicKey format. This key has never been seen before, so
	// will be ephemeral.
	ImportEphemeralKey(kid []byte) BoxPublicKey
}

// SigKeyring is an interface used during verification to find
// the public key for the signer of a message.
type SigKeyring interface {
	// LookupSigningPublicKey returns a public signing key for the specified key ID.
	LookupSigningPublicKey(kid []byte) SigningPublicKey
}

// SecretKeyEqual returns true if the two secret keys are equal.
func SecretKeyEqual(sk1, sk2 BoxSecretKey) bool {
	return PublicKeyEqual(sk1.GetPublicKey(), sk2.GetPublicKey())
}

// PublicKeyEqual returns true if the two public keys are equal.
func PublicKeyEqual(pk1, pk2 BoxPublicKey) bool {
	return KIDEqual(pk1, pk2)
}

// KIDEqual return true if the KIDs for two keys are equal.
func KIDEqual(k1, k2 KIDExtractor) bool {
	return hmac.Equal(k1.ToKID(), k2.ToKID())
}
<|endoftext|>


<commit_before>package testing

import (
	"context"
	"testing"

	"github.com/libp2p/go-libp2p-core/metrics"
	"github.com/libp2p/go-libp2p-core/network"
	"github.com/libp2p/go-libp2p-core/peerstore"
	"github.com/libp2p/go-libp2p-testing/net"
	"github.com/libp2p/go-tcp-transport"

	csms "github.com/libp2p/go-conn-security-multistream"
	pstoremem "github.com/libp2p/go-libp2p-peerstore/pstoremem"
	secio "github.com/libp2p/go-libp2p-secio"
	tptu "github.com/libp2p/go-libp2p-transport-upgrader"
	yamux "github.com/libp2p/go-libp2p-yamux"
	msmux "github.com/libp2p/go-stream-muxer-multistream"

	swarm "github.com/libp2p/go-libp2p-swarm"
)

type config struct {
	disableReuseport bool
	dialOnly         bool
}

// Option is an option that can be passed when constructing a test swarm.
type Option func(*testing.T, *config)

// OptDisableReuseport disables reuseport in this test swarm.
var OptDisableReuseport Option = func(_ *testing.T, c *config) {
	c.disableReuseport = true
}

// OptDialOnly prevents the test swarm from listening.
var OptDialOnly Option = func(_ *testing.T, c *config) {
	c.dialOnly = true
}

// GenUpgrader creates a new connection upgrader for use with this swarm.
func GenUpgrader(n *swarm.Swarm) *tptu.Upgrader {
	id := n.LocalPeer()
	pk := n.Peerstore().PrivKey(id)
	secMuxer := new(csms.SSMuxer)
	secMuxer.AddTransport(secio.ID, &secio.Transport{
		LocalID:    id,
		PrivateKey: pk,
	})

	stMuxer := msmux.NewBlankTransport()
	stMuxer.AddTransport("/yamux/1.0.0", yamux.DefaultTransport)

	return &tptu.Upgrader{
		Secure:  secMuxer,
		Muxer:   stMuxer,
		Filters: n.Filters,
	}

}

// GenSwarm generates a new test swarm.
func GenSwarm(t *testing.T, ctx context.Context, opts ...Option) *swarm.Swarm {
	var cfg config
	for _, o := range opts {
		o(t, &cfg)
	}

	p := tnet.RandPeerNetParamsOrFatal(t)

	ps := pstoremem.NewPeerstore()
	ps.AddPubKey(p.ID, p.PubKey)
	ps.AddPrivKey(p.ID, p.PrivKey)
	s := swarm.NewSwarm(ctx, p.ID, ps, metrics.NewBandwidthCounter())

	tcpTransport := tcp.NewTCPTransport(GenUpgrader(s))
	tcpTransport.DisableReuseport = cfg.disableReuseport

	if err := s.AddTransport(tcpTransport); err != nil {
		t.Fatal(err)
	}

	if !cfg.dialOnly {
		if err := s.Listen(p.Addr); err != nil {
			t.Fatal(err)
		}

		s.Peerstore().AddAddrs(p.ID, s.ListenAddresses(), peerstore.PermanentAddrTTL)
	}

	return s
}

// DivulgeAddresses adds swarm a's addresses to swarm b's peerstore.
func DivulgeAddresses(a, b network.Network) {
	id := a.LocalPeer()
	addrs := a.Peerstore().Addrs(id)
	b.Peerstore().AddAddrs(id, addrs, peerstore.PermanentAddrTTL)
}
<commit_msg>test: close peerstore when closing the test swarm<commit_after>package testing

import (
	"context"
	"testing"

	"github.com/libp2p/go-libp2p-core/metrics"
	"github.com/libp2p/go-libp2p-core/network"
	"github.com/libp2p/go-libp2p-core/peerstore"
	"github.com/libp2p/go-libp2p-testing/net"
	"github.com/libp2p/go-tcp-transport"

	goprocess "github.com/jbenet/goprocess"
	csms "github.com/libp2p/go-conn-security-multistream"
	pstoremem "github.com/libp2p/go-libp2p-peerstore/pstoremem"
	secio "github.com/libp2p/go-libp2p-secio"
	tptu "github.com/libp2p/go-libp2p-transport-upgrader"
	yamux "github.com/libp2p/go-libp2p-yamux"
	msmux "github.com/libp2p/go-stream-muxer-multistream"

	swarm "github.com/libp2p/go-libp2p-swarm"
)

type config struct {
	disableReuseport bool
	dialOnly         bool
}

// Option is an option that can be passed when constructing a test swarm.
type Option func(*testing.T, *config)

// OptDisableReuseport disables reuseport in this test swarm.
var OptDisableReuseport Option = func(_ *testing.T, c *config) {
	c.disableReuseport = true
}

// OptDialOnly prevents the test swarm from listening.
var OptDialOnly Option = func(_ *testing.T, c *config) {
	c.dialOnly = true
}

// GenUpgrader creates a new connection upgrader for use with this swarm.
func GenUpgrader(n *swarm.Swarm) *tptu.Upgrader {
	id := n.LocalPeer()
	pk := n.Peerstore().PrivKey(id)
	secMuxer := new(csms.SSMuxer)
	secMuxer.AddTransport(secio.ID, &secio.Transport{
		LocalID:    id,
		PrivateKey: pk,
	})

	stMuxer := msmux.NewBlankTransport()
	stMuxer.AddTransport("/yamux/1.0.0", yamux.DefaultTransport)

	return &tptu.Upgrader{
		Secure:  secMuxer,
		Muxer:   stMuxer,
		Filters: n.Filters,
	}

}

// GenSwarm generates a new test swarm.
func GenSwarm(t *testing.T, ctx context.Context, opts ...Option) *swarm.Swarm {
	var cfg config
	for _, o := range opts {
		o(t, &cfg)
	}

	p := tnet.RandPeerNetParamsOrFatal(t)

	ps := pstoremem.NewPeerstore()
	ps.AddPubKey(p.ID, p.PubKey)
	ps.AddPrivKey(p.ID, p.PrivKey)
	s := swarm.NewSwarm(ctx, p.ID, ps, metrics.NewBandwidthCounter())
	s.Process().AddChild(goprocess.WithTeardown(ps.Close))

	tcpTransport := tcp.NewTCPTransport(GenUpgrader(s))
	tcpTransport.DisableReuseport = cfg.disableReuseport

	if err := s.AddTransport(tcpTransport); err != nil {
		t.Fatal(err)
	}

	if !cfg.dialOnly {
		if err := s.Listen(p.Addr); err != nil {
			t.Fatal(err)
		}

		s.Peerstore().AddAddrs(p.ID, s.ListenAddresses(), peerstore.PermanentAddrTTL)
	}

	return s
}

// DivulgeAddresses adds swarm a's addresses to swarm b's peerstore.
func DivulgeAddresses(a, b network.Network) {
	id := a.LocalPeer()
	addrs := a.Peerstore().Addrs(id)
	b.Peerstore().AddAddrs(id, addrs, peerstore.PermanentAddrTTL)
}
<|endoftext|>


<commit_before>package identify

import (
	"context"
	"sync"
	"time"

	net "github.com/libp2p/go-libp2p-net"
	pstore "github.com/libp2p/go-libp2p-peerstore"
	ma "github.com/multiformats/go-multiaddr"
)

const ActivationThresh = 4

var GCInterval = 10 * time.Minute

type observation struct {
	seenTime      time.Time
	connDirection net.Direction
}

// ObservedAddr is an entry for an address reported by our peers.
// We only use addresses that:
// - have been observed at least 4 times in last 1h. (counter symmetric nats)
// - have been observed at least once recently (1h), because our position in the
//   network, or network port mapppings, may have changed.
type ObservedAddr struct {
	Addr     ma.Multiaddr
	SeenBy   map[string]observation // peer(observer) address -> observation info
	LastSeen time.Time
}

func (oa *ObservedAddr) activated(ttl time.Duration) bool {
	// cleanup SeenBy set
	now := time.Now()

	for k, ob := range oa.SeenBy {
		if now.Sub(ob.seenTime) > ttl*ActivationThresh {
			delete(oa.SeenBy, k)
		}
	}

	// We only activate if in the TTL other peers observed the same address
	// of ours at least 4 times.
	return len(oa.SeenBy) >= ActivationThresh
}

type newObservation struct {
	observed, local, observer ma.Multiaddr
	direction                 net.Direction
}

// ObservedAddrSet keeps track of a set of ObservedAddrs
// the zero-value is ready to be used.
type ObservedAddrSet struct {
	sync.RWMutex // guards whole datastruct.

	// local(internal) address -> list of observed(external) addresses
	addrs map[string][]*ObservedAddr
	ttl   time.Duration

	// this is the worker channel
	wch chan newObservation
}

func NewObservedAddrSet(ctx context.Context) *ObservedAddrSet {
	oas := &ObservedAddrSet{
		addrs: make(map[string][]*ObservedAddr),
		ttl:   pstore.OwnObservedAddrTTL,
		wch:   make(chan newObservation, 1),
	}
	go oas.worker(ctx)
	return oas
}

// AddrsFor return all activated observed addresses associated with the given
// (resolved) listen address.
func (oas *ObservedAddrSet) AddrsFor(addr ma.Multiaddr) (addrs []ma.Multiaddr) {
	oas.RLock()
	defer oas.RUnlock()

	if len(oas.addrs) == 0 {
		return nil
	}

	key := string(addr.Bytes())
	observedAddrs, ok := oas.addrs[key]
	if !ok {
		return
	}

	now := time.Now()
	for _, a := range observedAddrs {
		if now.Sub(a.LastSeen) <= oas.ttl && a.activated(oas.ttl) {
			addrs = append(addrs, a.Addr)
		}
	}

	return addrs
}

// Addrs return all activated observed addresses
func (oas *ObservedAddrSet) Addrs() (addrs []ma.Multiaddr) {
	oas.RLock()
	defer oas.RUnlock()

	if len(oas.addrs) == 0 {
		return nil
	}

	now := time.Now()
	for _, observedAddrs := range oas.addrs {
		for _, a := range observedAddrs {
			if now.Sub(a.LastSeen) <= oas.ttl && a.activated(oas.ttl) {
				addrs = append(addrs, a.Addr)
			}
		}
	}
	return addrs
}

func (oas *ObservedAddrSet) Add(observed, local, observer ma.Multiaddr,
	direction net.Direction) {
	select {
	case oas.wch <- newObservation{observed: observed, local: local, observer: observer, direction: direction}:
	default:
		log.Debugf("dropping address observation of %s; buffer full", observed)
	}
}

func (oas *ObservedAddrSet) worker(ctx context.Context) {
	ticker := time.NewTicker(GCInterval)
	defer ticker.Stop()

	for {
		select {
		case obs := <-oas.wch:
			oas.doAdd(obs.observed, obs.local, obs.observer, obs.direction)

		case <-ticker.C:
			oas.gc()

		case <-ctx.Done():
			return
		}
	}
}

func (oas *ObservedAddrSet) gc() {
	oas.Lock()
	defer oas.Unlock()

	now := time.Now()
	for local, observedAddrs := range oas.addrs {
		// TODO we can do this without allocating by compacting the array in place
		filteredAddrs := make([]*ObservedAddr, 0, len(observedAddrs))
		for _, a := range observedAddrs {
			// leave only alive observed addresses
			if now.Sub(a.LastSeen) <= oas.ttl {
				filteredAddrs = append(filteredAddrs, a)
			}
		}
		if len(filteredAddrs) > 0 {
			oas.addrs[local] = filteredAddrs
		} else {
			delete(oas.addrs, local)
		}
	}
}

func (oas *ObservedAddrSet) doAdd(observed, local, observer ma.Multiaddr,
	direction net.Direction) {

	now := time.Now()
	observerString := observerGroup(observer)
	localString := string(local.Bytes())
	ob := observation{
		seenTime:      now,
		connDirection: direction,
	}

	oas.Lock()
	defer oas.Unlock()

	observedAddrs := oas.addrs[localString]
	// check if observed address seen yet, if so, update it
	for i, previousObserved := range observedAddrs {
		if previousObserved.Addr.Equal(observed) {
			observedAddrs[i].SeenBy[observerString] = ob
			observedAddrs[i].LastSeen = now
			return
		}
	}
	// observed address not seen yet, append it
	oas.addrs[localString] = append(oas.addrs[localString], &ObservedAddr{
		Addr: observed,
		SeenBy: map[string]observation{
			observerString: ob,
		},
		LastSeen: now,
	})
}

// observerGroup is a function that determines what part of
// a multiaddr counts as a different observer. for example,
// two ipfs nodes at the same IP/TCP transport would get
// the exact same NAT mapping; they would count as the
// same observer. This may protect against NATs who assign
// different ports to addresses at different IP hosts, but
// not TCP ports.
//
// Here, we use the root multiaddr address. This is mostly
// IP addresses. In practice, this is what we want.
func observerGroup(m ma.Multiaddr) string {
	//TODO: If IPv6 rolls out we should mark /64 routing zones as one group
	first, _ := ma.SplitFirst(m)
	return string(first.Bytes())
}

func (oas *ObservedAddrSet) SetTTL(ttl time.Duration) {
	oas.Lock()
	defer oas.Unlock()
	oas.ttl = ttl
}

func (oas *ObservedAddrSet) TTL() time.Duration {
	oas.RLock()
	defer oas.RUnlock()
	return oas.ttl
}
<commit_msg>raise activation channel capacity to 16<commit_after>package identify

import (
	"context"
	"sync"
	"time"

	net "github.com/libp2p/go-libp2p-net"
	pstore "github.com/libp2p/go-libp2p-peerstore"
	ma "github.com/multiformats/go-multiaddr"
)

const ActivationThresh = 4

var GCInterval = 10 * time.Minute

type observation struct {
	seenTime      time.Time
	connDirection net.Direction
}

// ObservedAddr is an entry for an address reported by our peers.
// We only use addresses that:
// - have been observed at least 4 times in last 1h. (counter symmetric nats)
// - have been observed at least once recently (1h), because our position in the
//   network, or network port mapppings, may have changed.
type ObservedAddr struct {
	Addr     ma.Multiaddr
	SeenBy   map[string]observation // peer(observer) address -> observation info
	LastSeen time.Time
}

func (oa *ObservedAddr) activated(ttl time.Duration) bool {
	// cleanup SeenBy set
	now := time.Now()

	for k, ob := range oa.SeenBy {
		if now.Sub(ob.seenTime) > ttl*ActivationThresh {
			delete(oa.SeenBy, k)
		}
	}

	// We only activate if in the TTL other peers observed the same address
	// of ours at least 4 times.
	return len(oa.SeenBy) >= ActivationThresh
}

type newObservation struct {
	observed, local, observer ma.Multiaddr
	direction                 net.Direction
}

// ObservedAddrSet keeps track of a set of ObservedAddrs
// the zero-value is ready to be used.
type ObservedAddrSet struct {
	sync.RWMutex // guards whole datastruct.

	// local(internal) address -> list of observed(external) addresses
	addrs map[string][]*ObservedAddr
	ttl   time.Duration

	// this is the worker channel
	wch chan newObservation
}

func NewObservedAddrSet(ctx context.Context) *ObservedAddrSet {
	oas := &ObservedAddrSet{
		addrs: make(map[string][]*ObservedAddr),
		ttl:   pstore.OwnObservedAddrTTL,
		wch:   make(chan newObservation, 16),
	}
	go oas.worker(ctx)
	return oas
}

// AddrsFor return all activated observed addresses associated with the given
// (resolved) listen address.
func (oas *ObservedAddrSet) AddrsFor(addr ma.Multiaddr) (addrs []ma.Multiaddr) {
	oas.RLock()
	defer oas.RUnlock()

	if len(oas.addrs) == 0 {
		return nil
	}

	key := string(addr.Bytes())
	observedAddrs, ok := oas.addrs[key]
	if !ok {
		return
	}

	now := time.Now()
	for _, a := range observedAddrs {
		if now.Sub(a.LastSeen) <= oas.ttl && a.activated(oas.ttl) {
			addrs = append(addrs, a.Addr)
		}
	}

	return addrs
}

// Addrs return all activated observed addresses
func (oas *ObservedAddrSet) Addrs() (addrs []ma.Multiaddr) {
	oas.RLock()
	defer oas.RUnlock()

	if len(oas.addrs) == 0 {
		return nil
	}

	now := time.Now()
	for _, observedAddrs := range oas.addrs {
		for _, a := range observedAddrs {
			if now.Sub(a.LastSeen) <= oas.ttl && a.activated(oas.ttl) {
				addrs = append(addrs, a.Addr)
			}
		}
	}
	return addrs
}

func (oas *ObservedAddrSet) Add(observed, local, observer ma.Multiaddr,
	direction net.Direction) {
	select {
	case oas.wch <- newObservation{observed: observed, local: local, observer: observer, direction: direction}:
	default:
		log.Debugf("dropping address observation of %s; buffer full", observed)
	}
}

func (oas *ObservedAddrSet) worker(ctx context.Context) {
	ticker := time.NewTicker(GCInterval)
	defer ticker.Stop()

	for {
		select {
		case obs := <-oas.wch:
			oas.doAdd(obs.observed, obs.local, obs.observer, obs.direction)

		case <-ticker.C:
			oas.gc()

		case <-ctx.Done():
			return
		}
	}
}

func (oas *ObservedAddrSet) gc() {
	oas.Lock()
	defer oas.Unlock()

	now := time.Now()
	for local, observedAddrs := range oas.addrs {
		// TODO we can do this without allocating by compacting the array in place
		filteredAddrs := make([]*ObservedAddr, 0, len(observedAddrs))
		for _, a := range observedAddrs {
			// leave only alive observed addresses
			if now.Sub(a.LastSeen) <= oas.ttl {
				filteredAddrs = append(filteredAddrs, a)
			}
		}
		if len(filteredAddrs) > 0 {
			oas.addrs[local] = filteredAddrs
		} else {
			delete(oas.addrs, local)
		}
	}
}

func (oas *ObservedAddrSet) doAdd(observed, local, observer ma.Multiaddr,
	direction net.Direction) {

	now := time.Now()
	observerString := observerGroup(observer)
	localString := string(local.Bytes())
	ob := observation{
		seenTime:      now,
		connDirection: direction,
	}

	oas.Lock()
	defer oas.Unlock()

	observedAddrs := oas.addrs[localString]
	// check if observed address seen yet, if so, update it
	for i, previousObserved := range observedAddrs {
		if previousObserved.Addr.Equal(observed) {
			observedAddrs[i].SeenBy[observerString] = ob
			observedAddrs[i].LastSeen = now
			return
		}
	}
	// observed address not seen yet, append it
	oas.addrs[localString] = append(oas.addrs[localString], &ObservedAddr{
		Addr: observed,
		SeenBy: map[string]observation{
			observerString: ob,
		},
		LastSeen: now,
	})
}

// observerGroup is a function that determines what part of
// a multiaddr counts as a different observer. for example,
// two ipfs nodes at the same IP/TCP transport would get
// the exact same NAT mapping; they would count as the
// same observer. This may protect against NATs who assign
// different ports to addresses at different IP hosts, but
// not TCP ports.
//
// Here, we use the root multiaddr address. This is mostly
// IP addresses. In practice, this is what we want.
func observerGroup(m ma.Multiaddr) string {
	//TODO: If IPv6 rolls out we should mark /64 routing zones as one group
	first, _ := ma.SplitFirst(m)
	return string(first.Bytes())
}

func (oas *ObservedAddrSet) SetTTL(ttl time.Duration) {
	oas.Lock()
	defer oas.Unlock()
	oas.ttl = ttl
}

func (oas *ObservedAddrSet) TTL() time.Duration {
	oas.RLock()
	defer oas.RUnlock()
	return oas.ttl
}
<|endoftext|>


<commit_before>package network

import (
	"fmt"
	"errors"
	"github.com/yaricom/goNEAT/neat"
	"bytes"
)

// A NODE is either a NEURON or a SENSOR.
//   - If it's a sensor, it can be loaded with a value for output
//   - If it's a neuron, it has a list of its incoming input signals ([]*Link is used)
// Use an activation count to avoid flushing
type NNode struct {
	// The ID of the node
	Id                int

	// The type of node activation function (SIGMOID, ...)
	ActivationType    NodeActivationType
	// The neuron type for this node (HIDDEN, INPUT, OUTPUT, BIAS)
	NeuronType        NodeNeuronType

	// The node's activation value
	Activation        float64
	// The number of activations for current node
	ActivationsCount  int32
	// The activation sum
	ActivationSum     float64

	// The list of all incoming connections
	Incoming          []*Link
	// The list of all outgoing connections
	Outgoing          []*Link
	// The trait linked to the node
	Trait             *neat.Trait

	// Used for Gene decoding by referencing analogue to this node in organism phenotype
	PhenotypeAnalogue *NNode

	/* ************ LEARNING PARAMETERS *********** */
	// The following parameters are for use in neurons that learn through habituation,
	// sensitization, or Hebbian-type processes  */
	Params            []float64

	// Activation value of node at time t-1; Holds the previous step's activation for recurrency
	lastActivation    float64
	// Activation value of node at time t-2 Holds the activation before  the previous step's
	// This is necessary for a special recurrent case when the innode of a recurrent link is one time step ahead of the outnode.
	// The innode then needs to send from TWO time steps ago
	lastActivation2   float64

	// If true the node is active - used during node activation
	isActive          bool
}

// Creates new node with specified ID and neuron type associated (INPUT, HIDDEN, OUTPUT, BIAS)
func NewNNode(nodeid int, neuronType NodeNeuronType) *NNode {
	n := NewNetworkNode()
	n.Id = nodeid
	n.NeuronType = neuronType
	return n
}

// Construct a NNode off another NNode with given trait for genome purposes
func NewNNodeCopy(n *NNode, t *neat.Trait) *NNode {
	node := NewNetworkNode()
	node.Id = n.Id
	node.NeuronType = n.NeuronType
	node.ActivationType = n.ActivationType
	node.Trait = t
	return node
}

// The default constructor
func NewNetworkNode() *NNode {
	return &NNode{
		NeuronType:HiddenNeuron,
		ActivationType:SigmoidSteepenedActivation,
		Incoming:make([]*Link, 0),
		Outgoing:make([]*Link, 0),
	}
}

// Set new activation value to this node
func (n *NNode) setActivation(input float64) {
	// Keep a memory of activations for potential time delayed connections
	n.saveActivations()
	// Set new activation value
	n.Activation = input
	// Increment the activation_count
	n.ActivationsCount++
}

// Saves current node's activations for potential time delayed connections
func (n *NNode) saveActivations() {
	n.lastActivation2 = n.lastActivation
	n.lastActivation = n.Activation
}

// Returns activation for a current step
func (n *NNode) GetActiveOut() float64 {
	if n.ActivationsCount > 0 {
		return n.Activation
	} else {
		return 0.0
	}
}

// Returns activation from PREVIOUS time step
func (n *NNode) GetActiveOutTd() float64 {
	if n.ActivationsCount > 1 {
		return n.lastActivation
	} else {
		return 0.0
	}
}

// Returns true if this node is SENSOR
func (n *NNode) IsSensor() bool {
	return n.NeuronType == InputNeuron || n.NeuronType == BiasNeuron
}

// returns true if this node is NEURON
func (n *NNode) IsNeuron() bool {
	return n.NeuronType == HiddenNeuron || n.NeuronType == OutputNeuron
}

// If the node is a SENSOR, returns TRUE and loads the value
func (n *NNode) SensorLoad(load float64) bool {
	if n.IsSensor() {
		// Keep a memory of activations for potential time delayed connections
		n.saveActivations()
		// Puts sensor into next time-step
		n.ActivationsCount++
		n.Activation = load
		return true
	} else {
		return false
	}
}

// Adds a non recurrent outgoing link to this node
func (n *NNode) addOutgoing(out *NNode, weight float64) {
	newLink := NewLink(weight, n, out, false)
	n.Outgoing = append(n.Outgoing, newLink)
}

// Adds a NONRECURRENT Link to an incoming NNode in the incoming List
func (n *NNode) addIncoming(in *NNode, weight float64) {
	newLink := NewLink(weight, in, n, false)
	n.Incoming = append(n.Incoming, newLink)
}

// Recursively deactivate backwards through the network
func (n *NNode) Flushback() {
	n.ActivationsCount = 0
	n.Activation = 0
	n.lastActivation = 0
	n.lastActivation2 = 0
}

// Verify flushing for debuging
func (n *NNode) FlushbackCheck() error {
	if n.ActivationsCount > 0 {
		return errors.New(fmt.Sprintf("NNODE: %s has activation count %d", n, n.ActivationsCount))
	}
	if n.Activation > 0 {
		return errors.New(fmt.Sprintf("NNODE: %s has activation %f", n, n.Activation))
	}
	if n.lastActivation > 0 {
		return errors.New(fmt.Sprintf("NNODE: %s has last_activation %f", n, n.lastActivation))
	}
	if n.lastActivation2 > 0 {
		return errors.New(fmt.Sprintf("NNODE: %s has last_activation2 %f", n, n.lastActivation2))
	}
	return nil
}

// Find the greatest depth starting from this neuron at depth d
func (n *NNode) Depth(d int) (int, error) {
	if d > 100 {
		return 10, errors.New("NNode: Depth can not be determined for network with loop");
	}
	// Base Case
	if n.IsSensor() {
		return d, nil
	} else {
		// recursion
		max := d // The max depth
		for _, l := range n.Incoming {
			cur_depth, err := l.InNode.Depth(d + 1)
			if err != nil {
				return cur_depth, err
			}
			if cur_depth > max {
				max = cur_depth
			}
		}
		return max, nil
	}

}

// Convenient method to check network's node type (SENSOR, NEURON)
func (n *NNode) NodeType() NodeType {
	if n.IsSensor() {
		return SensorNode
	}
	return NeuronNode
}

func (n *NNode) String() string {
	activation, _ := NodeActivators.ActivationNameFromType(n.ActivationType)
	active := "active"
	if !n.isActive {
		active = "inactive"
	}
	return fmt.Sprintf("(%s id:%03d, %s, %s,\t%s -> step: %d = %.3f %.3f)",
		NodeTypeName(n.NodeType()), n.Id, NeuronTypeName(n.NeuronType), activation, active,
		n.ActivationsCount, n.Activation, n.Params)
}

// Prints all node's fields to the string
func (n *NNode) Print() string {
	str := "NNode fields\n"
	b := bytes.NewBufferString(str)
	fmt.Fprintf(b, "\tId: %d\n", n.Id)
	fmt.Fprintf(b, "\tIsActive: %t\n", n.isActive)
	fmt.Fprintf(b, "\tActivation: %f\n", n.Activation)
	activation, _ := NodeActivators.ActivationNameFromType(n.ActivationType)
	fmt.Fprintf(b, "\tActivation Type: %s\n", activation)
	fmt.Fprintf(b, "\tNeuronType: %d\n", n.NeuronType)
	fmt.Fprintf(b, "\tActivationsCount: %d\n", n.ActivationsCount)
	fmt.Fprintf(b, "\tActivationSum: %f\n", n.ActivationSum)
	fmt.Fprintf(b, "\tIncoming: %s\n", n.Incoming)
	fmt.Fprintf(b, "\tOutgoing: %s\n", n.Outgoing)
	fmt.Fprintf(b, "\tTrait: %s\n", n.Trait)
	fmt.Fprintf(b, "\tPhenotypeAnalogue: %s\n", n.PhenotypeAnalogue)
	fmt.Fprintf(b, "\tParams: %f\n", n.Params)
	fmt.Fprintf(b, "\tlastActivation: %f\n", n.lastActivation)
	fmt.Fprintf(b, "\tlastActivation2: %f\n", n.lastActivation2)

	return b.String()
}

<commit_msg>Added active status reset to flushback method<commit_after>package network

import (
	"fmt"
	"errors"
	"github.com/yaricom/goNEAT/neat"
	"bytes"
)

// A NODE is either a NEURON or a SENSOR.
//   - If it's a sensor, it can be loaded with a value for output
//   - If it's a neuron, it has a list of its incoming input signals ([]*Link is used)
// Use an activation count to avoid flushing
type NNode struct {
	// The ID of the node
	Id                int

	// The type of node activation function (SIGMOID, ...)
	ActivationType    NodeActivationType
	// The neuron type for this node (HIDDEN, INPUT, OUTPUT, BIAS)
	NeuronType        NodeNeuronType

	// The node's activation value
	Activation        float64
	// The number of activations for current node
	ActivationsCount  int32
	// The activation sum
	ActivationSum     float64

	// The list of all incoming connections
	Incoming          []*Link
	// The list of all outgoing connections
	Outgoing          []*Link
	// The trait linked to the node
	Trait             *neat.Trait

	// Used for Gene decoding by referencing analogue to this node in organism phenotype
	PhenotypeAnalogue *NNode

	/* ************ LEARNING PARAMETERS *********** */
	// The following parameters are for use in neurons that learn through habituation,
	// sensitization, or Hebbian-type processes  */
	Params            []float64

	// Activation value of node at time t-1; Holds the previous step's activation for recurrency
	lastActivation    float64
	// Activation value of node at time t-2 Holds the activation before  the previous step's
	// This is necessary for a special recurrent case when the innode of a recurrent link is one time step ahead of the outnode.
	// The innode then needs to send from TWO time steps ago
	lastActivation2   float64

	// If true the node is active - used during node activation
	isActive          bool
}

// Creates new node with specified ID and neuron type associated (INPUT, HIDDEN, OUTPUT, BIAS)
func NewNNode(nodeid int, neuronType NodeNeuronType) *NNode {
	n := NewNetworkNode()
	n.Id = nodeid
	n.NeuronType = neuronType
	return n
}

// Construct a NNode off another NNode with given trait for genome purposes
func NewNNodeCopy(n *NNode, t *neat.Trait) *NNode {
	node := NewNetworkNode()
	node.Id = n.Id
	node.NeuronType = n.NeuronType
	node.ActivationType = n.ActivationType
	node.Trait = t
	return node
}

// The default constructor
func NewNetworkNode() *NNode {
	return &NNode{
		NeuronType:HiddenNeuron,
		ActivationType:SigmoidSteepenedActivation,
		Incoming:make([]*Link, 0),
		Outgoing:make([]*Link, 0),
	}
}

// Set new activation value to this node
func (n *NNode) setActivation(input float64) {
	// Keep a memory of activations for potential time delayed connections
	n.saveActivations()
	// Set new activation value
	n.Activation = input
	// Increment the activation_count
	n.ActivationsCount++
}

// Saves current node's activations for potential time delayed connections
func (n *NNode) saveActivations() {
	n.lastActivation2 = n.lastActivation
	n.lastActivation = n.Activation
}

// Returns activation for a current step
func (n *NNode) GetActiveOut() float64 {
	if n.ActivationsCount > 0 {
		return n.Activation
	} else {
		return 0.0
	}
}

// Returns activation from PREVIOUS time step
func (n *NNode) GetActiveOutTd() float64 {
	if n.ActivationsCount > 1 {
		return n.lastActivation
	} else {
		return 0.0
	}
}

// Returns true if this node is SENSOR
func (n *NNode) IsSensor() bool {
	return n.NeuronType == InputNeuron || n.NeuronType == BiasNeuron
}

// returns true if this node is NEURON
func (n *NNode) IsNeuron() bool {
	return n.NeuronType == HiddenNeuron || n.NeuronType == OutputNeuron
}

// If the node is a SENSOR, returns TRUE and loads the value
func (n *NNode) SensorLoad(load float64) bool {
	if n.IsSensor() {
		// Keep a memory of activations for potential time delayed connections
		n.saveActivations()
		// Puts sensor into next time-step
		n.ActivationsCount++
		n.Activation = load
		return true
	} else {
		return false
	}
}

// Adds a non recurrent outgoing link to this node
func (n *NNode) addOutgoing(out *NNode, weight float64) {
	newLink := NewLink(weight, n, out, false)
	n.Outgoing = append(n.Outgoing, newLink)
}

// Adds a NONRECURRENT Link to an incoming NNode in the incoming List
func (n *NNode) addIncoming(in *NNode, weight float64) {
	newLink := NewLink(weight, in, n, false)
	n.Incoming = append(n.Incoming, newLink)
}

// Recursively deactivate backwards through the network
func (n *NNode) Flushback() {
	n.ActivationsCount = 0
	n.Activation = 0
	n.lastActivation = 0
	n.lastActivation2 = 0
	n.isActive = false
}

// Verify flushing for debuging
func (n *NNode) FlushbackCheck() error {
	if n.ActivationsCount > 0 {
		return errors.New(fmt.Sprintf("NNODE: %s has activation count %d", n, n.ActivationsCount))
	}
	if n.Activation > 0 {
		return errors.New(fmt.Sprintf("NNODE: %s has activation %f", n, n.Activation))
	}
	if n.lastActivation > 0 {
		return errors.New(fmt.Sprintf("NNODE: %s has last_activation %f", n, n.lastActivation))
	}
	if n.lastActivation2 > 0 {
		return errors.New(fmt.Sprintf("NNODE: %s has last_activation2 %f", n, n.lastActivation2))
	}
	return nil
}

// Find the greatest depth starting from this neuron at depth d
func (n *NNode) Depth(d int) (int, error) {
	if d > 100 {
		return 10, errors.New("NNode: Depth can not be determined for network with loop");
	}
	// Base Case
	if n.IsSensor() {
		return d, nil
	} else {
		// recursion
		max := d // The max depth
		for _, l := range n.Incoming {
			cur_depth, err := l.InNode.Depth(d + 1)
			if err != nil {
				return cur_depth, err
			}
			if cur_depth > max {
				max = cur_depth
			}
		}
		return max, nil
	}

}

// Convenient method to check network's node type (SENSOR, NEURON)
func (n *NNode) NodeType() NodeType {
	if n.IsSensor() {
		return SensorNode
	}
	return NeuronNode
}

func (n *NNode) String() string {
	activation, _ := NodeActivators.ActivationNameFromType(n.ActivationType)
	active := "active"
	if !n.isActive {
		active = "inactive"
	}
	return fmt.Sprintf("(%s id:%03d, %s, %s,\t%s -> step: %d = %.3f %.3f)",
		NodeTypeName(n.NodeType()), n.Id, NeuronTypeName(n.NeuronType), activation, active,
		n.ActivationsCount, n.Activation, n.Params)
}

// Prints all node's fields to the string
func (n *NNode) Print() string {
	str := "NNode fields\n"
	b := bytes.NewBufferString(str)
	fmt.Fprintf(b, "\tId: %d\n", n.Id)
	fmt.Fprintf(b, "\tIsActive: %t\n", n.isActive)
	fmt.Fprintf(b, "\tActivation: %f\n", n.Activation)
	activation, _ := NodeActivators.ActivationNameFromType(n.ActivationType)
	fmt.Fprintf(b, "\tActivation Type: %s\n", activation)
	fmt.Fprintf(b, "\tNeuronType: %d\n", n.NeuronType)
	fmt.Fprintf(b, "\tActivationsCount: %d\n", n.ActivationsCount)
	fmt.Fprintf(b, "\tActivationSum: %f\n", n.ActivationSum)
	fmt.Fprintf(b, "\tIncoming: %s\n", n.Incoming)
	fmt.Fprintf(b, "\tOutgoing: %s\n", n.Outgoing)
	fmt.Fprintf(b, "\tTrait: %s\n", n.Trait)
	fmt.Fprintf(b, "\tPhenotypeAnalogue: %s\n", n.PhenotypeAnalogue)
	fmt.Fprintf(b, "\tParams: %f\n", n.Params)
	fmt.Fprintf(b, "\tlastActivation: %f\n", n.lastActivation)
	fmt.Fprintf(b, "\tlastActivation2: %f\n", n.lastActivation2)

	return b.String()
}

<|endoftext|>


<commit_before>package rabbithole

import (
	"encoding/json"
	"net/http"
	"strconv"
)

type TimeUnit string

const (
	SECONDS TimeUnit = "seconds"
	DAYS    TimeUnit = "days"
	MONTHS  TimeUnit = "months"
	YEARS   TimeUnit = "years"
)

type Protocol string

const (
	AMQP091   Protocol = "amqp091"
	AMQP10    Protocol = "amqp10"
	MQTT      Protocol = "mqtt"
	STOMP     Protocol = "stomp"
	WEB_MQTT  Protocol = "web-mqtt"
	WEB_STOMP Protocol = "web-stomp"
)

type Check interface {
	// Returns true if the check is ok, otherwise false
	Ok() bool

	// Returns true if the check failed, otherwise false
	Failed() bool
}

// Health represents response from healthchecks endpoint
type Health struct {
	Check
	Status string `json:"status"`
	Reason string `json:"reason"`
}

func (h *Health) Ok() bool {
	return h.Status == "ok"
}

func (h *Health) Failed() bool {
	return !h.Ok()
}

// Responds a 200 OK if there are no alarms in effect in the cluster, otherwise responds with a 503 Service Unavailable.
func (c *Client) HealthCheckAlarms() (rec Health, err error) {
	err = executeCheck(c, "health/checks/alarms", &rec)
	return rec, err
}

// Responds a 200 OK if there are no local alarms in effect on the target node, otherwise responds with a 503 Service Unavailable.
func (c *Client) HealthCheckLocalAlarms() (rec Health, err error) {
	err = executeCheck(c, "health/checks/local-alarms", &rec)
	return rec, err
}

// Checks the expiration date on the certificates for every listener configured to use TLS.
// Responds a 200 OK if all certificates are valid (have not expired), otherwise responds with a 503 Service Unavailable.
// Valid units: days, weeks, months, years. The value of the within argument is the number of units.
// So, when within is 2 and unit is "months", the expiration period used by the check will be the next two months.
func (c *Client) HealthCheckCertificateExpiration(within uint, unit TimeUnit) (rec Health, err error) {
	err = executeCheck(c, "health/checks/certificate-expiration/"+strconv.Itoa(int(within))+"/"+string(unit), &rec)
	return rec, err
}

type PortListenerHealth struct {
	Check
	Status  string `json:"status"`
	Reason  string `json:"reason"`
	Missing string `json:"missing"'`
	Port    uint   `json:"port"`
	Ports   []uint `json:"ports"`
}

// Responds a 200 OK if there is an active listener on the give port, otherwise responds with a 503 Service Unavailable.
func (c *Client) HealthCheckPortListener(port uint) (rec PortListenerHealth, err error) {
	err = executeCheck(c, "health/checks/port-listener/"+strconv.Itoa(int(port)), &rec)
	return rec, err
}

type ProtocolListenerHealth struct {
	Check
	Status    string   `json:"status"`
	Reason    string   `json:"reason"`
	Missing   string   `json:"missing"`
	Protocols []string `json:"protocols"`
}

// Responds a 200 OK if there is an active listener for the given protocol, otherwise responds with a 503 Service Unavailable.
// Valid protocol names are: amqp091, amqp10, mqtt, stomp, web-mqtt, web-stomp.
func (c *Client) HealthCheckProtocolListener(protocol Protocol) (rec ProtocolListenerHealth, err error) {
	err = executeCheck(c, "health/checks/protocol-listener/"+string(protocol), &rec)
	return rec, err
}

// Responds a 200 OK if all virtual hosts and running on the target node, otherwise responds with a 503 Service Unavailable.
func (c *Client) HealthCheckVirtualHosts() (rec Health, err error) {
	err = executeCheck(c, "health/checks/virtual-hosts", &rec)
	return rec, err
}

// Checks if there are classic mirrored queues without synchronised mirrors online (queues that would potentially lose data if the target node is shut down).
// Responds a 200 OK if there are no such classic mirrored queues, otherwise responds with a 503 Service Unavailable.
func (c *Client) HealthCheckNodeIsMirrorSyncCritical() (rec Health, err error) {
	err = executeCheck(c, "health/checks/node-is-mirror-sync-critical", &rec)
	return rec, err
}

// Checks if there are quorum queues with minimum online quorum (queues that would lose their quorum and availability if the target node is shut down).
// Responds a 200 OK if there are no such quorum queues, otherwise responds with a 503 Service Unavailable.
func (c *Client) HealthCheckNodeIsQuorumCritical() (rec Health, err error) {
	err = executeCheck(c, "health/checks/node-is-quorum-critical", &rec)
	return rec, err
}

func executeCheck(client *Client, path string, rec interface{}) error {
	req, err := newGETRequest(client, path)
	httpc := &http.Client{
		Timeout: client.timeout,
	}
	if client.transport != nil {
		httpc.Transport = client.transport
	}
	resp, err := httpc.Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()

	if resp.StatusCode < http.StatusBadRequest || resp.StatusCode == http.StatusServiceUnavailable {
		if err = json.NewDecoder(resp.Body).Decode(&rec); err != nil {
			return err
		}

		return nil
	}

	if err = parseResponseErrors(resp); err != nil {
		return err
	}

	return nil
}
<commit_msg>update comments<commit_after>package rabbithole

import (
	"encoding/json"
	"net/http"
	"strconv"
)

type TimeUnit string

const (
	SECONDS TimeUnit = "seconds"
	DAYS    TimeUnit = "days"
	MONTHS  TimeUnit = "months"
	YEARS   TimeUnit = "years"
)

type Protocol string

const (
	AMQP091   Protocol = "amqp091"
	AMQP10    Protocol = "amqp10"
	MQTT      Protocol = "mqtt"
	STOMP     Protocol = "stomp"
	WEB_MQTT  Protocol = "web-mqtt"
	WEB_STOMP Protocol = "web-stomp"
)

type Check interface {
	// Returns true if the check is ok, otherwise false
	Ok() bool

	// Returns true if the check failed, otherwise false
	Failed() bool
}

// Represents general response from health check endpoints if no dedicated representation is defined
type Health struct {
	Check
	Status string `json:"status"`
	Reason string `json:"reason"`
}

func (h *Health) Ok() bool {
	return h.Status == "ok"
}

func (h *Health) Failed() bool {
	return !h.Ok()
}

// Checks if there are alarms in effect in the cluster
func (c *Client) HealthCheckAlarms() (rec Health, err error) {
	err = executeCheck(c, "health/checks/alarms", &rec)
	return rec, err
}

// Checks if there are local alarms in effect on the target node
func (c *Client) HealthCheckLocalAlarms() (rec Health, err error) {
	err = executeCheck(c, "health/checks/local-alarms", &rec)
	return rec, err
}

// Checks the expiration date on the certificates for every listener configured to use TLS.
// Valid units: days, weeks, months, years. The value of the within argument is the number of units.
// So, when within is 2 and unit is "months", the expiration period used by the check will be the next two months.
func (c *Client) HealthCheckCertificateExpiration(within uint, unit TimeUnit) (rec Health, err error) {
	err = executeCheck(c, "health/checks/certificate-expiration/"+strconv.Itoa(int(within))+"/"+string(unit), &rec)
	return rec, err
}

// Represents the response from HealthCheckPortListener
type PortListenerHealth struct {
	Check
	Status  string `json:"status"`
	Reason  string `json:"reason"`
	Missing string `json:"missing"`
	Port    uint   `json:"port"`
	Ports   []uint `json:"ports"`
}

// Checks if there is an active listener on the give port
func (c *Client) HealthCheckPortListener(port uint) (rec PortListenerHealth, err error) {
	err = executeCheck(c, "health/checks/port-listener/"+strconv.Itoa(int(port)), &rec)
	return rec, err
}

// Represents the response from HealthCheckProtocolListener
type ProtocolListenerHealth struct {
	Check
	Status    string   `json:"status"`
	Reason    string   `json:"reason"`
	Missing   string   `json:"missing"`
	Protocols []string `json:"protocols"`
}

// Checks if there is an active listener for the given protocol
// Valid protocol names are: amqp091, amqp10, mqtt, stomp, web-mqtt, web-stomp.
func (c *Client) HealthCheckProtocolListener(protocol Protocol) (rec ProtocolListenerHealth, err error) {
	err = executeCheck(c, "health/checks/protocol-listener/"+string(protocol), &rec)
	return rec, err
}

// Checks if all virtual hosts and running on the target node
func (c *Client) HealthCheckVirtualHosts() (rec Health, err error) {
	err = executeCheck(c, "health/checks/virtual-hosts", &rec)
	return rec, err
}

// Checks if there are classic mirrored queues without synchronised mirrors online (queues that would potentially lose data if the target node is shut down).
func (c *Client) HealthCheckNodeIsMirrorSyncCritical() (rec Health, err error) {
	err = executeCheck(c, "health/checks/node-is-mirror-sync-critical", &rec)
	return rec, err
}

// Checks if there are quorum queues with minimum online quorum (queues that would lose their quorum and availability if the target node is shut down).
func (c *Client) HealthCheckNodeIsQuorumCritical() (rec Health, err error) {
	err = executeCheck(c, "health/checks/node-is-quorum-critical", &rec)
	return rec, err
}

func executeCheck(client *Client, path string, rec interface{}) error {
	req, err := newGETRequest(client, path)
	httpc := &http.Client{
		Timeout: client.timeout,
	}
	if client.transport != nil {
		httpc.Transport = client.transport
	}
	resp, err := httpc.Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()

	if resp.StatusCode < http.StatusBadRequest || resp.StatusCode == http.StatusServiceUnavailable {
		if err = json.NewDecoder(resp.Body).Decode(&rec); err != nil {
			return err
		}

		return nil
	}

	if err = parseResponseErrors(resp); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package mocknet

import (
	"bytes"
	"io"
	"math/rand"
	"sync"
	"testing"

	inet "github.com/jbenet/go-ipfs/net"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
)

// func TestNetworkSetup(t *testing.T) {

// 	p1 := testutil.RandPeer()
// 	p2 := testutil.RandPeer()
// 	p3 := testutil.RandPeer()
// 	peers := []peer.Peer{p1, p2, p3}

// 	nets, err := MakeNetworks(context.Background(), peers)
// 	if err != nil {
// 		t.Fatal(err)
// 	}

// 	// check things

// 	if len(nets) != 3 {
// 		t.Error("nets must be 3")
// 	}

// 	for i, n := range nets {
// 		if n.local != peers[i] {
// 			t.Error("peer mismatch")
// 		}

// 		if len(n.conns) != len(nets) {
// 			t.Error("conn mismatch")
// 		}

// 		for _, c := range n.conns {
// 			if c.remote.conns[n.local] == nil {
// 				t.Error("conn other side fail")
// 			}
// 			if c.remote.conns[n.local].remote.local != n.local {
// 				t.Error("conn other side fail")
// 			}
// 		}

// 	}

// }

func TestStreams(t *testing.T) {

	mn, err := FullMeshConnected(context.Background(), 3)
	if err != nil {
		t.Fatal(err)
	}

	handler := func(s inet.Stream) {
		go func() {
			b := make([]byte, 4)
			if _, err := io.ReadFull(s, b); err != nil {
				panic(err)
			}
			if !bytes.Equal(b, []byte("beep")) {
				panic("bytes mismatch")
			}
			if _, err := s.Write([]byte("boop")); err != nil {
				panic(err)
			}
			s.Close()
		}()
	}

	nets := mn.Nets()
	for _, n := range nets {
		n.SetHandler(inet.ProtocolDHT, handler)
	}

	s, err := nets[0].NewStream(inet.ProtocolDHT, nets[1].LocalPeer())
	if err != nil {
		t.Fatal(err)
	}

	if _, err := s.Write([]byte("beep")); err != nil {
		panic(err)
	}
	b := make([]byte, 4)
	if _, err := io.ReadFull(s, b); err != nil {
		panic(err)
	}
	if !bytes.Equal(b, []byte("boop")) {
		panic("bytes mismatch 2")
	}

}

func makePinger(st string, n int) func(inet.Stream) {
	return func(s inet.Stream) {
		go func() {
			defer s.Close()

			for i := 0; i < n; i++ {
				b := make([]byte, 4+len(st))
				if _, err := s.Write([]byte("ping" + st)); err != nil {
					panic(err)
				}
				if _, err := io.ReadFull(s, b); err != nil {
					panic(err)
				}
				if !bytes.Equal(b, []byte("pong"+st)) {
					panic("bytes mismatch")
				}
			}
		}()
	}
}

func makePonger(st string) func(inet.Stream) {
	return func(s inet.Stream) {
		go func() {
			defer s.Close()

			for {
				b := make([]byte, 4+len(st))
				if _, err := io.ReadFull(s, b); err != nil {
					if err == io.EOF {
						return
					}
					panic(err)
				}
				if !bytes.Equal(b, []byte("ping"+st)) {
					panic("bytes mismatch")
				}
				if _, err := s.Write([]byte("pong" + st)); err != nil {
					panic(err)
				}
			}
		}()
	}
}

func TestStreamsStress(t *testing.T) {

	mn, err := FullMeshConnected(context.Background(), 100)
	if err != nil {
		t.Fatal(err)
	}

	protos := []inet.ProtocolID{
		inet.ProtocolDHT,
		inet.ProtocolBitswap,
		inet.ProtocolDiag,
	}

	nets := mn.Nets()
	for _, n := range nets {
		for _, p := range protos {
			n.SetHandler(p, makePonger(string(p)))
		}
	}

	var wg sync.WaitGroup
	for i := 0; i < 1000; i++ {
		wg.Add(1)
		go func(i int) {
			defer wg.Done()
			from := rand.Intn(len(nets))
			to := rand.Intn(len(nets))
			p := rand.Intn(3)
			proto := protos[p]
			// log.Debug("%d (%s) %d (%s) %d (%s)", from, nets[from], to, nets[to], p, protos[p])
			s, err := nets[from].NewStream(protos[p], nets[to].LocalPeer())
			if err != nil {
				panic(err)
			}

			log.Infof("%d start pinging", i)
			makePinger(string(proto), rand.Intn(100))(s)
			log.Infof("%d done pinging", i)
		}(i)
	}

	wg.Done()
}
<commit_msg>mocknet add test<commit_after>package mocknet

import (
	"bytes"
	"io"
	"math/rand"
	"sync"
	"testing"

	inet "github.com/jbenet/go-ipfs/net"
	peer "github.com/jbenet/go-ipfs/peer"
	testutil "github.com/jbenet/go-ipfs/util/testutil"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
)

// func TestNetworkSetup(t *testing.T) {

// 	p1 := testutil.RandPeer()
// 	p2 := testutil.RandPeer()
// 	p3 := testutil.RandPeer()
// 	peers := []peer.Peer{p1, p2, p3}

// 	nets, err := MakeNetworks(context.Background(), peers)
// 	if err != nil {
// 		t.Fatal(err)
// 	}

// 	// check things

// 	if len(nets) != 3 {
// 		t.Error("nets must be 3")
// 	}

// 	for i, n := range nets {
// 		if n.local != peers[i] {
// 			t.Error("peer mismatch")
// 		}

// 		if len(n.conns) != len(nets) {
// 			t.Error("conn mismatch")
// 		}

// 		for _, c := range n.conns {
// 			if c.remote.conns[n.local] == nil {
// 				t.Error("conn other side fail")
// 			}
// 			if c.remote.conns[n.local].remote.local != n.local {
// 				t.Error("conn other side fail")
// 			}
// 		}

// 	}

// }

func TestStreams(t *testing.T) {

	mn, err := FullMeshConnected(context.Background(), 3)
	if err != nil {
		t.Fatal(err)
	}

	handler := func(s inet.Stream) {
		go func() {
			b := make([]byte, 4)
			if _, err := io.ReadFull(s, b); err != nil {
				panic(err)
			}
			if !bytes.Equal(b, []byte("beep")) {
				panic("bytes mismatch")
			}
			if _, err := s.Write([]byte("boop")); err != nil {
				panic(err)
			}
			s.Close()
		}()
	}

	nets := mn.Nets()
	for _, n := range nets {
		n.SetHandler(inet.ProtocolDHT, handler)
	}

	s, err := nets[0].NewStream(inet.ProtocolDHT, nets[1].LocalPeer())
	if err != nil {
		t.Fatal(err)
	}

	if _, err := s.Write([]byte("beep")); err != nil {
		panic(err)
	}
	b := make([]byte, 4)
	if _, err := io.ReadFull(s, b); err != nil {
		panic(err)
	}
	if !bytes.Equal(b, []byte("boop")) {
		panic("bytes mismatch 2")
	}

}

func makePinger(st string, n int) func(inet.Stream) {
	return func(s inet.Stream) {
		go func() {
			defer s.Close()

			for i := 0; i < n; i++ {
				b := make([]byte, 4+len(st))
				if _, err := s.Write([]byte("ping" + st)); err != nil {
					panic(err)
				}
				if _, err := io.ReadFull(s, b); err != nil {
					panic(err)
				}
				if !bytes.Equal(b, []byte("pong"+st)) {
					panic("bytes mismatch")
				}
			}
		}()
	}
}

func makePonger(st string) func(inet.Stream) {
	return func(s inet.Stream) {
		go func() {
			defer s.Close()

			for {
				b := make([]byte, 4+len(st))
				if _, err := io.ReadFull(s, b); err != nil {
					if err == io.EOF {
						return
					}
					panic(err)
				}
				if !bytes.Equal(b, []byte("ping"+st)) {
					panic("bytes mismatch")
				}
				if _, err := s.Write([]byte("pong" + st)); err != nil {
					panic(err)
				}
			}
		}()
	}
}

func TestStreamsStress(t *testing.T) {

	mn, err := FullMeshConnected(context.Background(), 100)
	if err != nil {
		t.Fatal(err)
	}

	protos := []inet.ProtocolID{
		inet.ProtocolDHT,
		inet.ProtocolBitswap,
		inet.ProtocolDiag,
	}

	nets := mn.Nets()
	for _, n := range nets {
		for _, p := range protos {
			n.SetHandler(p, makePonger(string(p)))
		}
	}

	var wg sync.WaitGroup
	for i := 0; i < 1000; i++ {
		wg.Add(1)
		go func(i int) {
			defer wg.Done()
			from := rand.Intn(len(nets))
			to := rand.Intn(len(nets))
			p := rand.Intn(3)
			proto := protos[p]
			// log.Debug("%d (%s) %d (%s) %d (%s)", from, nets[from], to, nets[to], p, protos[p])
			s, err := nets[from].NewStream(protos[p], nets[to].LocalPeer())
			if err != nil {
				panic(err)
			}

			log.Infof("%d start pinging", i)
			makePinger(string(proto), rand.Intn(100))(s)
			log.Infof("%d done pinging", i)
		}(i)
	}

	wg.Done()
}

func TestAdding(t *testing.T) {

	mn := New(context.Background())

	p1 := testutil.RandPeer()
	p2 := testutil.RandPeer()
	p3 := testutil.RandPeer()
	peers := []peer.Peer{p1, p2, p3}

	for _, p := range peers {
		if _, err := mn.AddPeer(p.ID()); err != nil {
			t.Error(err)
		}
	}

	// link them
	for _, p1 := range peers {
		for _, p2 := range peers {
			if _, err := mn.LinkPeers(p1, p2); err != nil {
				t.Error(err)
			}
		}
	}

	// set the new stream handler on p2
	n2 := mn.Net(p2.ID())
	if n2 == nil {
		t.Fatalf("no network for %s", p2.ID())
	}
	n2.SetHandler(inet.ProtocolBitswap, func(s inet.Stream) {
		go func() {
			defer s.Close()

			b := make([]byte, 4)
			if _, err := io.ReadFull(s, b); err != nil {
				panic(err)
			}
			if string(b) != "beep" {
				panic("did not beep!")
			}

			if _, err := s.Write([]byte("boop")); err != nil {
				panic(err)
			}
		}()
	})

	// connect p1 to p2
	if err := mn.ConnectPeers(p1, p2); err != nil {
		t.Fatal(err)
	}

	// talk to p2
	n1 := mn.Net(p1.ID())
	if n1 == nil {
		t.Fatalf("no network for %s", p1.ID())
	}

	s, err := n1.NewStream(inet.ProtocolBitswap, p2)
	if err != nil {
		t.Fatal(err)
	}

	if _, err := s.Write([]byte("beep")); err != nil {
		t.Error(err)
	}
	b := make([]byte, 4)
	if _, err := io.ReadFull(s, b); err != nil {
		t.Error(err)
	}
	if !bytes.Equal(b, []byte("boop")) {
		t.Error("bytes mismatch 2")
	}

}
<|endoftext|>


<commit_before>// Command mealplanner is the main entry point of the application. It simply
// runs the *mux.Router provided by mphandlers.CreateMux() as an HTTP server.
package main

import (
	"flag"
	"fmt"
	"github.com/kierdavis/mealplanner/mpdb"
	"github.com/kierdavis/mealplanner/mphandlers"
	"github.com/kierdavis/mealplanner/mpresources"
	"log"
	"net/http"
	"os"

	_ "github.com/go-sql-driver/mysql"
)

var (
	dbSource    = flag.String("dbsource", "", "database source, in the form USER:PASS@unix(/PATH/TO/SOCKET)/DB or USER:PASS@tcp(HOST:PORT)/DB")
	host        = flag.String("host", "", "hostname to listen on")
	port        = flag.Int("port", 8080, "port to listen on")
	debug       = flag.Bool("debug", false, "debug mode")
	testdata    = flag.Bool("testdata", false, "clear the database and insert test data")
	resourceDir = flag.String("resourcedir", "", "path to directory containing the resources used by the application")
)

func main() {
	flag.Parse()

	source := *dbSource
	if source == "" {
		source = os.Getenv("MPDBSOURCE")
		if source == "" {
			fmt.Println("Please specify a non-empty -dbsource flag or set the MPDBSOURCE environment variable.")
			os.Exit(1)
		}
	}

	resDir := *resourceDir
	if resDir == "" {
		resDir = os.Getenv("MPRESDIR")
	}

	mpdb.DBSource = source
	mpresources.SetResourceDir(resDir)
	mpresources.GetTemplates() // Check that the templates load correctly

	err := mpdb.InitDB(*debug, *testdata)
	if err != nil {
		log.Printf("Database error during startup: %s\n", err)
		os.Exit(1)
	}

	listenAddr := fmt.Sprintf("%s:%d", *host, *port)
	m := mphandlers.CreateMux()

	app := http.Handler(m)
	if *debug {
		app = mphandlers.LoggingHandler{Handler: app}

		log.Printf("Listening on %s\n", listenAddr)
	}

	err = http.ListenAndServe(listenAddr, app)
	if err != nil {
		log.Printf("Server error in HTTP listener: %s\n", err)
		os.Exit(1)
	}
}
<commit_msg>Use port 80 by default instead of 8080<commit_after>// Command mealplanner is the main entry point of the application. It simply
// runs the *mux.Router provided by mphandlers.CreateMux() as an HTTP server.
package main

import (
	"flag"
	"fmt"
	"github.com/kierdavis/mealplanner/mpdb"
	"github.com/kierdavis/mealplanner/mphandlers"
	"github.com/kierdavis/mealplanner/mpresources"
	"log"
	"net/http"
	"os"

	_ "github.com/go-sql-driver/mysql"
)

var (
	dbSource    = flag.String("dbsource", "", "database source, in the form USER:PASS@unix(/PATH/TO/SOCKET)/DB or USER:PASS@tcp(HOST:PORT)/DB")
	host        = flag.String("host", "", "hostname to listen on")
	port        = flag.Int("port", 80, "port to listen on")
	debug       = flag.Bool("debug", false, "debug mode")
	testdata    = flag.Bool("testdata", false, "clear the database and insert test data")
	resourceDir = flag.String("resourcedir", "", "path to directory containing the resources used by the application")
)

func main() {
	flag.Parse()

	source := *dbSource
	if source == "" {
		source = os.Getenv("MPDBSOURCE")
		if source == "" {
			fmt.Println("Please specify a non-empty -dbsource flag or set the MPDBSOURCE environment variable.")
			os.Exit(1)
		}
	}

	resDir := *resourceDir
	if resDir == "" {
		resDir = os.Getenv("MPRESDIR")
	}

	mpdb.DBSource = source
	mpresources.SetResourceDir(resDir)
	mpresources.GetTemplates() // Check that the templates load correctly

	err := mpdb.InitDB(*debug, *testdata)
	if err != nil {
		log.Printf("Database error during startup: %s\n", err)
		os.Exit(1)
	}

	listenAddr := fmt.Sprintf("%s:%d", *host, *port)
	m := mphandlers.CreateMux()

	app := http.Handler(m)
	if *debug {
		app = mphandlers.LoggingHandler{Handler: app}

		log.Printf("Listening on %s\n", listenAddr)
	}

	err = http.ListenAndServe(listenAddr, app)
	if err != nil {
		log.Printf("Server error in HTTP listener: %s\n", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>//go:generate msgp
//msgp:ignore Parser
package turtle

// #cgo CFLAGS: -I ../raptor/src
// #cgo LDFLAGS: -lraptor2
// #include <stdio.h>
// #include <raptor2.h>
import "C"
import (
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"sync"
	"time"
)

var p *Parser

type Parser struct {
	dataset *DataSet
	sync.Mutex
}

type URI struct {
	Namespace string
	Value     string
}

func (u URI) String() string {
	if u.Namespace != "" {
		return u.Namespace + "#" + u.Value
	}
	return u.Value
}

func (u URI) Bytes() []byte {
	return []byte(u.Namespace + "#" + u.Value)
}

func (u URI) IsVariable() bool {
	return strings.HasPrefix(u.Value, "?")
}

func ParseURI(uri string) URI {
	uri = strings.TrimLeft(uri, "<")
	uri = strings.TrimRight(uri, ">")
	parts := strings.Split(uri, "#")
	parts[0] = strings.TrimRight(parts[0], "#")
	if len(parts) != 2 {
		// try to parse ":"
		parts = strings.SplitN(uri, ":", 2)
		if len(parts) > 1 {
			return URI{Namespace: parts[0], Value: parts[1]}
		}
		return URI{Value: uri}
	}
	return URI{Namespace: parts[0], Value: parts[1]}
}

type Triple struct {
	Subject   URI `msg:"s"`
	Predicate URI `msg:"p"`
	Object    URI `msg:"o"`
}

func MakeTriple(sub, pred, obj string) Triple {
	s := ParseURI(sub)
	p := ParseURI(pred)
	o := ParseURI(obj)
	return Triple{
		Subject:   s,
		Predicate: p,
		Object:    o,
	}
}

func init() {
	p = &Parser{}
}

//export transform
func transform(_subject, _predicate, _object *C.char, sub_len, pred_len, obj_len C.int) {
	subject := C.GoStringN(_subject, sub_len)
	predicate := C.GoStringN(_predicate, pred_len)
	object := C.GoStringN(_object, obj_len)
	p.dataset.addTriple(subject, predicate, object)
}

//export registerNamespace
func registerNamespace(_namespace, _prefix *C.char, ns_len, pfx_len C.int) {
	namespace := C.GoStringN(_namespace, ns_len)
	prefix := C.GoStringN(_prefix, pfx_len)
	p.dataset.addNamespace(prefix, namespace)
}

// Return Parser instance
func GetParser() *Parser {
	return p
}

// Parses the given filename using the turtle format.
// Returns the dataset, and the time elapsed in parsing
func (p *Parser) Parse(filename string) (DataSet, time.Duration) {
	p.Lock()
	defer p.Unlock()
	start := time.Now()
	p.dataset = newDataSet()
	p.parseFile(filename)
	took := time.Since(start)
	return *p.dataset, took
}

// Writes the contents of the reader to a temporary file, and then reads in that file
func (p *Parser) ParseReader(r io.Reader) (DataSet, time.Duration, error) {
	p.Lock()
	defer p.Unlock()
	start := time.Now()
	p.dataset = newDataSet()
	f, err := ioutil.TempFile(".", "_raptor")
	if err != nil {
		return *p.dataset, time.Since(start), err
	}
	defer func() {
		os.Remove(f.Name())
	}()
	n, err := io.Copy(f, r)
	if err != nil {
		return *p.dataset, time.Since(start), err
	}
	log.Printf("Wrote %d bytes", n)
	p.parseFile(f.Name())
	took := time.Since(start)
	return *p.dataset, took, nil
}
<commit_msg>Fix parsing of TTL "labels" to isolate the value<commit_after>//go:generate msgp
//msgp:ignore Parser
package turtle

// #cgo CFLAGS: -I ../raptor/src
// #cgo LDFLAGS: -lraptor2
// #include <stdio.h>
// #include <raptor2.h>
import "C"
import (
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"sync"
	"time"
)

var p *Parser

type Parser struct {
	dataset *DataSet
	sync.Mutex
}

type URI struct {
	Namespace string
	Value     string
}

func (u URI) String() string {
	if u.Namespace != "" {
		return u.Namespace + "#" + u.Value
	}
	return u.Value
}

func (u URI) Bytes() []byte {
	return []byte(u.Namespace + "#" + u.Value)
}

func (u URI) IsVariable() bool {
	return strings.HasPrefix(u.Value, "?")
}

func ParseURI(uri string) URI {
	uri = strings.TrimLeft(uri, "<")
	uri = strings.TrimRight(uri, ">")
	parts := strings.Split(uri, "#")
	parts[0] = strings.TrimRight(parts[0], "#")
	if len(parts) != 2 {
		// try to parse ":"
		parts = strings.SplitN(uri, ":", 2)
		if len(parts) > 1 {
			return URI{Namespace: parts[0], Value: parts[1]}
		}
		uri = strings.TrimLeft(uri, "\"")
		uri = strings.TrimRight(uri, "\"@en")
		return URI{Value: uri}
	}
	return URI{Namespace: parts[0], Value: parts[1]}
}

type Triple struct {
	Subject   URI `msg:"s"`
	Predicate URI `msg:"p"`
	Object    URI `msg:"o"`
}

func MakeTriple(sub, pred, obj string) Triple {
	s := ParseURI(sub)
	p := ParseURI(pred)
	o := ParseURI(obj)
	return Triple{
		Subject:   s,
		Predicate: p,
		Object:    o,
	}
}

func init() {
	p = &Parser{}
}

//export transform
func transform(_subject, _predicate, _object *C.char, sub_len, pred_len, obj_len C.int) {
	subject := C.GoStringN(_subject, sub_len)
	predicate := C.GoStringN(_predicate, pred_len)
	object := C.GoStringN(_object, obj_len)
	p.dataset.addTriple(subject, predicate, object)
}

//export registerNamespace
func registerNamespace(_namespace, _prefix *C.char, ns_len, pfx_len C.int) {
	namespace := C.GoStringN(_namespace, ns_len)
	prefix := C.GoStringN(_prefix, pfx_len)
	p.dataset.addNamespace(prefix, namespace)
}

// Return Parser instance
func GetParser() *Parser {
	return p
}

// Parses the given filename using the turtle format.
// Returns the dataset, and the time elapsed in parsing
func (p *Parser) Parse(filename string) (DataSet, time.Duration) {
	p.Lock()
	defer p.Unlock()
	start := time.Now()
	p.dataset = newDataSet()
	p.parseFile(filename)
	took := time.Since(start)
	return *p.dataset, took
}

// Writes the contents of the reader to a temporary file, and then reads in that file
func (p *Parser) ParseReader(r io.Reader) (DataSet, time.Duration, error) {
	p.Lock()
	defer p.Unlock()
	start := time.Now()
	p.dataset = newDataSet()
	f, err := ioutil.TempFile(".", "_raptor")
	if err != nil {
		return *p.dataset, time.Since(start), err
	}
	defer func() {
		os.Remove(f.Name())
	}()
	n, err := io.Copy(f, r)
	if err != nil {
		return *p.dataset, time.Since(start), err
	}
	log.Printf("Wrote %d bytes", n)
	p.parseFile(f.Name())
	took := time.Since(start)
	return *p.dataset, took, nil
}
<|endoftext|>