<commit_before>// Copyright 2015 Brett Vickers.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package ntp provides a simple mechanism for querying the current time from
// a remote NTP server. See RFC 5905. Approach inspired by go-nuts post by
// Michael Hofmann:
//
// https://groups.google.com/forum/?fromgroups#!topic/golang-nuts/FlcdMU5fkLQ
package ntp

import (
	"encoding/binary"
	"net"
	"time"
)

type mode uint8

const (
	reserved mode = 0 + iota
	symmetricActive
	symmetricPassive
	client
	server
	broadcast
	controlMessage
	reservedPrivate
)

const (
	// MaxStratum is the largest possible NTP stratum
	MaxStratum = 16

	// LeapNoWarning indicates no impending leap second
	LeapNoWarning = 0

	// LeapAddSecond indicates the last minute of the day has 61 seconds
	LeapAddSecond = 1

	// LeapDelSecond indicates the last minute of the day has 59 seconds
	LeapDelSecond = 2

	// LeapNotInSync indicates an unsynchronized leap second.
	LeapNotInSync = 3

	nanoPerSec = 1000000000
)

var (
	timeout  = 5 * time.Second
	ntpEpoch = time.Date(1900, 1, 1, 0, 0, 0, 0, time.UTC)
)

// An ntpTime is a 64-bit fixed-point (Q32.32) representation of the number of
// seconds elapsed since the NTP epoch.
type ntpTime uint64

// Duration interprets the fixed-point ntpTime as a number of elapsed seconds
// and returns the corresponding time.Duration value.
func (t ntpTime) Duration() time.Duration {
	sec := (t >> 32) * nanoPerSec
	frac := (t & 0xffffffff) * nanoPerSec >> 32
	return time.Duration(sec + frac)
}

// Time interprets the fixed-point ntpTime as a an absolute time and returns
// the corresponding time.Time value.
func (t ntpTime) Time() time.Time {
	return ntpEpoch.Add(t.Duration())
}

// toNtpTime converts the time.Time value t into its 64-bit fixed-point
// ntpTime representation.
func toNtpTime(t time.Time) ntpTime {
	nsec := uint64(t.Sub(ntpEpoch))
	sec := nsec / nanoPerSec
	frac := (nsec - sec*nanoPerSec) << 32 / nanoPerSec
	return ntpTime(sec<<32 | frac)
}

// An ntpTimeShort is a 32-bit fixed-point (Q16.16) representation of the
// number of seconds elapsed since the NTP epoch.
type ntpTimeShort uint32

// Duration interprets the fixed-point ntpTimeShort as a number of elapsed
// seconds and returns the corresponding time.Duration value.
func (t ntpTimeShort) Duration() time.Duration {
	t64 := uint64(t)
	sec := (t64 >> 16) * nanoPerSec
	frac := (t64 & 0xffff) * nanoPerSec >> 16
	return time.Duration(sec + frac)
}

// msg is an internal representation of an NTP packet.
type msg struct {
	LiVnMode       uint8 // Leap Indicator (2) + Version (3) + Mode (3)
	Stratum        uint8
	Poll           int8
	Precision      int8
	RootDelay      ntpTimeShort
	RootDispersion ntpTimeShort
	ReferenceID    uint32
	ReferenceTime  ntpTime
	OriginTime     ntpTime
	ReceiveTime    ntpTime
	TransmitTime   ntpTime
}

// setVersion sets the NTP protocol version on the message.
func (m *msg) setVersion(v int) {
	m.LiVnMode = (m.LiVnMode & 0xc7) | uint8(v)<<3
}

// setMode sets the NTP protocol mode on the message.
func (m *msg) setMode(md mode) {
	m.LiVnMode = (m.LiVnMode & 0xf8) | uint8(md)
}

// A Response contains time data, some of which is returned by the NTP server
// and some of which is calculated by the client.
type Response struct {
	Time           time.Time     // receive time reported by the server
	RTT            time.Duration // round-trip time between client and server
	ClockOffset    time.Duration // local clock offset relative to server
	Poll           time.Duration // maximum polling interval
	Precision      time.Duration // precision of server's system clock
	Stratum        uint8         // stratum level of NTP server's clock
	ReferenceID    uint32        // server's reference ID
	ReferenceTime  time.Time     // server's time of last clock update
	RootDelay      time.Duration // server's RTT to the reference clock
	RootDispersion time.Duration // server's dispersion to the reference clock
	Leap           uint8         // server's leap second indicator; see RFC 5905
}

// Query returns the current time from the remote server host using the
// requested version of the NTP protocol. It also returns additional
// information about the exchanged time information. The version may be 2, 3,
// or 4; although 4 is most typically used.
func Query(host string, version int) (*Response, error) {
	m, err := getTime(host, version)
	now := toNtpTime(time.Now())
	if err != nil {
		return nil, err
	}

	r := &Response{
		Time:           m.ReceiveTime.Time(),
		RTT:            rtt(m.OriginTime, m.ReceiveTime, m.TransmitTime, now),
		ClockOffset:    offset(m.OriginTime, m.ReceiveTime, m.TransmitTime, now),
		Poll:           toInterval(m.Poll),
		Precision:      toInterval(m.Precision),
		Stratum:        m.Stratum,
		ReferenceID:    m.ReferenceID,
		ReferenceTime:  m.ReferenceTime.Time(),
		RootDelay:      m.RootDelay.Duration(),
		RootDispersion: m.RootDispersion.Duration(),
		Leap:           (m.LiVnMode >> 6) & 0x03,
	}

	// https://tools.ietf.org/html/rfc5905#section-7.3
	if r.Stratum == 0 {
		r.Stratum = MaxStratum
	}

	return r, nil
}

// getTime returns the "receive time" from the remote NTP server host.
func getTime(host string, version int) (*msg, error) {
	if version < 2 || version > 4 {
		panic("ntp: invalid version number")
	}

	raddr, err := net.ResolveUDPAddr("udp", host+":123")
	if err != nil {
		return nil, err
	}

	con, err := net.DialUDP("udp", nil, raddr)
	if err != nil {
		return nil, err
	}
	defer con.Close()
	con.SetDeadline(time.Now().Add(timeout))

	m := new(msg)
	m.setMode(client)
	m.setVersion(version)
	m.TransmitTime = toNtpTime(time.Now())

	err = binary.Write(con, binary.BigEndian, m)
	if err != nil {
		return nil, err
	}

	err = binary.Read(con, binary.BigEndian, m)
	if err != nil {
		return nil, err
	}

	return m, nil
}

// TimeV returns the current time from the remote server host using the
// requested version of the NTP protocol. The version may be 2, 3, or 4;
// although 4 is most typically used.
func TimeV(host string, version int) (time.Time, error) {
	m, err := getTime(host, version)
	if err != nil {
		return time.Now(), err
	}
	return m.ReceiveTime.Time().Local(), nil
}

// Time returns the current time from the remote server host using version 4
// of the NTP protocol.
func Time(host string) (time.Time, error) {
	return TimeV(host, 4)
}

func rtt(t1, t2, t3, t4 ntpTime) time.Duration {
	// round trip delay time (https://tools.ietf.org/html/rfc5905#section-8)
	//   T1 = client send time
	//   T2 = server receive time
	//   T3 = server reply time
	//   T4 = client receive time
	//
	// RTT d:
	//   d = (T4-T1) - (T3-T2)
	a := t4.Time().Sub(t1.Time())
	b := t3.Time().Sub(t2.Time())
	return a - b
}

func offset(t1, t2, t3, t4 ntpTime) time.Duration {
	// local offset equation (https://tools.ietf.org/html/rfc5905#section-8)
	//   T1 = client send time
	//   T2 = server receive time
	//   T3 = server reply time
	//   T4 = client receive time
	//
	// Local clock offset t:
	//   t = ((T2-T1) + (T3-T4)) / 2
	a := t2.Time().Sub(t1.Time())
	b := t3.Time().Sub(t4.Time())
	return (a + b) / time.Duration(2)
}

func toInterval(t int8) time.Duration {
	switch {
	case t > 0:
		return time.Duration(uint64(time.Second) << uint(t))
	case t < 0:
		return time.Duration(uint64(time.Second) >> uint(-t))
	default:
		return time.Second
	}
}
<commit_msg>Add a LeapIndicator type<commit_after>// Copyright 2015 Brett Vickers.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package ntp provides a simple mechanism for querying the current time from
// a remote NTP server. See RFC 5905. Approach inspired by go-nuts post by
// Michael Hofmann:
//
// https://groups.google.com/forum/?fromgroups#!topic/golang-nuts/FlcdMU5fkLQ
package ntp

import (
	"encoding/binary"
	"net"
	"time"
)

type mode uint8

const (
	reserved mode = 0 + iota
	symmetricActive
	symmetricPassive
	client
	server
	broadcast
	controlMessage
	reservedPrivate
)

// The LeapIndicator is used to warn if a leap second should be inserted
// or deleted in the last minute of the current month.
type LeapIndicator uint8

const (
	// LeapNoWarning indicates no impending leap second
	LeapNoWarning LeapIndicator = 0

	// LeapAddSecond indicates the last minute of the day has 61 seconds
	LeapAddSecond = 1

	// LeapDelSecond indicates the last minute of the day has 59 seconds
	LeapDelSecond = 2

	// LeapNotInSync indicates an unsynchronized leap second.
	LeapNotInSync = 3
)

const (
	// MaxStratum is the largest allowable NTP stratum value
	MaxStratum = 16

	nanoPerSec = 1000000000
)

var (
	timeout  = 5 * time.Second
	ntpEpoch = time.Date(1900, 1, 1, 0, 0, 0, 0, time.UTC)
)

// An ntpTime is a 64-bit fixed-point (Q32.32) representation of the number of
// seconds elapsed since the NTP epoch.
type ntpTime uint64

// Duration interprets the fixed-point ntpTime as a number of elapsed seconds
// and returns the corresponding time.Duration value.
func (t ntpTime) Duration() time.Duration {
	sec := (t >> 32) * nanoPerSec
	frac := (t & 0xffffffff) * nanoPerSec >> 32
	return time.Duration(sec + frac)
}

// Time interprets the fixed-point ntpTime as a an absolute time and returns
// the corresponding time.Time value.
func (t ntpTime) Time() time.Time {
	return ntpEpoch.Add(t.Duration())
}

// toNtpTime converts the time.Time value t into its 64-bit fixed-point
// ntpTime representation.
func toNtpTime(t time.Time) ntpTime {
	nsec := uint64(t.Sub(ntpEpoch))
	sec := nsec / nanoPerSec
	frac := (nsec - sec*nanoPerSec) << 32 / nanoPerSec
	return ntpTime(sec<<32 | frac)
}

// An ntpTimeShort is a 32-bit fixed-point (Q16.16) representation of the
// number of seconds elapsed since the NTP epoch.
type ntpTimeShort uint32

// Duration interprets the fixed-point ntpTimeShort as a number of elapsed
// seconds and returns the corresponding time.Duration value.
func (t ntpTimeShort) Duration() time.Duration {
	t64 := uint64(t)
	sec := (t64 >> 16) * nanoPerSec
	frac := (t64 & 0xffff) * nanoPerSec >> 16
	return time.Duration(sec + frac)
}

// msg is an internal representation of an NTP packet.
type msg struct {
	LiVnMode       uint8 // Leap Indicator (2) + Version (3) + Mode (3)
	Stratum        uint8
	Poll           int8
	Precision      int8
	RootDelay      ntpTimeShort
	RootDispersion ntpTimeShort
	ReferenceID    uint32
	ReferenceTime  ntpTime
	OriginTime     ntpTime
	ReceiveTime    ntpTime
	TransmitTime   ntpTime
}

// setVersion sets the NTP protocol version on the message.
func (m *msg) setVersion(v int) {
	m.LiVnMode = (m.LiVnMode & 0xc7) | uint8(v)<<3
}

// setMode sets the NTP protocol mode on the message.
func (m *msg) setMode(md mode) {
	m.LiVnMode = (m.LiVnMode & 0xf8) | uint8(md)
}

// A Response contains time data, some of which is returned by the NTP server
// and some of which is calculated by the client.
type Response struct {
	Time           time.Time     // receive time reported by the server
	RTT            time.Duration // round-trip time between client and server
	ClockOffset    time.Duration // local clock offset relative to server
	Poll           time.Duration // maximum polling interval
	Precision      time.Duration // precision of server's system clock
	Stratum        uint8         // stratum level of NTP server's clock
	ReferenceID    uint32        // server's reference ID
	ReferenceTime  time.Time     // server's time of last clock update
	RootDelay      time.Duration // server's RTT to the reference clock
	RootDispersion time.Duration // server's dispersion to the reference clock
	Leap           LeapIndicator // server's leap second indicator; see RFC 5905
}

// Query returns the current time from the remote server host using the
// requested version of the NTP protocol. It also returns additional
// information about the exchanged time information. The version may be 2, 3,
// or 4; although 4 is most typically used.
func Query(host string, version int) (*Response, error) {
	m, err := getTime(host, version)
	now := toNtpTime(time.Now())
	if err != nil {
		return nil, err
	}

	r := &Response{
		Time:           m.ReceiveTime.Time(),
		RTT:            rtt(m.OriginTime, m.ReceiveTime, m.TransmitTime, now),
		ClockOffset:    offset(m.OriginTime, m.ReceiveTime, m.TransmitTime, now),
		Poll:           toInterval(m.Poll),
		Precision:      toInterval(m.Precision),
		Stratum:        m.Stratum,
		ReferenceID:    m.ReferenceID,
		ReferenceTime:  m.ReferenceTime.Time(),
		RootDelay:      m.RootDelay.Duration(),
		RootDispersion: m.RootDispersion.Duration(),
		Leap:           LeapIndicator((m.LiVnMode >> 6) & 0x03),
	}

	// https://tools.ietf.org/html/rfc5905#section-7.3
	if r.Stratum == 0 {
		r.Stratum = MaxStratum
	}

	return r, nil
}

// getTime returns the "receive time" from the remote NTP server host.
func getTime(host string, version int) (*msg, error) {
	if version < 2 || version > 4 {
		panic("ntp: invalid version number")
	}

	raddr, err := net.ResolveUDPAddr("udp", host+":123")
	if err != nil {
		return nil, err
	}

	con, err := net.DialUDP("udp", nil, raddr)
	if err != nil {
		return nil, err
	}
	defer con.Close()
	con.SetDeadline(time.Now().Add(timeout))

	m := new(msg)
	m.setMode(client)
	m.setVersion(version)
	m.TransmitTime = toNtpTime(time.Now())

	err = binary.Write(con, binary.BigEndian, m)
	if err != nil {
		return nil, err
	}

	err = binary.Read(con, binary.BigEndian, m)
	if err != nil {
		return nil, err
	}

	return m, nil
}

// TimeV returns the current time from the remote server host using the
// requested version of the NTP protocol. The version may be 2, 3, or 4;
// although 4 is most typically used.
func TimeV(host string, version int) (time.Time, error) {
	m, err := getTime(host, version)
	if err != nil {
		return time.Now(), err
	}
	return m.ReceiveTime.Time().Local(), nil
}

// Time returns the current time from the remote server host using version 4
// of the NTP protocol.
func Time(host string) (time.Time, error) {
	return TimeV(host, 4)
}

func rtt(t1, t2, t3, t4 ntpTime) time.Duration {
	// round trip delay time (https://tools.ietf.org/html/rfc5905#section-8)
	//   T1 = client send time
	//   T2 = server receive time
	//   T3 = server reply time
	//   T4 = client receive time
	//
	// RTT d:
	//   d = (T4-T1) - (T3-T2)
	a := t4.Time().Sub(t1.Time())
	b := t3.Time().Sub(t2.Time())
	return a - b
}

func offset(t1, t2, t3, t4 ntpTime) time.Duration {
	// local offset equation (https://tools.ietf.org/html/rfc5905#section-8)
	//   T1 = client send time
	//   T2 = server receive time
	//   T3 = server reply time
	//   T4 = client receive time
	//
	// Local clock offset t:
	//   t = ((T2-T1) + (T3-T4)) / 2
	a := t2.Time().Sub(t1.Time())
	b := t3.Time().Sub(t4.Time())
	return (a + b) / time.Duration(2)
}

func toInterval(t int8) time.Duration {
	switch {
	case t > 0:
		return time.Duration(uint64(time.Second) << uint(t))
	case t < 0:
		return time.Duration(uint64(time.Second) >> uint(-t))
	default:
		return time.Second
	}
}
<|endoftext|>


<commit_before>package git

/*
#include <git2.h>

extern int _go_git_odb_foreach(git_odb *db, void *payload);
extern void _go_git_odb_backend_free(git_odb_backend *backend);
*/
import "C"
import (
	"reflect"
	"runtime"
	"unsafe"
)

type Odb struct {
	ptr *C.git_odb
}

type OdbBackend struct {
	ptr *C.git_odb_backend
}

func NewOdb() (odb *Odb, err error) {
	odb = new(Odb)

	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	ret := C.git_odb_new(&odb.ptr)
	if ret < 0 {
		return nil, MakeGitError(ret)
	}

	runtime.SetFinalizer(odb, (*Odb).Free)
	return odb, nil
}

func NewOdbBackendFromC(ptr unsafe.Pointer) (backend *OdbBackend) {
	backend = &OdbBackend{(*C.git_odb_backend)(ptr)}
	return backend
}

func (v *Odb) AddBackend(backend *OdbBackend, priority int) (err error) {

	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	ret := C.git_odb_add_backend(v.ptr, backend.ptr, C.int(priority))
	runtime.KeepAlive(v)
	if ret < 0 {
		backend.Free()
		return MakeGitError(ret)
	}
	return nil
}

func (v *Odb) ReadHeader(oid *Oid) (uint64, ObjectType, error) {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	var sz C.size_t
	var cotype C.git_otype

	ret := C.git_odb_read_header(&sz, &cotype, v.ptr, oid.toC())
	runtime.KeepAlive(v)
	if ret < 0 {
		return 0, C.GIT_OBJ_BAD, MakeGitError(ret)
	}

	return uint64(sz), ObjectType(cotype), nil
}

func (v *Odb) Exists(oid *Oid) bool {
	ret := C.git_odb_exists(v.ptr, oid.toC())
	runtime.KeepAlive(v)
	runtime.KeepAlive(oid)
	return ret != 0
}

func (v *Odb) Write(data []byte, otype ObjectType) (oid *Oid, err error) {
	oid = new(Oid)
	var cptr unsafe.Pointer
	if len(data) > 0 {
		cptr = unsafe.Pointer(&data[0])
	}

	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	ret := C.git_odb_write(oid.toC(), v.ptr, cptr, C.size_t(len(data)), C.git_otype(otype))
	runtime.KeepAlive(v)
	if ret < 0 {
		return nil, MakeGitError(ret)
	}

	return oid, nil
}

func (v *Odb) Read(oid *Oid) (obj *OdbObject, err error) {
	obj = new(OdbObject)

	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	ret := C.git_odb_read(&obj.ptr, v.ptr, oid.toC())
	runtime.KeepAlive(v)
	runtime.KeepAlive(oid)
	if ret < 0 {
		return nil, MakeGitError(ret)
	}

	runtime.SetFinalizer(obj, (*OdbObject).Free)
	return obj, nil
}

type OdbForEachCallback func(id *Oid) error

type foreachData struct {
	callback OdbForEachCallback
	err      error
}

//export odbForEachCb
func odbForEachCb(id *C.git_oid, handle unsafe.Pointer) int {
	data, ok := pointerHandles.Get(handle).(*foreachData)

	if !ok {
		panic("could not retrieve handle")
	}

	err := data.callback(newOidFromC(id))
	if err != nil {
		data.err = err
		return C.GIT_EUSER
	}

	return 0
}

func (v *Odb) ForEach(callback OdbForEachCallback) error {
	data := foreachData{
		callback: callback,
		err:      nil,
	}

	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	handle := pointerHandles.Track(&data)
	defer pointerHandles.Untrack(handle)

	ret := C._go_git_odb_foreach(v.ptr, handle)
	runtime.KeepAlive(v)
	if ret == C.GIT_EUSER {
		return data.err
	} else if ret < 0 {
		return MakeGitError(ret)
	}

	return nil
}

// Hash determines the object-ID (sha1) of a data buffer.
func (v *Odb) Hash(data []byte, otype ObjectType) (oid *Oid, err error) {
	oid = new(Oid)
	header := (*reflect.SliceHeader)(unsafe.Pointer(&data))
	ptr := unsafe.Pointer(header.Data)

	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	ret := C.git_odb_hash(oid.toC(), ptr, C.size_t(header.Len), C.git_otype(otype))
	runtime.KeepAlive(data)
	if ret < 0 {
		return nil, MakeGitError(ret)
	}
	return oid, nil
}

// NewReadStream opens a read stream from the ODB. Reading from it will give you the
// contents of the object.
func (v *Odb) NewReadStream(id *Oid) (*OdbReadStream, error) {
	stream := new(OdbReadStream)
	var ctype C.git_otype
	var csize C.size_t

	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	ret := C.git_odb_open_rstream(&stream.ptr, &csize, &ctype, v.ptr, id.toC())
	runtime.KeepAlive(v)
	runtime.KeepAlive(id)
	if ret < 0 {
		return nil, MakeGitError(ret)
	}

	stream.Size = uint64(csize)
	stream.Type = ObjectType(ctype)
	runtime.SetFinalizer(stream, (*OdbReadStream).Free)
	return stream, nil
}

// NewWriteStream opens a write stream to the ODB, which allows you to
// create a new object in the database. The size and type must be
// known in advance
func (v *Odb) NewWriteStream(size int64, otype ObjectType) (*OdbWriteStream, error) {
	stream := new(OdbWriteStream)

	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	ret := C.git_odb_open_wstream(&stream.ptr, v.ptr, C.git_off_t(size), C.git_otype(otype))
	runtime.KeepAlive(v)
	if ret < 0 {
		return nil, MakeGitError(ret)
	}

	runtime.SetFinalizer(stream, (*OdbWriteStream).Free)
	return stream, nil
}

func (v *OdbBackend) Free() {
	C._go_git_odb_backend_free(v.ptr)
}

type OdbObject struct {
	ptr *C.git_odb_object
}

func (v *OdbObject) Free() {
	runtime.SetFinalizer(v, nil)
	C.git_odb_object_free(v.ptr)
}

func (object *OdbObject) Id() (oid *Oid) {
	ret := newOidFromC(C.git_odb_object_id(object.ptr))
	runtime.KeepAlive(object)
	return ret
}

func (object *OdbObject) Len() (len uint64) {
	ret := uint64(C.git_odb_object_size(object.ptr))
	runtime.KeepAlive(object)
	return ret
}

func (object *OdbObject) Type() ObjectType {
	ret := ObjectType(C.git_odb_object_type(object.ptr))
	runtime.KeepAlive(object)
	return ret
}

// Data returns a slice pointing to the unmanaged object memory. You must make
// sure the object is referenced for at least as long as the slice is used.
func (object *OdbObject) Data() (data []byte) {
	var c_blob unsafe.Pointer = C.git_odb_object_data(object.ptr)
	var blob []byte

	len := int(C.git_odb_object_size(object.ptr))

	sliceHeader := (*reflect.SliceHeader)((unsafe.Pointer(&blob)))
	sliceHeader.Cap = len
	sliceHeader.Len = len
	sliceHeader.Data = uintptr(c_blob)

	return blob
}

type OdbReadStream struct {
	ptr  *C.git_odb_stream
	Size uint64
	Type ObjectType
}

// Read reads from the stream
func (stream *OdbReadStream) Read(data []byte) (int, error) {
	header := (*reflect.SliceHeader)(unsafe.Pointer(&data))
	ptr := (*C.char)(unsafe.Pointer(header.Data))
	size := C.size_t(header.Cap)

	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	ret := C.git_odb_stream_read(stream.ptr, ptr, size)
	runtime.KeepAlive(stream)
	if ret < 0 {
		return 0, MakeGitError(ret)
	}

	header.Len = int(ret)

	return len(data), nil
}

// Close is a dummy function in order to implement the Closer and
// ReadCloser interfaces
func (stream *OdbReadStream) Close() error {
	return nil
}

func (stream *OdbReadStream) Free() {
	runtime.SetFinalizer(stream, nil)
	C.git_odb_stream_free(stream.ptr)
}

type OdbWriteStream struct {
	ptr *C.git_odb_stream
	Id  Oid
}

// Write writes to the stream
func (stream *OdbWriteStream) Write(data []byte) (int, error) {
	header := (*reflect.SliceHeader)(unsafe.Pointer(&data))
	ptr := (*C.char)(unsafe.Pointer(header.Data))
	size := C.size_t(header.Len)

	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	ret := C.git_odb_stream_write(stream.ptr, ptr, size)
	runtime.KeepAlive(stream)
	if ret < 0 {
		return 0, MakeGitError(ret)
	}

	return len(data), nil
}

// Close signals that all the data has been written and stores the
// resulting object id in the stream's Id field.
func (stream *OdbWriteStream) Close() error {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	ret := C.git_odb_stream_finalize_write(stream.Id.toC(), stream.ptr)
	runtime.KeepAlive(stream)
	if ret < 0 {
		return MakeGitError(ret)
	}

	return nil
}

func (stream *OdbWriteStream) Free() {
	runtime.SetFinalizer(stream, nil)
	C.git_odb_stream_free(stream.ptr)
}
<commit_msg>Return io.EOF on OdbReadStream.Read()<commit_after>package git

/*
#include <git2.h>

extern int _go_git_odb_foreach(git_odb *db, void *payload);
extern void _go_git_odb_backend_free(git_odb_backend *backend);
*/
import "C"
import (
	"io"
	"reflect"
	"runtime"
	"unsafe"
)

type Odb struct {
	ptr *C.git_odb
}

type OdbBackend struct {
	ptr *C.git_odb_backend
}

func NewOdb() (odb *Odb, err error) {
	odb = new(Odb)

	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	ret := C.git_odb_new(&odb.ptr)
	if ret < 0 {
		return nil, MakeGitError(ret)
	}

	runtime.SetFinalizer(odb, (*Odb).Free)
	return odb, nil
}

func NewOdbBackendFromC(ptr unsafe.Pointer) (backend *OdbBackend) {
	backend = &OdbBackend{(*C.git_odb_backend)(ptr)}
	return backend
}

func (v *Odb) AddBackend(backend *OdbBackend, priority int) (err error) {

	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	ret := C.git_odb_add_backend(v.ptr, backend.ptr, C.int(priority))
	runtime.KeepAlive(v)
	if ret < 0 {
		backend.Free()
		return MakeGitError(ret)
	}
	return nil
}

func (v *Odb) ReadHeader(oid *Oid) (uint64, ObjectType, error) {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	var sz C.size_t
	var cotype C.git_otype

	ret := C.git_odb_read_header(&sz, &cotype, v.ptr, oid.toC())
	runtime.KeepAlive(v)
	if ret < 0 {
		return 0, C.GIT_OBJ_BAD, MakeGitError(ret)
	}

	return uint64(sz), ObjectType(cotype), nil
}

func (v *Odb) Exists(oid *Oid) bool {
	ret := C.git_odb_exists(v.ptr, oid.toC())
	runtime.KeepAlive(v)
	runtime.KeepAlive(oid)
	return ret != 0
}

func (v *Odb) Write(data []byte, otype ObjectType) (oid *Oid, err error) {
	oid = new(Oid)
	var cptr unsafe.Pointer
	if len(data) > 0 {
		cptr = unsafe.Pointer(&data[0])
	}

	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	ret := C.git_odb_write(oid.toC(), v.ptr, cptr, C.size_t(len(data)), C.git_otype(otype))
	runtime.KeepAlive(v)
	if ret < 0 {
		return nil, MakeGitError(ret)
	}

	return oid, nil
}

func (v *Odb) Read(oid *Oid) (obj *OdbObject, err error) {
	obj = new(OdbObject)

	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	ret := C.git_odb_read(&obj.ptr, v.ptr, oid.toC())
	runtime.KeepAlive(v)
	runtime.KeepAlive(oid)
	if ret < 0 {
		return nil, MakeGitError(ret)
	}

	runtime.SetFinalizer(obj, (*OdbObject).Free)
	return obj, nil
}

type OdbForEachCallback func(id *Oid) error

type foreachData struct {
	callback OdbForEachCallback
	err      error
}

//export odbForEachCb
func odbForEachCb(id *C.git_oid, handle unsafe.Pointer) int {
	data, ok := pointerHandles.Get(handle).(*foreachData)

	if !ok {
		panic("could not retrieve handle")
	}

	err := data.callback(newOidFromC(id))
	if err != nil {
		data.err = err
		return C.GIT_EUSER
	}

	return 0
}

func (v *Odb) ForEach(callback OdbForEachCallback) error {
	data := foreachData{
		callback: callback,
		err:      nil,
	}

	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	handle := pointerHandles.Track(&data)
	defer pointerHandles.Untrack(handle)

	ret := C._go_git_odb_foreach(v.ptr, handle)
	runtime.KeepAlive(v)
	if ret == C.GIT_EUSER {
		return data.err
	} else if ret < 0 {
		return MakeGitError(ret)
	}

	return nil
}

// Hash determines the object-ID (sha1) of a data buffer.
func (v *Odb) Hash(data []byte, otype ObjectType) (oid *Oid, err error) {
	oid = new(Oid)
	header := (*reflect.SliceHeader)(unsafe.Pointer(&data))
	ptr := unsafe.Pointer(header.Data)

	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	ret := C.git_odb_hash(oid.toC(), ptr, C.size_t(header.Len), C.git_otype(otype))
	runtime.KeepAlive(data)
	if ret < 0 {
		return nil, MakeGitError(ret)
	}
	return oid, nil
}

// NewReadStream opens a read stream from the ODB. Reading from it will give you the
// contents of the object.
func (v *Odb) NewReadStream(id *Oid) (*OdbReadStream, error) {
	stream := new(OdbReadStream)
	var ctype C.git_otype
	var csize C.size_t

	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	ret := C.git_odb_open_rstream(&stream.ptr, &csize, &ctype, v.ptr, id.toC())
	runtime.KeepAlive(v)
	runtime.KeepAlive(id)
	if ret < 0 {
		return nil, MakeGitError(ret)
	}

	stream.Size = uint64(csize)
	stream.Type = ObjectType(ctype)
	runtime.SetFinalizer(stream, (*OdbReadStream).Free)
	return stream, nil
}

// NewWriteStream opens a write stream to the ODB, which allows you to
// create a new object in the database. The size and type must be
// known in advance
func (v *Odb) NewWriteStream(size int64, otype ObjectType) (*OdbWriteStream, error) {
	stream := new(OdbWriteStream)

	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	ret := C.git_odb_open_wstream(&stream.ptr, v.ptr, C.git_off_t(size), C.git_otype(otype))
	runtime.KeepAlive(v)
	if ret < 0 {
		return nil, MakeGitError(ret)
	}

	runtime.SetFinalizer(stream, (*OdbWriteStream).Free)
	return stream, nil
}

func (v *OdbBackend) Free() {
	C._go_git_odb_backend_free(v.ptr)
}

type OdbObject struct {
	ptr *C.git_odb_object
}

func (v *OdbObject) Free() {
	runtime.SetFinalizer(v, nil)
	C.git_odb_object_free(v.ptr)
}

func (object *OdbObject) Id() (oid *Oid) {
	ret := newOidFromC(C.git_odb_object_id(object.ptr))
	runtime.KeepAlive(object)
	return ret
}

func (object *OdbObject) Len() (len uint64) {
	ret := uint64(C.git_odb_object_size(object.ptr))
	runtime.KeepAlive(object)
	return ret
}

func (object *OdbObject) Type() ObjectType {
	ret := ObjectType(C.git_odb_object_type(object.ptr))
	runtime.KeepAlive(object)
	return ret
}

// Data returns a slice pointing to the unmanaged object memory. You must make
// sure the object is referenced for at least as long as the slice is used.
func (object *OdbObject) Data() (data []byte) {
	var c_blob unsafe.Pointer = C.git_odb_object_data(object.ptr)
	var blob []byte

	len := int(C.git_odb_object_size(object.ptr))

	sliceHeader := (*reflect.SliceHeader)((unsafe.Pointer(&blob)))
	sliceHeader.Cap = len
	sliceHeader.Len = len
	sliceHeader.Data = uintptr(c_blob)

	return blob
}

type OdbReadStream struct {
	ptr  *C.git_odb_stream
	Size uint64
	Type ObjectType
}

// Read reads from the stream
func (stream *OdbReadStream) Read(data []byte) (int, error) {
	header := (*reflect.SliceHeader)(unsafe.Pointer(&data))
	ptr := (*C.char)(unsafe.Pointer(header.Data))
	size := C.size_t(header.Cap)

	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	ret := C.git_odb_stream_read(stream.ptr, ptr, size)
	runtime.KeepAlive(stream)
	if ret < 0 {
		return 0, MakeGitError(ret)
	}
	if ret == 0 {
		return 0, io.EOF
	}

	header.Len = int(ret)

	return len(data), nil
}

// Close is a dummy function in order to implement the Closer and
// ReadCloser interfaces
func (stream *OdbReadStream) Close() error {
	return nil
}

func (stream *OdbReadStream) Free() {
	runtime.SetFinalizer(stream, nil)
	C.git_odb_stream_free(stream.ptr)
}

type OdbWriteStream struct {
	ptr *C.git_odb_stream
	Id  Oid
}

// Write writes to the stream
func (stream *OdbWriteStream) Write(data []byte) (int, error) {
	header := (*reflect.SliceHeader)(unsafe.Pointer(&data))
	ptr := (*C.char)(unsafe.Pointer(header.Data))
	size := C.size_t(header.Len)

	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	ret := C.git_odb_stream_write(stream.ptr, ptr, size)
	runtime.KeepAlive(stream)
	if ret < 0 {
		return 0, MakeGitError(ret)
	}

	return len(data), nil
}

// Close signals that all the data has been written and stores the
// resulting object id in the stream's Id field.
func (stream *OdbWriteStream) Close() error {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	ret := C.git_odb_stream_finalize_write(stream.Id.toC(), stream.ptr)
	runtime.KeepAlive(stream)
	if ret < 0 {
		return MakeGitError(ret)
	}

	return nil
}

func (stream *OdbWriteStream) Free() {
	runtime.SetFinalizer(stream, nil)
	C.git_odb_stream_free(stream.ptr)
}
<|endoftext|>


<commit_before>package ofx

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"io"
	"log"
	"math/big"
	"strings"
	"time"
)

// AccountType indicates type of account represented by OFX document.
type AccountType int

const (
	// UNKNOWN - Account type could not be determined
	UNKNOWN AccountType = iota
	// CHECKING - Checking account
	CHECKING AccountType = iota
	// SAVING - Savings account
	SAVING AccountType = iota
)

//go:generate stringer -type=TransactionType
// TransactionType indicates type of transaction (Debit/Credit).
type TransactionType int

const (
	DEBIT  TransactionType = iota
	CREDIT TransactionType = iota
)

type nextKey int

const (
	none            nextKey = iota
	acctID          nextKey = iota
	branchID        nextKey = iota
	bankID          nextKey = iota
	transAmount     nextKey = iota
	transDatePosted nextKey = iota
	transUserDate   nextKey = iota
	transID         nextKey = iota
	transDesc       nextKey = iota
)

type Amount struct {
	Value big.Rat
}

func (a *Amount) ParseFromString(s string) error {
	_, ok := a.Value.SetString(s)
	if !ok {
		return fmt.Errorf("Unable to parse string '%s' as an amount\n", s)
	}

	return nil
}

type Transaction struct {
	Type        TransactionType
	Description string
	PostedDate  time.Time
	UserDate    time.Time
	ID          string
	Amount      Amount
}

func (t Transaction) String() string {
	return fmt.Sprintf("T: %s Desc: %s Post Date: %s ID: %s Amount: %s", t.Type, t.Description, t.PostedDate, t.ID, t.Amount.Value.String())
}

// Ofx contains a parsed Ofx document.
type Ofx struct {
	Type          AccountType
	BankCode      string
	BranchCode    string
	AccountNumber string
	Transactions  []*Transaction
}

func (o Ofx) String() string {
	var buf bytes.Buffer

	buf.WriteString(fmt.Sprintf("Account Type: %s\nBank Code: %s\nBranch Code: %s\nAccount Number: %s\n", o.Type, o.BankCode, o.BranchCode, o.AccountNumber))

	for _, t := range o.Transactions {
		buf.WriteString(fmt.Sprintf("%s\n", t))
	}

	return buf.String()
}

// Parse parses an input stream and produces an Ofx instance summarizing it. In case of any errors
// during the parse, a non-nil error is returned.
func Parse(f io.Reader) (*Ofx, error) {
	ofx := &Ofx{Transactions: []*Transaction{}}
	stack := make([]string, 1000)
	stackPos := 0

	next := none
	var trans *Transaction = nil

	dec := xml.NewDecoder(f)

	tok, err := dec.RawToken()
	for err == nil {
		switch t := tok.(type) {
		case xml.StartElement:
			stack[stackPos] = t.Name.Local
			stackPos++

			switch t.Name.Local {
			case "ACCTID":
				next = acctID

			case "BRANCHID":
				next = branchID

			case "BANKID":
				next = bankID

			case "STMTTRN":
				trans = &Transaction{}

			case "DTPOSTED":
				next = transDatePosted

			case "FITID":
				next = transID

			case "TRNAMT":
				next = transAmount

			case "NAME":
				next = transDesc
			}

		case xml.CharData:
			var b bytes.Buffer
			if _, err := b.Write(t); err != nil {
				return nil, err
			}
			res := strings.TrimSpace(b.String())

			switch next {
			case acctID:
				ofx.AccountNumber = res

			case branchID:
				ofx.BranchCode = res

			case bankID:
				ofx.BankCode = res

			case transDesc:
				trans.Description = res

			case transID:
				trans.ID = res

			case transDatePosted:
				if len(res) < 8 {
					return nil, fmt.Errorf("Invalid date posted string: '%s'", res)
				}
				res = res[:8]
				// YYYYMMDD
				if t, err := time.Parse("20060102", res); err != nil {
					return nil, err
				} else {
					trans.PostedDate = t
				}

			case transAmount:
				if err := trans.Amount.ParseFromString(res); err != nil {
					return nil, err
				}

				if trans.Amount.Value.Sign() == 1 {
					trans.Type = CREDIT
				} else {
					trans.Type = DEBIT
				}
			}

			next = none

		case xml.EndElement:
			for stackPos != 0 {
				if stack[stackPos-1] == "STMTTRN" {
					ofx.Transactions = append(ofx.Transactions, trans)
					trans = nil
				}

				if stack[stackPos-1] == t.Name.Local {
					stackPos--
					break
				}
				stackPos--
			}

		default:
			log.Printf("Unknown: %T %s\n", t, t)
		}

		tok, err = dec.RawToken()

		if err != nil && err != io.EOF {
			log.Printf("Error: %s\n", err)
		}
	}

	return ofx, nil
}
<commit_msg>Add memo support to the ofx parser<commit_after>package ofx

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"io"
	"log"
	"math/big"
	"strings"
	"time"
)

// AccountType indicates type of account represented by OFX document.
type AccountType int

const (
	// UNKNOWN - Account type could not be determined
	UNKNOWN AccountType = iota
	// CHECKING - Checking account
	CHECKING AccountType = iota
	// SAVING - Savings account
	SAVING AccountType = iota
)

//go:generate stringer -type=TransactionType
// TransactionType indicates type of transaction (Debit/Credit).
type TransactionType int

const (
	DEBIT  TransactionType = iota
	CREDIT TransactionType = iota
)

type nextKey int

const (
	none            nextKey = iota
	acctID          nextKey = iota
	branchID        nextKey = iota
	bankID          nextKey = iota
	transAmount     nextKey = iota
	transDatePosted nextKey = iota
	transUserDate   nextKey = iota
	transID         nextKey = iota
	transDesc       nextKey = iota
	transMemo       nextKey = iota
)

type Amount struct {
	Value big.Rat
}

func (a *Amount) ParseFromString(s string) error {
	_, ok := a.Value.SetString(s)
	if !ok {
		return fmt.Errorf("Unable to parse string '%s' as an amount\n", s)
	}

	return nil
}

type Transaction struct {
	Type        TransactionType
	Description string
	Memo        string
	PostedDate  time.Time
	UserDate    time.Time
	ID          string
	Amount      Amount
}

func (t Transaction) String() string {
	return fmt.Sprintf("T: %s Desc: %s Post Date: %s ID: %s Amount: %s", t.Type, t.Description, t.PostedDate, t.ID, t.Amount.Value.String())
}

// Ofx contains a parsed Ofx document.
type Ofx struct {
	Type          AccountType
	BankCode      string
	BranchCode    string
	AccountNumber string
	Transactions  []*Transaction
}

func (o Ofx) String() string {
	var buf bytes.Buffer

	buf.WriteString(fmt.Sprintf("Account Type: %s\nBank Code: %s\nBranch Code: %s\nAccount Number: %s\n", o.Type, o.BankCode, o.BranchCode, o.AccountNumber))

	for _, t := range o.Transactions {
		buf.WriteString(fmt.Sprintf("%s\n", t))
	}

	return buf.String()
}

// Parse parses an input stream and produces an Ofx instance summarizing it. In case of any errors
// during the parse, a non-nil error is returned.
func Parse(f io.Reader) (*Ofx, error) {
	ofx := &Ofx{Transactions: []*Transaction{}}
	stack := make([]string, 1000)
	stackPos := 0

	next := none
	var trans *Transaction = nil

	dec := xml.NewDecoder(f)

	tok, err := dec.RawToken()
	for err == nil {
		switch t := tok.(type) {
		case xml.StartElement:
			stack[stackPos] = t.Name.Local
			stackPos++

			switch t.Name.Local {
			case "ACCTID":
				next = acctID

			case "BRANCHID":
				next = branchID

			case "BANKID":
				next = bankID

			case "STMTTRN":
				trans = &Transaction{}

			case "DTPOSTED":
				next = transDatePosted

			case "FITID":
				next = transID

			case "TRNAMT":
				next = transAmount

			case "NAME":
				next = transDesc
			case "MEMO":
				next = transMemo
			}

		case xml.CharData:
			var b bytes.Buffer
			if _, err := b.Write(t); err != nil {
				return nil, err
			}
			res := strings.TrimSpace(b.String())

			switch next {
			case acctID:
				ofx.AccountNumber = res

			case branchID:
				ofx.BranchCode = res

			case bankID:
				ofx.BankCode = res

			case transDesc:
				trans.Description = res

			case transMemo:
				trans.Memo = res

			case transID:
				trans.ID = res

			case transDatePosted:
				if len(res) < 8 {
					return nil, fmt.Errorf("Invalid date posted string: '%s'", res)
				}
				res = res[:8]
				// YYYYMMDD
				if t, err := time.Parse("20060102", res); err != nil {
					return nil, err
				} else {
					trans.PostedDate = t
				}

			case transAmount:
				if err := trans.Amount.ParseFromString(res); err != nil {
					return nil, err
				}

				if trans.Amount.Value.Sign() == 1 {
					trans.Type = CREDIT
				} else {
					trans.Type = DEBIT
				}
			}

			next = none

		case xml.EndElement:
			for stackPos != 0 {
				if stack[stackPos-1] == "STMTTRN" {
					ofx.Transactions = append(ofx.Transactions, trans)
					trans = nil
				}

				if stack[stackPos-1] == t.Name.Local {
					stackPos--
					break
				}
				stackPos--
			}

		default:
			log.Printf("Unknown: %T %s\n", t, t)
		}

		tok, err = dec.RawToken()

		if err != nil && err != io.EOF {
			log.Printf("Error: %s\n", err)
		}
	}

	return ofx, nil
}
<|endoftext|>


<commit_before>// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package consumers

import (
	"context"
	"database/sql"
	"encoding/json"
	"fmt"

	"github.com/getsentry/sentry-go"
	"github.com/matrix-org/dendrite/internal/eventutil"
	"github.com/matrix-org/dendrite/setup/config"
	"github.com/matrix-org/dendrite/setup/jetstream"
	"github.com/matrix-org/dendrite/setup/process"
	"github.com/matrix-org/dendrite/syncapi/notifier"
	"github.com/matrix-org/dendrite/syncapi/producers"
	"github.com/matrix-org/dendrite/syncapi/storage"
	"github.com/matrix-org/dendrite/syncapi/types"
	"github.com/matrix-org/gomatrixserverlib"
	"github.com/nats-io/nats.go"
	"github.com/sirupsen/logrus"
	log "github.com/sirupsen/logrus"
)

// OutputClientDataConsumer consumes events that originated in the client API server.
type OutputClientDataConsumer struct {
	ctx        context.Context
	jetstream  nats.JetStreamContext
	durable    string
	topic      string
	db         storage.Database
	stream     types.StreamProvider
	notifier   *notifier.Notifier
	serverName gomatrixserverlib.ServerName
	producer   *producers.UserAPIReadProducer
}

// NewOutputClientDataConsumer creates a new OutputClientData consumer. Call Start() to begin consuming from room servers.
func NewOutputClientDataConsumer(
	process *process.ProcessContext,
	cfg *config.SyncAPI,
	js nats.JetStreamContext,
	store storage.Database,
	notifier *notifier.Notifier,
	stream types.StreamProvider,
	producer *producers.UserAPIReadProducer,
) *OutputClientDataConsumer {
	return &OutputClientDataConsumer{
		ctx:        process.Context(),
		jetstream:  js,
		topic:      cfg.Matrix.JetStream.Prefixed(jetstream.OutputClientData),
		durable:    cfg.Matrix.JetStream.Durable("SyncAPIAccountDataConsumer"),
		db:         store,
		notifier:   notifier,
		stream:     stream,
		serverName: cfg.Matrix.ServerName,
		producer:   producer,
	}
}

// Start consuming from room servers
func (s *OutputClientDataConsumer) Start() error {
	return jetstream.JetStreamConsumer(
		s.ctx, s.jetstream, s.topic, s.durable, s.onMessage,
		nats.DeliverAll(), nats.ManualAck(),
	)
}

// onMessage is called when the sync server receives a new event from the client API server output log.
// It is not safe for this function to be called from multiple goroutines, or else the
// sync stream position may race and be incorrectly calculated.
func (s *OutputClientDataConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
	// Parse out the event JSON
	userID := msg.Header.Get(jetstream.UserID)
	var output eventutil.AccountData
	if err := json.Unmarshal(msg.Data, &output); err != nil {
		// If the message was invalid, log it and move on to the next message in the stream
		log.WithError(err).Errorf("client API server output log: message parse failure")
		sentry.CaptureException(err)
		return true
	}

	log.WithFields(log.Fields{
		"type":    output.Type,
		"room_id": output.RoomID,
	}).Debug("Received data from client API server")

	streamPos, err := s.db.UpsertAccountData(
		s.ctx, userID, output.RoomID, output.Type,
	)
	if err != nil {
		sentry.CaptureException(err)
		log.WithFields(log.Fields{
			"type":       output.Type,
			"room_id":    output.RoomID,
			log.ErrorKey: err,
		}).Panicf("could not save account data")
	}

	if err = s.sendReadUpdate(ctx, userID, output); err != nil {
		log.WithError(err).WithFields(logrus.Fields{
			"user_id": userID,
			"room_id": output.RoomID,
		}).Errorf("Failed to generate read update")
		sentry.CaptureException(err)
		return false
	}

	if output.IgnoredUsers != nil {
		if err := s.db.UpdateIgnoresForUser(ctx, userID, output.IgnoredUsers); err != nil {
			log.WithError(err).WithFields(logrus.Fields{
				"user_id": userID,
			}).Errorf("Failed to update ignored users")
			sentry.CaptureException(err)
		}
	}

	s.stream.Advance(streamPos)
	s.notifier.OnNewAccountData(userID, types.StreamingToken{AccountDataPosition: streamPos})

	return true
}

func (s *OutputClientDataConsumer) sendReadUpdate(ctx context.Context, userID string, output eventutil.AccountData) error {
	if output.Type != "m.fully_read" || output.ReadMarker == nil {
		return nil
	}
	_, serverName, err := gomatrixserverlib.SplitID('@', userID)
	if err != nil {
		return fmt.Errorf("gomatrixserverlib.SplitID: %w", err)
	}
	if serverName != s.serverName {
		return nil
	}
	var readPos types.StreamPosition
	var fullyReadPos types.StreamPosition
	if output.ReadMarker.Read != "" {
		if _, readPos, err = s.db.PositionInTopology(ctx, output.ReadMarker.Read); err != nil && err != sql.ErrNoRows {
			return fmt.Errorf("s.db.PositionInTopology (Read): %w", err)
		}
	}
	if output.ReadMarker.FullyRead != "" {
		if _, fullyReadPos, err = s.db.PositionInTopology(ctx, output.ReadMarker.FullyRead); err != nil && err != sql.ErrNoRows {
			return fmt.Errorf("s.db.PositionInTopology (FullyRead): %w", err)
		}
	}
	if readPos > 0 || fullyReadPos > 0 {
		if err := s.producer.SendReadUpdate(userID, output.RoomID, readPos, fullyReadPos); err != nil {
			return fmt.Errorf("s.producer.SendReadUpdate: %w", err)
		}
	}
	return nil
}
<commit_msg>Don't panic if we fail to upsert account data<commit_after>// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package consumers

import (
	"context"
	"database/sql"
	"encoding/json"
	"fmt"

	"github.com/getsentry/sentry-go"
	"github.com/matrix-org/gomatrixserverlib"
	"github.com/nats-io/nats.go"
	"github.com/sirupsen/logrus"
	log "github.com/sirupsen/logrus"

	"github.com/matrix-org/dendrite/internal/eventutil"
	"github.com/matrix-org/dendrite/setup/config"
	"github.com/matrix-org/dendrite/setup/jetstream"
	"github.com/matrix-org/dendrite/setup/process"
	"github.com/matrix-org/dendrite/syncapi/notifier"
	"github.com/matrix-org/dendrite/syncapi/producers"
	"github.com/matrix-org/dendrite/syncapi/storage"
	"github.com/matrix-org/dendrite/syncapi/types"
)

// OutputClientDataConsumer consumes events that originated in the client API server.
type OutputClientDataConsumer struct {
	ctx        context.Context
	jetstream  nats.JetStreamContext
	durable    string
	topic      string
	db         storage.Database
	stream     types.StreamProvider
	notifier   *notifier.Notifier
	serverName gomatrixserverlib.ServerName
	producer   *producers.UserAPIReadProducer
}

// NewOutputClientDataConsumer creates a new OutputClientData consumer. Call Start() to begin consuming from room servers.
func NewOutputClientDataConsumer(
	process *process.ProcessContext,
	cfg *config.SyncAPI,
	js nats.JetStreamContext,
	store storage.Database,
	notifier *notifier.Notifier,
	stream types.StreamProvider,
	producer *producers.UserAPIReadProducer,
) *OutputClientDataConsumer {
	return &OutputClientDataConsumer{
		ctx:        process.Context(),
		jetstream:  js,
		topic:      cfg.Matrix.JetStream.Prefixed(jetstream.OutputClientData),
		durable:    cfg.Matrix.JetStream.Durable("SyncAPIAccountDataConsumer"),
		db:         store,
		notifier:   notifier,
		stream:     stream,
		serverName: cfg.Matrix.ServerName,
		producer:   producer,
	}
}

// Start consuming from room servers
func (s *OutputClientDataConsumer) Start() error {
	return jetstream.JetStreamConsumer(
		s.ctx, s.jetstream, s.topic, s.durable, s.onMessage,
		nats.DeliverAll(), nats.ManualAck(),
	)
}

// onMessage is called when the sync server receives a new event from the client API server output log.
// It is not safe for this function to be called from multiple goroutines, or else the
// sync stream position may race and be incorrectly calculated.
func (s *OutputClientDataConsumer) onMessage(ctx context.Context, msg *nats.Msg) bool {
	// Parse out the event JSON
	userID := msg.Header.Get(jetstream.UserID)
	var output eventutil.AccountData
	if err := json.Unmarshal(msg.Data, &output); err != nil {
		// If the message was invalid, log it and move on to the next message in the stream
		log.WithError(err).Errorf("client API server output log: message parse failure")
		sentry.CaptureException(err)
		return true
	}

	log.WithFields(log.Fields{
		"type":    output.Type,
		"room_id": output.RoomID,
	}).Debug("Received data from client API server")

	streamPos, err := s.db.UpsertAccountData(
		s.ctx, userID, output.RoomID, output.Type,
	)
	if err != nil {
		sentry.CaptureException(err)
		log.WithFields(log.Fields{
			"type":       output.Type,
			"room_id":    output.RoomID,
			log.ErrorKey: err,
		}).Errorf("could not save account data")
		return false
	}

	if err = s.sendReadUpdate(ctx, userID, output); err != nil {
		log.WithError(err).WithFields(logrus.Fields{
			"user_id": userID,
			"room_id": output.RoomID,
		}).Errorf("Failed to generate read update")
		sentry.CaptureException(err)
		return false
	}

	if output.IgnoredUsers != nil {
		if err := s.db.UpdateIgnoresForUser(ctx, userID, output.IgnoredUsers); err != nil {
			log.WithError(err).WithFields(logrus.Fields{
				"user_id": userID,
			}).Errorf("Failed to update ignored users")
			sentry.CaptureException(err)
		}
	}

	s.stream.Advance(streamPos)
	s.notifier.OnNewAccountData(userID, types.StreamingToken{AccountDataPosition: streamPos})

	return true
}

func (s *OutputClientDataConsumer) sendReadUpdate(ctx context.Context, userID string, output eventutil.AccountData) error {
	if output.Type != "m.fully_read" || output.ReadMarker == nil {
		return nil
	}
	_, serverName, err := gomatrixserverlib.SplitID('@', userID)
	if err != nil {
		return fmt.Errorf("gomatrixserverlib.SplitID: %w", err)
	}
	if serverName != s.serverName {
		return nil
	}
	var readPos types.StreamPosition
	var fullyReadPos types.StreamPosition
	if output.ReadMarker.Read != "" {
		if _, readPos, err = s.db.PositionInTopology(ctx, output.ReadMarker.Read); err != nil && err != sql.ErrNoRows {
			return fmt.Errorf("s.db.PositionInTopology (Read): %w", err)
		}
	}
	if output.ReadMarker.FullyRead != "" {
		if _, fullyReadPos, err = s.db.PositionInTopology(ctx, output.ReadMarker.FullyRead); err != nil && err != sql.ErrNoRows {
			return fmt.Errorf("s.db.PositionInTopology (FullyRead): %w", err)
		}
	}
	if readPos > 0 || fullyReadPos > 0 {
		if err := s.producer.SendReadUpdate(userID, output.RoomID, readPos, fullyReadPos); err != nil {
			return fmt.Errorf("s.producer.SendReadUpdate: %w", err)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package syncutil

import (
	"fmt"
	"sync"
)

// A weighted counted semaphore. Can be created as part of larger structs. Must
// be initialized with Init.
type WeightedSemaphore struct {
	// Constant after Init returns.
	capacity uint64

	mu InvariantMutex

	// The remaining capacity.
	//
	// INVARIANT: remaining <= capacity
	//
	// GUARDED_BY(mu)
	remaining uint64

	// Signalled when remaining is changed.
	remainingChanged sync.Cond
}

// LOCKS_REQUIRED(ws.mu)
func (ws *WeightedSemaphore) checkInvariants() {
	// INVARIANT: remaining <= capacity
	if !(ws.remaining <= ws.capacity) {
		panic(fmt.Sprintf("remaining > capacity: %v, %v", ws.remaining, ws.capacity))
	}
}

// Initialize the semaphore with the given capacity.
func (ws *WeightedSemaphore) Init(capacity uint64) {
	ws.capacity = capacity
	ws.remaining = capacity
	ws.mu = NewInvariantMutex(ws.checkInvariants)
	ws.remainingChanged.L = &ws.mu
}

// Return the capacity with which the semaphore was initialized.
func (ws *WeightedSemaphore) Capacity() uint64

// Block until the supplied capacity is available.
//
// REQUIRES: c <= ws.Capacity()
func (ws *WeightedSemaphore) Acquire(c uint64)

// Release previously-acquired capacity.
func (ws *WeightedSemaphore) Release(c uint64)
<commit_msg>WeightedSemaphore.Acquire<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package syncutil

import (
	"fmt"
	"sync"
)

// A weighted counted semaphore. Can be created as part of larger structs. Must
// be initialized with Init.
type WeightedSemaphore struct {
	// Constant after Init returns.
	capacity uint64

	mu InvariantMutex

	// The remaining capacity.
	//
	// INVARIANT: remaining <= capacity
	//
	// GUARDED_BY(mu)
	remaining uint64

	// Signalled when remaining is changed.
	remainingChanged sync.Cond
}

// LOCKS_REQUIRED(ws.mu)
func (ws *WeightedSemaphore) checkInvariants() {
	// INVARIANT: remaining <= capacity
	if !(ws.remaining <= ws.capacity) {
		panic(fmt.Sprintf("remaining > capacity: %v, %v", ws.remaining, ws.capacity))
	}
}

// Initialize the semaphore with the given capacity.
func (ws *WeightedSemaphore) Init(capacity uint64) {
	ws.capacity = capacity
	ws.remaining = capacity
	ws.mu = NewInvariantMutex(ws.checkInvariants)
	ws.remainingChanged.L = &ws.mu
}

// Return the capacity with which the semaphore was initialized.
func (ws *WeightedSemaphore) Capacity() uint64 {
	return ws.capacity
}

// Block until the supplied capacity is available.
//
// REQUIRES: c <= ws.Capacity()
func (ws *WeightedSemaphore) Acquire(c uint64) {
	if c > ws.Capacity() {
		panic(fmt.Sprintf("Bad cost: %v", c))
	}

	// Wait for capacity to be available.
	ws.mu.Lock()
	defer ws.mu.Unlock()

	for c > ws.remaining {
		ws.remainingChanged.Wait()
	}

	// Mark it as ours.
	ws.remaining -= c
}

// Release previously-acquired capacity.
func (ws *WeightedSemaphore) Release(c uint64)
<|endoftext|>


<commit_before>package srvr

import (
	"encoding/json"
	"fmt"
	"math/rand"
	"os"
	"strings"
	"sync"
	"testing"
	"time"
)

var tmpFileCount int
var randVal int64

var dirLock = &sync.RWMutex{}

func GenerateFileName() string {
	tmpFileCount++
	return fmt.Sprintf("tmp%d_%d.json", randVal, tmpFileCount)
}

type TestPersistable struct {
	TimestampHolder
	Data     string `json:"data"`
	Filename string `json:"-"`
}

func (tp *TestPersistable) Filepath() string {
	return tp.Filename
}

func LockfileName(filename string) string {
	return filename + ".lk"
}

func NewTestPersistable() *TestPersistable {
	tmpFileCount++
	if randVal == 0 {
		randVal = time.Now().UnixNano()
	}
	tp := &TestPersistable{
		Data:     "Some information here",
		Filename: GenerateFileName(),
	}
	return tp
}

func TestDelete(t *testing.T) {
	tp := NewTestPersistable()
	filename := tp.Filepath()
	create(filename, t)
	write(filename, tp, t)
	defer remove(filename)

	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	err := fd.Delete()
	AssertSuccess(t, err)
	FileDoesNotExist(t, filename)
}

func TestUpdateFileFailsWhenLkFileExists(t *testing.T) {
	tp := NewTestPersistable()
	filename := tp.Filepath()
	create(filename, t)
	write(filename, tp, t)
	defer remove(filename)
	create(LockfileName(filename), t)
	defer remove(LockfileName(filename))

	tpr := &TestPersistable{
		Data: "Some new data here",
	}
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	err := fd.Update(tpr)
	IsTrue(t, err != nil, "Update should fail since lk file exists")
}

func TestUpdateFileFailsWhenFileDoesNotExists(t *testing.T) {
	tpr := NewTestPersistable()
	filename := tpr.Filepath()
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	err := fd.Update(tpr)
	IsTrue(t, err != nil, "Update should fail since target file does not exist")
}

func TestUpdateFileFailsWhenPersistableOutOfDate(t *testing.T) {
	tp := NewTestPersistable()
	filename := tp.Filepath()
	create(filename, t)
	write(filename, tp, t)
	defer remove(filename)

	tpr := &TestPersistable{
		Data: "Some new data here",
	}
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	err := fd.Update(tpr)
	IsTrue(t, err != nil, "Update should fail since timestamp is out of date")
}

func TestUpdateFileSucceeds(t *testing.T) {
	tp := NewTestPersistable()
	filename := tp.Filepath()
	create(filename, t)
	write(filename, tp, t)
	st, err := stat(filename)
	AssertSuccess(t, err)
	stamp := st.ModTime()
	defer remove(filename)

	tpr := &TestPersistable{
		TimestampHolder: TimestampHolder{
			Stamp: stamp,
		},
		Data: "Some new data here",
	}
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	err = fd.Update(tpr)
	AssertSuccess(t, err)
	IsTrue(t, tpr.Stamp != stamp, "Update should have changed the timestamp")
}

func TestConcurrentUpdateToFileSucceedsOrFailsWithLockOrStaleErrors(t *testing.T) {
	tp := NewTestPersistable()
	filename := tp.Filepath()
	create(filename, t)
	write(filename, tp, t)
	defer remove(filename)

	hold := make(chan bool)
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	result := make(chan int)

	for i := 0; i < 100; i++ {
		go func() {
			<-hold
			r := rand.Intn(10)
			time.Sleep(time.Duration(r) * time.Nanosecond)

			tpr := &TestPersistable{}
			fd.Read(tpr)
			rs := tpr.Timestamp()
			tpr.Data = fmt.Sprintf("Data %d", i)
			err := fd.Update(tpr)
			chk := &TestPersistable{}
			e := fd.Read(chk)
			if e != nil {
				fd.Read(chk)
			}
			switch {
			case err == nil && tpr.Timestamp() != rs && tpr.Data == chk.Data:
				result <- 1
			case err != nil && strings.Contains(err.Error(), "Unable to lock file"):
				result <- 2
			case err != nil && strings.Contains(err.Error(), "Stale data"):
				result <- 3
			default:
				result <- 0
			}
		}()
	}
	close(hold)
	updateCount := 0
	staleCount := 0
	lockCount := 0
	for i := 0; i < 100; i++ {
		val := <-result
		IsTrue(t, val > 0, fmt.Sprintf("%d update failed", i))
		switch {
		case val == 1:
			updateCount++
		case val == 2:
			lockCount++
		case val == 3:
			staleCount++
		}
	}
	IsTrue(t, updateCount > 0, "Expected at least 1 update to succeed")
	IsTrue(t, staleCount > 0, "Expected at least 1 update to be see stale data")
	close(result)
}

func TestConcurrentReadFromFileSucceeds(t *testing.T) {
	tp := NewTestPersistable()
	filename := tp.Filepath()
	create(filename, t)
	write(filename, tp, t)
	st, err := stat(filename)
	AssertSuccess(t, err)
	stamp := st.ModTime()
	defer remove(filename)

	hold := make(chan bool)
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	success := make(chan bool)

	for i := 0; i < 100; i++ {
		go func() {
			<-hold
			r := rand.Intn(10)
			time.Sleep(time.Duration(r) * time.Nanosecond)

			tpr := &TestPersistable{}
			err := fd.Read(tpr)
			success <- err == nil && tpr.Stamp == stamp && tpr.Data == "Some information here"
		}()
	}
	close(hold)
	for i := 0; i < 100; i++ {
		IsTrue(t, <-success, fmt.Sprintf("%d read failed", i))
	}
	close(success)
}

func TestReadFromFileSucceeds(t *testing.T) {
	tp := NewTestPersistable()
	filename := tp.Filepath()
	create(filename, t)
	write(filename, tp, t)
	st, err := stat(filename)
	AssertSuccess(t, err)
	defer remove(filename)

	tpr := &TestPersistable{}
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	err = fd.Read(tpr)
	AssertSuccess(t, err)
	AreEqual(t, "Some information here", tpr.Data, "Data was not read correctly from file")
	AreEqual(t, st.ModTime(), tpr.Timestamp(), "Timestamp value not updated during read")
}

func TestReadFromFileFailsWhenLkFileExists(t *testing.T) {
	tp := NewTestPersistable()
	filename := tp.Filepath()
	create(filename, t)
	write(filename, tp, t)
	defer remove(filename)
	create(LockfileName(filename), t)
	defer remove(LockfileName(filename))

	tpr := &TestPersistable{}
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	err := fd.Read(tpr)
	IsTrue(t, err != nil, "Read should fail since lk file exists")
	AreEqual(t, "", tpr.Data, "Data should not be read from file")
}

func TestReadDoesNotReadFileIfSameTimestamp(t *testing.T) {
	tp := NewTestPersistable()
	filename := tp.Filepath()
	create(filename, t)
	write(filename, tp, t)
	st, err := stat(filename)
	AssertSuccess(t, err)
	defer remove(filename)

	tpr := &TestPersistable{}
	tpr.Stamp = st.ModTime()
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	err = fd.Read(tpr)
	AssertSuccess(t, err)
	AreEqual(t, "", tpr.Data, "Data should not be read from file because Timestamps are same")
}

func TestCreate(t *testing.T) {
	obj := NewTestPersistable()
	filename := obj.Filepath()
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	err := fd.Create(obj)
	defer remove(filename)
	AssertSuccess(t, err)
	FileExists(t, fd)
}

func TestCreateUpdatesTimeStamp(t *testing.T) {
	obj := NewTestPersistable()
	filename := obj.Filepath()
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	err := fd.Create(obj)
	defer remove(filename)
	AssertSuccess(t, err)
	FileExists(t, fd)
	st, err := stat(filename)
	AssertSuccess(t, err)
	AreEqual(t, st.ModTime(), obj.Stamp, "Timestamp has not been updated")
}

func TestCreateReturnsErrorWhenFileExists(t *testing.T) {
	obj := NewTestPersistable()
	filename := obj.Filepath()
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	err := fd.Create(obj)
	defer remove(filename)
	AssertSuccess(t, err)
	FileExists(t, fd)
	err = fd.Create(obj)
	IsTrue(t, err != nil, "CreateNewFile should return error when File exists")
}

func TestCreateReturnsErrorWhenLKFileExists(t *testing.T) {
	obj := NewTestPersistable()
	filename := obj.Filepath()
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}

	create(LockfileName(filename), t)
	defer remove(LockfileName(filename))
	err := fd.Create(obj)
	IsTrue(t, err != nil, "CreateNewFile should return error when lock file exists")
	FileDoesNotExist(t, filename)
}

func TestCreateReturnsErrorInvalidFilename(t *testing.T) {
	obj := NewTestPersistable()
	fd := &FileDetails{
		Filepath: "TestCreate///.json",
		DirLock:  dirLock,
	}
	err := fd.Create(obj)
	IsTrue(t, err != nil, "CreateNewFile should return error when invalid filename")
}

func TestCreateReturnsErrorEmptyFilename(t *testing.T) {
	obj := &TestPersistable{}
	fd := &FileDetails{
		DirLock: dirLock,
	}
	err := fd.Create(obj)
	IsTrue(t, err != nil, "CreateNewFile should return error when empty filename")
}

func FileExists(t *testing.T, fu FileUpdater) {
	var path string
	if fu != nil {
		path = fu.Path()
	} else {
		t.Errorf("FileUpdater is nil")
		return
	}
	_, err := stat(path)
	AssertSuccess(t, err)
}

func FileDoesNotExist(t *testing.T, path string) {
	var err error
	//Try twice because file system can return access denied on TestDelete
	for i := 0; i < 10; i++ {
		_, err = stat(path)
		if "Access is denied" == err.Error() {
			continue
		}
		if os.IsNotExist(err) {
			return
		}
	}
	t.Errorf("File should not exist: %v", err)
}

func stat(path string) (os.FileInfo, error) {
	dirLock.RLock()
	defer dirLock.RUnlock()
	return os.Stat(path)
}

func remove(path string) error {
	dirLock.Lock()
	defer dirLock.Unlock()
	return os.Remove(path)
}

func create(path string, t *testing.T) {
	dirLock.Lock()
	defer dirLock.Unlock()
	fl, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL, 0644)
	if err != nil {
		t.Error(err)
	}
	err = fl.Close()
	if err != nil {
		t.Error(err)
	}
}

func write(path string, p Persistable, t *testing.T) {
	fl, err := os.OpenFile(path, os.O_TRUNC|os.O_WRONLY, 0644)
	if err != nil {
		t.Error(err)
	}
	b, err := json.Marshal(p)
	if err != nil {
		t.Error(err)
	}
	_, err = fl.Write(b)
	if err != nil {
		t.Error(err)
	}
	err = fl.Close()
	if err != nil {
		t.Error(err)
	}
}
<commit_msg>Retry FileDoesNotExist check in fileupdater test<commit_after>package srvr

import (
	"encoding/json"
	"fmt"
	"math/rand"
	"os"
	"strings"
	"sync"
	"testing"
	"time"
)

var tmpFileCount int
var randVal int64

var dirLock = &sync.RWMutex{}

func GenerateFileName() string {
	tmpFileCount++
	return fmt.Sprintf("tmp%d_%d.json", randVal, tmpFileCount)
}

type TestPersistable struct {
	TimestampHolder
	Data     string `json:"data"`
	Filename string `json:"-"`
}

func (tp *TestPersistable) Filepath() string {
	return tp.Filename
}

func LockfileName(filename string) string {
	return filename + ".lk"
}

func NewTestPersistable() *TestPersistable {
	tmpFileCount++
	if randVal == 0 {
		randVal = time.Now().UnixNano()
	}
	tp := &TestPersistable{
		Data:     "Some information here",
		Filename: GenerateFileName(),
	}
	return tp
}

func TestDelete(t *testing.T) {
	tp := NewTestPersistable()
	filename := tp.Filepath()
	create(filename, t)
	write(filename, tp, t)
	defer remove(filename)

	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	err := fd.Delete()
	AssertSuccess(t, err)
	FileDoesNotExist(t, filename)
}

func TestUpdateFileFailsWhenLkFileExists(t *testing.T) {
	tp := NewTestPersistable()
	filename := tp.Filepath()
	create(filename, t)
	write(filename, tp, t)
	defer remove(filename)
	create(LockfileName(filename), t)
	defer remove(LockfileName(filename))

	tpr := &TestPersistable{
		Data: "Some new data here",
	}
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	err := fd.Update(tpr)
	IsTrue(t, err != nil, "Update should fail since lk file exists")
}

func TestUpdateFileFailsWhenFileDoesNotExists(t *testing.T) {
	tpr := NewTestPersistable()
	filename := tpr.Filepath()
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	err := fd.Update(tpr)
	IsTrue(t, err != nil, "Update should fail since target file does not exist")
}

func TestUpdateFileFailsWhenPersistableOutOfDate(t *testing.T) {
	tp := NewTestPersistable()
	filename := tp.Filepath()
	create(filename, t)
	write(filename, tp, t)
	defer remove(filename)

	tpr := &TestPersistable{
		Data: "Some new data here",
	}
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	err := fd.Update(tpr)
	IsTrue(t, err != nil, "Update should fail since timestamp is out of date")
}

func TestUpdateFileSucceeds(t *testing.T) {
	tp := NewTestPersistable()
	filename := tp.Filepath()
	create(filename, t)
	write(filename, tp, t)
	st, err := stat(filename)
	AssertSuccess(t, err)
	stamp := st.ModTime()
	defer remove(filename)

	tpr := &TestPersistable{
		TimestampHolder: TimestampHolder{
			Stamp: stamp,
		},
		Data: "Some new data here",
	}
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	err = fd.Update(tpr)
	AssertSuccess(t, err)
	NotEqual(t, stamp, tpr.Stamp, "Update should have changed the timestamp")
}

func TestConcurrentUpdateToFileSucceedsOrFailsWithLockOrStaleErrors(t *testing.T) {
	tp := NewTestPersistable()
	filename := tp.Filepath()
	create(filename, t)
	write(filename, tp, t)
	defer remove(filename)

	hold := make(chan bool)
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	result := make(chan int)

	for i := 0; i < 100; i++ {
		go func() {
			<-hold
			r := rand.Intn(10)
			time.Sleep(time.Duration(r) * time.Nanosecond)

			tpr := &TestPersistable{}
			fd.Read(tpr)
			rs := tpr.Timestamp()
			tpr.Data = fmt.Sprintf("Data %d", i)
			err := fd.Update(tpr)
			chk := &TestPersistable{}
			e := fd.Read(chk)
			if e != nil {
				fd.Read(chk)
			}
			switch {
			case err == nil && tpr.Timestamp() != rs && tpr.Data == chk.Data:
				result <- 1
			case err != nil && strings.Contains(err.Error(), "Unable to lock file"):
				result <- 2
			case err != nil && strings.Contains(err.Error(), "Stale data"):
				result <- 3
			default:
				result <- 0
			}
		}()
	}
	close(hold)
	updateCount := 0
	staleCount := 0
	lockCount := 0
	for i := 0; i < 100; i++ {
		val := <-result
		IsTrue(t, val > 0, fmt.Sprintf("%d update failed", i))
		switch {
		case val == 1:
			updateCount++
		case val == 2:
			lockCount++
		case val == 3:
			staleCount++
		}
	}
	IsTrue(t, updateCount > 0, "Expected at least 1 update to succeed")
	IsTrue(t, staleCount > 0, "Expected at least 1 update to be see stale data")
	close(result)
}

func TestConcurrentReadFromFileSucceeds(t *testing.T) {
	tp := NewTestPersistable()
	filename := tp.Filepath()
	create(filename, t)
	write(filename, tp, t)
	st, err := stat(filename)
	AssertSuccess(t, err)
	stamp := st.ModTime()
	defer remove(filename)

	hold := make(chan bool)
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	success := make(chan bool)

	for i := 0; i < 100; i++ {
		go func() {
			<-hold
			r := rand.Intn(10)
			time.Sleep(time.Duration(r) * time.Nanosecond)

			tpr := &TestPersistable{}
			err := fd.Read(tpr)
			success <- err == nil && tpr.Stamp == stamp && tpr.Data == "Some information here"
		}()
	}
	close(hold)
	for i := 0; i < 100; i++ {
		IsTrue(t, <-success, fmt.Sprintf("%d read failed", i))
	}
	close(success)
}

func TestReadFromFileSucceeds(t *testing.T) {
	tp := NewTestPersistable()
	filename := tp.Filepath()
	create(filename, t)
	write(filename, tp, t)
	st, err := stat(filename)
	AssertSuccess(t, err)
	defer remove(filename)

	tpr := &TestPersistable{}
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	err = fd.Read(tpr)
	AssertSuccess(t, err)
	AreEqual(t, "Some information here", tpr.Data, "Data was not read correctly from file")
	AreEqual(t, st.ModTime(), tpr.Timestamp(), "Timestamp value not updated during read")
}

func TestReadFromFileFailsWhenLkFileExists(t *testing.T) {
	tp := NewTestPersistable()
	filename := tp.Filepath()
	create(filename, t)
	write(filename, tp, t)
	defer remove(filename)
	create(LockfileName(filename), t)
	defer remove(LockfileName(filename))

	tpr := &TestPersistable{}
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	err := fd.Read(tpr)
	IsTrue(t, err != nil, "Read should fail since lk file exists")
	AreEqual(t, "", tpr.Data, "Data should not be read from file")
}

func TestReadDoesNotReadFileIfSameTimestamp(t *testing.T) {
	tp := NewTestPersistable()
	filename := tp.Filepath()
	create(filename, t)
	write(filename, tp, t)
	st, err := stat(filename)
	AssertSuccess(t, err)
	defer remove(filename)

	tpr := &TestPersistable{}
	tpr.Stamp = st.ModTime()
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	err = fd.Read(tpr)
	AssertSuccess(t, err)
	AreEqual(t, "", tpr.Data, "Data should not be read from file because Timestamps are same")
}

func TestCreate(t *testing.T) {
	obj := NewTestPersistable()
	filename := obj.Filepath()
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	err := fd.Create(obj)
	defer remove(filename)
	AssertSuccess(t, err)
	FileExists(t, fd)
}

func TestCreateUpdatesTimeStamp(t *testing.T) {
	obj := NewTestPersistable()
	filename := obj.Filepath()
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	err := fd.Create(obj)
	defer remove(filename)
	AssertSuccess(t, err)
	FileExists(t, fd)
	st, err := stat(filename)
	AssertSuccess(t, err)
	AreEqual(t, st.ModTime(), obj.Stamp, "Timestamp has not been updated")
}

func TestCreateReturnsErrorWhenFileExists(t *testing.T) {
	obj := NewTestPersistable()
	filename := obj.Filepath()
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}
	err := fd.Create(obj)
	defer remove(filename)
	AssertSuccess(t, err)
	FileExists(t, fd)
	err = fd.Create(obj)
	IsTrue(t, err != nil, "CreateNewFile should return error when File exists")
}

func TestCreateReturnsErrorWhenLKFileExists(t *testing.T) {
	obj := NewTestPersistable()
	filename := obj.Filepath()
	fd := &FileDetails{
		Filepath: filename,
		DirLock:  dirLock,
	}

	create(LockfileName(filename), t)
	defer remove(LockfileName(filename))
	err := fd.Create(obj)
	IsTrue(t, err != nil, "CreateNewFile should return error when lock file exists")
	FileDoesNotExist(t, filename)
}

func TestCreateReturnsErrorInvalidFilename(t *testing.T) {
	obj := NewTestPersistable()
	fd := &FileDetails{
		Filepath: "TestCreate///.json",
		DirLock:  dirLock,
	}
	err := fd.Create(obj)
	IsTrue(t, err != nil, "CreateNewFile should return error when invalid filename")
}

func TestCreateReturnsErrorEmptyFilename(t *testing.T) {
	obj := &TestPersistable{}
	fd := &FileDetails{
		DirLock: dirLock,
	}
	err := fd.Create(obj)
	IsTrue(t, err != nil, "CreateNewFile should return error when empty filename")
}

func FileExists(t *testing.T, fu FileUpdater) {
	var path string
	if fu != nil {
		path = fu.Path()
	} else {
		t.Errorf("FileUpdater is nil")
		return
	}
	_, err := stat(path)
	AssertSuccess(t, err)
}

func FileDoesNotExist(t *testing.T, path string) {
	var err error
	//Try twice because file system can return access denied on TestDelete
	for i := 0; i < 10; i++ {
		_, err = stat(path)
		if os.IsNotExist(err) {
			return
		}
		time.Sleep(5 * time.Nanosecond)
	}
	t.Errorf("File should not exist: %v", err)
}

func stat(path string) (os.FileInfo, error) {
	dirLock.RLock()
	defer dirLock.RUnlock()
	return os.Stat(path)
}

func remove(path string) error {
	dirLock.Lock()
	defer dirLock.Unlock()
	return os.Remove(path)
}

func create(path string, t *testing.T) {
	dirLock.Lock()
	defer dirLock.Unlock()
	fl, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL, 0644)
	if err != nil {
		t.Error(err)
	}
	err = fl.Close()
	if err != nil {
		t.Error(err)
	}
}

func write(path string, p Persistable, t *testing.T) {
	fl, err := os.OpenFile(path, os.O_TRUNC|os.O_WRONLY, 0644)
	if err != nil {
		t.Error(err)
	}
	b, err := json.Marshal(p)
	if err != nil {
		t.Error(err)
	}
	_, err = fl.Write(b)
	if err != nil {
		t.Error(err)
	}
	err = fl.Close()
	if err != nil {
		t.Error(err)
	}
}
<|endoftext|>


<commit_before>package tarantool

import (
	"context"
)

// the Result type is used to return write errors here
func (conn *Connection) writeRequest(ctx context.Context, q Query, opaque interface{}, replyChan chan *AsyncResult) (*request, *Result) {
	var err error

	requestID := conn.nextID()
	request := &request{
		opaque:    opaque,
		replyChan: replyChan,
	}

	pp := packetPool.GetWithID(requestID)

	if err = pp.packMsg(q, conn.packData); err != nil {
		return nil, &Result{
			Error:     NewQueryError(ErrInvalidMsgpack, err.Error()),
			ErrorCode: ErrInvalidMsgpack,
		}
	}

	if oldRequest := conn.requests.Put(requestID, request); oldRequest != nil {
		select {
		case oldRequest.replyChan <- &AsyncResult{
			Error:     ConnectionClosedError(conn),
			ErrorCode: ErrNoConnection,
		}:
		default:
		}
	}

	writeChan := conn.writeChan
	if writeChan == nil {
		return nil, &Result{
			Error:     ConnectionClosedError(conn),
			ErrorCode: ErrNoConnection,
		}
	}

	select {
	case writeChan <- pp:
	case <-ctx.Done():
		if conn.perf.QueryTimeouts != nil {
			conn.perf.QueryTimeouts.Add(1)
		}
		conn.requests.Pop(requestID)
		return nil, &Result{
			Error:     NewContextError(ctx, conn, "Send error"),
			ErrorCode: ErrTimeout,
		}
	case <-conn.exit:
		return nil, &Result{
			Error:     ConnectionClosedError(conn),
			ErrorCode: ErrNoConnection,
		}
	}

	return request, nil
}

func (conn *Connection) readResult(ctx context.Context, arc chan *AsyncResult) *AsyncResult {
	select {
	case ar := <-arc:
		if ar == nil {
			return &AsyncResult{
				Error:     ConnectionClosedError(conn),
				ErrorCode: ErrNoConnection,
			}
		}
		return ar
	case <-ctx.Done():
		if conn.perf.QueryTimeouts != nil {
			conn.perf.QueryTimeouts.Add(1)
		}
		return &AsyncResult{
			Error:     NewContextError(ctx, conn, "Recv error"),
			ErrorCode: ErrTimeout,
		}
	case <-conn.exit:
		return &AsyncResult{
			Error:     ConnectionClosedError(conn),
			ErrorCode: ErrNoConnection,
		}
	}
}

func (conn *Connection) Exec(ctx context.Context, q Query) (result *Result) {
	var cancel context.CancelFunc = func() {}

	if _, ok := ctx.Deadline(); !ok && conn.queryTimeout != 0 {
		ctx, cancel = context.WithTimeout(ctx, conn.queryTimeout)
	}

	replyChan := make(chan *AsyncResult, 1)

	if _, rerr := conn.writeRequest(ctx, q, nil, replyChan); rerr != nil {
		cancel()
		return rerr
	}

	ar := conn.readResult(ctx, replyChan)
	cancel()

	if rerr := ar.Error; rerr != nil {
		return &Result{
			Error:     rerr,
			ErrorCode: ar.ErrorCode,
		}
	}

	pp := ar.BinaryPacket
	if pp == nil {
		return &Result{
			Error:     ConnectionClosedError(conn),
			ErrorCode: ErrNoConnection,
		}
	}

	if err := pp.Unmarshal(); err != nil {
		result = &Result{
			Error:     err,
			ErrorCode: ErrInvalidMsgpack,
		}
	} else {
		result = pp.Result()
		if result == nil {
			result = &Result{}
		}
	}
	pp.Release()

	return result
}

func (conn *Connection) ExecAsync(ctx context.Context, q Query, opaque interface{}, replyChan chan *AsyncResult) error {
	if _, rerr := conn.writeRequest(ctx, q, opaque, replyChan); rerr != nil {
		return rerr.Error
	}
	return nil
}

func (conn *Connection) Execute(q Query) ([][]interface{}, error) {
	res := conn.Exec(context.Background(), q)
	return res.Data, res.Error
}
<commit_msg>Fix another case of opaque pointer not being passed back<commit_after>package tarantool

import (
	"context"
)

// the Result type is used to return write errors here
func (conn *Connection) writeRequest(ctx context.Context, q Query, opaque interface{}, replyChan chan *AsyncResult) (*request, *Result) {
	var err error

	requestID := conn.nextID()
	request := &request{
		opaque:    opaque,
		replyChan: replyChan,
	}

	pp := packetPool.GetWithID(requestID)

	if err = pp.packMsg(q, conn.packData); err != nil {
		return nil, &Result{
			Error:     NewQueryError(ErrInvalidMsgpack, err.Error()),
			ErrorCode: ErrInvalidMsgpack,
		}
	}

	if oldRequest := conn.requests.Put(requestID, request); oldRequest != nil {
		select {
		case oldRequest.replyChan <- &AsyncResult{
			Error:     ConnectionClosedError(conn),
			ErrorCode: ErrNoConnection,
			Opaque:    oldRequest.opaque,
		}:
		default:
		}
	}

	writeChan := conn.writeChan
	if writeChan == nil {
		return nil, &Result{
			Error:     ConnectionClosedError(conn),
			ErrorCode: ErrNoConnection,
		}
	}

	select {
	case writeChan <- pp:
	case <-ctx.Done():
		if conn.perf.QueryTimeouts != nil {
			conn.perf.QueryTimeouts.Add(1)
		}
		conn.requests.Pop(requestID)
		return nil, &Result{
			Error:     NewContextError(ctx, conn, "Send error"),
			ErrorCode: ErrTimeout,
		}
	case <-conn.exit:
		return nil, &Result{
			Error:     ConnectionClosedError(conn),
			ErrorCode: ErrNoConnection,
		}
	}

	return request, nil
}

func (conn *Connection) readResult(ctx context.Context, arc chan *AsyncResult) *AsyncResult {
	select {
	case ar := <-arc:
		if ar == nil {
			return &AsyncResult{
				Error:     ConnectionClosedError(conn),
				ErrorCode: ErrNoConnection,
			}
		}
		return ar
	case <-ctx.Done():
		if conn.perf.QueryTimeouts != nil {
			conn.perf.QueryTimeouts.Add(1)
		}
		return &AsyncResult{
			Error:     NewContextError(ctx, conn, "Recv error"),
			ErrorCode: ErrTimeout,
		}
	case <-conn.exit:
		return &AsyncResult{
			Error:     ConnectionClosedError(conn),
			ErrorCode: ErrNoConnection,
		}
	}
}

func (conn *Connection) Exec(ctx context.Context, q Query) (result *Result) {
	var cancel context.CancelFunc = func() {}

	if _, ok := ctx.Deadline(); !ok && conn.queryTimeout != 0 {
		ctx, cancel = context.WithTimeout(ctx, conn.queryTimeout)
	}

	replyChan := make(chan *AsyncResult, 1)

	if _, rerr := conn.writeRequest(ctx, q, nil, replyChan); rerr != nil {
		cancel()
		return rerr
	}

	ar := conn.readResult(ctx, replyChan)
	cancel()

	if rerr := ar.Error; rerr != nil {
		return &Result{
			Error:     rerr,
			ErrorCode: ar.ErrorCode,
		}
	}

	pp := ar.BinaryPacket
	if pp == nil {
		return &Result{
			Error:     ConnectionClosedError(conn),
			ErrorCode: ErrNoConnection,
		}
	}

	if err := pp.Unmarshal(); err != nil {
		result = &Result{
			Error:     err,
			ErrorCode: ErrInvalidMsgpack,
		}
	} else {
		result = pp.Result()
		if result == nil {
			result = &Result{}
		}
	}
	pp.Release()

	return result
}

func (conn *Connection) ExecAsync(ctx context.Context, q Query, opaque interface{}, replyChan chan *AsyncResult) error {
	if _, rerr := conn.writeRequest(ctx, q, opaque, replyChan); rerr != nil {
		return rerr.Error
	}
	return nil
}

func (conn *Connection) Execute(q Query) ([][]interface{}, error) {
	res := conn.Exec(context.Background(), q)
	return res.Data, res.Error
}
<|endoftext|>


<commit_before>package infermedica

import (
	"encoding/json"
	"net/http"
)

type ExplainReq struct {
	Sex       Sex        `json:"sex"`
	Age       int        `json:"age"`
	Target    string     `json:"target"`
	Evidences []Evidence `json:"evidence"`
}

type ExplainRes struct {
	SupportingEvidence  []EvidenceItem `json:"supporting_evidence"`
	ConflictingEvidence []EvidenceItem `json:"conflicting_evidence"`
}

type EvidenceItem struct {
	ID         string `json:"id"`
	Name       string `json:"name"`
	CommonName string `json:"common_name"`
}

func (a *App) Explain(er ExplainReq) (*ExplainRes, error) {
	req, err := a.prepareRequest("POST", "explain", er)
	if err != nil {
		return nil, err
	}
	client := &http.Client{}
	res, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	r := ExplainRes{}
	err = json.NewDecoder(res.Body).Decode(&r)
	if err != nil {
		return nil, err
	}
	return &r, nil
}
<commit_msg>added missing UnconfirmedEvidence to explain<commit_after>package infermedica

import (
	"encoding/json"
	"net/http"
)

type ExplainReq struct {
	Sex       Sex        `json:"sex"`
	Age       int        `json:"age"`
	Target    string     `json:"target"`
	Evidences []Evidence `json:"evidence"`
}

type ExplainRes struct {
	SupportingEvidence  []EvidenceItem `json:"supporting_evidence"`
	ConflictingEvidence []EvidenceItem `json:"conflicting_evidence"`
	UnconfirmedEvidence []EvidenceItem `json:"unconfirmed_evidence"`
}

type EvidenceItem struct {
	ID         string `json:"id"`
	Name       string `json:"name"`
	CommonName string `json:"common_name"`
}

func (a *App) Explain(er ExplainReq) (*ExplainRes, error) {
	req, err := a.prepareRequest("POST", "explain", er)
	if err != nil {
		return nil, err
	}
	client := &http.Client{}
	res, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	r := ExplainRes{}
	err = json.NewDecoder(res.Body).Decode(&r)
	if err != nil {
		return nil, err
	}
	return &r, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2012 Joseph D Poirier
// Distributable under the terms of The New BSD License
// that can be found in the LICENSE file.

// Package gortlsdr wraps librtlsdr, which turns your Realtek RTL2832 based
// DVB dongle into a SDR receiver.
//
package rtlsdr

import (
	"reflect"
	"unsafe"
)

/*
#include <rtl-sdr.h>
*/
import "C"

//export go_callback
func go_callback(p1 *C.char, p2 C.uint32_t, p3 unsafe.Pointer) {
	// c buffer to go slice without copying
	var buf []int8
	length := int(p2)
	b := (*reflect.SliceHeader)((unsafe.Pointer(&buf)))
	b.Cap = length
	b.Len = length
	b.Data = uintptr(unsafe.Pointer((*int8)(p1)))
	clientCb(buf)
}

<commit_msg>add missing parameter<commit_after>// Copyright (c) 2012 Joseph D Poirier
// Distributable under the terms of The New BSD License
// that can be found in the LICENSE file.

// Package gortlsdr wraps librtlsdr, which turns your Realtek RTL2832 based
// DVB dongle into a SDR receiver.
//
package rtlsdr

import (
	"reflect"
	"unsafe"
)

/*
#include <rtl-sdr.h>
*/
import "C"

//export go_callback
func go_callback(p1 *C.char, p2 C.uint32_t, p3 unsafe.Pointer) {
	// c buffer to go slice without copying
	var buf []int8
	length := int(p2)
	b := (*reflect.SliceHeader)((unsafe.Pointer(&buf)))
	b.Cap = length
	b.Len = length
	b.Data = uintptr(unsafe.Pointer((*int8)(p1)))
	clientCb(buf, (*UserCtx)(p3))
}

<|endoftext|>