<commit_before>package git

/*
#include <git2.h>
*/
import "C"
import (
	"errors"
	"runtime"
	"unsafe"
)

// RebaseOperationType is the type of rebase operation
type RebaseOperationType uint

const (
	// RebaseOperationPick The given commit is to be cherry-picked.  The client should commit the changes and continue if there are no conflicts.
	RebaseOperationPick RebaseOperationType = C.GIT_REBASE_OPERATION_PICK
	// RebaseOperationReword The given commit is to be cherry-picked, but the client should prompt the user to provide an updated commit message.
	RebaseOperationReword RebaseOperationType = C.GIT_REBASE_OPERATION_REWORD
	// RebaseOperationEdit The given commit is to be cherry-picked, but the client should stop to allow the user to edit the changes before committing them.
	RebaseOperationEdit RebaseOperationType = C.GIT_REBASE_OPERATION_EDIT
	// RebaseOperationSquash The given commit is to be squashed into the previous commit.  The commit message will be merged with the previous message.
	RebaseOperationSquash RebaseOperationType = C.GIT_REBASE_OPERATION_SQUASH
	// RebaseOperationFixup No commit will be cherry-picked.  The client should run the given command and (if successful) continue.
	RebaseOperationFixup RebaseOperationType = C.GIT_REBASE_OPERATION_FIXUP
	// RebaseOperationExec No commit will be cherry-picked.  The client should run the given command and (if successful) continue.
	RebaseOperationExec RebaseOperationType = C.GIT_REBASE_OPERATION_EXEC
)

// Special value indicating that there is no currently active operation
var RebaseNoOperation uint = ^uint(0)

// Error returned if there is no current rebase operation
var ErrRebaseNoOperation = errors.New("no current rebase operation")

// RebaseOperation describes a single instruction/operation to be performed during the rebase.
type RebaseOperation struct {
	Type RebaseOperationType
	Id   *Oid
	Exec string
}

func newRebaseOperationFromC(c *C.git_rebase_operation) *RebaseOperation {
	operation := &RebaseOperation{}
	operation.Type = RebaseOperationType(c._type)
	operation.Id = newOidFromC(&c.id)
	operation.Exec = C.GoString(c.exec)

	return operation
}

// RebaseOptions are used to tell the rebase machinery how to operate
type RebaseOptions struct {
	Version         uint
	Quiet           int
	InMemory        int
	RewriteNotesRef string
	MergeOptions    MergeOptions
	CheckoutOptions CheckoutOpts
}

// DefaultRebaseOptions returns a RebaseOptions with default values.
func DefaultRebaseOptions() (RebaseOptions, error) {
	opts := C.git_rebase_options{}

	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	ecode := C.git_rebase_init_options(&opts, C.GIT_REBASE_OPTIONS_VERSION)
	if ecode < 0 {
		return RebaseOptions{}, MakeGitError(ecode)
	}
	return rebaseOptionsFromC(&opts), nil
}

func rebaseOptionsFromC(opts *C.git_rebase_options) RebaseOptions {
	return RebaseOptions{
		Version:         uint(opts.version),
		Quiet:           int(opts.quiet),
		InMemory:        int(opts.inmemory),
		RewriteNotesRef: C.GoString(opts.rewrite_notes_ref),
		MergeOptions:    mergeOptionsFromC(&opts.merge_options),
		CheckoutOptions: checkoutOptionsFromC(&opts.checkout_options),
	}
}

func (ro *RebaseOptions) toC() *C.git_rebase_options {
	if ro == nil {
		return nil
	}
	return &C.git_rebase_options{
		version:           C.uint(ro.Version),
		quiet:             C.int(ro.Quiet),
		inmemory:          C.int(ro.InMemory),
		rewrite_notes_ref: mapEmptyStringToNull(ro.RewriteNotesRef),
		merge_options:     *ro.MergeOptions.toC(),
		checkout_options:  *ro.CheckoutOptions.toC(),
	}
}

func mapEmptyStringToNull(ref string) *C.char {
	if ref == "" {
		return nil
	}
	return C.CString(ref)
}

// Rebase is the struct representing a Rebase object.
type Rebase struct {
	ptr *C.git_rebase
	r   *Repository
}

// InitRebase initializes a rebase operation to rebase the changes in branch relative to upstream onto another branch.
func (r *Repository) InitRebase(branch *AnnotatedCommit, upstream *AnnotatedCommit, onto *AnnotatedCommit, opts *RebaseOptions) (*Rebase, error) {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	if branch == nil {
		branch = &AnnotatedCommit{ptr: nil}
	}

	if upstream == nil {
		upstream = &AnnotatedCommit{ptr: nil}
	}

	if onto == nil {
		onto = &AnnotatedCommit{ptr: nil}
	}

	var ptr *C.git_rebase
	err := C.git_rebase_init(&ptr, r.ptr, branch.ptr, upstream.ptr, onto.ptr, opts.toC())
	runtime.KeepAlive(branch)
	runtime.KeepAlive(upstream)
	runtime.KeepAlive(onto)
	if err < 0 {
		return nil, MakeGitError(err)
	}

	return newRebaseFromC(ptr), nil
}

// OpenRebase opens an existing rebase that was previously started by either an invocation of InitRebase or by another client.
func (r *Repository) OpenRebase(opts *RebaseOptions) (*Rebase, error) {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	var ptr *C.git_rebase
	err := C.git_rebase_open(&ptr, r.ptr, opts.toC())
	runtime.KeepAlive(r)
	if err < 0 {
		return nil, MakeGitError(err)
	}

	return newRebaseFromC(ptr), nil
}

// OperationAt gets the rebase operation specified by the given index.
func (rebase *Rebase) OperationAt(index uint) *RebaseOperation {
	operation := C.git_rebase_operation_byindex(rebase.ptr, C.size_t(index))

	return newRebaseOperationFromC(operation)
}

// CurrentOperationIndex gets the index of the rebase operation that is
// currently being applied. There is also an error returned for API
// compatibility.
func (rebase *Rebase) CurrentOperationIndex() (uint, error) {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	var err error
	operationIndex := uint(C.git_rebase_operation_current(rebase.ptr))
	runtime.KeepAlive(rebase)
	if operationIndex == RebaseNoOperation {
		err = ErrRebaseNoOperation
	}

	return uint(operationIndex), err
}

// OperationCount gets the count of rebase operations that are to be applied.
func (rebase *Rebase) OperationCount() uint {
	ret := uint(C.git_rebase_operation_entrycount(rebase.ptr))
	runtime.KeepAlive(rebase)
	return ret
}

// Next performs the next rebase operation and returns the information about it.
// If the operation is one that applies a patch (which is any operation except RebaseOperationExec)
// then the patch will be applied and the index and working directory will be updated with the changes.
// If there are conflicts, you will need to address those before committing the changes.
func (rebase *Rebase) Next() (*RebaseOperation, error) {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	var ptr *C.git_rebase_operation
	err := C.git_rebase_next(&ptr, rebase.ptr)
	runtime.KeepAlive(rebase)
	if err < 0 {
		return nil, MakeGitError(err)
	}

	return newRebaseOperationFromC(ptr), nil
}

// Commit commits the current patch.
// You must have resolved any conflicts that were introduced during the patch application from the Next() invocation.
func (rebase *Rebase) Commit(ID *Oid, author, committer *Signature, message string) error {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	authorSig, err := author.toC()
	if err != nil {
		return err
	}
	defer C.git_signature_free(authorSig)

	committerSig, err := committer.toC()
	if err != nil {
		return err
	}
	defer C.git_signature_free(committerSig)

	cmsg := C.CString(message)
	defer C.free(unsafe.Pointer(cmsg))

	cerr := C.git_rebase_commit(ID.toC(), rebase.ptr, authorSig, committerSig, nil, cmsg)
	runtime.KeepAlive(ID)
	runtime.KeepAlive(rebase)
	if cerr < 0 {
		return MakeGitError(cerr)
	}

	return nil
}

// Finish finishes a rebase that is currently in progress once all patches have been applied.
func (rebase *Rebase) Finish() error {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	err := C.git_rebase_finish(rebase.ptr, nil)
	runtime.KeepAlive(rebase)
	if err < 0 {
		return MakeGitError(err)
	}

	return nil
}

// Abort aborts a rebase that is currently in progress, resetting the repository and working directory to their state before rebase began.
func (rebase *Rebase) Abort() error {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	err := C.git_rebase_abort(rebase.ptr)
	runtime.KeepAlive(rebase)
	if err < 0 {
		return MakeGitError(err)
	}
	return nil
}

// Free frees the Rebase object.
func (rebase *Rebase) Free() {
	runtime.SetFinalizer(rebase, nil)
	C.git_rebase_free(rebase.ptr)
}

func newRebaseFromC(ptr *C.git_rebase) *Rebase {
	rebase := &Rebase{ptr: ptr}
	runtime.SetFinalizer(rebase, (*Rebase).Free)
	return rebase
}
<commit_msg>rebase: make RebaseOperationType a fmt.Stringer<commit_after>package git

/*
#include <git2.h>
*/
import "C"
import (
	"errors"
	"fmt"
	"runtime"
	"unsafe"
)

// RebaseOperationType is the type of rebase operation
type RebaseOperationType uint

const (
	// RebaseOperationPick The given commit is to be cherry-picked.  The client should commit the changes and continue if there are no conflicts.
	RebaseOperationPick RebaseOperationType = C.GIT_REBASE_OPERATION_PICK
	// RebaseOperationReword The given commit is to be cherry-picked, but the client should prompt the user to provide an updated commit message.
	RebaseOperationReword RebaseOperationType = C.GIT_REBASE_OPERATION_REWORD
	// RebaseOperationEdit The given commit is to be cherry-picked, but the client should stop to allow the user to edit the changes before committing them.
	RebaseOperationEdit RebaseOperationType = C.GIT_REBASE_OPERATION_EDIT
	// RebaseOperationSquash The given commit is to be squashed into the previous commit.  The commit message will be merged with the previous message.
	RebaseOperationSquash RebaseOperationType = C.GIT_REBASE_OPERATION_SQUASH
	// RebaseOperationFixup No commit will be cherry-picked.  The client should run the given command and (if successful) continue.
	RebaseOperationFixup RebaseOperationType = C.GIT_REBASE_OPERATION_FIXUP
	// RebaseOperationExec No commit will be cherry-picked.  The client should run the given command and (if successful) continue.
	RebaseOperationExec RebaseOperationType = C.GIT_REBASE_OPERATION_EXEC
)

func (t RebaseOperationType) String() string {
	switch t {
	case RebaseOperationPick:
		return "pick"
	case RebaseOperationReword:
		return "reword"
	case RebaseOperationEdit:
		return "edit"
	case RebaseOperationSquash:
		return "squash"
	case RebaseOperationFixup:
		return "fixup"
	case RebaseOperationExec:
		return "exec"
	}
	return fmt.Sprintf("RebaseOperationType(%d)", t)
}

// Special value indicating that there is no currently active operation
var RebaseNoOperation uint = ^uint(0)

// Error returned if there is no current rebase operation
var ErrRebaseNoOperation = errors.New("no current rebase operation")

// RebaseOperation describes a single instruction/operation to be performed during the rebase.
type RebaseOperation struct {
	Type RebaseOperationType
	Id   *Oid
	Exec string
}

func newRebaseOperationFromC(c *C.git_rebase_operation) *RebaseOperation {
	operation := &RebaseOperation{}
	operation.Type = RebaseOperationType(c._type)
	operation.Id = newOidFromC(&c.id)
	operation.Exec = C.GoString(c.exec)

	return operation
}

// RebaseOptions are used to tell the rebase machinery how to operate
type RebaseOptions struct {
	Version         uint
	Quiet           int
	InMemory        int
	RewriteNotesRef string
	MergeOptions    MergeOptions
	CheckoutOptions CheckoutOpts
}

// DefaultRebaseOptions returns a RebaseOptions with default values.
func DefaultRebaseOptions() (RebaseOptions, error) {
	opts := C.git_rebase_options{}

	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	ecode := C.git_rebase_init_options(&opts, C.GIT_REBASE_OPTIONS_VERSION)
	if ecode < 0 {
		return RebaseOptions{}, MakeGitError(ecode)
	}
	return rebaseOptionsFromC(&opts), nil
}

func rebaseOptionsFromC(opts *C.git_rebase_options) RebaseOptions {
	return RebaseOptions{
		Version:         uint(opts.version),
		Quiet:           int(opts.quiet),
		InMemory:        int(opts.inmemory),
		RewriteNotesRef: C.GoString(opts.rewrite_notes_ref),
		MergeOptions:    mergeOptionsFromC(&opts.merge_options),
		CheckoutOptions: checkoutOptionsFromC(&opts.checkout_options),
	}
}

func (ro *RebaseOptions) toC() *C.git_rebase_options {
	if ro == nil {
		return nil
	}
	return &C.git_rebase_options{
		version:           C.uint(ro.Version),
		quiet:             C.int(ro.Quiet),
		inmemory:          C.int(ro.InMemory),
		rewrite_notes_ref: mapEmptyStringToNull(ro.RewriteNotesRef),
		merge_options:     *ro.MergeOptions.toC(),
		checkout_options:  *ro.CheckoutOptions.toC(),
	}
}

func mapEmptyStringToNull(ref string) *C.char {
	if ref == "" {
		return nil
	}
	return C.CString(ref)
}

// Rebase is the struct representing a Rebase object.
type Rebase struct {
	ptr *C.git_rebase
	r   *Repository
}

// InitRebase initializes a rebase operation to rebase the changes in branch relative to upstream onto another branch.
func (r *Repository) InitRebase(branch *AnnotatedCommit, upstream *AnnotatedCommit, onto *AnnotatedCommit, opts *RebaseOptions) (*Rebase, error) {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	if branch == nil {
		branch = &AnnotatedCommit{ptr: nil}
	}

	if upstream == nil {
		upstream = &AnnotatedCommit{ptr: nil}
	}

	if onto == nil {
		onto = &AnnotatedCommit{ptr: nil}
	}

	var ptr *C.git_rebase
	err := C.git_rebase_init(&ptr, r.ptr, branch.ptr, upstream.ptr, onto.ptr, opts.toC())
	runtime.KeepAlive(branch)
	runtime.KeepAlive(upstream)
	runtime.KeepAlive(onto)
	if err < 0 {
		return nil, MakeGitError(err)
	}

	return newRebaseFromC(ptr), nil
}

// OpenRebase opens an existing rebase that was previously started by either an invocation of InitRebase or by another client.
func (r *Repository) OpenRebase(opts *RebaseOptions) (*Rebase, error) {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	var ptr *C.git_rebase
	err := C.git_rebase_open(&ptr, r.ptr, opts.toC())
	runtime.KeepAlive(r)
	if err < 0 {
		return nil, MakeGitError(err)
	}

	return newRebaseFromC(ptr), nil
}

// OperationAt gets the rebase operation specified by the given index.
func (rebase *Rebase) OperationAt(index uint) *RebaseOperation {
	operation := C.git_rebase_operation_byindex(rebase.ptr, C.size_t(index))

	return newRebaseOperationFromC(operation)
}

// CurrentOperationIndex gets the index of the rebase operation that is
// currently being applied. There is also an error returned for API
// compatibility.
func (rebase *Rebase) CurrentOperationIndex() (uint, error) {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	var err error
	operationIndex := uint(C.git_rebase_operation_current(rebase.ptr))
	runtime.KeepAlive(rebase)
	if operationIndex == RebaseNoOperation {
		err = ErrRebaseNoOperation
	}

	return uint(operationIndex), err
}

// OperationCount gets the count of rebase operations that are to be applied.
func (rebase *Rebase) OperationCount() uint {
	ret := uint(C.git_rebase_operation_entrycount(rebase.ptr))
	runtime.KeepAlive(rebase)
	return ret
}

// Next performs the next rebase operation and returns the information about it.
// If the operation is one that applies a patch (which is any operation except RebaseOperationExec)
// then the patch will be applied and the index and working directory will be updated with the changes.
// If there are conflicts, you will need to address those before committing the changes.
func (rebase *Rebase) Next() (*RebaseOperation, error) {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	var ptr *C.git_rebase_operation
	err := C.git_rebase_next(&ptr, rebase.ptr)
	runtime.KeepAlive(rebase)
	if err < 0 {
		return nil, MakeGitError(err)
	}

	return newRebaseOperationFromC(ptr), nil
}

// Commit commits the current patch.
// You must have resolved any conflicts that were introduced during the patch application from the Next() invocation.
func (rebase *Rebase) Commit(ID *Oid, author, committer *Signature, message string) error {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	authorSig, err := author.toC()
	if err != nil {
		return err
	}
	defer C.git_signature_free(authorSig)

	committerSig, err := committer.toC()
	if err != nil {
		return err
	}
	defer C.git_signature_free(committerSig)

	cmsg := C.CString(message)
	defer C.free(unsafe.Pointer(cmsg))

	cerr := C.git_rebase_commit(ID.toC(), rebase.ptr, authorSig, committerSig, nil, cmsg)
	runtime.KeepAlive(ID)
	runtime.KeepAlive(rebase)
	if cerr < 0 {
		return MakeGitError(cerr)
	}

	return nil
}

// Finish finishes a rebase that is currently in progress once all patches have been applied.
func (rebase *Rebase) Finish() error {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	err := C.git_rebase_finish(rebase.ptr, nil)
	runtime.KeepAlive(rebase)
	if err < 0 {
		return MakeGitError(err)
	}

	return nil
}

// Abort aborts a rebase that is currently in progress, resetting the repository and working directory to their state before rebase began.
func (rebase *Rebase) Abort() error {
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	err := C.git_rebase_abort(rebase.ptr)
	runtime.KeepAlive(rebase)
	if err < 0 {
		return MakeGitError(err)
	}
	return nil
}

// Free frees the Rebase object.
func (rebase *Rebase) Free() {
	runtime.SetFinalizer(rebase, nil)
	C.git_rebase_free(rebase.ptr)
}

func newRebaseFromC(ptr *C.git_rebase) *Rebase {
	rebase := &Rebase{ptr: ptr}
	runtime.SetFinalizer(rebase, (*Rebase).Free)
	return rebase
}
<|endoftext|>


<commit_before>/*
 * Copyright 2014 Xuyuan Pang <xuyuanp@gmail.com>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package glogger

import (
	"regexp"
	"time"
)

// Record is a struct contains all the logging information
type Record struct {
	Name    string
	Level   LogLevel
	Time    time.Time
	LFile   string
	SFile   string
	Line    int
	Func    string
	Message string
}

var pathReg = regexp.MustCompile("/.*/")

// NewRecord return a new Record
func NewRecord(name string, t time.Time, level LogLevel, file string, funcname string, line int, msg string) *Record {
	rec := &Record{
		Name:    name,
		Level:   level,
		Time:    t,
		LFile:   file,
		Line:    line,
		Func:    funcname,
		Message: msg,
	}
	rec.SFile = pathReg.ReplaceAllString(file, "")
	return rec
}
<commit_msg>add comment<commit_after>/*
 * Copyright 2014 Xuyuan Pang <xuyuanp@gmail.com>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package glogger

import (
	"regexp"
	"time"
)

// Record is a struct contains all the logging information
type Record struct {
	Name    string    // logger name
	Level   LogLevel  // log level
	Time    time.Time // log time
	LFile   string    // full file name
	SFile   string    // final file name
	Line    int       // line number
	Func    string    // function name
	Message string    // log message
}

var pathReg = regexp.MustCompile("/.*/")

// NewRecord return a new Record
func NewRecord(name string, t time.Time, level LogLevel, file string, funcname string, line int, msg string) *Record {
	rec := &Record{
		Name:    name,
		Level:   level,
		Time:    t,
		LFile:   file,
		Line:    line,
		Func:    funcname,
		Message: msg,
	}
	rec.SFile = pathReg.ReplaceAllString(file, "")
	return rec
}
<|endoftext|>


<commit_before>// Copyright ©2012 Dan Kortschak <dan.kortschak@adelaide.edu.au>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package boom

type Record struct {
	*bamRecord
	filled   bool
	cigarStr string
	nameStr  string
	seqStr   string
	qualStr  string
	auxStr   string
}

func (self *Record) ID() string {
	self.fillData()
	return self.nameStr
}

func (self *Record) Aux() string {
	self.fillData()
	return self.auxStr
}

func (self *Record) Start() int {
	return int(self.pos())
}

func (self *Record) Len() int {
	return int(self.lQseq())
}

func (self *Record) End() int {
	return int(self.pos() + self.lQseq())
}

func (self *Record) Quality() byte {
	return self.qual()
}

func (self *Record) Flags() uint {
	return self.flag()
}

func (self *Record) Strand() int8 {
	if self.Flags()&Reverse == Reverse {
		return -1
	}
	return 1
}

func (self *Record) fillData() (n int) {
	if self.filled {
		return
	}

	d := self.dataUnsafe()
	var s, e int

	s, e = 0, int(self.lQname())
	self.nameStr = string(d[s : e-1])
	self.auxStr = string(d[len(d)-int(self.lAux()):])
	// TODO: Not clear how CIGAR, seq and qual are encoded.

	self.filled = true
	return
}
<commit_msg>Reading and reporting Tags and CIGAR strings<commit_after>// Copyright ©2012 Dan Kortschak <dan.kortschak@adelaide.edu.au>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package boom

import (
	"bytes"
	"encoding/binary"
	"encoding/hex"
	"fmt"
	"unsafe"
)

type Record struct {
	*bamRecord
	filled     bool
	cigar      []CigarOp
	nameStr    string
	seqBytes   []byte
	qualScores []int8
	auxBytes   []byte
	auxTags    []Aux
}

func (self *Record) Name() string {
	self.fillData()
	return self.nameStr
}

func (self *Record) Seq() []byte {
	self.fillData()
	return self.seqBytes
}

func (self *Record) Quality() []int8 {
	self.fillData()
	return self.qualScores
}

func (self *Record) Cigar() []CigarOp {
	self.fillData()
	return self.cigar
}

func (self *Record) Tag(tag []byte) (v Aux, ok bool) {
	self.fillData()
	for i := range self.auxTags {
		if bytes.Compare(self.auxTags[i][:2], tag) == 0 {
			return self.auxTags[i], true
		}
	}
	return
}

func (self *Record) Tags() []Aux {
	self.fillData()
	return self.auxTags
}

func (self *Record) Start() int {
	return int(self.pos())
}

func (self *Record) Len() int {
	return int(self.lQseq())
}

func (self *Record) End() int {
	return int(self.pos() + self.lQseq())
}

func (self *Record) Score() byte {
	return self.qual()
}

func (self *Record) Flags() uint {
	return self.flag()
}

func (self *Record) Strand() int8 {
	if self.Flags()&Reverse == Reverse {
		return -1
	}
	return 1
}

var (
	bamNT16TableRev = [16]byte{'=', 'A', 'C', 'M', 'G', 'R', 'S', 'V', 'T', 'W', 'Y', 'H', 'K', 'D', 'B', 'N'}
	bamNT16Table    = [256]byte{
		0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf,
		0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf,
		0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf,
		0x1, 0x2, 0x4, 0x8, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0x0, 0xf, 0xf,
		0xf, 0x1, 0xe, 0x2, 0xd, 0xf, 0xf, 0x4, 0xb, 0xf, 0xf, 0xc, 0xf, 0x3, 0xf, 0xf,
		0xf, 0xf, 0x5, 0x6, 0x8, 0xf, 0x7, 0x9, 0xf, 0xa, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf,
		0xf, 0x1, 0xe, 0x2, 0xd, 0xf, 0xf, 0x4, 0xb, 0xf, 0xf, 0xc, 0xf, 0x3, 0xf, 0xf,
		0xf, 0xf, 0x5, 0x6, 0x8, 0xf, 0x7, 0x9, 0xf, 0xa, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf,
		0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf,
		0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf,
		0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf,
		0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf,
		0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf,
		0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf,
		0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf,
		0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf,
	}
)

func (self *Record) fillData() (n int) {
	if self.filled {
		return
	}

	d := self.dataUnsafe()
	var s, e int

	// Get query name.
	s, e = 0, int(self.lQname())
	self.nameStr = string(d[s : e-1])

	// Get CIGAR data.
	nCigar := self.nCigar()
	s, e = e, e+int(nCigar<<2) // CIGAR represented as C.uint32 so length is 4*n_cigar
	cigarCodes := make([]uint32, nCigar)
	err := binary.Read(bytes.NewBuffer(d[s:e]), endian, &cigarCodes)
	if err != nil {
		panic(fmt.Sprintf("boom: binary.Read failed: %v", err))
	}
	for _, c := range cigarCodes {
		self.cigar = append(self.cigar, CigarOp{
			op:     CigarOpType(c & 0xf),
			length: c >> 4,
		})
	}

	// Get sequence data.
	lQqual := int(self.lQseq())
	lQseq := (lQqual + 1) >> 1
	// Extract nucleotide nybbles.
	s, e = e, e+lQseq
	self.seqBytes = make([]byte, lQqual)
	for i, c := range d[s:e] {
		i2 := i << 1
		self.seqBytes[i2] = bamNT16TableRev[c>>4]
		if i2++; i2 == len(self.seqBytes) {
			break
		}
		self.seqBytes[i2] = bamNT16TableRev[c&0xf]
	}
	// Get quality scores.
	s, e = e, e+lQqual
	self.qualScores = make([]int8, lQqual)
	q := d[s:e]
	copy(self.qualScores, *(*[]int8)(unsafe.Pointer(&q)))

	// Get auxilliary tags.
	lAux := self.lAux()
	s, e = e, e+lAux
	self.auxBytes = make([]byte, lAux)
	copy(self.auxBytes, d[s:e])
	self.auxTags = parseAux(self.auxBytes)

	self.filled = true
	return
}

type CigarOpType byte

const (
	CigarMatch CigarOpType = iota
	CigarInsert
	CigarDeletion
	CigarSkipReference
	CigarSoftClip
	CigarHardClip
	CigarPad
	CigarEqual
	CigarDifferent
	lastCigar
)

var cigarOps = []string{"M", "I", "D", "N", "S", "H", "P", "=", "X", "?"}

func (ct CigarOpType) String() string {
	if ct < 0 || ct > lastCigar {
		ct = lastCigar
	}
	return cigarOps[ct]
}

// Compact Idiosyncratic Gapped Alignment Report
type CigarOp struct {
	op     CigarOpType
	length uint32
}

func (co CigarOp) String() string {
	return fmt.Sprintf("%d%s", co.length, co.op.String())
}

type Aux []byte

var (
	jumps = [256]int{
		'A': 1,
		'c': 1, 'C': 1,
		's': 2, 'S': 2,
		'i': 4, 'I': 4,
		'f': 4,
		'Z': -1,
		'H': -1,
		'B': -1,
	}
	auxTypes = [256]byte{
		'A': 'A',
		'c': 'i', 'C': 'i',
		's': 'i', 'S': 'i',
		'i': 'i', 'I': 'i',
		'f': 'f',
		'Z': 'Z',
		'H': 'H',
		'B': 'B',
	}
)

func parseAux(aux []byte) (aa []Aux) {
	for i := 0; i+2 < len(aux); {
		t := aux[i+2]
		switch j := jumps[t]; {
		case j > 0:
			j += 3
			aa = append(aa, Aux(aux[i:i+j]))
			i += j
		case j < 0:
			switch t {
			case 'Z', 'H':
				var (
					j int
					v byte
				)
				for j, v = range aux[i:] {
					if v == 0 { // C string termination
						break // Truncate terminal zero.
					}
				}
				aa = append(aa, Aux(aux[i:i+j]))
				i += j + 1
			case 'B':
				var length int32
				err := binary.Read(bytes.NewBuffer([]byte(aux[i+4:i+8])), endian, &length)
				if err != nil {
					panic(fmt.Sprintf("boom: binary.Read failed: %v", err))
				}
				j = int(length)*jumps[aux[i+3]] + int(unsafe.Sizeof(length)) + 4
				aa = append(aa, Aux(aux[i:i+j]))
				i += j
			}
		default:
			panic(fmt.Sprintf("boom: unrecognised optional field type: %q", t))
		}
	}
	return
}

func buildAux(aa []Aux) (aux []byte) {
	for _, a := range aa {
		// TODO: validate each a
		aux = append(aux, []byte(a)...)
	}
	return
}

func (self Aux) String() string {
	return fmt.Sprintf("%s:%c:%v", []byte(self[:2]), auxTypes[self.Type()], self.Value())
}

func (self Aux) Tag() string { return string(self[:2]) }
func (self Aux) Type() byte  { return self[2] }
func (self Aux) Value() (v interface{}) {
	switch t := self.Type(); t {
	case 'A':
		return self[3]
	case 'c':
		return int8(self[3])
	case 'C':
		return uint(self[3])
	case 's':
		s := int16(0)
		err := binary.Read(bytes.NewBuffer([]byte(self[4:6])), endian, &s)
		if err != nil {
			panic(fmt.Sprintf("boom: binary.Read failed: %v", err))
		}
		return s
	case 'S':
		S := uint16(0)
		err := binary.Read(bytes.NewBuffer([]byte(self[4:6])), endian, &S)
		if err != nil {
			panic(fmt.Sprintf("boom: binary.Read failed: %v", err))
		}
		return S
	case 'i':
		i := int32(0)
		err := binary.Read(bytes.NewBuffer([]byte(self[4:8])), endian, &i)
		if err != nil {
			panic(fmt.Sprintf("boom: binary.Read failed: %v", err))
		}
		return i
	case 'I':
		I := uint32(0)
		err := binary.Read(bytes.NewBuffer([]byte(self[4:8])), endian, &I)
		if err != nil {
			panic(fmt.Sprintf("boom: binary.Read failed: %v", err))
		}
		return I
	case 'f':
		f := float32(0)
		err := binary.Read(bytes.NewBuffer([]byte(self[4:8])), endian, &f)
		if err != nil {
			panic(fmt.Sprintf("boom: binary.Read failed: %v", err))
		}
		return f
	case 'Z': // Z and H Require that parsing stops before the terminating zero.
		return string(self[3:])
	case 'H':
		h := make([]byte, hex.DecodedLen(len(self[3:])))
		_, err := hex.Decode(h, []byte(self[3:]))
		if err != nil {
			panic(fmt.Sprintf("boom: hex decoding error: %v", err))
		}
		return h
	case 'B':
		var length int32
		err := binary.Read(bytes.NewBuffer([]byte(self[4:8])), endian, &length)
		if err != nil {
			panic(fmt.Sprintf("boom: binary.Read failed: %v", err))
		}
		switch t := self[3]; t {
		case 'c':
			c := self[4:]
			return *(*[]int8)(unsafe.Pointer(&c))
		case 'C':
			return []uint8(self[4:])
		case 's':
			Bs := make([]int16, length)
			err := binary.Read(bytes.NewBuffer([]byte(self[8:])), endian, &Bs)
			if err != nil {
				panic(fmt.Sprintf("boom: binary.Read failed: %v", err))
			}
			return Bs
		case 'S':
			BS := make([]uint16, length)
			err := binary.Read(bytes.NewBuffer([]byte(self[8:])), endian, &BS)
			if err != nil {
				panic(fmt.Sprintf("boom: binary.Read failed: %v", err))
			}
			return BS
		case 'i':
			Bi := make([]int32, length)
			err := binary.Read(bytes.NewBuffer([]byte(self[8:])), endian, &Bi)
			if err != nil {
				panic(fmt.Sprintf("boom: binary.Read failed: %v", err))
			}
			return Bi
		case 'I':
			BI := make([]uint32, length)
			err := binary.Read(bytes.NewBuffer([]byte(self[8:])), endian, &BI)
			if err != nil {
				panic(fmt.Sprintf("boom: binary.Read failed: %v", err))
			}
			return BI
		case 'f':
			Bf := make([]float32, length)
			err := binary.Read(bytes.NewBuffer([]byte(self[8:])), endian, &Bf)
			if err != nil {
				panic(fmt.Sprintf("boom: binary.Read failed: %v", err))
			}
			return Bf
		default:
			panic(fmt.Sprintf("boom: unknown array type %q", t))
		}
	default:
		panic(fmt.Sprintf("boom: unknown type %q", t))
	}
	return
}
<|endoftext|>


<commit_before>//
// Copyright (c) 2018
// Mainflux
//
// SPDX-License-Identifier: Apache-2.0
//

package nats

import (
	"fmt"

	"github.com/gogo/protobuf/proto"
	"github.com/mainflux/mainflux"
	log "github.com/mainflux/mainflux/logger"
	"github.com/mainflux/mainflux/normalizer"
	"github.com/nats-io/go-nats"
)

const (
	queue         = "normalizers"
	input         = "channel.>"
	outputUnknown = "out.unknown"
	senML         = "application/senml+json"
)

type pubsub struct {
	nc     *nats.Conn
	svc    normalizer.Service
	logger log.Logger
}

// Subscribe to appropriate NATS topic and normalizes received messages.
func Subscribe(svc normalizer.Service, nc *nats.Conn, logger log.Logger) {
	ps := pubsub{
		nc:     nc,
		svc:    svc,
		logger: logger,
	}
	ps.nc.QueueSubscribe(input, queue, ps.handleMsg)
}

func (ps pubsub) handleMsg(m *nats.Msg) {
	var msg mainflux.RawMessage
	if err := proto.Unmarshal(m.Data, &msg); err != nil {
		ps.logger.Warn(fmt.Sprintf("Unmarshalling failed: %s", err))
		return
	}

	if err := ps.publish(msg); err != nil {
		ps.logger.Warn(fmt.Sprintf("Publishing failed: %s", err))
		return
	}
}

func (ps pubsub) publish(msg mainflux.RawMessage) error {
	output := mainflux.OutputSenML
	normalized, err := ps.svc.Normalize(msg)
	if err != nil {
		switch ct := normalized.ContentType; ct {
		case senML:
			return err
		case "":
			output = outputUnknown
		default:
			output = fmt.Sprintf("out.%s", ct)
		}
	}

	for _, v := range normalized.Messages {
		data, err := proto.Marshal(&v)
		if err != nil {
			ps.logger.Warn(fmt.Sprintf("Marshalling failed: %s", err))
			return err
		}

		if err := ps.nc.Publish(output, data); err != nil {
			ps.logger.Warn(fmt.Sprintf("Publishing failed: %s", err))
			return err
		}
	}

	return nil
}
<commit_msg>Fix non-SenML message routing in normalizer (#573)<commit_after>//
// Copyright (c) 2018
// Mainflux
//
// SPDX-License-Identifier: Apache-2.0
//

package nats

import (
	"fmt"

	"github.com/gogo/protobuf/proto"
	"github.com/mainflux/mainflux"
	log "github.com/mainflux/mainflux/logger"
	"github.com/mainflux/mainflux/normalizer"
	"github.com/nats-io/go-nats"
)

const (
	queue         = "normalizers"
	input         = "channel.>"
	outputUnknown = "out.unknown"
	senML         = "application/senml+json"
)

type pubsub struct {
	nc     *nats.Conn
	svc    normalizer.Service
	logger log.Logger
}

// Subscribe to appropriate NATS topic and normalizes received messages.
func Subscribe(svc normalizer.Service, nc *nats.Conn, logger log.Logger) {
	ps := pubsub{
		nc:     nc,
		svc:    svc,
		logger: logger,
	}
	ps.nc.QueueSubscribe(input, queue, ps.handleMsg)
}

func (ps pubsub) handleMsg(m *nats.Msg) {
	var msg mainflux.RawMessage
	if err := proto.Unmarshal(m.Data, &msg); err != nil {
		ps.logger.Warn(fmt.Sprintf("Unmarshalling failed: %s", err))
		return
	}

	if err := ps.publish(msg); err != nil {
		ps.logger.Warn(fmt.Sprintf("Publishing failed: %s", err))
		return
	}
}

func (ps pubsub) publish(msg mainflux.RawMessage) error {
	output := mainflux.OutputSenML
	normalized, err := ps.svc.Normalize(msg)
	if err != nil {
		switch ct := msg.ContentType; ct {
		case senML:
			return err
		case "":
			output = outputUnknown
		default:
			output = fmt.Sprintf("out.%s", ct)
		}

		if err := ps.nc.Publish(output, msg.GetPayload()); err != nil {
			ps.logger.Warn(fmt.Sprintf("Publishing failed: %s", err))
			return err
		}
	}

	for _, v := range normalized.Messages {
		data, err := proto.Marshal(&v)
		if err != nil {
			ps.logger.Warn(fmt.Sprintf("Marshalling failed: %s", err))
			return err
		}

		if err := ps.nc.Publish(output, data); err != nil {
			ps.logger.Warn(fmt.Sprintf("Publishing failed: %s", err))
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package xpfunds

import (
	"fmt"
	"io/ioutil"
	"math"
	"strconv"
	"strings"
	"xpfunds/binarysearch"
	"xpfunds/check"
	"xpfunds/median"
)

type Fund struct {
	name string

	active string

	min string

	// The monthly return of the fund, starting from the last month.
	monthly []float64

	// The position of the first slice determines the dimension. The position of
	// the second slice indicates an end time of a period and the third position
	// the difference from the start time to the end time of a period. Arbitrary
	// range.
	features [][][]float64

	// Same as fieldValues, but holds the ratio of the value in this fund to the
	// value in the fund with the highest value of this field. Range: 0-1.
	ratio [][][]float64
}

func NewFund(monthly []float64) *Fund {
	f := &Fund{
		monthly: monthly,
	}
	f.setFeatures()
	f.makeRatio()
	return f
}

func (f *Fund) setFeatures() {
	f.setReturn()
	f.setMedian()
	f.setStdDev()
	f.setNegativeMonthRatio()
	f.setGreatestFall()
}

func (f *Fund) setReturn() {
	ret := make([][]float64, len(f.monthly))
	for end, monthly := range f.monthly {
		ret[end] = make([]float64, len(f.monthly)-end)
		ret[end][0] = monthly
		for diff := 1; diff < len(f.monthly)-end; diff++ {
			ret[end][diff] = ret[end][diff-1] * f.monthly[end+diff]
		}
	}
	f.features = append(f.features, ret)
}

func (f *Fund) setMedian() {
	med := make([][]float64, len(f.monthly))
	for end, monthly := range f.monthly {
		med[end] = make([]float64, len(f.monthly)-end)
		med[end][0] = monthly
		returns := []float64{monthly}
		for diff := 1; diff < len(f.monthly)-end; diff++ {
			returns = binarysearch.InsertInSorted(returns, f.monthly[end+diff])
			med[end][diff] = median.MedianFromSorted(returns)
		}
	}
	f.features = append(f.features, med)
}

func (f *Fund) setStdDev() {
	stdDev := make([][]float64, len(f.monthly))
	for end, monthly := range f.monthly {
		stdDev[end] = make([]float64, len(f.monthly)-end)
		stdDev[end][0] = 0
		total := monthly
		for diff := 1; diff < len(f.monthly)-end; diff++ {
			total += f.monthly[end+diff]
			count := float64(diff + 1)
			avg := total / count
			sumDiffs := 0.0
			for i := end; i <= end+diff; i++ {
				diff := f.monthly[i] - avg
				sumDiffs += diff * diff
			}
			stdDev[end][diff] = math.Sqrt(sumDiffs / count)
		}
	}
	f.features = append(f.features, stdDev)
}

func (f *Fund) setNegativeMonthRatio() {
	nmr := make([][]float64, len(f.monthly))
	for end := range f.monthly {
		nmr[end] = make([]float64, len(f.monthly)-end)
		negative := 0
		nonNegative := 0
		for diff := 0; diff < len(f.monthly)-end; diff++ {
			if f.monthly[end+diff] < 1 {
				negative++
			} else {
				nonNegative++
			}
			nmr[end][diff] = float64(negative) / float64(negative+nonNegative)
		}
	}
	f.features = append(f.features, nmr)
}

func (f *Fund) setGreatestFall() {
	gf := make([][]float64, len(f.monthly))
	gfl := make([][]float64, len(f.monthly))
	for end := range f.monthly {
		gf[end] = make([]float64, len(f.monthly)-end)
		gfl[end] = make([]float64, len(f.monthly)-end)
		greatestFall := 1.0
		greatestFallLen := 0
		curr := 1.0
		currLen := 0
		for diff := 0; diff < len(f.monthly)-end; diff++ {
			curr *= f.monthly[end+diff]
			currLen++
			if f.monthly[end+diff] < curr {
				curr = f.monthly[end+diff]
				currLen = 1
			}
			if curr < greatestFall {
				greatestFall = curr
				greatestFallLen = currLen
			}
			gf[end][diff] = greatestFall
			gfl[end][diff] = float64(greatestFallLen)
		}
	}
	f.features = append(f.features, gf, gfl)
}

func (f *Fund) makeRatio() {
	f.ratio = make([][][]float64, f.FeatureCount())
	for feature := range f.ratio {
		f.ratio[feature] = make([][]float64, f.Duration())
		for end := range f.ratio[feature] {
			f.ratio[feature][end] = make([]float64, f.Duration()-end)
		}
	}
}

func ReadFunds() []*Fund {
	text, err := ioutil.ReadFile("get.tsv")
	check.Check(err)
	var funds []*Fund
	for _, line := range strings.Split(string(text), "\n") {
		f := fundFromLine(line)
		if f == nil {
			continue
		}
		funds = append(funds, f)
	}
	SetRatio(funds)
	return funds
}

func fundFromLine(line string) *Fund {
	fields := strings.Split(strings.Trim(line, "\n"), "\t")
	if len(fields) < 6 {
		return nil
	}

	var monthly []float64
	for i := 5; i < len(fields); i++ {
		v, err := strconv.ParseFloat(strings.Replace(fields[i], ",", ".", 1), 64)
		check.Check(err)
		monthly = append(monthly, 1.0+v/100.0)
	}
	f := NewFund(monthly)
	f.name = fields[0]
	f.active = fields[4]
	f.min = fields[1]
	return f
}

func (f *Fund) FeatureCount() int {
	return len(f.features)
}

func (f *Fund) Duration() int {
	return len(f.monthly)
}

// End is inclusive, start is exclusive
func (f *Fund) Weighted(weight []float64, end, start int) float64 {
	total := 0.0
	for i, w := range weight {
		total += f.ratio[i][end][start-1-end] * w
	}
	return total
}

func (f *Fund) Return(end, start int) float64 {
	return f.Weighted([]float64{1}, end, start)
}

func (f *Fund) Print() string {
	return fmt.Sprintf("%v\t%v\t%v", f.name, f.active, f.min)
}

func SetRatio(funds []*Fund) {
	duration := MaxDuration(funds)
	for feature := 0; feature < funds[0].FeatureCount(); feature++ {
		for end := range funds[0].features[feature] {
			for diff := 0; diff < duration-end; diff++ {
				highest := -999999.99
				for _, f := range funds {
					if f.Duration() <= end+diff {
						continue
					}
					if f.features[feature][end][diff] > highest {
						highest = f.features[feature][end][diff]
					}
				}
				for _, f := range funds {
					if f.Duration() <= end+diff {
						continue
					}
					if highest == 0 {
						f.ratio[feature][end][diff] = 1
						continue
					}
					f.ratio[feature][end][diff] = f.features[feature][end][diff] / highest
				}
			}
		}
	}
}

func MaxDuration(funds []*Fund) int {
	duration := 0
	for _, f := range funds {
		if f.Duration() > duration {
			duration = f.Duration()
		}
	}
	return duration
}
<commit_msg>duration<commit_after>package xpfunds

import (
	"fmt"
	"io/ioutil"
	"math"
	"strconv"
	"strings"
	"xpfunds/binarysearch"
	"xpfunds/check"
	"xpfunds/median"
)

type Fund struct {
	name string

	active string

	min string

	// The monthly return of the fund, starting from the last month.
	monthly []float64

	// The position of the first slice determines the dimension. The position of
	// the second slice indicates an end time of a period and the third position
	// the difference from the start time to the end time of a period. Arbitrary
	// range.
	features [][][]float64

	// Same as fieldValues, but holds the ratio of the value in this fund to the
	// value in the fund with the highest value of this field. Range: 0-1.
	ratio [][][]float64
}

func NewFund(monthly []float64) *Fund {
	f := &Fund{
		monthly: monthly,
	}
	f.setFeatures()
	f.makeRatio()
	return f
}

func (f *Fund) setFeatures() {
	f.setReturn()
	f.setMedian()
	f.setStdDev()
	f.setNegativeMonthRatio()
	f.setGreatestFall()
}

func (f *Fund) setReturn() {
	ret := make([][]float64, len(f.monthly))
	for end, monthly := range f.monthly {
		ret[end] = make([]float64, len(f.monthly)-end)
		ret[end][0] = monthly
		for diff := 1; diff < len(f.monthly)-end; diff++ {
			ret[end][diff] = ret[end][diff-1] * f.monthly[end+diff]
		}
	}
	f.features = append(f.features, ret)
}

func (f *Fund) setMedian() {
	med := make([][]float64, len(f.monthly))
	for end, monthly := range f.monthly {
		med[end] = make([]float64, len(f.monthly)-end)
		med[end][0] = monthly
		returns := []float64{monthly}
		for diff := 1; diff < len(f.monthly)-end; diff++ {
			returns = binarysearch.InsertInSorted(returns, f.monthly[end+diff])
			med[end][diff] = median.MedianFromSorted(returns)
		}
	}
	f.features = append(f.features, med)
}

func (f *Fund) setStdDev() {
	stdDev := make([][]float64, len(f.monthly))
	for end, monthly := range f.monthly {
		stdDev[end] = make([]float64, len(f.monthly)-end)
		stdDev[end][0] = 0
		total := monthly
		for diff := 1; diff < len(f.monthly)-end; diff++ {
			total += f.monthly[end+diff]
			count := float64(diff + 1)
			avg := total / count
			sumDiffs := 0.0
			for i := end; i <= end+diff; i++ {
				diff := f.monthly[i] - avg
				sumDiffs += diff * diff
			}
			stdDev[end][diff] = math.Sqrt(sumDiffs / count)
		}
	}
	f.features = append(f.features, stdDev)
}

func (f *Fund) setNegativeMonthRatio() {
	nmr := make([][]float64, len(f.monthly))
	for end := range f.monthly {
		nmr[end] = make([]float64, len(f.monthly)-end)
		negative := 0
		nonNegative := 0
		for diff := 0; diff < len(f.monthly)-end; diff++ {
			if f.monthly[end+diff] < 1 {
				negative++
			} else {
				nonNegative++
			}
			nmr[end][diff] = float64(negative) / float64(negative+nonNegative)
		}
	}
	f.features = append(f.features, nmr)
}

func (f *Fund) setGreatestFall() {
	gf := make([][]float64, len(f.monthly))
	gfl := make([][]float64, len(f.monthly))
	for end := range f.monthly {
		gf[end] = make([]float64, len(f.monthly)-end)
		gfl[end] = make([]float64, len(f.monthly)-end)
		greatestFall := 1.0
		greatestFallLen := 0
		curr := 1.0
		currLen := 0
		for diff := 0; diff < len(f.monthly)-end; diff++ {
			curr *= f.monthly[end+diff]
			currLen++
			if f.monthly[end+diff] < curr {
				curr = f.monthly[end+diff]
				currLen = 1
			}
			if curr < greatestFall {
				greatestFall = curr
				greatestFallLen = currLen
			}
			gf[end][diff] = greatestFall
			gfl[end][diff] = float64(greatestFallLen)
		}
	}
	f.features = append(f.features, gf, gfl)
}

func (f *Fund) makeRatio() {
	f.ratio = make([][][]float64, f.FeatureCount())
	for feature := range f.ratio {
		f.ratio[feature] = make([][]float64, f.Duration())
		for end := range f.ratio[feature] {
			f.ratio[feature][end] = make([]float64, f.Duration()-end)
		}
	}
}

func ReadFunds() []*Fund {
	text, err := ioutil.ReadFile("get.tsv")
	check.Check(err)
	var funds []*Fund
	for _, line := range strings.Split(string(text), "\n") {
		f := fundFromLine(line)
		if f == nil {
			continue
		}
		funds = append(funds, f)
	}
	SetRatio(funds)
	return funds
}

func fundFromLine(line string) *Fund {
	fields := strings.Split(strings.Trim(line, "\n"), "\t")
	if len(fields) < 6 {
		return nil
	}

	var monthly []float64
	for i := 5; i < len(fields); i++ {
		v, err := strconv.ParseFloat(strings.Replace(fields[i], ",", ".", 1), 64)
		check.Check(err)
		monthly = append(monthly, 1.0+v/100.0)
	}
	f := NewFund(monthly)
	f.name = fields[0]
	f.active = fields[4]
	f.min = fields[1]
	return f
}

func (f *Fund) FeatureCount() int {
	return len(f.features)
}

func (f *Fund) Duration() int {
	return len(f.monthly)
}

// End is inclusive, start is exclusive
func (f *Fund) Weighted(weight []float64, end, start int) float64 {
	total := 0.0
	for i, w := range weight {
		total += f.ratio[i][end][start-1-end] * w
	}
	return total
}

func (f *Fund) Return(end, start int) float64 {
	return f.Weighted([]float64{1}, end, start)
}

func (f *Fund) Print() string {
	return fmt.Sprintf("%v\t%v\t%v", f.name, f.active, f.min)
}

func SetRatio(funds []*Fund) {
	duration := MaxDuration(funds)
	for feature := 0; feature < funds[0].FeatureCount(); feature++ {
		for end := 0; end < duration; end++ {
			for diff := 0; diff < duration-end; diff++ {
				highest := -999999.99
				for _, f := range funds {
					if f.Duration() <= end+diff {
						continue
					}
					if f.features[feature][end][diff] > highest {
						highest = f.features[feature][end][diff]
					}
				}
				for _, f := range funds {
					if f.Duration() <= end+diff {
						continue
					}
					if highest == 0 {
						f.ratio[feature][end][diff] = 1
						continue
					}
					f.ratio[feature][end][diff] = f.features[feature][end][diff] / highest
				}
			}
		}
	}
}

func MaxDuration(funds []*Fund) int {
	duration := 0
	for _, f := range funds {
		if f.Duration() > duration {
			duration = f.Duration()
		}
	}
	return duration
}
<|endoftext|>


<commit_before>package hyperloglog

import (
	"unsafe"
)

// This file implements the murmur3 32-bit hash on 32bit and 64bit integers
// for little endian machines only with no heap allocation.  If you are using
// HLL to count integer IDs on intel machines, this is your huckleberry.

// MurmurString implements a fast version of the murmur hash function for strings
// for little endian machines.  Suitable for adding strings to HLL counter.
func MurmurString(key string) uint32 {
	var c1, c2 uint32 = 0xcc9e2d51, 0x1b873593
	var h, k uint32

	// Reinterpret the string as bytes. This is safe because we don't write into the byte array.
	bkey := *(*[]byte)(unsafe.Pointer(&key))
	blen := len(bkey)

	l := blen / 4 // chunk length
	tail := bkey[l*4:]

	// for each 4 byte chunk of `key'
	for i := 0; i < l; i++ {
		// next 4 byte chunk of `key'
		k = *(*uint32)(unsafe.Pointer(&bkey[i*4]))

		// encode next 4 byte chunk of `key'
		k *= c1
		k = (k << 15) | (k >> (32 - 15))
		k *= c2
		h ^= k
		h = (h << 13) | (h >> (32 - 13))
		h = (h * 5) + 0xe6546b64
	}

	k = 0
	// remainder
	switch len(tail) {
	case 3:
		k ^= uint32(tail[2]) << 16
		fallthrough
	case 2:
		k ^= uint32(tail[1]) << 8
		fallthrough
	case 1:
		k ^= uint32(tail[0])
		k *= c1
		k = (k << 15) | (k >> (32 - 15))
		k *= c2
		h ^= k
	}

	h ^= uint32(blen)
	h ^= (h >> 16)
	h *= 0x85ebca6b
	h ^= (h >> 13)
	h *= 0xc2b2ae35
	h ^= (h >> 16)

	return h
}

// Murmur32 implements a fast version of the murmur hash function for uint32 for
// little endian machines.  Suitable for adding 32bit integers to a HLL counter.
func Murmur32(i uint32) uint32 {
	var c1, c2 uint32 = 0xcc9e2d51, 0x1b873593
	var h, k uint32
	k = i
	k *= c1
	k = (k << 15) | (k >> (32 - 15))
	k *= c2
	h ^= k
	h = (h << 13) | (h >> (32 - 13))
	h = (h * 5) + 0xe6546b64
	// second part
	h ^= 4
	h ^= h >> 16
	h *= 0x85ebca6b
	h ^= h >> 13
	h *= 0xc2b2ae35
	h ^= h >> 16
	return h
}

// Murmur64 implements a fast version of the murmur hash function for uint64 for
// little endian machines.  Suitable for adding 64bit integers to a HLL counter.
func Murmur64(i uint64) uint32 {
	var c1, c2 uint32 = 0xcc9e2d51, 0x1b873593
	var h, k uint32
	//first 4-byte chunk
	k = uint32(i)
	k *= c1
	k = (k << 15) | (k >> (32 - 15))
	k *= c2
	h ^= k
	h = (h << 13) | (h >> (32 - 13))
	h = (h * 5) + 0xe6546b64
	// second 4-byte chunk
	k = uint32(i >> 32)
	k *= c1
	k = (k << 15) | (k >> (32 - 15))
	k *= c2
	h ^= k
	h = (h << 13) | (h >> (32 - 13))
	h = (h * 5) + 0xe6546b64
	// second part
	h ^= 8
	h ^= h >> 16
	h *= 0x85ebca6b
	h ^= h >> 13
	h *= 0xc2b2ae35
	h ^= h >> 16
	return h
}

// Murmur128 implements a fast version of the murmur hash function for two uint64s
// for little endian machines.  Suitable for adding a 128bit value to an HLL counter.
func Murmur128(i, j uint64) uint32 {
	var c1, c2 uint32 = 0xcc9e2d51, 0x1b873593
	var h, k uint32
	//first 4-byte chunk
	k = uint32(i)
	k *= c1
	k = (k << 15) | (k >> (32 - 15))
	k *= c2
	h ^= k
	h = (h << 13) | (h >> (32 - 13))
	h = (h * 5) + 0xe6546b64
	// second 4-byte chunk
	k = uint32(i >> 32)
	k *= c1
	k = (k << 15) | (k >> (32 - 15))
	k *= c2
	h ^= k
	h = (h << 13) | (h >> (32 - 13))
	h = (h * 5) + 0xe6546b64
	// third 4-byte chunk
	k = uint32(j)
	k *= c1
	k = (k << 15) | (k >> (32 - 15))
	k *= c2
	h ^= k
	h = (h << 13) | (h >> (32 - 13))
	h = (h * 5) + 0xe6546b64
	// fourth 4-byte chunk
	k = uint32(j >> 32)
	k *= c1
	k = (k << 15) | (k >> (32 - 15))
	k *= c2
	h ^= k
	h = (h << 13) | (h >> (32 - 13))
	h = (h * 5) + 0xe6546b64
	// second part
	h ^= 16
	h ^= h >> 16
	h *= 0x85ebca6b
	h ^= h >> 13
	h *= 0xc2b2ae35
	h ^= h >> 16
	return h

}
<commit_msg>[murmur] Add MurmurBytes method to not have to cast to string<commit_after>package hyperloglog

import (
	"unsafe"
)

// This file implements the murmur3 32-bit hash on 32bit and 64bit integers
// for little endian machines only with no heap allocation.  If you are using
// HLL to count integer IDs on intel machines, this is your huckleberry.

// MurmurString implements a fast version of the murmur hash function for strings
// for little endian machines.  Suitable for adding strings to HLL counter.
func MurmurString(key string) uint32 {
	// Reinterpret the string as bytes. This is safe because we don't write into the byte array.
	bkey := *(*[]byte)(unsafe.Pointer(&key))
	return MurmurBytes(bkey)
}

// MurmurBytes implements a fast version of the murmur hash function for bytes
// for little endian machines.  Suitable for adding strings to HLL counter.
func MurmurBytes(bkey []byte) uint32 {
	var c1, c2 uint32 = 0xcc9e2d51, 0x1b873593
	var h, k uint32

	blen := len(bkey)
	l := blen / 4 // chunk length
	tail := bkey[l*4:]

	// for each 4 byte chunk of `key'
	for i := 0; i < l; i++ {
		// next 4 byte chunk of `key'
		k = *(*uint32)(unsafe.Pointer(&bkey[i*4]))

		// encode next 4 byte chunk of `key'
		k *= c1
		k = (k << 15) | (k >> (32 - 15))
		k *= c2
		h ^= k
		h = (h << 13) | (h >> (32 - 13))
		h = (h * 5) + 0xe6546b64
	}

	k = 0
	// remainder
	switch len(tail) {
	case 3:
		k ^= uint32(tail[2]) << 16
		fallthrough
	case 2:
		k ^= uint32(tail[1]) << 8
		fallthrough
	case 1:
		k ^= uint32(tail[0])
		k *= c1
		k = (k << 15) | (k >> (32 - 15))
		k *= c2
		h ^= k
	}

	h ^= uint32(blen)
	h ^= (h >> 16)
	h *= 0x85ebca6b
	h ^= (h >> 13)
	h *= 0xc2b2ae35
	h ^= (h >> 16)

	return h
}

// Murmur32 implements a fast version of the murmur hash function for uint32 for
// little endian machines.  Suitable for adding 32bit integers to a HLL counter.
func Murmur32(i uint32) uint32 {
	var c1, c2 uint32 = 0xcc9e2d51, 0x1b873593
	var h, k uint32
	k = i
	k *= c1
	k = (k << 15) | (k >> (32 - 15))
	k *= c2
	h ^= k
	h = (h << 13) | (h >> (32 - 13))
	h = (h * 5) + 0xe6546b64
	// second part
	h ^= 4
	h ^= h >> 16
	h *= 0x85ebca6b
	h ^= h >> 13
	h *= 0xc2b2ae35
	h ^= h >> 16
	return h
}

// Murmur64 implements a fast version of the murmur hash function for uint64 for
// little endian machines.  Suitable for adding 64bit integers to a HLL counter.
func Murmur64(i uint64) uint32 {
	var c1, c2 uint32 = 0xcc9e2d51, 0x1b873593
	var h, k uint32
	//first 4-byte chunk
	k = uint32(i)
	k *= c1
	k = (k << 15) | (k >> (32 - 15))
	k *= c2
	h ^= k
	h = (h << 13) | (h >> (32 - 13))
	h = (h * 5) + 0xe6546b64
	// second 4-byte chunk
	k = uint32(i >> 32)
	k *= c1
	k = (k << 15) | (k >> (32 - 15))
	k *= c2
	h ^= k
	h = (h << 13) | (h >> (32 - 13))
	h = (h * 5) + 0xe6546b64
	// second part
	h ^= 8
	h ^= h >> 16
	h *= 0x85ebca6b
	h ^= h >> 13
	h *= 0xc2b2ae35
	h ^= h >> 16
	return h
}

// Murmur128 implements a fast version of the murmur hash function for two uint64s
// for little endian machines.  Suitable for adding a 128bit value to an HLL counter.
func Murmur128(i, j uint64) uint32 {
	var c1, c2 uint32 = 0xcc9e2d51, 0x1b873593
	var h, k uint32
	//first 4-byte chunk
	k = uint32(i)
	k *= c1
	k = (k << 15) | (k >> (32 - 15))
	k *= c2
	h ^= k
	h = (h << 13) | (h >> (32 - 13))
	h = (h * 5) + 0xe6546b64
	// second 4-byte chunk
	k = uint32(i >> 32)
	k *= c1
	k = (k << 15) | (k >> (32 - 15))
	k *= c2
	h ^= k
	h = (h << 13) | (h >> (32 - 13))
	h = (h * 5) + 0xe6546b64
	// third 4-byte chunk
	k = uint32(j)
	k *= c1
	k = (k << 15) | (k >> (32 - 15))
	k *= c2
	h ^= k
	h = (h << 13) | (h >> (32 - 13))
	h = (h * 5) + 0xe6546b64
	// fourth 4-byte chunk
	k = uint32(j >> 32)
	k *= c1
	k = (k << 15) | (k >> (32 - 15))
	k *= c2
	h ^= k
	h = (h << 13) | (h >> (32 - 13))
	h = (h * 5) + 0xe6546b64
	// second part
	h ^= 16
	h ^= h >> 16
	h *= 0x85ebca6b
	h ^= h >> 13
	h *= 0xc2b2ae35
	h ^= h >> 16
	return h

}
<|endoftext|>


<commit_before>// Package static provides go.h compatible hashed static asset
// URIs. This allows for providing long lived cache headers for
// resources which change URLs as their content changes.
//
// TODO
// - actually dont cache in memory when disabled
// - defer reading files
// - gzip support + caching of gzipped data
// - css processor for handling url()
// - css minification
// - js minification
// - css sprites
// - support referencing http urls
package static

import (
	"bytes"
	"crypto/md5"
	"flag"
	"fmt"
	"github.com/daaku/go.h"
	"io/ioutil"
	"net/http"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

const errHandlerRequired = "go.static: a handler is required for static HTML: %+v"

type cacheEntry struct {
	Content []byte
	ModTime time.Time
}

type Handler struct {
	sync.RWMutex
	HttpPath    string        // prefix path for static files
	MaxAge      time.Duration // max-age for HTTP headers
	MemoryCache bool          // enable in memory cache
	DiskPath    string        // path on disk to serve
	cache       map[string]cacheEntry
}

func HandlerFlag(name string) *Handler {
	h := &Handler{cache: make(map[string]cacheEntry)}
	flag.StringVar(
		&h.HttpPath,
		name+".http-path",
		"/static/",
		name+": http path prefix for handler")
	flag.DurationVar(
		&h.MaxAge,
		name+".max-age",
		time.Hour*87658,
		name+": max age to use in the cache headers")
	flag.BoolVar(
		&h.MemoryCache,
		name+".memory-cache",
		true,
		name+": enable in-memory cache")
	flag.StringVar(
		&h.DiskPath,
		name+".disk-path",
		"",
		name+": the directory to serve static files from")
	return h
}

func notFound(w http.ResponseWriter, r *http.Request) {
	header := w.Header()
	header.Set("Cache-Control", "no-cache")
	header.Set("Pragma", "no-cache")
	w.WriteHeader(404)
	w.Write([]byte("static resource not found!"))
}

func joinBasenames(names []string) string {
	basenames := make([]string, len(names))
	for i, name := range names {
		basenames[i] = filepath.Base(name)
	}
	return strings.Join(basenames, "-")
}

func (h *Handler) fileSystem() http.FileSystem {
	return http.Dir(h.DiskPath)
}

// Get a hashed URL for a single file.
func (h *Handler) URL(name string) (string, error) {
	return h.CombinedURL([]string{name})
}

// Get a hashed combined URL for all named files.
func (h *Handler) CombinedURL(names []string) (string, error) {
	hash := md5.New()
	var ce cacheEntry
	for _, name := range names {
		f, err := h.fileSystem().Open(name)
		if err != nil {
			return "", err
		}
		defer f.Close()

		stat, err := f.Stat()
		if err != nil {
			return "", err
		}
		modTime := stat.ModTime()
		if ce.ModTime.Before(modTime) {
			ce.ModTime = modTime
		}

		content, err := ioutil.ReadAll(f)
		if err != nil {
			return "", err
		}
		ce.Content = append(ce.Content, content...)
		_, err = hash.Write(content)
		if err != nil {
			return "", err
		}
	}
	hex := fmt.Sprintf("%x", hash.Sum(nil))
	hexS := hex[:10]
	url := path.Join(h.HttpPath, hexS, joinBasenames(names))
	h.Lock()
	defer h.Unlock()
	h.cache[hexS] = ce
	return url, nil
}

// Serves the static resource.
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	path := r.URL.Path
	if !strings.HasPrefix(path, h.HttpPath) {
		notFound(w, r)
		return
	}
	parts := strings.Split(path, "/")
	if len(parts) < 3 {
		notFound(w, r)
		return
	}

	h.RLock()
	defer h.RUnlock()
	ce, ok := h.cache[parts[2]]
	if !ok {
		notFound(w, r)
		return
	}

	header := w.Header()
	header.Set(
		"Cache-Control",
		fmt.Sprintf("public, max-age=%d", int(h.MaxAge.Seconds())))
	http.ServeContent(w, r, path, ce.ModTime, bytes.NewReader(ce.Content))
}

type LinkStyle struct {
	HREF    []string
	Handler *Handler
	cache   h.HTML
}

func (l *LinkStyle) HTML() (h.HTML, error) {
	if l.Handler == nil {
		return nil, fmt.Errorf(errHandlerRequired, l)
	}
	if !l.Handler.MemoryCache || l.cache == nil {
		url, err := l.Handler.CombinedURL(l.HREF)
		if err != nil {
			return nil, err
		}
		l.cache = &h.LinkStyle{HREF: url}
	}
	return l.cache, nil
}

type Script struct {
	Src     []string
	Async   bool
	Handler *Handler
	cache   h.HTML
}

func (l *Script) HTML() (h.HTML, error) {
	if l.Handler == nil {
		return nil, fmt.Errorf(errHandlerRequired, l)
	}
	if !l.Handler.MemoryCache || l.cache == nil {
		url, err := l.Handler.CombinedURL(l.Src)
		if err != nil {
			return nil, err
		}
		l.cache = &h.Script{
			Src:   url,
			Async: l.Async,
		}
	}
	return l.cache, nil
}

// For github.com/daaku/go.h.js.loader compatibility.
func (l *Script) URLs() []string {
	url, err := l.Handler.CombinedURL(l.Src)
	if err != nil {
		panic(err)
	}
	return []string{url}
}

// For github.com/daaku/go.h.js.loader compatibility.
func (l *Script) Script() string {
	return ""
}

type Img struct {
	ID      string
	Class   string
	Style   string
	Src     string
	Alt     string
	Handler *Handler
	cache   h.HTML
}

func (i *Img) HTML() (h.HTML, error) {
	if i.Handler == nil {
		return nil, fmt.Errorf(errHandlerRequired, i)
	}
	if !i.Handler.MemoryCache || i.cache == nil {
		src, err := i.Handler.URL(i.Src)
		if err != nil {
			return nil, err
		}
		i.cache = &h.Node{
			Tag:         "img",
			SelfClosing: true,
			Attributes: h.Attributes{
				"id":    i.ID,
				"class": i.Class,
				"style": i.Style,
				"src":   src,
				"alt":   i.Alt,
			},
		}
	}
	return i.cache, nil
}
<commit_msg>switch to go.rice, remove flag, loader<commit_after>// Package static provides go.h compatible hashed static asset
// URIs. This allows for providing long lived cache headers for
// resources which change URLs as their content changes.
package static

import (
	"bytes"
	"crypto/md5"
	"fmt"
	"io/ioutil"
	"net/http"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/GeertJohan/go.rice"
	"github.com/daaku/go.h"
)

const errHandlerRequired = "go.static: a handler is required for static HTML: %+v"

type cacheEntry struct {
	Content []byte
	ModTime time.Time
}

type Handler struct {
	sync.RWMutex
	HttpPath    string        // prefix path for static files
	MaxAge      time.Duration // max-age for HTTP headers
	MemoryCache bool          // enable in memory cache
	Box         *rice.Box
	cache       map[string]cacheEntry
}

func notFound(w http.ResponseWriter, r *http.Request) {
	header := w.Header()
	header.Set("Cache-Control", "no-cache")
	header.Set("Pragma", "no-cache")
	w.WriteHeader(404)
	w.Write([]byte("static resource not found!"))
}

func joinBasenames(names []string) string {
	basenames := make([]string, len(names))
	for i, name := range names {
		basenames[i] = filepath.Base(name)
	}
	return strings.Join(basenames, "-")
}

// Get a hashed URL for a single file.
func (h *Handler) URL(name string) (string, error) {
	return h.CombinedURL([]string{name})
}

// Get a hashed combined URL for all named files.
func (h *Handler) CombinedURL(names []string) (string, error) {
	hash := md5.New()
	var ce cacheEntry
	for _, name := range names {
		f, err := h.Box.HTTPBox().Open(name)
		if err != nil {
			return "", err
		}
		defer f.Close()

		stat, err := f.Stat()
		if err != nil {
			return "", err
		}
		modTime := stat.ModTime()
		if ce.ModTime.Before(modTime) {
			ce.ModTime = modTime
		}

		content, err := ioutil.ReadAll(f)
		if err != nil {
			return "", err
		}
		ce.Content = append(ce.Content, content...)
		_, err = hash.Write(content)
		if err != nil {
			return "", err
		}
	}
	hex := fmt.Sprintf("%x", hash.Sum(nil))
	hexS := hex[:10]
	url := path.Join(h.HttpPath, hexS, joinBasenames(names))
	h.Lock()
	defer h.Unlock()
	if h.cache == nil {
		h.cache = make(map[string]cacheEntry)
	}
	h.cache[hexS] = ce
	return url, nil
}

// Serves the static resource.
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	path := r.URL.Path
	if !strings.HasPrefix(path, h.HttpPath) {
		notFound(w, r)
		return
	}
	parts := strings.Split(path, "/")
	if len(parts) < 3 {
		notFound(w, r)
		return
	}

	h.RLock()
	defer h.RUnlock()
	if h.cache == nil {
		notFound(w, r)
		return
	}
	ce, ok := h.cache[parts[2]]
	if !ok {
		notFound(w, r)
		return
	}

	header := w.Header()
	header.Set(
		"Cache-Control",
		fmt.Sprintf("public, max-age=%d", int(h.MaxAge.Seconds())))
	http.ServeContent(w, r, path, ce.ModTime, bytes.NewReader(ce.Content))
}

type LinkStyle struct {
	HREF    []string
	Handler *Handler
	cache   h.HTML
}

func (l *LinkStyle) HTML() (h.HTML, error) {
	if l.Handler == nil {
		return nil, fmt.Errorf(errHandlerRequired, l)
	}
	if !l.Handler.MemoryCache || l.cache == nil {
		url, err := l.Handler.CombinedURL(l.HREF)
		if err != nil {
			return nil, err
		}
		l.cache = &h.LinkStyle{HREF: url}
	}
	return l.cache, nil
}

type Script struct {
	Src     []string
	Async   bool
	Handler *Handler
	cache   h.HTML
}

func (l *Script) HTML() (h.HTML, error) {
	if l.Handler == nil {
		return nil, fmt.Errorf(errHandlerRequired, l)
	}
	if !l.Handler.MemoryCache || l.cache == nil {
		url, err := l.Handler.CombinedURL(l.Src)
		if err != nil {
			return nil, err
		}
		l.cache = &h.Script{
			Src:   url,
			Async: l.Async,
		}
	}
	return l.cache, nil
}

type Img struct {
	ID      string
	Class   string
	Style   string
	Src     string
	Alt     string
	Handler *Handler
	cache   h.HTML
}

func (i *Img) HTML() (h.HTML, error) {
	if i.Handler == nil {
		return nil, fmt.Errorf(errHandlerRequired, i)
	}
	if !i.Handler.MemoryCache || i.cache == nil {
		src, err := i.Handler.URL(i.Src)
		if err != nil {
			return nil, err
		}
		i.cache = &h.Node{
			Tag:         "img",
			SelfClosing: true,
			Attributes: h.Attributes{
				"id":    i.ID,
				"class": i.Class,
				"style": i.Style,
				"src":   src,
				"alt":   i.Alt,
			},
		}
	}
	return i.cache, nil
}
<|endoftext|>


<commit_before>/*
Copyright IBM Corp. 2017 All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

		 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package msp

import (
	"fmt"

	"github.com/golang/protobuf/proto"
	"github.com/hyperledger/fabric/msp"
	"github.com/hyperledger/fabric/protos/common"
	mspprotos "github.com/hyperledger/fabric/protos/msp"
)

// MSPConfigHandler
type MSPConfigHandler struct {
	msp.MSPManager
	config      []*mspprotos.MSPConfig
	proposedMgr msp.MSPManager
}

// BeginConfig called when a config proposal is begun
func (bh *MSPConfigHandler) BeginConfig() {
	if bh.config != nil || bh.proposedMgr != nil {
		panic("Programming error, called BeginConfig while a proposal was in process")
	}
	bh.config = make([]*mspprotos.MSPConfig, 0)
}

// RollbackConfig called when a config proposal is abandoned
func (bh *MSPConfigHandler) RollbackConfig() {
	bh.config = nil
	bh.proposedMgr = nil
}

// CommitConfig called when a config proposal is committed
func (bh *MSPConfigHandler) CommitConfig() {
	if bh.config == nil {
		panic("Programming error, called CommitConfig with no proposal in process")
	}

	bh.MSPManager = bh.proposedMgr
	bh.config = nil
	bh.proposedMgr = nil
}

// ProposeConfig called when config is added to a proposal
func (bh *MSPConfigHandler) ProposeConfig(configItem *common.ConfigItem) error {
	mspconfig := &mspprotos.MSPConfig{}
	err := proto.Unmarshal(configItem.Value, mspconfig)
	if err != nil {
		return fmt.Errorf("Error unmarshalling msp config item, err %s", err)
	}

	bh.config = append(bh.config, []*mspprotos.MSPConfig{mspconfig}...)
	// the only way to make sure that I have a
	// workable config is to toss the proposed
	// manager, create a new one, call setup on
	// it and return whatever error setup gives me
	bh.proposedMgr = msp.NewMSPManager()
	return bh.proposedMgr.Setup(bh.config)
}

// DesierializeIdentity allows *MSPConfigHandler to implement the msp.Common interface
func (bh *MSPConfigHandler) DeserializeIdentity(serializedIdentity []byte) (msp.Identity, error) {
	return bh.DeserializeIdentity(serializedIdentity)
}
<commit_msg>[FAB-2195] Fix infinite loop in mspconf<commit_after>/*
Copyright IBM Corp. 2017 All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

		 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package msp

import (
	"fmt"

	"github.com/golang/protobuf/proto"
	"github.com/hyperledger/fabric/msp"
	"github.com/hyperledger/fabric/protos/common"
	mspprotos "github.com/hyperledger/fabric/protos/msp"
)

// MSPConfigHandler
type MSPConfigHandler struct {
	msp.MSPManager
	config      []*mspprotos.MSPConfig
	proposedMgr msp.MSPManager
}

// BeginConfig called when a config proposal is begun
func (bh *MSPConfigHandler) BeginConfig() {
	if bh.config != nil || bh.proposedMgr != nil {
		panic("Programming error, called BeginConfig while a proposal was in process")
	}
	bh.config = make([]*mspprotos.MSPConfig, 0)
}

// RollbackConfig called when a config proposal is abandoned
func (bh *MSPConfigHandler) RollbackConfig() {
	bh.config = nil
	bh.proposedMgr = nil
}

// CommitConfig called when a config proposal is committed
func (bh *MSPConfigHandler) CommitConfig() {
	if bh.config == nil {
		panic("Programming error, called CommitConfig with no proposal in process")
	}

	bh.MSPManager = bh.proposedMgr
	bh.config = nil
	bh.proposedMgr = nil
}

// ProposeConfig called when config is added to a proposal
func (bh *MSPConfigHandler) ProposeConfig(configItem *common.ConfigItem) error {
	mspconfig := &mspprotos.MSPConfig{}
	err := proto.Unmarshal(configItem.Value, mspconfig)
	if err != nil {
		return fmt.Errorf("Error unmarshalling msp config item, err %s", err)
	}

	bh.config = append(bh.config, []*mspprotos.MSPConfig{mspconfig}...)
	// the only way to make sure that I have a
	// workable config is to toss the proposed
	// manager, create a new one, call setup on
	// it and return whatever error setup gives me
	bh.proposedMgr = msp.NewMSPManager()
	return bh.proposedMgr.Setup(bh.config)
}
<|endoftext|>


<commit_before>// Based on https://github.com/martini-contrib/staticbin
package staticbin

import (
	"bytes"
	"log"
	"net/http"
	"path"
	"strings"
	"time"

	"github.com/gin-gonic/gin"
)

type Options struct {
	// SkipLogging will disable [Static] log messages when a static file is served.
	SkipLogging bool
	// IndexFile defines which file to serve as index if it exists.
	IndexFile string
	// Path prefix
	Dir string
}

func (o *Options) init() {
	if o.IndexFile == "" {
		o.IndexFile = "index.html"
	}
}

// Static returns a middleware handler that serves static files in the given directory.
func Static(asset func(string) ([]byte, error), options ...Options) gin.HandlerFunc {
	if asset == nil {
		panic("asset is nil")
	}

	opt := Options{}
	for _, o := range options {
		opt = o
		break
	}
	opt.init()

	modtime := time.Now()

	return func(c *gin.Context) {
		if c.Request.Method != "GET" && c.Request.Method != "HEAD" {
			// Request is not correct. Go farther.
			return
		}

		url := c.Request.URL.Path
		if !strings.HasPrefix(url, opt.Dir) {
			return
		}
		file := strings.TrimPrefix(
			strings.TrimPrefix(url, opt.Dir),
			"/",
		)
		b, err := asset(file)

		if err != nil {
			// Try to serve the index file.
			b, err = asset(path.Join(file, opt.IndexFile))

			if err != nil {
				// Go farther if the asset could not be found.
				return
			}
		}

		if !opt.SkipLogging {
			log.Println("[Static] Serving " + url)
		}

		http.ServeContent(c.Writer, c.Request, url, modtime, bytes.NewReader(b))
		c.Abort(-1)
	}
}
<commit_msg>test fixes<commit_after>// Based on https://github.com/martini-contrib/staticbin
package staticbin

import (
	"bytes"
	"log"
	"net/http"
	"path"
	"strings"
	"time"

	"github.com/gin-gonic/gin"
)

type Options struct {
	// SkipLogging will disable [Static] log messages when a static file is served.
	SkipLogging bool
	// IndexFile defines which file to serve as index if it exists.
	IndexFile string
	// Path prefix
	Dir string
}

func (o *Options) init() {
	if o.IndexFile == "" {
		o.IndexFile = "index.html"
	}
}

// Static returns a middleware handler that serves static files in the given directory.
func Static(asset func(string) ([]byte, error), options ...Options) gin.HandlerFunc {
	if asset == nil {
		panic("asset is nil")
	}

	opt := Options{}
	for _, o := range options {
		opt = o
		break
	}
	opt.init()

	modtime := time.Now()

	return func(c *gin.Context) {
		if c.Request.Method != "GET" && c.Request.Method != "HEAD" {
			// Request is not correct. Go farther.
			return
		}

		url := c.Request.URL.Path
		if !strings.HasPrefix(url, opt.Dir) {
			return
		}
		file := strings.TrimPrefix(
			strings.TrimPrefix(url, opt.Dir),
			"/",
		)
		b, err := asset(file)

		if err != nil {
			// Try to serve the index file.
			b, err = asset(path.Join(file, opt.IndexFile))

			if err != nil {
				// Go farther if the asset could not be found.
				return
			}
		}

		if !opt.SkipLogging {
			log.Println("[Static] Serving " + url)
		}

		http.ServeContent(c.Writer, c.Request, url, modtime, bytes.NewReader(b))
		c.Abort()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math"
	"os"
	"runtime"
	"time"

	"github.com/thinkofdeath/steven/platform"
	"github.com/thinkofdeath/steven/protocol"
	"github.com/thinkofdeath/steven/protocol/mojang"
	"github.com/thinkofdeath/steven/render"
)

var loadChan = make(chan struct{})

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU() * 2)

	// Can't use flags as we need to support a weird flag
	// format
	var username, uuid, accessToken, server string

	for i, arg := range os.Args {
		switch arg {
		case "--username":
			username = os.Args[i+1]
		case "--uuid":
			uuid = os.Args[i+1]
		case "--accessToken":
			accessToken = os.Args[i+1]
		case "--server":
			server = os.Args[i+1]
		}
	}

	// Start connecting whilst starting the renderer
	go startConnection(mojang.Profile{
		Username:    username,
		ID:          uuid,
		AccessToken: accessToken,
	}, server)

	go func() {
		render.LoadTextures()
		initBlocks()
		loadChan <- struct{}{}
	}()

	platform.Init(platform.Handler{
		Start:  start,
		Draw:   draw,
		Move:   move,
		Rotate: rotate,
		Action: action,
	})
}

func start() {
	<-loadChan
	render.Start()
}

func rotate(x, y float64) {
	render.Camera.Yaw -= x
	render.Camera.Pitch -= y
}

var mf, ms float64

func move(f, s float64) {
	mf, ms = f, s
}

func action(action platform.Action) {
	fmt.Println("action:", action)
	switch action {
	case platform.Debug:
	}
}

var maxBuilders = runtime.NumCPU() * 2

var (
	ready            bool
	freeBuilders     = maxBuilders
	completeBuilders = make(chan buildPos, maxBuilders)
	syncChan         = make(chan func(), 200)
	ticker           = time.NewTicker(time.Second / 20)
)

func draw() {
handle:
	for {
		select {
		case err := <-errorChan:
			panic(err)
		case packet := <-readChan:
			defaultHandler.Handle(packet)
		case pos := <-completeBuilders:
			c := chunkMap[chunkPosition{pos.X, pos.Z}]
			freeBuilders++
			if c != nil {
				s := c.Sections[pos.Y]
				if s != nil {
					s.building = false
				}
			}
		case f := <-syncChan:
			f()
		default:
			break handle
		}
	}

	// TODO(Think) Tidy up
	render.Camera.X += mf * math.Cos(render.Camera.Yaw-math.Pi/2) * -math.Cos(render.Camera.Pitch) * (1.0 / 7.0)
	render.Camera.Z -= mf * math.Sin(render.Camera.Yaw-math.Pi/2) * -math.Cos(render.Camera.Pitch) * (1.0 / 7.0)
	render.Camera.Y -= mf * math.Sin(render.Camera.Pitch) * (1.0 / 7.0)
	if ready {
		select {
		case <-ticker.C:
			tick()
		default:
		}
	}

	// Search for 'dirty' chunk sections and start building
	// them if we have any builders free. To prevent race conditions
	// two flags are used, dirty and building, to allow a second
	// build to be requested whilst the chunk is still building
	// without either losing the change or having two builds
	// for the same section going on at once (where the second
	// could finish quicker causing the old version to be
	// displayed.
dirtyClean:
	for _, c := range sortedChunks() {
		for _, s := range c.Sections {
			if s == nil {
				continue
			}
			if freeBuilders <= 0 {
				break dirtyClean
			}
			if s.dirty && !s.building {
				freeBuilders--
				s.dirty = false
				s.building = true
				s.build(completeBuilders)
			}
		}
	}

	render.Draw()
}

// tick is called 20 times a second (bar any preformance issues).
// Minecraft is built around this fact so we have to follow it
// as well.
func tick() {
	// Now you may be wondering why we have to spam movement
	// packets (any of the Player* move/look packets) 20 times
	// a second instead of only sending when something changes.
	// This is because the server only ticks certain parts of
	// the player when a movement packet is recieved meaning
	// if we sent them any slower health regen would be slowed
	// down as well and various other things too (potions, speed
	// hack check). This also has issues if we send them too
	// fast as well since we will regen health at much faster
	// rates than normal players and some modded servers will
	// (correctly) detect this as cheating. Its Minecraft
	// what did you expect?
	// TODO(Think) Use the smaller packets when possible
	writeChan <- &protocol.PlayerPositionLook{
		X:     render.Camera.X,
		Y:     render.Camera.Y,
		Z:     render.Camera.Z,
		Yaw:   float32(-render.Camera.Yaw * (180 / math.Pi)),
		Pitch: float32((-render.Camera.Pitch - math.Pi) * (180 / math.Pi)),
	}
}
<commit_msg>/: require arguments to be passed into steven to prevent running without the launcher<commit_after>package main

import (
	"fmt"
	"math"
	"os"
	"runtime"
	"time"

	"github.com/thinkofdeath/steven/platform"
	"github.com/thinkofdeath/steven/protocol"
	"github.com/thinkofdeath/steven/protocol/mojang"
	"github.com/thinkofdeath/steven/render"
)

var loadChan = make(chan struct{})

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU() * 2)

	if len(os.Args) == 0 {
		fmt.Println("steven must be run via the mojang launcher")
		return
	}

	// Can't use flags as we need to support a weird flag
	// format
	var username, uuid, accessToken, server string

	for i, arg := range os.Args {
		switch arg {
		case "--username":
			username = os.Args[i+1]
		case "--uuid":
			uuid = os.Args[i+1]
		case "--accessToken":
			accessToken = os.Args[i+1]
		case "--server":
			server = os.Args[i+1]
		}
	}

	// Start connecting whilst starting the renderer
	go startConnection(mojang.Profile{
		Username:    username,
		ID:          uuid,
		AccessToken: accessToken,
	}, server)

	go func() {
		render.LoadTextures()
		initBlocks()
		loadChan <- struct{}{}
	}()

	platform.Init(platform.Handler{
		Start:  start,
		Draw:   draw,
		Move:   move,
		Rotate: rotate,
		Action: action,
	})
}

func start() {
	<-loadChan
	render.Start()
}

func rotate(x, y float64) {
	render.Camera.Yaw -= x
	render.Camera.Pitch -= y
}

var mf, ms float64

func move(f, s float64) {
	mf, ms = f, s
}

func action(action platform.Action) {
	fmt.Println("action:", action)
	switch action {
	case platform.Debug:
	}
}

var maxBuilders = runtime.NumCPU() * 2

var (
	ready            bool
	freeBuilders     = maxBuilders
	completeBuilders = make(chan buildPos, maxBuilders)
	syncChan         = make(chan func(), 200)
	ticker           = time.NewTicker(time.Second / 20)
)

func draw() {
handle:
	for {
		select {
		case err := <-errorChan:
			panic(err)
		case packet := <-readChan:
			defaultHandler.Handle(packet)
		case pos := <-completeBuilders:
			c := chunkMap[chunkPosition{pos.X, pos.Z}]
			freeBuilders++
			if c != nil {
				s := c.Sections[pos.Y]
				if s != nil {
					s.building = false
				}
			}
		case f := <-syncChan:
			f()
		default:
			break handle
		}
	}

	// TODO(Think) Tidy up
	render.Camera.X += mf * math.Cos(render.Camera.Yaw-math.Pi/2) * -math.Cos(render.Camera.Pitch) * (1.0 / 7.0)
	render.Camera.Z -= mf * math.Sin(render.Camera.Yaw-math.Pi/2) * -math.Cos(render.Camera.Pitch) * (1.0 / 7.0)
	render.Camera.Y -= mf * math.Sin(render.Camera.Pitch) * (1.0 / 7.0)
	if ready {
		select {
		case <-ticker.C:
			tick()
		default:
		}
	}

	// Search for 'dirty' chunk sections and start building
	// them if we have any builders free. To prevent race conditions
	// two flags are used, dirty and building, to allow a second
	// build to be requested whilst the chunk is still building
	// without either losing the change or having two builds
	// for the same section going on at once (where the second
	// could finish quicker causing the old version to be
	// displayed.
dirtyClean:
	for _, c := range sortedChunks() {
		for _, s := range c.Sections {
			if s == nil {
				continue
			}
			if freeBuilders <= 0 {
				break dirtyClean
			}
			if s.dirty && !s.building {
				freeBuilders--
				s.dirty = false
				s.building = true
				s.build(completeBuilders)
			}
		}
	}

	render.Draw()
}

// tick is called 20 times a second (bar any preformance issues).
// Minecraft is built around this fact so we have to follow it
// as well.
func tick() {
	// Now you may be wondering why we have to spam movement
	// packets (any of the Player* move/look packets) 20 times
	// a second instead of only sending when something changes.
	// This is because the server only ticks certain parts of
	// the player when a movement packet is recieved meaning
	// if we sent them any slower health regen would be slowed
	// down as well and various other things too (potions, speed
	// hack check). This also has issues if we send them too
	// fast as well since we will regen health at much faster
	// rates than normal players and some modded servers will
	// (correctly) detect this as cheating. Its Minecraft
	// what did you expect?
	// TODO(Think) Use the smaller packets when possible
	writeChan <- &protocol.PlayerPositionLook{
		X:     render.Camera.X,
		Y:     render.Camera.Y,
		Z:     render.Camera.Z,
		Yaw:   float32(-render.Camera.Yaw * (180 / math.Pi)),
		Pitch: float32((-render.Camera.Pitch - math.Pi) * (180 / math.Pi)),
	}
}
<|endoftext|>


<commit_before>package store

import (
	"github.com/smancke/guble/protocol"

	"errors"
	"os"
	"path"
	"sync"
	"syscall"
)

// FileMessageStore is an implementation of the MessageStore interface based on files
type FileMessageStore struct {
	partitions map[string]*MessagePartition
	basedir    string
	mutex      sync.RWMutex
}

func NewFileMessageStore(basedir string) *FileMessageStore {
	return &FileMessageStore{
		partitions: make(map[string]*MessagePartition),
		basedir:    basedir,
	}
}

func (fms *FileMessageStore) MaxMessageId(partition string) (uint64, error) {
	p, err := fms.partitionStore(partition)
	if err != nil {
		return 0, err
	}
	return p.MaxMessageId()
}

func (fms *FileMessageStore) Stop() error {
	fms.mutex.Lock()
	defer fms.mutex.Unlock()

	var returnError error
	for key, partition := range fms.partitions {
		if err := partition.Close(); err != nil {
			returnError = err
			protocol.Err("error on closing message store partition %q: %v", key, err)
		}
		delete(fms.partitions, key)
	}
	return returnError
}

func (fms *FileMessageStore) StoreTx(partition string,
	callback func(msgId uint64) (msg []byte)) error {

	p, err := fms.partitionStore(partition)
	if err != nil {
		return err
	}
	return p.StoreTx(partition, callback)
}

// Store stores a message within a partition
func (fms *FileMessageStore) Store(partition string, msgId uint64, msg []byte) error {
	p, err := fms.partitionStore(partition)
	if err != nil {
		return err
	}
	return p.Store(msgId, msg)
}

// Fetch asynchronously fetches a set of messages defined by the fetch request
func (fms *FileMessageStore) Fetch(req FetchRequest) {
	p, err := fms.partitionStore(req.Partition)
	if err != nil {
		req.ErrorCallback <- err
		return
	}
	p.Fetch(req)
}

func (fms *FileMessageStore) DoInTx(partition string, fnToExecute func(maxMessageId uint64) error) error {
	p, err := fms.partitionStore(partition)
	if err != nil {
		return err
	}
	return p.DoInTx(fnToExecute)
}

func (fms *FileMessageStore) partitionStore(partition string) (*MessagePartition, error) {
	fms.mutex.Lock()
	defer fms.mutex.Unlock()

	partitionStore, exist := fms.partitions[partition]
	if !exist {
		dir := path.Join(fms.basedir, partition)
		if _, err := os.Stat(dir); err != nil {
			if os.IsNotExist(err) {
				if err := os.MkdirAll(dir, 0700); err != nil {
					return nil, err
				}
			} else {
				return nil, err
			}
		}
		var err error
		partitionStore, err = NewMessagePartition(dir, partition)
		if err != nil {
			return nil, err
		}
		fms.partitions[partition] = partitionStore
	}
	return partitionStore, nil
}

func (fms *FileMessageStore) Check() error {
	var stat syscall.Statfs_t
	wd, err := os.Getwd()
	if err != nil {
		protocol.Err("MessageStore check returned error", err)
		return err
	}
	syscall.Statfs(wd, &stat)

	// Available blocks * size per block = available space in bytes
	freeSpace := stat.Bavail * uint64(stat.Bsize)
	// total system blocks * size per block = total space in bytes
	totalSpace := stat.Blocks * uint64(stat.Bsize)

	usedSpacePercentage := 1 - (float64(freeSpace) / float64(totalSpace))
	if usedSpacePercentage > 0.95 {
		protocol.Err("Disk space is used in a proportion more than 95%.Actual space used %.2f", usedSpacePercentage)
		return errors.New("HDD Disk is almost full .")
	}

	return nil
}
<commit_msg>Refactor<commit_after>package store

import (
	"github.com/smancke/guble/protocol"

	"errors"
	"os"
	"path"
	"sync"
	"syscall"
)

// FileMessageStore is an implementation of the MessageStore interface based on files
type FileMessageStore struct {
	partitions map[string]*MessagePartition
	basedir    string
	mutex      sync.RWMutex
}

func NewFileMessageStore(basedir string) *FileMessageStore {
	return &FileMessageStore{
		partitions: make(map[string]*MessagePartition),
		basedir:    basedir,
	}
}

func (fms *FileMessageStore) MaxMessageId(partition string) (uint64, error) {
	p, err := fms.partitionStore(partition)
	if err != nil {
		return 0, err
	}
	return p.MaxMessageId()
}

func (fms *FileMessageStore) Stop() error {
	fms.mutex.Lock()
	defer fms.mutex.Unlock()

	var returnError error
	for key, partition := range fms.partitions {
		if err := partition.Close(); err != nil {
			returnError = err
			protocol.Err("error on closing message store partition %q: %v", key, err)
		}
		delete(fms.partitions, key)
	}
	return returnError
}

func (fms *FileMessageStore) StoreTx(partition string,
	callback func(msgId uint64) (msg []byte)) error {

	p, err := fms.partitionStore(partition)
	if err != nil {
		return err
	}
	return p.StoreTx(partition, callback)
}

// Store stores a message within a partition
func (fms *FileMessageStore) Store(partition string, msgId uint64, msg []byte) error {
	p, err := fms.partitionStore(partition)
	if err != nil {
		return err
	}
	return p.Store(msgId, msg)
}

// Fetch asynchronously fetches a set of messages defined by the fetch request
func (fms *FileMessageStore) Fetch(req FetchRequest) {
	p, err := fms.partitionStore(req.Partition)
	if err != nil {
		req.ErrorCallback <- err
		return
	}
	p.Fetch(req)
}

func (fms *FileMessageStore) DoInTx(partition string, fnToExecute func(maxMessageId uint64) error) error {
	p, err := fms.partitionStore(partition)
	if err != nil {
		return err
	}
	return p.DoInTx(fnToExecute)
}

func (fms *FileMessageStore) partitionStore(partition string) (*MessagePartition, error) {
	fms.mutex.Lock()
	defer fms.mutex.Unlock()

	partitionStore, exist := fms.partitions[partition]
	if !exist {
		dir := path.Join(fms.basedir, partition)
		if _, err := os.Stat(dir); err != nil {
			if os.IsNotExist(err) {
				if err := os.MkdirAll(dir, 0700); err != nil {
					return nil, err
				}
			} else {
				return nil, err
			}
		}
		var err error
		partitionStore, err = NewMessagePartition(dir, partition)
		if err != nil {
			return nil, err
		}
		fms.partitions[partition] = partitionStore
	}
	return partitionStore, nil
}

func (fms *FileMessageStore) Check() error {
	var stat syscall.Statfs_t
	wd, err := os.Getwd()
	if err != nil {
		protocol.Err("MessageStore check returned error", err)
		return err
	}
	syscall.Statfs(wd, &stat)

	// Available blocks * size per block = available space in bytes
	freeSpace := stat.Bavail * uint64(stat.Bsize)
	// total system blocks * size per block = total space in bytes
	totalSpace := stat.Blocks * uint64(stat.Bsize)

	usedSpacePercentage := 1 - (float64(freeSpace) / float64(totalSpace))

	if usedSpacePercentage > 0.95 {
		protocol.Err("Disk space is used  more than 95 percent. Actual percent of space used %.2f", usedSpacePercentage)
		return errors.New("HDD Disk is almost full .")
	}

	return nil
}
<|endoftext|>


<commit_before>// Package stream implements stream algorithms.
package stream

import "sync"

// Counter represents a counted item in a Summary.
type Counter struct {
	Item      string
	Count     uint64
	ErrorRate uint64
}

// Summary represents a Stream-Summary data structure as described in "Efficient
// Computation of Frequent and Top-k Elements in Data Streams".
type Summary struct {
	observed uint64
	capacity int
	counters []*Counter
	index    map[string]int
	rw       sync.RWMutex
}

// NewSummary returns a new Summary with the given max capacity.
func NewSummary(capacity int) *Summary {
	return &Summary{
		capacity: capacity,
		counters: make([]*Counter, 0, capacity),
		index:    make(map[string]int),
	}
}

// Observed returns the total number of observations which have occurred.
func (s *Summary) Observed() uint64 {
	s.rw.RLock()
	defer s.rw.RUnlock()
	return s.observed
}

// Top returns the top n Counters in the Summary.  If the Summary contains less
// than n Counters, all Counters in the Summary are returned.
func (s *Summary) Top(n int) []*Counter {
	s.rw.RLock()
	defer s.rw.RUnlock()

	if n > len(s.counters) {
		return s.counters[0:len(s.counters)]
	}

	return s.counters[0:n]
}

// Observe adds an observation of an item to the Summary.
func (s *Summary) Observe(item string) {
	s.rw.Lock()
	defer s.rw.Unlock()

	s.observed++

	i, exists := s.index[item]
	if exists {
		s.counters[i].Count++

		// Slide this counter forward in the array to keep it in sorted order.
		for i > 0 && s.counters[i].Count > s.counters[i-1].Count {
			s.swap(i, i-1)
			i--
		}
	} else {
		if len(s.counters) < s.capacity {
			// Add the new counter since the summary is below capacity.
			s.counters = append(s.counters, &Counter{Item: item, Count: 1})
			s.index[item] = len(s.counters) - 1
		} else {
			// Replace the lowest counter with a new counter.
			minCounter := s.counters[len(s.counters)-1]
			delete(s.index, minCounter.Item)

			counter := &Counter{
				Item:      item,
				Count:     minCounter.Count + 1,
				ErrorRate: minCounter.Count,
			}
			s.counters[len(s.counters)-1] = counter
			s.index[item] = len(s.counters) - 1
		}
	}
}

func (s *Summary) swap(i, j int) {
	s.index[s.counters[i].Item] = j
	s.index[s.counters[j].Item] = i
	s.counters[j], s.counters[i] = s.counters[i], s.counters[j]
}
<commit_msg>Observe is now easier to reason about<commit_after>// Package stream implements stream algorithms.
package stream

import "sync"

// Counter represents a counted item in a Summary.
type Counter struct {
	Item      string
	Count     uint64
	ErrorRate uint64
}

// Summary represents a Stream-Summary data structure as described in "Efficient
// Computation of Frequent and Top-k Elements in Data Streams".
type Summary struct {
	observed uint64
	capacity int
	counters []*Counter
	index    map[string]int
	rw       sync.RWMutex
}

// NewSummary returns a new Summary with the given max capacity.
func NewSummary(capacity int) *Summary {
	return &Summary{
		capacity: capacity,
		counters: make([]*Counter, 0, capacity),
		index:    make(map[string]int),
	}
}

// Observed returns the total number of observations which have occurred.
func (s *Summary) Observed() uint64 {
	s.rw.RLock()
	defer s.rw.RUnlock()
	return s.observed
}

// Top returns the top n Counters in the Summary.  If the Summary contains less
// than n Counters, all Counters in the Summary are returned.
func (s *Summary) Top(n int) []*Counter {
	s.rw.RLock()
	defer s.rw.RUnlock()

	if n > len(s.counters) {
		return s.counters[0:len(s.counters)]
	}

	return s.counters[0:n]
}

// Observe adds an observation of an item to the Summary.
func (s *Summary) Observe(item string) {
	s.rw.Lock()
	defer s.rw.Unlock()

	s.observed++

	i, exists := s.index[item]
	if exists {
		s.counters[i].Count++

		// Slide this counter forward in the array to keep it in sorted order.
		for ; i > 0 && s.counters[i].Count > s.counters[i-1].Count; i-- {
			s.swap(i, i-1)
		}
	} else {
		if len(s.counters) < s.capacity {
			s.append(&Counter{Item: item, Count: 1})
		} else {
			lastIndex := len(s.counters) - 1
			minCounter := s.deleteAt(lastIndex)
			counter := &Counter{
				Item:      item,
				Count:     minCounter.Count + 1,
				ErrorRate: minCounter.Count,
			}
			s.insertAt(lastIndex, counter)
		}
	}
}

func (s *Summary) append(counter *Counter) {
	s.counters = append(s.counters, counter)
	s.index[counter.Item] = len(s.counters) - 1
}

func (s *Summary) deleteAt(i int) *Counter {
	counter := s.counters[len(s.counters)-1]
	delete(s.index, counter.Item)
	return counter
}

func (s *Summary) insertAt(i int, counter *Counter) {
	s.counters[i] = counter
	s.index[counter.Item] = i
}

func (s *Summary) swap(i, j int) {
	s.index[s.counters[i].Item] = j
	s.index[s.counters[j].Item] = i
	s.counters[j], s.counters[i] = s.counters[i], s.counters[j]
}
<|endoftext|>


<commit_before>package gofast

// Stream for a newly started stream on the transport. Refer to
// Stream() method on the transport.
type Stream struct {
	transport         *Transport
	Rxch              chan Message
	opaque            uint64
	remote            bool
	out, data, tagout []byte
}

// constructor used for remote streams.
func (t *Transport) newremotestream(opaque uint64) *Stream {
	stream := t.fromrxstrm()
	//fmsg := "%v ##%d(remote:%v) stream created ...\n"
	//log.Verbosef(fmsg, t.logprefix, opaque, remote)
	// reset all fields (it is coming from a pool)
	stream.transport, stream.remote, stream.opaque = t, true, opaque
	stream.Rxch = nil
	return stream
}

func (t *Transport) getlocalstream(ch chan Message) *Stream { // called only be tx.
	stream := <-t.p_strms
	stream.Rxch = ch
	t.putch(t.rxch, rxpacket{stream: stream})
	return stream
}

func (t *Transport) putstream(opaque uint64, stream *Stream, tellrx bool) {
	// Rxch could also be closed when transport is closed...
	// Rxch could also be nil in case of post...
	defer func() {
		if r := recover(); r != nil {
			fmsg := "%v ##%v putstream recovered: %v\n"
			log.Debugf(fmsg, t.logprefix, opaque, r)
		}
	}()
	if stream == nil {
		log.Errorf("%v ##%v unkown stream\n", t.logprefix, opaque)
		return
	}
	if stream.remote == false {
		t.p_strms <- stream // don't collect remote streams
	}
	if stream.Rxch != nil {
		close(stream.Rxch)
	}
	stream.Rxch = nil
	if tellrx {
		t.putch(t.rxch, rxpacket{stream: stream})
	}
}

// Response to a request, to batch the response pass flush as false.
func (s *Stream) Response(msg Message, flush bool) error {
	n := s.transport.response(msg, s, s.out)
	return s.transport.txasync(s.out[:n], flush)
}

// Stream a single message, to batch the message pass flush as false.
func (s *Stream) Stream(msg Message, flush bool) (err error) {
	n := s.transport.stream(msg, s, s.out)
	return s.transport.txasync(s.out[:n], flush)
}

// Close this stream.
func (s *Stream) Close() error {
	n := s.transport.finish(s, s.out)
	return s.transport.txasync(s.out[:n], true /*flush*/)
}

// Transport return the underlying transport carrying this stream.
func (s *Stream) Transport() *Transport {
	return s.transport
}
<commit_msg>fix race situation in putstream()<commit_after>package gofast

// Stream for a newly started stream on the transport. Refer to
// Stream() method on the transport.
type Stream struct {
	transport         *Transport
	Rxch              chan Message
	opaque            uint64
	remote            bool
	out, data, tagout []byte
}

// constructor used for remote streams.
func (t *Transport) newremotestream(opaque uint64) *Stream {
	stream := t.fromrxstrm()
	//fmsg := "%v ##%d(remote:%v) stream created ...\n"
	//log.Verbosef(fmsg, t.logprefix, opaque, remote)
	// reset all fields (it is coming from a pool)
	stream.transport, stream.remote, stream.opaque = t, true, opaque
	stream.Rxch = nil
	return stream
}

func (t *Transport) getlocalstream(ch chan Message) *Stream { // called only be tx.
	stream := <-t.p_strms
	stream.Rxch = ch
	t.putch(t.rxch, rxpacket{stream: stream})
	return stream
}

func (t *Transport) putstream(opaque uint64, stream *Stream, tellrx bool) {
	// Rxch could also be closed when transport is closed...
	// Rxch could also be nil in case of post...
	defer func() {
		if r := recover(); r != nil {
			fmsg := "%v ##%v putstream recovered: %v\n"
			log.Debugf(fmsg, t.logprefix, opaque, r)
		}
	}()
	if stream == nil {
		log.Errorf("%v ##%v unkown stream\n", t.logprefix, opaque)
		return
	}
	if stream.remote == false {
		t.p_strms <- stream // don't collect remote streams
	}
	if stream.Rxch != nil {
		close(stream.Rxch)
		stream.Rxch = nil
	}
	if tellrx {
		t.putch(t.rxch, rxpacket{stream: stream})
	}
}

// Response to a request, to batch the response pass flush as false.
func (s *Stream) Response(msg Message, flush bool) error {
	n := s.transport.response(msg, s, s.out)
	return s.transport.txasync(s.out[:n], flush)
}

// Stream a single message, to batch the message pass flush as false.
func (s *Stream) Stream(msg Message, flush bool) (err error) {
	n := s.transport.stream(msg, s, s.out)
	return s.transport.txasync(s.out[:n], flush)
}

// Close this stream.
func (s *Stream) Close() error {
	n := s.transport.finish(s, s.out)
	return s.transport.txasync(s.out[:n], true /*flush*/)
}

// Transport return the underlying transport carrying this stream.
func (s *Stream) Transport() *Transport {
	return s.transport
}
<|endoftext|>