<commit_before>// Copyright 2016 The Mangos Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use file except in compliance with the License.
// You may obtain a copy of the license at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package ws implements a simple WebSocket transport for mangos.
// This transport is considered EXPERIMENTAL.
package ws

import (
	"crypto/tls"
	"github.com/gorilla/websocket"
	"net"
	"net/http"
	"net/url"
	"strings"

	"github.com/go-mangos/mangos"
	"sync"
)

// Some special options
const (
	// OptionWebSocketMux is a retrieve-only property used to obtain
	// the *http.ServeMux instance associated with the server.  This
	// can be used to subsequently register additional handlers for
	// different URIs.  This option is only valid on a Listener.
	// Generally you use this option when you want to use the standard
	// mangos Listen() method to start up the server.
	OptionWebSocketMux = "WEBSOCKET-MUX"

	// OptionWebSocketHandler is used to obtain the underlying
	// http.Handler (websocket.Server) object, so you can use this
	// on your own http.Server instances.  It is a gross error to use
	// the value returned by this method on an http server if the
	// server is also started with mangos Listen().  This means that you
	// will use at most either this option, or OptionWebSocketMux, but
	// never both.  This option is only valid on a listener.
	OptionWebSocketHandler = "WEBSOCKET-HANDLER"
)

type options map[string]interface{}

// GetOption retrieves an option value.
func (o options) get(name string) (interface{}, error) {
	if o == nil {
		return nil, mangos.ErrBadOption
	}
	v, ok := o[name]
	if !ok {
		return nil, mangos.ErrBadOption
	}
	return v, nil
}

// SetOption sets an option.  We have none, so just ErrBadOption.
func (o options) set(name string, val interface{}) error {
	switch name {
	case mangos.OptionNoDelay:
		fallthrough
	case mangos.OptionKeepAlive:
		switch v := val.(type) {
		case bool:
			o[name] = v
			return nil
		default:
			return mangos.ErrBadValue
		}
	case mangos.OptionTLSConfig:
		switch v := val.(type) {
		case *tls.Config:
			o[name] = v
			return nil
		default:
			return mangos.ErrBadValue
		}
	}
	return mangos.ErrBadOption
}

// wsPipe implements the Pipe interface on a websocket
type wsPipe struct {
	ws    *websocket.Conn
	proto mangos.Protocol
	addr  string
	open  bool
	wg    sync.WaitGroup
	props map[string]interface{}
	iswss bool
	dtype int
}

type wsTran int

func (w *wsPipe) Recv() (*mangos.Message, error) {

	// We ignore the message type for receive.
	_, body, err := w.ws.ReadMessage()
	if err != nil {
		return nil, err
	}
	msg := mangos.NewMessage(0)
	msg.Body = body
	return msg, nil
}

func (w *wsPipe) Send(m *mangos.Message) error {

	var buf []byte

	if m.Expired() {
		m.Free()
		return nil
	}
	if len(m.Header) > 0 {
		buf = make([]byte, 0, len(m.Header)+len(m.Body))
		buf = append(buf, m.Header...)
		buf = append(buf, m.Body...)
	} else {
		buf = m.Body
	}
	if err := w.ws.WriteMessage(w.dtype, buf); err != nil {
		return err
	}
	m.Free()
	return nil
}

func (w *wsPipe) LocalProtocol() uint16 {
	return w.proto.Number()
}

func (w *wsPipe) RemoteProtocol() uint16 {
	return w.proto.PeerNumber()
}

func (w *wsPipe) Close() error {
	w.open = false
	w.ws.Close()
	w.wg.Done()
	return nil
}

func (w *wsPipe) IsOpen() bool {
	return w.open
}

func (w *wsPipe) GetProp(name string) (interface{}, error) {
	if v, ok := w.props[name]; ok {
		return v, nil
	}
	return nil, mangos.ErrBadProperty
}

type dialer struct {
	addr  string // url
	proto mangos.Protocol
	opts  options
	iswss bool
	maxrx int
}

func (d *dialer) Dial() (mangos.Pipe, error) {
	var w *wsPipe

	wd := &websocket.Dialer{}

	wd.Subprotocols = []string{d.proto.PeerName() + ".sp.nanomsg.org"}
	if v, ok := d.opts[mangos.OptionTLSConfig]; ok {
		wd.TLSClientConfig = v.(*tls.Config)
	}

	w = &wsPipe{proto: d.proto, addr: d.addr, open: true}
	w.dtype = websocket.BinaryMessage
	w.props = make(map[string]interface{})

	var err error
	if w.ws, _, err = wd.Dial(d.addr, nil); err != nil {
		return nil, err
	}
	w.ws.SetReadLimit(int64(d.maxrx))
	w.props[mangos.PropLocalAddr] = w.ws.LocalAddr()
	w.props[mangos.PropRemoteAddr] = w.ws.RemoteAddr()

	w.wg.Add(1)
	return w, nil
}

func (d *dialer) SetOption(n string, v interface{}) error {
	return d.opts.set(n, v)
}

func (d *dialer) GetOption(n string) (interface{}, error) {
	return d.opts.get(n)
}

type listener struct {
	pending  []*wsPipe
	lock     sync.Mutex
	cv       sync.Cond
	running  bool
	addr     string
	ug       websocket.Upgrader
	htsvr    *http.Server
	mux      *http.ServeMux
	url      *url.URL
	listener net.Listener
	proto    mangos.Protocol
	opts     options
	iswss    bool
	maxrx    int
}

func (l *listener) SetOption(n string, v interface{}) error {
	return l.opts.set(n, v)
}

func (l *listener) GetOption(n string) (interface{}, error) {
	switch n {
	case OptionWebSocketMux:
		return l.mux, nil
	case OptionWebSocketHandler:
		// Caller intends to use use in his own server, so mark
		// us running.  If he didn't mean this, the side effect is
		// that Accept() will appear to hang, even though Listen()
		// is not called yet.
		l.running = true
		return l, nil
	}
	return l.opts.get(n)
}

func (l *listener) Listen() error {
	var taddr *net.TCPAddr
	var err error
	var tcfg *tls.Config

	if l.iswss {
		v, ok := l.opts[mangos.OptionTLSConfig]
		if !ok || v == nil {
			return mangos.ErrTLSNoConfig
		}
		tcfg = v.(*tls.Config)
		if tcfg.Certificates == nil || len(tcfg.Certificates) == 0 {
			return mangos.ErrTLSNoCert
		}
	}

	// We listen separately, that way we can catch and deal with the
	// case of a port already in use.  This also lets us configure
	// properties of the underlying TCP connection.

	if taddr, err = mangos.ResolveTCPAddr(l.url.Host); err != nil {
		return err
	}

	if tlist, err := net.ListenTCP("tcp", taddr); err != nil {
		return err
	} else if l.iswss {
		l.listener = tls.NewListener(tlist, tcfg)
	} else {
		l.listener = tlist
	}
	l.pending = nil
	l.running = true

	l.htsvr = &http.Server{Addr: l.url.Host, Handler: l.mux}

	go l.htsvr.Serve(l.listener)

	return nil
}

func (l *listener) Accept() (mangos.Pipe, error) {
	var w *wsPipe

	l.lock.Lock()
	defer l.lock.Unlock()

	for {
		if !l.running {
			return nil, mangos.ErrClosed
		}
		if len(l.pending) == 0 {
			l.cv.Wait()
			continue
		}
		w = l.pending[len(l.pending)-1]
		l.pending = l.pending[:len(l.pending)-1]
		break
	}

	return w, nil
}

func (l *listener) handler(ws *websocket.Conn, req *http.Request) {
	l.lock.Lock()

	if !l.running {
		ws.Close()
		l.lock.Unlock()
		return
	}

	if ws.Subprotocol() != l.proto.Name()+".sp.nanomsg.org" {
		ws.Close()
		l.lock.Unlock()
		return
	}

	w := &wsPipe{ws: ws, addr: l.addr, proto: l.proto, open: true}
	w.dtype = websocket.BinaryMessage
	w.iswss = l.iswss
	w.ws.SetReadLimit(int64(l.maxrx))

	w.props = make(map[string]interface{})
	w.props[mangos.PropLocalAddr] = ws.LocalAddr()
	w.props[mangos.PropRemoteAddr] = ws.RemoteAddr()

	if req.TLS != nil {
		w.props[mangos.PropTLSConnState] = *req.TLS
	}

	w.wg.Add(1)
	l.pending = append(l.pending, w)
	l.cv.Broadcast()
	l.lock.Unlock()

	// We must not return before the socket is closed, because
	// our caller will close the websocket on our return.
	w.wg.Wait()
}

func (l *listener) Handle(pattern string, handler http.Handler) {
	l.mux.Handle(pattern, handler)
}

func (l *listener) HandleFunc(pattern string, handler http.HandlerFunc) {
	l.mux.HandleFunc(pattern, handler)
}

func (l *listener) Close() error {
	l.lock.Lock()
	defer l.lock.Unlock()
	if !l.running {
		return mangos.ErrClosed
	}
	if l.listener != nil {
		l.listener.Close()
	}
	l.running = false
	l.cv.Broadcast()
	for _, ws := range l.pending {
		ws.Close()
	}
	l.pending = nil
	return nil
}

func (l *listener) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	ws, err := l.ug.Upgrade(w, r, nil)
	if err != nil {
		return
	}
	l.handler(ws, r)
}

func (l *listener) Address() string {
	return l.url.String()
}

func (wsTran) Scheme() string {
	return "ws"
}

func (wsTran) NewDialer(addr string, sock mangos.Socket) (mangos.PipeDialer, error) {
	iswss := strings.HasPrefix(addr, "wss://")
	opts := make(map[string]interface{})

	opts[mangos.OptionNoDelay] = true
	opts[mangos.OptionKeepAlive] = true
	proto := sock.GetProtocol()
	maxrx := 0
	if v, e := sock.GetOption(mangos.OptionMaxRecvSize); e == nil {
		maxrx = v.(int)
	}

	return &dialer{addr: addr, proto: proto, iswss: iswss, opts: opts, maxrx: maxrx}, nil
}

func (t wsTran) NewListener(addr string, sock mangos.Socket) (mangos.PipeListener, error) {
	proto := sock.GetProtocol()
	l, e := t.listener(addr, proto)
	if e == nil {
		if v, e := sock.GetOption(mangos.OptionMaxRecvSize); e == nil {
			l.maxrx = v.(int)
		}
		l.mux.Handle(l.url.Path, l)
	}
	return l, e
}

func (wsTran) listener(addr string, proto mangos.Protocol) (*listener, error) {
	var err error
	l := &listener{proto: proto, opts: make(map[string]interface{})}
	l.cv.L = &l.lock
	l.ug.Subprotocols = []string{proto.Name() + ".sp.nanomsg.org"}

	if strings.HasPrefix(addr, "wss://") {
		l.iswss = true
	}
	l.url, err = url.ParseRequestURI(addr)
	if err != nil {
		return nil, err
	}
	if len(l.url.Path) == 0 {
		l.url.Path = "/"
	}
	l.mux = http.NewServeMux()

	l.htsvr = &http.Server{Addr: l.url.Host, Handler: l.mux}

	return l, nil
}

// NewTransport allocates a new inproc:// transport.
func NewTransport() mangos.Transport {
	return wsTran(0)
}
<commit_msg>fixes #258 Cannot use returned http.Handler properly<commit_after>// Copyright 2016 The Mangos Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use file except in compliance with the License.
// You may obtain a copy of the license at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package ws implements a simple WebSocket transport for mangos.
// This transport is considered EXPERIMENTAL.
package ws

import (
	"crypto/tls"
	"github.com/gorilla/websocket"
	"net"
	"net/http"
	"net/url"
	"strings"

	"github.com/go-mangos/mangos"
	"sync"
)

// Some special options
const (
	// OptionWebSocketMux is a retrieve-only property used to obtain
	// the *http.ServeMux instance associated with the server.  This
	// can be used to subsequently register additional handlers for
	// different URIs.  This option is only valid on a Listener.
	// Generally you use this option when you want to use the standard
	// mangos Listen() method to start up the server.
	OptionWebSocketMux = "WEBSOCKET-MUX"

	// OptionWebSocketHandler is used to obtain the underlying
	// http.Handler (websocket.Server) object, so you can use this
	// on your own http.Server instances.  It is a gross error to use
	// the value returned by this method on an http server if the
	// server is also started with mangos Listen().  This means that you
	// will use at most either this option, or OptionWebSocketMux, but
	// never both.  This option is only valid on a listener.
	OptionWebSocketHandler = "WEBSOCKET-HANDLER"
)

type options map[string]interface{}

// GetOption retrieves an option value.
func (o options) get(name string) (interface{}, error) {
	if o == nil {
		return nil, mangos.ErrBadOption
	}
	v, ok := o[name]
	if !ok {
		return nil, mangos.ErrBadOption
	}
	return v, nil
}

// SetOption sets an option.  We have none, so just ErrBadOption.
func (o options) set(name string, val interface{}) error {
	switch name {
	case mangos.OptionNoDelay:
		fallthrough
	case mangos.OptionKeepAlive:
		switch v := val.(type) {
		case bool:
			o[name] = v
			return nil
		default:
			return mangos.ErrBadValue
		}
	case mangos.OptionTLSConfig:
		switch v := val.(type) {
		case *tls.Config:
			o[name] = v
			return nil
		default:
			return mangos.ErrBadValue
		}
	}
	return mangos.ErrBadOption
}

// wsPipe implements the Pipe interface on a websocket
type wsPipe struct {
	ws    *websocket.Conn
	proto mangos.Protocol
	addr  string
	open  bool
	wg    sync.WaitGroup
	props map[string]interface{}
	iswss bool
	dtype int
}

type wsTran int

func (w *wsPipe) Recv() (*mangos.Message, error) {

	// We ignore the message type for receive.
	_, body, err := w.ws.ReadMessage()
	if err != nil {
		return nil, err
	}
	msg := mangos.NewMessage(0)
	msg.Body = body
	return msg, nil
}

func (w *wsPipe) Send(m *mangos.Message) error {

	var buf []byte

	if m.Expired() {
		m.Free()
		return nil
	}
	if len(m.Header) > 0 {
		buf = make([]byte, 0, len(m.Header)+len(m.Body))
		buf = append(buf, m.Header...)
		buf = append(buf, m.Body...)
	} else {
		buf = m.Body
	}
	if err := w.ws.WriteMessage(w.dtype, buf); err != nil {
		return err
	}
	m.Free()
	return nil
}

func (w *wsPipe) LocalProtocol() uint16 {
	return w.proto.Number()
}

func (w *wsPipe) RemoteProtocol() uint16 {
	return w.proto.PeerNumber()
}

func (w *wsPipe) Close() error {
	w.open = false
	w.ws.Close()
	w.wg.Done()
	return nil
}

func (w *wsPipe) IsOpen() bool {
	return w.open
}

func (w *wsPipe) GetProp(name string) (interface{}, error) {
	if v, ok := w.props[name]; ok {
		return v, nil
	}
	return nil, mangos.ErrBadProperty
}

type dialer struct {
	addr  string // url
	proto mangos.Protocol
	opts  options
	iswss bool
	maxrx int
}

func (d *dialer) Dial() (mangos.Pipe, error) {
	var w *wsPipe

	wd := &websocket.Dialer{}

	wd.Subprotocols = []string{d.proto.PeerName() + ".sp.nanomsg.org"}
	if v, ok := d.opts[mangos.OptionTLSConfig]; ok {
		wd.TLSClientConfig = v.(*tls.Config)
	}

	w = &wsPipe{proto: d.proto, addr: d.addr, open: true}
	w.dtype = websocket.BinaryMessage
	w.props = make(map[string]interface{})

	var err error
	if w.ws, _, err = wd.Dial(d.addr, nil); err != nil {
		return nil, err
	}
	w.ws.SetReadLimit(int64(d.maxrx))
	w.props[mangos.PropLocalAddr] = w.ws.LocalAddr()
	w.props[mangos.PropRemoteAddr] = w.ws.RemoteAddr()

	w.wg.Add(1)
	return w, nil
}

func (d *dialer) SetOption(n string, v interface{}) error {
	return d.opts.set(n, v)
}

func (d *dialer) GetOption(n string) (interface{}, error) {
	return d.opts.get(n)
}

type listener struct {
	pending  []*wsPipe
	lock     sync.Mutex
	cv       sync.Cond
	running  bool
	noserve  bool
	addr     string
	ug       websocket.Upgrader
	htsvr    *http.Server
	mux      *http.ServeMux
	url      *url.URL
	listener net.Listener
	proto    mangos.Protocol
	opts     options
	iswss    bool
	maxrx    int
}

func (l *listener) SetOption(n string, v interface{}) error {
	return l.opts.set(n, v)
}

func (l *listener) GetOption(n string) (interface{}, error) {
	switch n {
	case OptionWebSocketMux:
		return l.mux, nil
	case OptionWebSocketHandler:
		// Caller intends to use use in his own server, so mark
		// us running.  If he didn't mean this, the side effect is
		// that Accept() will appear to hang, even though Listen()
		// is not called yet.
		l.running = true
		l.noserve = true
		return l, nil
	}
	return l.opts.get(n)
}

func (l *listener) Listen() error {
	var taddr *net.TCPAddr
	var err error
	var tcfg *tls.Config

	if l.noserve {
		// The HTTP framework is going to call us, so we use that rather than
		// listening on our own.  We just fake this out.
		return nil
	}
	if l.iswss {
		v, ok := l.opts[mangos.OptionTLSConfig]
		if !ok || v == nil {
			return mangos.ErrTLSNoConfig
		}
		tcfg = v.(*tls.Config)
		if tcfg.Certificates == nil || len(tcfg.Certificates) == 0 {
			return mangos.ErrTLSNoCert
		}
	}

	// We listen separately, that way we can catch and deal with the
	// case of a port already in use.  This also lets us configure
	// properties of the underlying TCP connection.

	if taddr, err = mangos.ResolveTCPAddr(l.url.Host); err != nil {
		return err
	}

	if tlist, err := net.ListenTCP("tcp", taddr); err != nil {
		return err
	} else if l.iswss {
		l.listener = tls.NewListener(tlist, tcfg)
	} else {
		l.listener = tlist
	}
	l.pending = nil
	l.running = true

	l.htsvr = &http.Server{Addr: l.url.Host, Handler: l.mux}

	go l.htsvr.Serve(l.listener)

	return nil
}

func (l *listener) Accept() (mangos.Pipe, error) {
	var w *wsPipe

	l.lock.Lock()
	defer l.lock.Unlock()

	for {
		if !l.running {
			return nil, mangos.ErrClosed
		}
		if len(l.pending) == 0 {
			l.cv.Wait()
			continue
		}
		w = l.pending[len(l.pending)-1]
		l.pending = l.pending[:len(l.pending)-1]
		break
	}

	return w, nil
}

func (l *listener) handler(ws *websocket.Conn, req *http.Request) {
	l.lock.Lock()

	if !l.running {
		ws.Close()
		l.lock.Unlock()
		return
	}

	if ws.Subprotocol() != l.proto.Name()+".sp.nanomsg.org" {
		ws.Close()
		l.lock.Unlock()
		return
	}

	w := &wsPipe{ws: ws, addr: l.addr, proto: l.proto, open: true}
	w.dtype = websocket.BinaryMessage
	w.iswss = l.iswss
	w.ws.SetReadLimit(int64(l.maxrx))

	w.props = make(map[string]interface{})
	w.props[mangos.PropLocalAddr] = ws.LocalAddr()
	w.props[mangos.PropRemoteAddr] = ws.RemoteAddr()

	if req.TLS != nil {
		w.props[mangos.PropTLSConnState] = *req.TLS
	}

	w.wg.Add(1)
	l.pending = append(l.pending, w)
	l.cv.Broadcast()
	l.lock.Unlock()

	// We must not return before the socket is closed, because
	// our caller will close the websocket on our return.
	w.wg.Wait()
}

func (l *listener) Handle(pattern string, handler http.Handler) {
	l.mux.Handle(pattern, handler)
}

func (l *listener) HandleFunc(pattern string, handler http.HandlerFunc) {
	l.mux.HandleFunc(pattern, handler)
}

func (l *listener) Close() error {
	l.lock.Lock()
	defer l.lock.Unlock()
	if !l.running {
		return mangos.ErrClosed
	}
	if l.listener != nil {
		l.listener.Close()
	}
	l.running = false
	l.cv.Broadcast()
	for _, ws := range l.pending {
		ws.Close()
	}
	l.pending = nil
	return nil
}

func (l *listener) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	ws, err := l.ug.Upgrade(w, r, nil)
	if err != nil {
		return
	}
	l.handler(ws, r)
}

func (l *listener) Address() string {
	return l.url.String()
}

func (wsTran) Scheme() string {
	return "ws"
}

func (wsTran) NewDialer(addr string, sock mangos.Socket) (mangos.PipeDialer, error) {
	iswss := strings.HasPrefix(addr, "wss://")
	opts := make(map[string]interface{})

	opts[mangos.OptionNoDelay] = true
	opts[mangos.OptionKeepAlive] = true
	proto := sock.GetProtocol()
	maxrx := 0
	if v, e := sock.GetOption(mangos.OptionMaxRecvSize); e == nil {
		maxrx = v.(int)
	}

	return &dialer{addr: addr, proto: proto, iswss: iswss, opts: opts, maxrx: maxrx}, nil
}

func (t wsTran) NewListener(addr string, sock mangos.Socket) (mangos.PipeListener, error) {
	proto := sock.GetProtocol()
	l, e := t.listener(addr, proto)
	if e == nil {
		if v, e := sock.GetOption(mangos.OptionMaxRecvSize); e == nil {
			l.maxrx = v.(int)
		}
		l.mux.Handle(l.url.Path, l)
	}
	return l, e
}

func (wsTran) listener(addr string, proto mangos.Protocol) (*listener, error) {
	var err error
	l := &listener{proto: proto, opts: make(map[string]interface{})}
	l.cv.L = &l.lock
	l.ug.Subprotocols = []string{proto.Name() + ".sp.nanomsg.org"}

	if strings.HasPrefix(addr, "wss://") {
		l.iswss = true
	}
	l.url, err = url.ParseRequestURI(addr)
	if err != nil {
		return nil, err
	}
	if len(l.url.Path) == 0 {
		l.url.Path = "/"
	}
	l.mux = http.NewServeMux()

	l.htsvr = &http.Server{Addr: l.url.Host, Handler: l.mux}

	return l, nil
}

// NewTransport allocates a new ws:// transport.
func NewTransport() mangos.Transport {
	return wsTran(0)
}
<|endoftext|>


<commit_before>/*
Copyright (c) 2016, Theodore Butler
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

* Redistributions of source code must retain the above copyright notice, this
  list of conditions and the following disclaimer.

* Redistributions in binary form must reproduce the above copyright notice,
  this list of conditions and the following disclaimer in the documentation
  and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

package dtrie

import (
	"fmt"
	"hash/fnv"
)

func mask(hash, level uint32) uint32 {
	return (hash >> (5 * level)) & 0x01f
}

func setBit(bitmap uint32, pos uint32) uint32 {
	return bitmap | (1 << pos)
}

func clearBit(bitmap uint32, pos uint32) uint32 {
	return bitmap & ^(1 << pos)
}

func hasBit(bitmap uint32, pos uint32) bool {
	return (bitmap & (1 << pos)) != 0
}

func popCount(bitmap uint32) int {
	// bit population count, see
	// http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
	bitmap -= (bitmap >> 1) & 0x55555555
	bitmap = (bitmap>>2)&0x33333333 + bitmap&0x33333333
	bitmap += bitmap >> 4
	bitmap &= 0x0f0f0f0f
	bitmap *= 0x01010101
	return int(byte(bitmap >> 24))
}

func defaultHasher(value interface{}) uint32 {
	switch value.(type) {
	case uint8:
		return uint32(value.(uint8))
	case uint16:
		return uint32(value.(uint16))
	case uint32:
		return value.(uint32)
	case uint64:
		return uint32(value.(uint64))
	case int8:
		return uint32(value.(int8))
	case int16:
		return uint32(value.(int16))
	case int32:
		return uint32(value.(int32))
	case int64:
		return uint32(value.(int64))
	case uint:
		return uint32(value.(uint))
	case int:
		return uint32(value.(int))
	case uintptr:
		return uint32(value.(uintptr))
	case float32:
		return uint32(value.(float32))
	case float64:
		return uint32(value.(float64))
	}
	hasher := fnv.New32a()
	hasher.Write([]byte(fmt.Sprintf("%#v", value)))
	return hasher.Sum32()
}
<commit_msg>simplify default hasher<commit_after>/*
Copyright (c) 2016, Theodore Butler
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

* Redistributions of source code must retain the above copyright notice, this
  list of conditions and the following disclaimer.

* Redistributions in binary form must reproduce the above copyright notice,
  this list of conditions and the following disclaimer in the documentation
  and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

package dtrie

import (
	"fmt"
	"hash/fnv"
)

func mask(hash, level uint32) uint32 {
	return (hash >> (5 * level)) & 0x01f
}

func setBit(bitmap uint32, pos uint32) uint32 {
	return bitmap | (1 << pos)
}

func clearBit(bitmap uint32, pos uint32) uint32 {
	return bitmap & ^(1 << pos)
}

func hasBit(bitmap uint32, pos uint32) bool {
	return (bitmap & (1 << pos)) != 0
}

func popCount(bitmap uint32) int {
	// bit population count, see
	// http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
	bitmap -= (bitmap >> 1) & 0x55555555
	bitmap = (bitmap>>2)&0x33333333 + bitmap&0x33333333
	bitmap += bitmap >> 4
	bitmap &= 0x0f0f0f0f
	bitmap *= 0x01010101
	return int(byte(bitmap >> 24))
}

func defaultHasher(value interface{}) uint32 {
	switch v := value.(type) {
	case uint8:
		return uint32(v)
	case uint16:
		return uint32(v)
	case uint32:
		return v
	case uint64:
		return uint32(v)
	case int8:
		return uint32(v)
	case int16:
		return uint32(v)
	case int32:
		return uint32(v)
	case int64:
		return uint32(v)
	case uint:
		return uint32(v)
	case int:
		return uint32(v)
	case uintptr:
		return uint32(v)
	case float32:
		return uint32(v)
	case float64:
		return uint32(v)
	}
	hasher := fnv.New32a()
	hasher.Write([]byte(fmt.Sprintf("%#v", value)))
	return hasher.Sum32()
}
<|endoftext|>


<commit_before>package gateway

import (
	"errors"
	"net"
	"time"

	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
)

const (
	maxSharedNodes = 10
	maxAddrLength  = 100
	minPeers       = 3
)

// addNode adds an address to the set of nodes on the network.
func (g *Gateway) addNode(addr modules.NetAddress) error {
	if _, exists := g.nodes[addr]; exists {
		return errors.New("node already added")
	} else if net.ParseIP(addr.Host()) == nil {
		return errors.New("address is not routable: " + string(addr))
	} else if net.ParseIP(addr.Host()).IsLoopback() {
		return errors.New("cannot add loopback address")
	}
	g.nodes[addr] = struct{}{}
	return nil
}

func (g *Gateway) removeNode(addr modules.NetAddress) error {
	if _, exists := g.nodes[addr]; !exists {
		return errors.New("no record of that node")
	}
	delete(g.nodes, addr)
	g.log.Println("INFO: removed node", addr)
	return nil
}

func (g *Gateway) randomNode() (modules.NetAddress, error) {
	if len(g.nodes) > 0 {
		r, _ := crypto.RandIntn(len(g.nodes))
		for node := range g.nodes {
			if r <= 0 {
				return node, nil
			}
			r--
		}
	}

	return "", errNoPeers
}

// shareNodes is the receiving end of the ShareNodes RPC. It writes up to 10
// randomly selected nodes to the caller.
func (g *Gateway) shareNodes(conn modules.PeerConn) error {
	id := g.mu.RLock()
	var nodes []modules.NetAddress
	for node := range g.nodes {
		if len(nodes) == maxSharedNodes {
			break
		}
		nodes = append(nodes, node)
	}
	g.mu.RUnlock(id)
	return encoding.WriteObject(conn, nodes)
}

// requestNodes is the calling end of the ShareNodes RPC.
func (g *Gateway) requestNodes(conn modules.PeerConn) error {
	var nodes []modules.NetAddress
	if err := encoding.ReadObject(conn, &nodes, maxSharedNodes*maxAddrLength); err != nil {
		return err
	}
	g.log.Printf("INFO: %v sent us %v nodes", conn.RemoteAddr(), len(nodes))
	id := g.mu.Lock()
	for _, node := range nodes {
		g.addNode(node)
	}
	g.save()
	g.mu.Unlock(id)
	return nil
}

// relayNode is the recipient end of the RelayNode RPC. It reads a node, adds
// it to the Gateway's node list, and relays it to each of the Gateway's
// peers. If the node is already in the node list, it is not relayed.
func (g *Gateway) relayNode(conn modules.PeerConn) error {
	// read address
	var addr modules.NetAddress
	if err := encoding.ReadObject(conn, &addr, maxAddrLength); err != nil {
		return err
	}
	// add node
	id := g.mu.Lock()
	defer g.mu.Unlock(id)
	if err := g.addNode(addr); err != nil {
		return err
	}
	g.save()
	// relay
	go g.Broadcast("RelayNode", addr)
	return nil
}

// sendAddress is the calling end of the RelayNode RPC.
func (g *Gateway) sendAddress(conn modules.PeerConn) error {
	// don't send if we aren't connectible
	if g.Address().Host() == "::1" {
		return errors.New("can't send address without knowing external IP")
	}
	return encoding.WriteObject(conn, g.Address())
}

// nodeManager tries to keep the Gateway's node list healthy. As long as the
// Gateway has fewer than minNodeListSize nodes, it asks a random peer for
// more nodes. It also continually pings nodes in order to establish their
// connectivity. Unresponsive nodes are aggressively removed.
func (g *Gateway) nodeManager() {
	for {
		time.Sleep(5 * time.Second)

		id := g.mu.RLock()
		numNodes := len(g.nodes)
		peer, err := g.randomPeer()
		g.mu.RUnlock(id)
		if err != nil {
			// can't do much until we have peers
			continue
		}

		if numNodes < minNodeListLen {
			g.RPC(peer, "ShareNodes", g.requestNodes)
		}

		// find an untested node to check
		id = g.mu.RLock()
		node, err := g.randomNode()
		g.mu.RUnlock(id)
		if err != nil {
			continue
		}

		// try to connect
		conn, err := net.DialTimeout("tcp", string(node), dialTimeout)
		if err != nil {
			id = g.mu.Lock()
			g.removeNode(node)
			g.save()
			g.mu.Unlock(id)
			continue
		}
		// if connection succeeds, supply an unacceptable version to ensure
		// they won't try to add us as a peer
		encoding.WriteObject(conn, "0.0.0")
		conn.Close()
	}
}
<commit_msg>sleep longer after successful node probe<commit_after>package gateway

import (
	"errors"
	"net"
	"time"

	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
)

const (
	maxSharedNodes = 10
	maxAddrLength  = 100
	minPeers       = 3
)

// addNode adds an address to the set of nodes on the network.
func (g *Gateway) addNode(addr modules.NetAddress) error {
	if _, exists := g.nodes[addr]; exists {
		return errors.New("node already added")
	} else if net.ParseIP(addr.Host()) == nil {
		return errors.New("address is not routable: " + string(addr))
	} else if net.ParseIP(addr.Host()).IsLoopback() {
		return errors.New("cannot add loopback address")
	}
	g.nodes[addr] = struct{}{}
	return nil
}

func (g *Gateway) removeNode(addr modules.NetAddress) error {
	if _, exists := g.nodes[addr]; !exists {
		return errors.New("no record of that node")
	}
	delete(g.nodes, addr)
	g.log.Println("INFO: removed node", addr)
	return nil
}

func (g *Gateway) randomNode() (modules.NetAddress, error) {
	if len(g.nodes) > 0 {
		r, _ := crypto.RandIntn(len(g.nodes))
		for node := range g.nodes {
			if r <= 0 {
				return node, nil
			}
			r--
		}
	}

	return "", errNoPeers
}

// shareNodes is the receiving end of the ShareNodes RPC. It writes up to 10
// randomly selected nodes to the caller.
func (g *Gateway) shareNodes(conn modules.PeerConn) error {
	id := g.mu.RLock()
	var nodes []modules.NetAddress
	for node := range g.nodes {
		if len(nodes) == maxSharedNodes {
			break
		}
		nodes = append(nodes, node)
	}
	g.mu.RUnlock(id)
	return encoding.WriteObject(conn, nodes)
}

// requestNodes is the calling end of the ShareNodes RPC.
func (g *Gateway) requestNodes(conn modules.PeerConn) error {
	var nodes []modules.NetAddress
	if err := encoding.ReadObject(conn, &nodes, maxSharedNodes*maxAddrLength); err != nil {
		return err
	}
	g.log.Printf("INFO: %v sent us %v nodes", conn.RemoteAddr(), len(nodes))
	id := g.mu.Lock()
	for _, node := range nodes {
		g.addNode(node)
	}
	g.save()
	g.mu.Unlock(id)
	return nil
}

// relayNode is the recipient end of the RelayNode RPC. It reads a node, adds
// it to the Gateway's node list, and relays it to each of the Gateway's
// peers. If the node is already in the node list, it is not relayed.
func (g *Gateway) relayNode(conn modules.PeerConn) error {
	// read address
	var addr modules.NetAddress
	if err := encoding.ReadObject(conn, &addr, maxAddrLength); err != nil {
		return err
	}
	// add node
	id := g.mu.Lock()
	defer g.mu.Unlock(id)
	if err := g.addNode(addr); err != nil {
		return err
	}
	g.save()
	// relay
	go g.Broadcast("RelayNode", addr)
	return nil
}

// sendAddress is the calling end of the RelayNode RPC.
func (g *Gateway) sendAddress(conn modules.PeerConn) error {
	// don't send if we aren't connectible
	if g.Address().Host() == "::1" {
		return errors.New("can't send address without knowing external IP")
	}
	return encoding.WriteObject(conn, g.Address())
}

// nodeManager tries to keep the Gateway's node list healthy. As long as the
// Gateway has fewer than minNodeListSize nodes, it asks a random peer for
// more nodes. It also continually pings nodes in order to establish their
// connectivity. Unresponsive nodes are aggressively removed.
func (g *Gateway) nodeManager() {
	for {
		time.Sleep(5 * time.Second)

		id := g.mu.RLock()
		numNodes := len(g.nodes)
		peer, err := g.randomPeer()
		g.mu.RUnlock(id)
		if err != nil {
			// can't do much until we have peers
			continue
		}

		if numNodes < minNodeListLen {
			g.RPC(peer, "ShareNodes", g.requestNodes)
		}

		// find an untested node to check
		id = g.mu.RLock()
		node, err := g.randomNode()
		g.mu.RUnlock(id)
		if err != nil {
			continue
		}

		// try to connect
		conn, err := net.DialTimeout("tcp", string(node), dialTimeout)
		if err != nil {
			id = g.mu.Lock()
			g.removeNode(node)
			g.save()
			g.mu.Unlock(id)
			continue
		}
		// if connection succeeds, supply an unacceptable version to ensure
		// they won't try to add us as a peer
		encoding.WriteObject(conn, "0.0.0")
		conn.Close()
		// sleep for an extra 10 minutes after success; we don't want to spam
		// connectable nodes
		time.Sleep(10 * time.Minute)
	}
}
<|endoftext|>


<commit_before>package spectre

import (
	"bytes"
	"errors"
	"fmt"
	"sync"
	"time"
)

// Link is a node in circular doubly linked list that stores information about the
// key usage and time to live
// structure is like :
//
//
// link1 prev\				  /link2 next
//			link1		   link2
//	 link1 next\        	/link2 prev
//				\		   /
//				 \		  /
//	    root prev \      /root next
//				    ROOT
//
type Link struct {
	key        string
	ExpireTime time.Time
	size       int
	ttlPrev    *Link
	ttlNext    *Link
	lruPrev    *Link
	lruNext    *Link
}

// isLinkTTLExpired tells in boolean about the key expiration.
// true if expired or false.
func (l *Link) isLinkTTLExpired() bool {
	//fmt.Printf("current time= %v \n", time.Now())
	//fmt.Printf("local time = %v \n", l.ExpireTime)
	//fmt.Printf("local expired = %v \n", l.ExpireTime.Before(time.Now()))
	return l.ExpireTime.Before(time.Now())
}

// addLRULink adds a lru link in the circular doubly link list between root
// and a node left of it.
func (l *Link) addLRULink(temp *Link) {
	l.lruNext = temp
	l.lruPrev = temp.lruPrev
	temp.lruPrev.lruNext = l
	temp.lruPrev = l
}

// addLRULink adds a ttl link in the circular doubly link list between root
// and a node left of it.
func (l *Link) addTTLLink(temp *Link) {
	l.ttlNext = temp
	l.ttlPrev = temp.ttlPrev
	temp.ttlPrev.ttlNext = l
	temp.ttlPrev = l
}

// unlinkLRULink unlinks the link from its lru pointers in the
// doubly link list
func (temp *Link) unlinkLRULink() {
	nextLink := temp.lruNext
	prevLink := temp.lruPrev
	nextLink.lruPrev = prevLink
	prevLink.lruNext = nextLink
}

// unlinkTTLLink unlinks the link from its ttl pointers in the
// doubly link list
func (temp *Link) unlinkTTLLink() {
	nextLink := temp.ttlNext
	prevLink := temp.ttlPrev
	nextLink.ttlPrev = prevLink
	prevLink.ttlNext = nextLink
}

// unlink removes a link in circular doubly link list.
func (temp *Link) unlink() {
	temp.unlinkLRULink()
	temp.unlinkTTLLink()
}

// add insert a new link in circular doubly link list
func (l *Link) add(temp *Link) {
	l.addLRULink(temp)
	l.addTTLLink(temp)
}

// VolatileLRUCache is a cache wrapper on top of Cache.
// so still the maximum size of the cache is controlled
// by Cache only. This wrapper just adds an algorithm for
// key eviction policy .
// In case of memory unavailability VolatileLRUCache deletes
// the keys in the following order :
// 		*** keys which has been expired then the keys which are
//		*** keys which are least recently used
// VolatileLRUCache maintains a circular doubly link list in memory
// to have the meta data of the keys ready. Also this structure is
// thread safe; meaning several goroutine can operate concurrently.
type VolatileLRUCache struct {
	cache         *Cache
	root          *Link
	isMakingSpace bool
	linkMap       map[string]*Link
	globalTTL     time.Duration
	sync.RWMutex  // to make double linked list thread safe
}

// GetCurrentSize is a wrapper on top of Cache GetCurrentSize
// which returns the current VolatileLRUCache size in bytes.
func (vlruCache *VolatileLRUCache) VolatileLRUCacheCurrentSize() int {
	vlruCache.RLocker().Lock()
	defer vlruCache.RLocker().Unlock()
	return vlruCache.cache.GetCurrentSize()
}

func (vlruCache *VolatileLRUCache) String() string {
	vlruCache.RLocker().Lock()
	defer vlruCache.RLocker().Unlock()
	var buffer bytes.Buffer
	buffer.WriteString(fmt.Sprintf("currentsize:%v\n", vlruCache.cache.CurrentSize))
	buffer.WriteString(vlruCache.GetLRUInfo())
	buffer.WriteString(vlruCache.GetTTLInfo())
	return buffer.String()
}

// GetLRUInfo return the lru information of the keys in VolatileLRUCache.
func (vlruCache *VolatileLRUCache) GetLRUInfo() string {
	vlruCache.RLocker().Lock()
	defer vlruCache.RLocker().Unlock()
	rootLink := vlruCache.root
	startingLink := rootLink.lruNext
	var keyList []string
	for startingLink != rootLink {
		keyList = append(keyList, startingLink.key)
		nextLink := startingLink.lruNext
		startingLink = nextLink
	}
	var buffer bytes.Buffer
	buffer.WriteString(fmt.Sprintf("key order in lru fashion with old first stratgy\n"))
	for i, key := range keyList {
		buffer.WriteString(fmt.Sprintf("{position:%v, key:%v}\t", i, key))
	}
	return buffer.String()
}

// GetLRUInfo return the ttl information of the keys in VolatileLRUCache.
func (vlruCache *VolatileLRUCache) GetTTLInfo() string {
	vlruCache.RLocker().Lock()
	defer vlruCache.RLocker().Unlock()
	rootLink := vlruCache.root
	startingLink := rootLink.ttlNext
	var keyList []string
	for startingLink != rootLink {
		keyList = append(keyList, startingLink.key)
		nextLink := startingLink.ttlNext
		startingLink = nextLink
	}
	var buffer bytes.Buffer
	buffer.WriteString(fmt.Sprintf("key order in ttl fashion with old first stratgy\n"))
	for i, key := range keyList {
		buffer.WriteString(fmt.Sprintf("{position:%v, key:%v}\t", i, key))
	}
	return buffer.String()
}

func (vlruCache *VolatileLRUCache) VolatileLRUCacheIterator(outputChannel chan CacheRow) {
	go func() {
		//panic handlling at goroutine level
		defer func() {
			if r := recover(); r != nil {
				close(outputChannel)
			}
		}()

		vlruCache.RLocker().Lock()
		defer vlruCache.RLocker().Unlock()
		rootLink := vlruCache.root
		// taking ttl pointer to start with the oldest key
		startingLink := rootLink.ttlNext
		for startingLink != rootLink {
			if !startingLink.isLinkTTLExpired() {
				val, ok := vlruCache.cache.CacheGet(startingLink.key)
				if ok {
					outputChannel <- CacheRow{Key: startingLink.key, Value: val}
				}
			}
			nextLink := startingLink.ttlNext
			startingLink = nextLink
		}
		close(outputChannel)
		//fmt.Printf("spawaned go routine finishes")
	}()
}

// VolatileLRUCacheGet returns the value corresponding to a key present in Cache.
// This also modify internal doubly link list to maintain the updated ttl and lru info
// of the keys preset in Cache.
// return values :
//		value: value corresponding to the key
//		ok: true if success else false
func (vlruCache *VolatileLRUCache) VolatileLRUCacheGet(key string) (interface{}, bool) {
	// lower level is thread safe so making write lock after this.
	value, ok := vlruCache.cache.CacheGet(key)
	// changing the link so grabbing write lock
	vlruCache.Lock()
	defer vlruCache.Unlock()
	keyLink := vlruCache.linkMap[key]
	if !ok || keyLink.isLinkTTLExpired() {
		return nil, false
	}
	keyLink.unlinkLRULink()
	keyLink.addLRULink(vlruCache.root)
	return value, ok
}

func (vlruCache *VolatileLRUCache) VolatileLRUCacheSet(key string, value interface{}, size int, keyExpire time.Duration) (bool, error) {
	go vlruCache.goVolatileLRUCacheSet(key, value, size, keyExpire)
	return true, nil
}

// VolatileLRUCacheSet sets the value corresponding to a key in Cache.
// Setting operation also removes the keys which are already expired ; so as to
// make the rem free as much as possible. In case of memory is not available
// even after removing expired keys it removes the lru keys.
// This also modify internal doubly link list to maintain the updated ttl and lru info
// of the keys preset in Cache.
//
// input params :
//				key: key to hold the value in cache (string type)
//				value: struct having the data to cache.
//				keyExpire: time duration for the current key expire.
// return values :
//		ok: true if operation is successful else false
//		error: error in case of occurred error else nil
func (vlruCache *VolatileLRUCache) goVolatileLRUCacheSet(key string, value interface{}, size int, keyExpire time.Duration) (bool, error) {
	// Check here to avoid race condition with makeSpace()
	if vlruCache.isMakingSpace {
		return false, LowSpaceError
	}

	//free memory from expired keys
	vlruCache.Lock()
	defer vlruCache.Unlock()
	vlruCache.RemoveVolatileKey()
	success, error := vlruCache.cache.SetData(key, value, size)
	for error == LowSpaceError {
		if !vlruCache.isMakingSpace {
			vlruCache.isMakingSpace = true
			vlruCache.makeSpace()
		}
		return false, error
		// success, error = vlruCache.cache.SetData(key, value, size)
	}
	if !success {
		return success, error
	}
	link, ok := vlruCache.linkMap[key]
	if !ok {
		link = &Link{}
		vlruCache.linkMap[key] = link
	} else {
		link.unlink()
	}
	link.key = key
	if keyExpire.Seconds() <= 0 {
		link.ExpireTime = time.Now().Add(vlruCache.globalTTL)
	} else {
		link.ExpireTime = time.Now().Add(keyExpire)
	}
	link.size = size
	link.add(vlruCache.root)
	return true, nil
}

// RemoveVolatileKey removes the keys which are already expired in VolatileLRUCache.
func (vlruCache *VolatileLRUCache) RemoveVolatileKey() {
	rootLink := vlruCache.root
	startingLink := rootLink.ttlNext
	for startingLink != rootLink && startingLink.isLinkTTLExpired() {
		vlruCache.cache.CacheDelete(startingLink.key)
		delete(vlruCache.linkMap, startingLink.key)
		nextLink := startingLink.ttlNext
		startingLink.unlink()
		startingLink = nextLink
		// to free memory # golang garbage collector
		//runtime.GC()
	}
}

// VolatileLRUCacheDelete deletes a key present in VolatileLRUCache.
func (vlruCache *VolatileLRUCache) VolatileLRUCacheDelete(key string) {
	// lower level is thread safe so making write lock after this.
	_, ok := vlruCache.cache.CacheGet(key)
	if ok {
		// changing the link so grabbing write lock
		vlruCache.Lock()
		defer vlruCache.Unlock()
		vlruCache.RemoveVolatileKey()
		vlruCache.cache.CacheDelete(key)
		deletedLink := vlruCache.linkMap[key]
		if deletedLink != nil {
			deletedLink.unlink()
			delete(vlruCache.linkMap, key)
		}
	}
}

// makeSpace frees the space with least recently key.
// return values :
//		ok: true if operation is successful else false
//		error: error in case of occurred error else nil
func (vlruCache *VolatileLRUCache) makeSpace() (bool, error) {
	deleteCount := vlruCache.cache.MaxSize * 10 / 100
	for deleteCount > 0 {

		// linkTBE means link to be evicted with its data(key, value) in cache
		linkTBE := vlruCache.root.lruNext
		if linkTBE == vlruCache.root {
			return false, errors.New("VolatileLRUCache is empty ... May be the memory is less")
		}
		key := linkTBE.key
		vlruCache.cache.CacheDelete(key)
		linkTBE.unlink()
		delete(vlruCache.linkMap, key)
		deleteCount = deleteCount - 1
	}
	vlruCache.isMakingSpace = false
	return true, nil
}

// VolatileLRUCacheClear clears all the keys in the cache.
func (vlruCache *VolatileLRUCache) VolatileLRUCacheClear() {
	vlruCache.Lock()
	defer vlruCache.Unlock()
	vlruCache.cache.ClearCache()
	vlruCache.root = &Link{}
	vlruCache.linkMap = make(map[string]*Link)
	vlruCache.root.lruNext = vlruCache.root
	vlruCache.root.lruPrev = vlruCache.root
	vlruCache.root.ttlNext = vlruCache.root
	vlruCache.root.ttlPrev = vlruCache.root
}

// GetVolatileLRUCache returns an instance of VolatileLRUCache with the specified
// input params:
//			cacheSize: size of the cache in bytes
//			cachePartitions: total number map participating in internal cache.
//			ttl: a global time duration for each key expiration.
func GetVolatileLRUCache(cacheSize int, cachePartitions int, ttl time.Duration) *VolatileLRUCache {
	newVolatileCache := &VolatileLRUCache{
		cache:   GetDefaultCache(cacheSize, cachePartitions),
		root:    &Link{},
		linkMap: make(map[string]*Link),
	}
	//converting ttl to seconds for microseconds
	ttl = ttl * time.Second
	newVolatileCache.globalTTL = ttl
	newVolatileCache.root.lruNext = newVolatileCache.root
	newVolatileCache.root.lruPrev = newVolatileCache.root
	newVolatileCache.root.ttlNext = newVolatileCache.root
	newVolatileCache.root.ttlPrev = newVolatileCache.root
	newVolatileCache.isMakingSpace = false
	return newVolatileCache
}
<commit_msg>adding fallback at set<commit_after>package spectre

import (
	"bytes"
	"errors"
	"fmt"
	"sync"
	"time"
)

// Link is a node in circular doubly linked list that stores information about the
// key usage and time to live
// structure is like :
//
//
// link1 prev\				  /link2 next
//			link1		   link2
//	 link1 next\        	/link2 prev
//				\		   /
//				 \		  /
//	    root prev \      /root next
//				    ROOT
//
type Link struct {
	key        string
	ExpireTime time.Time
	size       int
	ttlPrev    *Link
	ttlNext    *Link
	lruPrev    *Link
	lruNext    *Link
}

// isLinkTTLExpired tells in boolean about the key expiration.
// true if expired or false.
func (l *Link) isLinkTTLExpired() bool {
	//fmt.Printf("current time= %v \n", time.Now())
	//fmt.Printf("local time = %v \n", l.ExpireTime)
	//fmt.Printf("local expired = %v \n", l.ExpireTime.Before(time.Now()))
	return l.ExpireTime.Before(time.Now())
}

// addLRULink adds a lru link in the circular doubly link list between root
// and a node left of it.
func (l *Link) addLRULink(temp *Link) {
	l.lruNext = temp
	l.lruPrev = temp.lruPrev
	temp.lruPrev.lruNext = l
	temp.lruPrev = l
}

// addLRULink adds a ttl link in the circular doubly link list between root
// and a node left of it.
func (l *Link) addTTLLink(temp *Link) {
	l.ttlNext = temp
	l.ttlPrev = temp.ttlPrev
	temp.ttlPrev.ttlNext = l
	temp.ttlPrev = l
}

// unlinkLRULink unlinks the link from its lru pointers in the
// doubly link list
func (temp *Link) unlinkLRULink() {
	nextLink := temp.lruNext
	prevLink := temp.lruPrev
	nextLink.lruPrev = prevLink
	prevLink.lruNext = nextLink
}

// unlinkTTLLink unlinks the link from its ttl pointers in the
// doubly link list
func (temp *Link) unlinkTTLLink() {
	nextLink := temp.ttlNext
	prevLink := temp.ttlPrev
	nextLink.ttlPrev = prevLink
	prevLink.ttlNext = nextLink
}

// unlink removes a link in circular doubly link list.
func (temp *Link) unlink() {
	temp.unlinkLRULink()
	temp.unlinkTTLLink()
}

// add insert a new link in circular doubly link list
func (l *Link) add(temp *Link) {
	l.addLRULink(temp)
	l.addTTLLink(temp)
}

// VolatileLRUCache is a cache wrapper on top of Cache.
// so still the maximum size of the cache is controlled
// by Cache only. This wrapper just adds an algorithm for
// key eviction policy .
// In case of memory unavailability VolatileLRUCache deletes
// the keys in the following order :
// 		*** keys which has been expired then the keys which are
//		*** keys which are least recently used
// VolatileLRUCache maintains a circular doubly link list in memory
// to have the meta data of the keys ready. Also this structure is
// thread safe; meaning several goroutine can operate concurrently.
type VolatileLRUCache struct {
	cache         *Cache
	root          *Link
	isMakingSpace bool
	linkMap       map[string]*Link
	globalTTL     time.Duration
	sync.RWMutex  // to make double linked list thread safe
}

// GetCurrentSize is a wrapper on top of Cache GetCurrentSize
// which returns the current VolatileLRUCache size in bytes.
func (vlruCache *VolatileLRUCache) VolatileLRUCacheCurrentSize() int {
	vlruCache.RLocker().Lock()
	defer vlruCache.RLocker().Unlock()
	return vlruCache.cache.GetCurrentSize()
}

func (vlruCache *VolatileLRUCache) String() string {
	vlruCache.RLocker().Lock()
	defer vlruCache.RLocker().Unlock()
	var buffer bytes.Buffer
	buffer.WriteString(fmt.Sprintf("currentsize:%v\n", vlruCache.cache.CurrentSize))
	buffer.WriteString(vlruCache.GetLRUInfo())
	buffer.WriteString(vlruCache.GetTTLInfo())
	return buffer.String()
}

// GetLRUInfo return the lru information of the keys in VolatileLRUCache.
func (vlruCache *VolatileLRUCache) GetLRUInfo() string {
	vlruCache.RLocker().Lock()
	defer vlruCache.RLocker().Unlock()
	rootLink := vlruCache.root
	startingLink := rootLink.lruNext
	var keyList []string
	for startingLink != rootLink {
		keyList = append(keyList, startingLink.key)
		nextLink := startingLink.lruNext
		startingLink = nextLink
	}
	var buffer bytes.Buffer
	buffer.WriteString(fmt.Sprintf("key order in lru fashion with old first stratgy\n"))
	for i, key := range keyList {
		buffer.WriteString(fmt.Sprintf("{position:%v, key:%v}\t", i, key))
	}
	return buffer.String()
}

// GetLRUInfo return the ttl information of the keys in VolatileLRUCache.
func (vlruCache *VolatileLRUCache) GetTTLInfo() string {
	vlruCache.RLocker().Lock()
	defer vlruCache.RLocker().Unlock()
	rootLink := vlruCache.root
	startingLink := rootLink.ttlNext
	var keyList []string
	for startingLink != rootLink {
		keyList = append(keyList, startingLink.key)
		nextLink := startingLink.ttlNext
		startingLink = nextLink
	}
	var buffer bytes.Buffer
	buffer.WriteString(fmt.Sprintf("key order in ttl fashion with old first stratgy\n"))
	for i, key := range keyList {
		buffer.WriteString(fmt.Sprintf("{position:%v, key:%v}\t", i, key))
	}
	return buffer.String()
}

func (vlruCache *VolatileLRUCache) VolatileLRUCacheIterator(outputChannel chan CacheRow) {
	go func() {
		//panic handlling at goroutine level
		defer func() {
			if r := recover(); r != nil {
				close(outputChannel)
			}
		}()

		vlruCache.RLocker().Lock()
		defer vlruCache.RLocker().Unlock()
		rootLink := vlruCache.root
		// taking ttl pointer to start with the oldest key
		startingLink := rootLink.ttlNext
		for startingLink != rootLink {
			if !startingLink.isLinkTTLExpired() {
				val, ok := vlruCache.cache.CacheGet(startingLink.key)
				if ok {
					outputChannel <- CacheRow{Key: startingLink.key, Value: val}
				}
			}
			nextLink := startingLink.ttlNext
			startingLink = nextLink
		}
		close(outputChannel)
		//fmt.Printf("spawaned go routine finishes")
	}()
}

// VolatileLRUCacheGet returns the value corresponding to a key present in Cache.
// This also modify internal doubly link list to maintain the updated ttl and lru info
// of the keys preset in Cache.
// return values :
//		value: value corresponding to the key
//		ok: true if success else false
func (vlruCache *VolatileLRUCache) VolatileLRUCacheGet(key string) (interface{}, bool) {
	// lower level is thread safe so making write lock after this.
	value, ok := vlruCache.cache.CacheGet(key)
	// changing the link so grabbing write lock
	vlruCache.Lock()
	defer vlruCache.Unlock()
	keyLink := vlruCache.linkMap[key]
	if !ok || keyLink.isLinkTTLExpired() {
		return nil, false
	}
	keyLink.unlinkLRULink()
	keyLink.addLRULink(vlruCache.root)
	return value, ok
}

func (vlruCache *VolatileLRUCache) VolatileLRUCacheSet(key string, value interface{}, size int, keyExpire time.Duration) (bool, error) {
	// Check here to avoid race condition with makeSpace()
	if vlruCache.isMakingSpace {
		return false, LowSpaceError
	}
	go vlruCache.goVolatileLRUCacheSet(key, value, size, keyExpire)
	return true, nil
}

// VolatileLRUCacheSet sets the value corresponding to a key in Cache.
// Setting operation also removes the keys which are already expired ; so as to
// make the rem free as much as possible. In case of memory is not available
// even after removing expired keys it removes the lru keys.
// This also modify internal doubly link list to maintain the updated ttl and lru info
// of the keys preset in Cache.
//
// input params :
//				key: key to hold the value in cache (string type)
//				value: struct having the data to cache.
//				keyExpire: time duration for the current key expire.
// return values :
//		ok: true if operation is successful else false
//		error: error in case of occurred error else nil
func (vlruCache *VolatileLRUCache) goVolatileLRUCacheSet(key string, value interface{}, size int, keyExpire time.Duration) (bool, error) {
	//free memory from expired keys
	vlruCache.Lock()
	defer vlruCache.Unlock()
	vlruCache.RemoveVolatileKey()
	success, error := vlruCache.cache.SetData(key, value, size)
	for error == LowSpaceError {
		if !vlruCache.isMakingSpace {
			vlruCache.isMakingSpace = true
			vlruCache.makeSpace()
		}
		return false, error
		// success, error = vlruCache.cache.SetData(key, value, size)
	}
	if !success {
		return success, error
	}
	link, ok := vlruCache.linkMap[key]
	if !ok {
		link = &Link{}
		vlruCache.linkMap[key] = link
	} else {
		link.unlink()
	}
	link.key = key
	if keyExpire.Seconds() <= 0 {
		link.ExpireTime = time.Now().Add(vlruCache.globalTTL)
	} else {
		link.ExpireTime = time.Now().Add(keyExpire)
	}
	link.size = size
	link.add(vlruCache.root)
	return true, nil
}

// RemoveVolatileKey removes the keys which are already expired in VolatileLRUCache.
func (vlruCache *VolatileLRUCache) RemoveVolatileKey() {
	rootLink := vlruCache.root
	startingLink := rootLink.ttlNext
	for startingLink != rootLink && startingLink.isLinkTTLExpired() {
		vlruCache.cache.CacheDelete(startingLink.key)
		delete(vlruCache.linkMap, startingLink.key)
		nextLink := startingLink.ttlNext
		startingLink.unlink()
		startingLink = nextLink
		// to free memory # golang garbage collector
		//runtime.GC()
	}
}

// VolatileLRUCacheDelete deletes a key present in VolatileLRUCache.
func (vlruCache *VolatileLRUCache) VolatileLRUCacheDelete(key string) {
	// lower level is thread safe so making write lock after this.
	_, ok := vlruCache.cache.CacheGet(key)
	if ok {
		// changing the link so grabbing write lock
		vlruCache.Lock()
		defer vlruCache.Unlock()
		vlruCache.RemoveVolatileKey()
		vlruCache.cache.CacheDelete(key)
		deletedLink := vlruCache.linkMap[key]
		if deletedLink != nil {
			deletedLink.unlink()
			delete(vlruCache.linkMap, key)
		}
	}
}

// makeSpace frees the space with least recently key.
// return values :
//		ok: true if operation is successful else false
//		error: error in case of occurred error else nil
func (vlruCache *VolatileLRUCache) makeSpace() (bool, error) {
	deleteCount := vlruCache.cache.MaxSize * 10 / 100
	for deleteCount > 0 {

		// linkTBE means link to be evicted with its data(key, value) in cache
		linkTBE := vlruCache.root.lruNext
		if linkTBE == vlruCache.root {
			return false, errors.New("VolatileLRUCache is empty ... May be the memory is less")
		}
		key := linkTBE.key
		vlruCache.cache.CacheDelete(key)
		linkTBE.unlink()
		delete(vlruCache.linkMap, key)
		deleteCount = deleteCount - 1
	}
	vlruCache.isMakingSpace = false
	return true, nil
}

// VolatileLRUCacheClear clears all the keys in the cache.
func (vlruCache *VolatileLRUCache) VolatileLRUCacheClear() {
	vlruCache.Lock()
	defer vlruCache.Unlock()
	vlruCache.cache.ClearCache()
	vlruCache.root = &Link{}
	vlruCache.linkMap = make(map[string]*Link)
	vlruCache.root.lruNext = vlruCache.root
	vlruCache.root.lruPrev = vlruCache.root
	vlruCache.root.ttlNext = vlruCache.root
	vlruCache.root.ttlPrev = vlruCache.root
}

// GetVolatileLRUCache returns an instance of VolatileLRUCache with the specified
// input params:
//			cacheSize: size of the cache in bytes
//			cachePartitions: total number map participating in internal cache.
//			ttl: a global time duration for each key expiration.
func GetVolatileLRUCache(cacheSize int, cachePartitions int, ttl time.Duration) *VolatileLRUCache {
	newVolatileCache := &VolatileLRUCache{
		cache:   GetDefaultCache(cacheSize, cachePartitions),
		root:    &Link{},
		linkMap: make(map[string]*Link),
	}
	//converting ttl to seconds for microseconds
	ttl = ttl * time.Second
	newVolatileCache.globalTTL = ttl
	newVolatileCache.root.lruNext = newVolatileCache.root
	newVolatileCache.root.lruPrev = newVolatileCache.root
	newVolatileCache.root.ttlNext = newVolatileCache.root
	newVolatileCache.root.ttlPrev = newVolatileCache.root
	newVolatileCache.isMakingSpace = false
	return newVolatileCache
}
<|endoftext|>


<commit_before>package main

import (
	. "github.com/conclave/pcduino/core"
)

func init() {
	Init()
	setup()
}

func main() {
	for {
		loop()
	}
}

func setup() {
}

func loop() {
	Delay(100)
}
<commit_msg>update test/linker_magnetic_sensor_test<commit_after>package main

import (
	. "github.com/conclave/pcduino/core"
)

func init() {
	Init()
	setup()
}

func main() {
	for {
		loop()
	}
}

var magneticPin byte = 1
var ledPin byte = 0

func setup() {
	println("Magnetic sensor test code!")
	println("Using I/O_0=Drive LED, I/O_1=Sensor output.")
	PinMode(magneticPin, INPUT)
	PinMode(ledPin, OUTPUT)
}

func loop() {
	value := DigitalRead(magneticPin)
	DigitalWrite(ledPin, value)
}
<|endoftext|>


<commit_before>package receiver

import (
	"context"
	"io/ioutil"
	"math"
	"net"
	"net/http"
	"sync/atomic"
	"time"

	"github.com/gogo/protobuf/proto"
	"github.com/golang/snappy"
	"go.uber.org/zap"

	"github.com/lomik/carbon-clickhouse/helper/RowBinary"
	"github.com/lomik/carbon-clickhouse/helper/pb"
	"github.com/lomik/carbon-clickhouse/helper/prompb"
	"github.com/lomik/carbon-clickhouse/helper/tags"
)

var nameLabel = []byte("\n\b__name__\x12")

type PrometheusRemoteWrite struct {
	Base
	listener *net.TCPListener
}

func (rcv *PrometheusRemoteWrite) unpackFast(ctx context.Context, bufBody []byte) error {

	b := bufBody
	var err error
	var ts []byte
	var sample []byte

	metricBuffer := newPrometheusMetricBuffer()

	var metric []string
	var samplesOffset int

	var value float64
	var timestamp int64

	writer := RowBinary.NewWriter(ctx, rcv.writeChan)

TimeSeriesLoop:
	for len(b) > 0 {
		if b[0] != 0x0a { // repeated prometheus.TimeSeries timeseries = 1;
			if b, err = pb.Skip(b); err != nil {
				break TimeSeriesLoop
			}
			continue TimeSeriesLoop
		}

		if ts, b, err = pb.Bytes(b[1:]); err != nil {
			break TimeSeriesLoop
		}

		if metric, samplesOffset, err = metricBuffer.timeSeries(ts); err != nil {
			break TimeSeriesLoop
		}

		ts = ts[samplesOffset:]
	SamplesLoop:
		for len(ts) > 0 {
			if ts[0] != 0x12 { // repeated Sample samples = 2;
				if ts, err = pb.Skip(ts); err != nil {
					break TimeSeriesLoop
				}
				continue SamplesLoop
			}

			if sample, ts, err = pb.Bytes(ts[1:]); err != nil {
				break TimeSeriesLoop
			}

			timestamp = 0
			value = 0

			for len(sample) > 0 {
				switch sample[0] {
				case 0x09: // double value    = 1;
					if value, sample, err = pb.Double(sample[1:]); err != nil {
						break TimeSeriesLoop
					}
				case 0x10: // int64 timestamp = 2;
					if timestamp, sample, err = pb.Int64(sample[1:]); err != nil {
						break TimeSeriesLoop
					}
				default:
					if sample, err = pb.Skip(sample); err != nil {
						break TimeSeriesLoop
					}
				}
			}

			if math.IsNaN(value) {
				continue SamplesLoop
			}

			if rcv.isDropString("", writer.Now(), uint32(timestamp/1000), value) {
				continue
			}

			writer.WritePointTagged(metric, value, timestamp/1000)
		}
	}

	if err != nil {
		return err
	}

	writer.Flush()

	if samplesCount := writer.PointsWritten(); samplesCount > 0 {
		atomic.AddUint64(&rcv.stat.samplesReceived, uint64(samplesCount))
	}

	if writeErrors := writer.WriteErrors(); writeErrors > 0 {
		atomic.AddUint64(&rcv.stat.errors, uint64(writeErrors))
	}

	return nil
}

func (rcv *PrometheusRemoteWrite) unpackDefault(ctx context.Context, bufBody []byte) error {
	var req prompb.WriteRequest
	if err := proto.Unmarshal(bufBody, &req); err != nil {
		return err
	}

	writer := RowBinary.NewWriter(ctx, rcv.writeChan)

	series := req.GetTimeseries()
	for i := 0; i < len(series); i++ {
		metric, err := tags.Prometheus(series[i].GetLabels())

		if err != nil {
			return err
		}

		samples := series[i].GetSamples()

		for j := 0; j < len(samples); j++ {
			if samples[j] == nil {
				continue
			}
			if math.IsNaN(samples[j].Value) {
				continue
			}
			if rcv.isDropString(metric, writer.Now(), uint32(samples[j].Timestamp/1000), samples[j].Value) {
				continue
			}

			writer.WritePoint(metric, samples[j].Value, samples[j].Timestamp/1000)
		}
	}

	writer.Flush()

	if samplesCount := writer.PointsWritten(); samplesCount > 0 {
		atomic.AddUint64(&rcv.stat.samplesReceived, uint64(samplesCount))
	}

	if writeErrors := writer.WriteErrors(); writeErrors > 0 {
		atomic.AddUint64(&rcv.stat.errors, uint64(writeErrors))
	}

	return nil
}

func (rcv *PrometheusRemoteWrite) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	compressed, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	reqBuf, err := snappy.Decode(nil, compressed)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	var req prompb.WriteRequest
	if err := proto.Unmarshal(reqBuf, &req); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	writer := RowBinary.NewWriter(r.Context(), rcv.writeChan)

	series := req.GetTimeseries()
	for i := 0; i < len(series); i++ {
		metric, err := tags.Prometheus(series[i].GetLabels())

		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}

		samples := series[i].GetSamples()

		for j := 0; j < len(samples); j++ {
			if samples[j] == nil {
				continue
			}
			if math.IsNaN(samples[j].Value) {
				continue
			}
			if rcv.isDropString(metric, writer.Now(), uint32(samples[j].Timestamp/1000), samples[j].Value) {
				continue
			}

			writer.WritePoint(metric, samples[j].Value, samples[j].Timestamp/1000)
		}
	}

	writer.Flush()

	if samplesCount := writer.PointsWritten(); samplesCount > 0 {
		atomic.AddUint64(&rcv.stat.samplesReceived, uint64(samplesCount))
	}

	if writeErrors := writer.WriteErrors(); writeErrors > 0 {
		atomic.AddUint64(&rcv.stat.errors, uint64(writeErrors))
	}
}

// Addr returns binded socket address. For bind port 0 in tests
func (rcv *PrometheusRemoteWrite) Addr() net.Addr {
	if rcv.listener == nil {
		return nil
	}
	return rcv.listener.Addr()
}

func (rcv *PrometheusRemoteWrite) Stat(send func(metric string, value float64)) {
	rcv.SendStat(send, "samplesReceived", "errors", "futureDropped", "pastDropped")
}

// Listen bind port. Receive messages and send to out channel
func (rcv *PrometheusRemoteWrite) Listen(addr *net.TCPAddr) error {
	return rcv.StartFunc(func() error {

		tcpListener, err := net.ListenTCP("tcp", addr)
		if err != nil {
			return err
		}

		s := &http.Server{
			Handler:        rcv,
			ReadTimeout:    10 * time.Second,
			WriteTimeout:   10 * time.Second,
			MaxHeaderBytes: 1 << 20,
		}

		rcv.Go(func(ctx context.Context) {
			<-ctx.Done()
			tcpListener.Close()
		})

		rcv.Go(func(ctx context.Context) {
			if err := s.Serve(tcpListener); err != nil {
				rcv.logger.Fatal("failed to serve", zap.Error(err))
			}

		})

		rcv.listener = tcpListener

		return nil
	})
}
<commit_msg>enable fast parser<commit_after>package receiver

import (
	"context"
	"io/ioutil"
	"math"
	"net"
	"net/http"
	"sync/atomic"
	"time"

	"github.com/gogo/protobuf/proto"
	"github.com/golang/snappy"
	"go.uber.org/zap"

	"github.com/lomik/carbon-clickhouse/helper/RowBinary"
	"github.com/lomik/carbon-clickhouse/helper/pb"
	"github.com/lomik/carbon-clickhouse/helper/prompb"
	"github.com/lomik/carbon-clickhouse/helper/tags"
)

var nameLabel = []byte("\n\b__name__\x12")

type PrometheusRemoteWrite struct {
	Base
	listener *net.TCPListener
}

func (rcv *PrometheusRemoteWrite) unpackFast(ctx context.Context, bufBody []byte) error {

	b := bufBody
	var err error
	var ts []byte
	var sample []byte

	metricBuffer := newPrometheusMetricBuffer()

	var metric []string
	var samplesOffset int

	var value float64
	var timestamp int64

	writer := RowBinary.NewWriter(ctx, rcv.writeChan)

TimeSeriesLoop:
	for len(b) > 0 {
		if b[0] != 0x0a { // repeated prometheus.TimeSeries timeseries = 1;
			if b, err = pb.Skip(b); err != nil {
				break TimeSeriesLoop
			}
			continue TimeSeriesLoop
		}

		if ts, b, err = pb.Bytes(b[1:]); err != nil {
			break TimeSeriesLoop
		}

		if metric, samplesOffset, err = metricBuffer.timeSeries(ts); err != nil {
			break TimeSeriesLoop
		}

		ts = ts[samplesOffset:]
	SamplesLoop:
		for len(ts) > 0 {
			if ts[0] != 0x12 { // repeated Sample samples = 2;
				if ts, err = pb.Skip(ts); err != nil {
					break TimeSeriesLoop
				}
				continue SamplesLoop
			}

			if sample, ts, err = pb.Bytes(ts[1:]); err != nil {
				break TimeSeriesLoop
			}

			timestamp = 0
			value = 0

			for len(sample) > 0 {
				switch sample[0] {
				case 0x09: // double value    = 1;
					if value, sample, err = pb.Double(sample[1:]); err != nil {
						break TimeSeriesLoop
					}
				case 0x10: // int64 timestamp = 2;
					if timestamp, sample, err = pb.Int64(sample[1:]); err != nil {
						break TimeSeriesLoop
					}
				default:
					if sample, err = pb.Skip(sample); err != nil {
						break TimeSeriesLoop
					}
				}
			}

			if math.IsNaN(value) {
				continue SamplesLoop
			}

			if rcv.isDropString("", writer.Now(), uint32(timestamp/1000), value) {
				continue
			}

			writer.WritePointTagged(metric, value, timestamp/1000)
		}
	}

	if err != nil {
		return err
	}

	writer.Flush()

	if samplesCount := writer.PointsWritten(); samplesCount > 0 {
		atomic.AddUint64(&rcv.stat.samplesReceived, uint64(samplesCount))
	}

	if writeErrors := writer.WriteErrors(); writeErrors > 0 {
		atomic.AddUint64(&rcv.stat.errors, uint64(writeErrors))
	}

	return nil
}

func (rcv *PrometheusRemoteWrite) unpackDefault(ctx context.Context, bufBody []byte) error {
	var req prompb.WriteRequest
	if err := proto.Unmarshal(bufBody, &req); err != nil {
		return err
	}

	writer := RowBinary.NewWriter(ctx, rcv.writeChan)

	series := req.GetTimeseries()
	for i := 0; i < len(series); i++ {
		metric, err := tags.Prometheus(series[i].GetLabels())

		if err != nil {
			return err
		}

		samples := series[i].GetSamples()

		for j := 0; j < len(samples); j++ {
			if samples[j] == nil {
				continue
			}
			if math.IsNaN(samples[j].Value) {
				continue
			}
			if rcv.isDropString(metric, writer.Now(), uint32(samples[j].Timestamp/1000), samples[j].Value) {
				continue
			}

			writer.WritePoint(metric, samples[j].Value, samples[j].Timestamp/1000)
		}
	}

	writer.Flush()

	if samplesCount := writer.PointsWritten(); samplesCount > 0 {
		atomic.AddUint64(&rcv.stat.samplesReceived, uint64(samplesCount))
	}

	if writeErrors := writer.WriteErrors(); writeErrors > 0 {
		atomic.AddUint64(&rcv.stat.errors, uint64(writeErrors))
	}

	return nil
}

func (rcv *PrometheusRemoteWrite) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	compressed, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	reqBuf, err := snappy.Decode(nil, compressed)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	err = rcv.unpackFast(r.Context(), reqBuf)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
}

// Addr returns binded socket address. For bind port 0 in tests
func (rcv *PrometheusRemoteWrite) Addr() net.Addr {
	if rcv.listener == nil {
		return nil
	}
	return rcv.listener.Addr()
}

func (rcv *PrometheusRemoteWrite) Stat(send func(metric string, value float64)) {
	rcv.SendStat(send, "samplesReceived", "errors", "futureDropped", "pastDropped")
}

// Listen bind port. Receive messages and send to out channel
func (rcv *PrometheusRemoteWrite) Listen(addr *net.TCPAddr) error {
	return rcv.StartFunc(func() error {

		tcpListener, err := net.ListenTCP("tcp", addr)
		if err != nil {
			return err
		}

		s := &http.Server{
			Handler:        rcv,
			ReadTimeout:    10 * time.Second,
			WriteTimeout:   10 * time.Second,
			MaxHeaderBytes: 1 << 20,
		}

		rcv.Go(func(ctx context.Context) {
			<-ctx.Done()
			tcpListener.Close()
		})

		rcv.Go(func(ctx context.Context) {
			if err := s.Serve(tcpListener); err != nil {
				rcv.logger.Fatal("failed to serve", zap.Error(err))
			}

		})

		rcv.listener = tcpListener

		return nil
	})
}
<|endoftext|>


<commit_before>package util

import (
	"os"
	"fmt"
	"os/exec"
	"ugot/logger"
	"path/filepath"
	"strings"
	"io/ioutil"
	"runtime"
	"strconv"
	"bytes"
	"flag"
	"log"
)

func TestAndAnalyzePackageCoverage(path string) {
	resultFile := getCoverageResultFilePath(PathAdapterSystem(path))
	os.Remove(resultFile)
	filepath.Walk(PathAdapterSystem(path), analyzePackageCoverage)
	ParseAnalysisFile(PathAdapterSystem(resultFile))
	CleanAnalyzedPackageFile(PathAdapterSystem(path))
}

func CleanAnalyzedPackageFile(path string) {
	filepath.Walk(path, cleanAnalyzedOutFiles)
}

func cleanAnalyzedOutFiles(path string, info os.FileInfo, err error) error {
	err = matchAndDeleteFiles(err, info, path, `*.out`)
	err = matchAndDeleteFiles(err, info, path, `*.result`)
	return err
}
func matchAndDeleteFiles(err error, info os.FileInfo, path string, pattern string) error {
	ok, err := filepath.Match(pattern, info.Name())
	if ok {
		os.Remove(path)
	}
	return err
}

func analyzePackageCoverage(path string, info os.FileInfo, err error) error {
	if info.IsDir() {
		if hasSpecificFiles(path, "_test.go") {
			resultFile := getCoverageResultFilePath(path)
			//go test to generate .out file for analysis
			execGoTestCoverProfile(path, info.Name())
			if hasSpecificFiles(path, ".out") {
				//return code line count of current package
				_, packageLineCountStr := GetGoFilesLineCount(PathAdapterSystem(path+"/"+info.Name()+".out"), info.Name())
				//go tool generate every file coverage and write the result for analysis
				execGoToolCover(path, info.Name())
				packageLineCount := float64(packageLineCountStr["total"])
				if hasSpecificFiles(path, ".result") {
					//every package coverage
					packageLineCoverage := GetGoCovResultTotalCoverage(PathAdapterSystem(path + "/" + info.Name() + ".result"))
					coveredLineCount := packageLineCount * (packageLineCoverage / 100)
					WriteStringFile(resultFile, SplitPath(path, "src/")[1] + ":"+
						strconv.Itoa(packageLineCountStr["total"])+ ":"+
						strconv.FormatFloat(coveredLineCount, 'f', 0, 64)+ ":"+
						strconv.FormatFloat(packageLineCoverage, 'f', 1, 64)+ "%")
				}

			} else {
				logger.GetLogger().Error("Analysis Failed in package [" + path + "]")
			}
		}
	}
	return err
}

func execGoTestCoverProfile(path string, covName string) {
	if runtime.GOOS == "windows" {
		executeGoTestProfileCmd("cmd", PathAdapterSystem(path), "/C", `go`, "test", SplitPath(path, "src/")[1], "-coverprofile="+covName+".out")
	} else if runtime.GOOS == "linux" {
		executeGoTestProfileCmd("go", PathAdapterSystem(path), "test", SplitPath(path, "src/")[1], "-coverprofile="+covName+".out")
	}
}

func execGoToolCover(path string, covName string) {
	if runtime.GOOS == "windows" {
		execGoToolCmdAndWriteResultAfterClean("cmd", PathAdapterSystem(path), PathAdapterSystem(path+"/"+covName+".result"), "/C", `go`, "tool", "cover", "-func="+covName+".out")
	} else if runtime.GOOS == "linux" {
		execGoToolCmdAndWriteResultAfterClean("go", PathAdapterSystem(path), PathAdapterSystem(path+"/"+covName+".result"), "tool", "cover", "-func="+covName+".out")
	}
}

func executeGoTestProfileCmd(cmdName string, cmdExePath string, args ... string) bool {
	cmd := exec.Command(cmdName, args...)
	cmd.Dir = cmdExePath
	output, err := cmd.CombinedOutput()
	ignore := flag.Arg(1)
	if len(ignore) != 0 && ignore == "--ignore" {
		printIgnoreIfTestFails(output)
	} else {
		printPanicIfTestFails(output, cmdExePath)
	}
	if err != nil {
		//logger.CheckError(err, "Failed to execute command ["+cmdName+"] ")
		return false
	}
	return true
}

func execGoToolCmdAndWriteResultAfterClean(cmdName string, cmdExcPath string, file_path string, args ... string) bool {
	cmd := exec.Command(cmdName, args...)
	cmd.Dir = cmdExcPath
	output, err := cmd.CombinedOutput()
	WriteBytesFileAfterClean(file_path, output)
	printOutput(output)
	if err != nil {
		logger.CheckError(err, "Failed to execute command ["+cmdName+"] ")
		return false
	}
	return true
}

func getCoverageResultFilePath(path string) string {
	packagePathes := SplitPath(path, "src/")
	moduleName := SplitPath(packagePathes[1], "/")[0]
	resultFile := PathAdapterSystem(packagePathes[0] + "src/" + moduleName + "_analysis")
	return resultFile
}

func hasSpecificFiles(path string, suffix string) bool {
	dir, err := ioutil.ReadDir(path)
	if err != nil {
		fmt.Println(err)
		return false
	}
	for _, fi := range dir {
		if fi.IsDir() {
			continue
		}
		if strings.HasSuffix(fi.Name(), suffix) {
			return true
		}
	}
	return false
}

func PathAdapterSystem(path string) string {
	if runtime.GOOS == "windows" {
		return filepath.FromSlash(path)
	} else if runtime.GOOS == "linux" {
		return filepath.ToSlash(path)
	} else {
		return path
	}
}

func SplitPath(path string, sep string) []string {
	if runtime.GOOS == "windows" {
		return strings.Split(path, filepath.FromSlash(sep))
	} else if runtime.GOOS == "linux" {
		return strings.Split(path, filepath.ToSlash(sep))
	} else {
		return strings.Split(path, sep)
	}
}

func PathAppend(path ... string) string {
	var buffer bytes.Buffer
	for _, v := range path {
		buffer.WriteString(v)
	}
	return buffer.String()
}

func panicIfTestFails(outs []byte, cmdExePath string) {
	b := bytes.NewBuffer(outs)
	line, err := b.ReadString('\n')
	for ; err == nil; line, err = b.ReadString('\n') {
		outline := string(outs)
		if !strings.Contains(outline, "ok") && !strings.Contains(outline, cmdExePath) {
			if strings.Contains(line, "--- FAIL") {
				log.Fatalf("UT failed: %s", line)
			}
		}
	}
}

func printIgnoreIfTestFails(outs []byte) {
	if len(outs) > 0 {
		fmt.Printf("%s\n", string(outs))
	}
}

func printPanicIfTestFails(outs []byte, cmdExePath string) {
	if len(outs) > 0 {
		fmt.Printf("%s\n", string(outs))
		panicIfTestFails(outs, cmdExePath)
	}
}

func printOutput(outs []byte) {
	if len(outs) > 0 {
		fmt.Printf("%s\n", string(outs))
	}
}
<commit_msg>modify test faile judgement process<commit_after>package util

import (
	"os"
	"fmt"
	"os/exec"
	"ugot/logger"
	"path/filepath"
	"strings"
	"io/ioutil"
	"runtime"
	"strconv"
	"bytes"
	"flag"
	"log"
)

func TestAndAnalyzePackageCoverage(path string) {
	resultFile := getCoverageResultFilePath(PathAdapterSystem(path))
	os.Remove(resultFile)
	filepath.Walk(PathAdapterSystem(path), analyzePackageCoverage)
	ParseAnalysisFile(PathAdapterSystem(resultFile))
	CleanAnalyzedPackageFile(PathAdapterSystem(path))
}

func CleanAnalyzedPackageFile(path string) {
	filepath.Walk(path, cleanAnalyzedOutFiles)
}

func cleanAnalyzedOutFiles(path string, info os.FileInfo, err error) error {
	err = matchAndDeleteFiles(err, info, path, `*.out`)
	err = matchAndDeleteFiles(err, info, path, `*.result`)
	return err
}
func matchAndDeleteFiles(err error, info os.FileInfo, path string, pattern string) error {
	ok, err := filepath.Match(pattern, info.Name())
	if ok {
		os.Remove(path)
	}
	return err
}

func analyzePackageCoverage(path string, info os.FileInfo, err error) error {
	if info.IsDir() {
		if hasSpecificFiles(path, "_test.go") {
			resultFile := getCoverageResultFilePath(path)
			//go test to generate .out file for analysis
			execGoTestCoverProfile(path, info.Name())
			if hasSpecificFiles(path, ".out") {
				//return code line count of current package
				_, packageLineCountStr := GetGoFilesLineCount(PathAdapterSystem(path+"/"+info.Name()+".out"), info.Name())
				//go tool generate every file coverage and write the result for analysis
				execGoToolCover(path, info.Name())
				packageLineCount := float64(packageLineCountStr["total"])
				if hasSpecificFiles(path, ".result") {
					//every package coverage
					packageLineCoverage := GetGoCovResultTotalCoverage(PathAdapterSystem(path + "/" + info.Name() + ".result"))
					coveredLineCount := packageLineCount * (packageLineCoverage / 100)
					WriteStringFile(resultFile, SplitPath(path, "src/")[1] + ":"+
						strconv.Itoa(packageLineCountStr["total"])+ ":"+
						strconv.FormatFloat(coveredLineCount, 'f', 0, 64)+ ":"+
						strconv.FormatFloat(packageLineCoverage, 'f', 1, 64)+ "%")
				}

			} else {
				logger.GetLogger().Error("Analysis Failed in package [" + path + "]")
			}
		}
	}
	return err
}

func execGoTestCoverProfile(path string, covName string) {
	if runtime.GOOS == "windows" {
		executeGoTestProfileCmd("cmd", PathAdapterSystem(path), "/C", `go`, "test", "-v", "-coverprofile="+covName+".out")
	} else if runtime.GOOS == "linux" {
		executeGoTestProfileCmd("go", PathAdapterSystem(path), "test", "-v", "-coverprofile="+covName+".out")
	}
}

func execGoToolCover(path string, covName string) {
	if runtime.GOOS == "windows" {
		execGoToolCmdAndWriteResultAfterClean("cmd", PathAdapterSystem(path), PathAdapterSystem(path+"/"+covName+".result"), "/C", `go`, "tool", "cover", "-func="+covName+".out")
	} else if runtime.GOOS == "linux" {
		execGoToolCmdAndWriteResultAfterClean("go", PathAdapterSystem(path), PathAdapterSystem(path+"/"+covName+".result"), "tool", "cover", "-func="+covName+".out")
	}
}

func executeGoTestProfileCmd(cmdName string, cmdExePath string, args ... string) bool {
	cmd := exec.Command(cmdName, args...)
	cmd.Dir = cmdExePath
	output, err := cmd.CombinedOutput()
	ignore := flag.Arg(1)
	if len(ignore) != 0 && ignore == "--ignore" {
		printIgnoreIfTestFails(output)
	} else {
		printPanicIfTestFails(output, cmdExePath)
	}
	if err != nil {
		//logger.CheckError(err, "Failed to execute command ["+cmdName+"] ")
		return false
	}
	return true
}

func execGoToolCmdAndWriteResultAfterClean(cmdName string, cmdExcPath string, file_path string, args ... string) bool {
	cmd := exec.Command(cmdName, args...)
	cmd.Dir = cmdExcPath
	output, err := cmd.CombinedOutput()
	WriteBytesFileAfterClean(file_path, output)
	printOutput(output)
	if err != nil {
		logger.CheckError(err, "Failed to execute command ["+cmdName+"] ")
		return false
	}
	return true
}

func getCoverageResultFilePath(path string) string {
	packagePathes := SplitPath(path, "src/")
	moduleName := SplitPath(packagePathes[1], "/")[0]
	resultFile := PathAdapterSystem(packagePathes[0] + "src/" + moduleName + "_analysis")
	return resultFile
}

func hasSpecificFiles(path string, suffix string) bool {
	dir, err := ioutil.ReadDir(path)
	if err != nil {
		fmt.Println(err)
		return false
	}
	for _, fi := range dir {
		if fi.IsDir() {
			continue
		}
		if strings.HasSuffix(fi.Name(), suffix) {
			return true
		}
	}
	return false
}

func PathAdapterSystem(path string) string {
	if runtime.GOOS == "windows" {
		return filepath.FromSlash(path)
	} else if runtime.GOOS == "linux" {
		return filepath.ToSlash(path)
	} else {
		return path
	}
}

func SplitPath(path string, sep string) []string {
	if runtime.GOOS == "windows" {
		return strings.Split(path, filepath.FromSlash(sep))
	} else if runtime.GOOS == "linux" {
		return strings.Split(path, filepath.ToSlash(sep))
	} else {
		return strings.Split(path, sep)
	}
}

func PathAppend(path ... string) string {
	var buffer bytes.Buffer
	for _, v := range path {
		buffer.WriteString(v)
	}
	return buffer.String()
}

func panicIfTestFails(outs []byte, cmdExePath string) {
	b := bytes.NewBuffer(outs)
	line, err := b.ReadString('\n')
	fmt.Println(bytes.NewReader(outs).ReadAt(outs, int64(1)))
	for ; err == nil; line, err = b.ReadString('\n') {
		if !strings.Contains(line, "ok") && !strings.Contains(line, cmdExePath) {
			failUts(line)
		}
	}
}

func failUts(line string) {
	if strings.Contains(line, "--- FAIL") {
		log.Fatalf("UT failed: %s", line)
	}
}

func printIgnoreIfTestFails(outs []byte) {
	if len(outs) > 0 {
		fmt.Printf("%s\n", string(outs))
	}
}

func printPanicIfTestFails(outs []byte, cmdExePath string) {
	if len(outs) > 0 {
		fmt.Printf("%s\n", string(outs))
		panicIfTestFails(outs, cmdExePath)
	}
}

func printOutput(outs []byte) {
	if len(outs) > 0 {
		fmt.Printf("%s\n", string(outs))
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2011-2013 Frederic Langlet
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
you may obtain a copy of the License at

                http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package entropy

import (
	"errors"
	"kanzi"
)

type Predictor interface {
	// Used to update the probability model
	Update(bit byte)

	// Return the split value representing the  probability for each symbol
	// in the [0..4095] range.
	// E.G. 410 represents roughly a probability of 10% for 0
	Get() uint
}

type BinaryEntropyEncoder struct {
	predictor Predictor
	low       uint64
	high      uint64
	bitstream kanzi.OutputBitStream
}

func NewBinaryEntropyEncoder(bs kanzi.OutputBitStream, predictor Predictor) (*BinaryEntropyEncoder, error) {
	if bs == nil {
		return nil, errors.New("Bit stream parameter cannot be null")
	}

	if predictor == nil {
		return nil, errors.New("Predictor parameter cannot be null")
	}

	this := new(BinaryEntropyEncoder)
	this.predictor = predictor
	this.low = 0
	this.high = uint64(0xFFFFFFFF)
	this.bitstream = bs
	return this, nil
}

func (this *BinaryEntropyEncoder) EncodeByte(val byte) error {
	for i := 7; i >= 0; i-- {
		err := this.EncodeBit((val >> uint(i)) & 1)

		if err != nil {
			return err
		}
	}

	return nil
}

func (this *BinaryEntropyEncoder) EncodeBit(bit byte) error {
	// Compute prediction
	prediction := this.predictor.Get()

	// Calculate interval split
	xmid := this.low + ((this.high-this.low)>>12)*uint64(prediction)

	// Update fields with new interval bounds
	if (bit & 1) == 1 {
		this.high = xmid
	} else {
		this.low = xmid + 1
	}

	// Update predictor
	this.predictor.Update(bit)

	// Write unchanged first 8 bits to bitstream
	for ((this.low ^ this.high) & uint64(0xFF000000)) == 0 {
		this.Flush()
	}

	return nil
}

func (this *BinaryEntropyEncoder) Encode(block []byte) (int, error) {
	return EntropyEncodeArray(this, block)
}

func (this *BinaryEntropyEncoder) Flush() {
	this.bitstream.WriteBits(this.high>>24, 8)
	this.low <<= 8
	this.high = (this.high << 8) | uint64(255)
}

func (this *BinaryEntropyEncoder) BitStream() kanzi.OutputBitStream {
	return this.bitstream
}

func (this *BinaryEntropyEncoder) Dispose() {
	this.bitstream.WriteBits(this.low|uint64(0xFFFFFF), 32)
	this.bitstream.Flush()
}

type BinaryEntropyDecoder struct {
	predictor   Predictor
	low         uint64
	high        uint64
	current     uint64
	initialized bool
	bitstream   kanzi.InputBitStream
}

func NewBinaryEntropyDecoder(bs kanzi.InputBitStream, predictor Predictor) (*BinaryEntropyDecoder, error) {
	if bs == nil {
		return nil, errors.New("Bit stream parameter cannot be null")
	}

	if predictor == nil {
		return nil, errors.New("Predictor parameter cannot be null")
	}

	// Defer stream reading. We are creating the object, we should not do any I/O
	this := new(BinaryEntropyDecoder)
	this.predictor = predictor
	this.low = 0
	this.high = uint64(0xFFFFFFFF)
	this.current = 0
	this.initialized = false
	this.bitstream = bs
	return this, nil
}

func (this *BinaryEntropyDecoder) DecodeByte() (byte, error) {
	// Deferred initialization: the bistream may not be ready at build time
	// Initialize 'current' with bytes read from the bitstream
	if this.Initialized() == false {
		this.Initialize()
	}

	return this.decodeByte_()
}

func (this *BinaryEntropyDecoder) decodeByte_() (byte, error) {
	res := 0

	for i := 7; i >= 0; i-- {
		bit, err := this.DecodeBit()

		if err != nil {
			return 0, err
		}

		res |= bit << uint(i)
	}

	return byte(res), nil
}

func (this *BinaryEntropyDecoder) Initialized() bool {
	return this.initialized
}

func (this *BinaryEntropyDecoder) Initialize() error {
	if this.initialized == false {
		read, err := this.bitstream.ReadBits(32)

		if err != nil {
			return err
		}

		this.current = uint64(read)
		this.initialized = true
	}

	return nil
}

func (this *BinaryEntropyDecoder) DecodeBit() (int, error) {
	// Compute prediction
	prediction := this.predictor.Get()

	// Calculate interval split
	xmid := this.low + ((this.high-this.low)>>12)*uint64(prediction)
	var bit int

	if this.current <= xmid {
		bit = 1
		this.high = xmid
	} else {
		bit = 0
		this.low = xmid + 1
	}

	// Update predictor
	this.predictor.Update(byte(bit))

	// Read from bitstream
	for ((this.low ^ this.high) & uint64(0xFF000000)) == 0 {
		err := this.Read()

		if err != nil {
			return 0, err
		}
	}

	return bit, nil
}

func (this *BinaryEntropyDecoder) Read() error {
	this.low = uint64(this.low << 8)
	this.high = uint64((this.high << 8) | 255)
	read, err := this.bitstream.ReadBits(8)

	if err != nil {
		return err
	}

	this.current = uint64((this.current << 8) | read)
	return nil
}

func (this *BinaryEntropyDecoder) Decode(block []byte) (int, error) {
	err := error(nil)

	// Deferred initialization: the bistream may not be ready at build time
	// Initialize 'current' with bytes read from the bitstream
	if this.Initialized() == false {
		this.Initialize()
	}

	for i := range block {
		if block[i], err = this.decodeByte_(); err != nil {
			return i, err
		}
	}

	return len(block), err
}

func (this *BinaryEntropyDecoder) BitStream() kanzi.InputBitStream {
	return this.bitstream
}

func (this *BinaryEntropyDecoder) Dispose() {
}
<commit_msg>Improve error handling during initialization<commit_after>/*
Copyright 2011-2013 Frederic Langlet
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
you may obtain a copy of the License at

                http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package entropy

import (
	"errors"
	"kanzi"
)

type Predictor interface {
	// Used to update the probability model
	Update(bit byte)

	// Return the split value representing the  probability for each symbol
	// in the [0..4095] range.
	// E.G. 410 represents roughly a probability of 10% for 0
	Get() uint
}

type BinaryEntropyEncoder struct {
	predictor Predictor
	low       uint64
	high      uint64
	bitstream kanzi.OutputBitStream
}

func NewBinaryEntropyEncoder(bs kanzi.OutputBitStream, predictor Predictor) (*BinaryEntropyEncoder, error) {
	if bs == nil {
		return nil, errors.New("Bit stream parameter cannot be null")
	}

	if predictor == nil {
		return nil, errors.New("Predictor parameter cannot be null")
	}

	this := new(BinaryEntropyEncoder)
	this.predictor = predictor
	this.low = 0
	this.high = uint64(0xFFFFFFFF)
	this.bitstream = bs
	return this, nil
}

func (this *BinaryEntropyEncoder) EncodeByte(val byte) error {
	for i := 7; i >= 0; i-- {
		err := this.EncodeBit((val >> uint(i)) & 1)

		if err != nil {
			return err
		}
	}

	return nil
}

func (this *BinaryEntropyEncoder) EncodeBit(bit byte) error {
	// Compute prediction
	prediction := this.predictor.Get()

	// Calculate interval split
	xmid := this.low + ((this.high-this.low)>>12)*uint64(prediction)

	// Update fields with new interval bounds
	if (bit & 1) == 1 {
		this.high = xmid
	} else {
		this.low = xmid + 1
	}

	// Update predictor
	this.predictor.Update(bit)

	// Write unchanged first 8 bits to bitstream
	for ((this.low ^ this.high) & uint64(0xFF000000)) == 0 {
		this.Flush()
	}

	return nil
}

func (this *BinaryEntropyEncoder) Encode(block []byte) (int, error) {
	return EntropyEncodeArray(this, block)
}

func (this *BinaryEntropyEncoder) Flush() {
	this.bitstream.WriteBits(this.high>>24, 8)
	this.low <<= 8
	this.high = (this.high << 8) | uint64(255)
}

func (this *BinaryEntropyEncoder) BitStream() kanzi.OutputBitStream {
	return this.bitstream
}

func (this *BinaryEntropyEncoder) Dispose() {
	this.bitstream.WriteBits(this.low|uint64(0xFFFFFF), 32)
	this.bitstream.Flush()
}

type BinaryEntropyDecoder struct {
	predictor   Predictor
	low         uint64
	high        uint64
	current     uint64
	initialized bool
	bitstream   kanzi.InputBitStream
}

func NewBinaryEntropyDecoder(bs kanzi.InputBitStream, predictor Predictor) (*BinaryEntropyDecoder, error) {
	if bs == nil {
		return nil, errors.New("Bit stream parameter cannot be null")
	}

	if predictor == nil {
		return nil, errors.New("Predictor parameter cannot be null")
	}

	// Defer stream reading. We are creating the object, we should not do any I/O
	this := new(BinaryEntropyDecoder)
	this.predictor = predictor
	this.low = 0
	this.high = uint64(0xFFFFFFFF)
	this.current = 0
	this.initialized = false
	this.bitstream = bs
	return this, nil
}

func (this *BinaryEntropyDecoder) DecodeByte() (byte, error) {
	// Deferred initialization: the bistream may not be ready at build time
	// Initialize 'current' with bytes read from the bitstream
	if this.Initialized() == false {
		this.Initialize()
	}

	return this.decodeByte_()
}

func (this *BinaryEntropyDecoder) decodeByte_() (byte, error) {
	res := 0

	for i := 7; i >= 0; i-- {
		bit, err := this.DecodeBit()

		if err != nil {
			return 0, err
		}

		res |= bit << uint(i)
	}

	return byte(res), nil
}

func (this *BinaryEntropyDecoder) Initialized() bool {
	return this.initialized
}

func (this *BinaryEntropyDecoder) Initialize() error {
	if this.initialized == false {
		read, err := this.bitstream.ReadBits(32)

		if err != nil {
			return err
		}

		this.current = uint64(read)
		this.initialized = true
	}

	return nil
}

func (this *BinaryEntropyDecoder) DecodeBit() (int, error) {
	// Compute prediction
	prediction := this.predictor.Get()

	// Calculate interval split
	xmid := this.low + ((this.high-this.low)>>12)*uint64(prediction)
	var bit int

	if this.current <= xmid {
		bit = 1
		this.high = xmid
	} else {
		bit = 0
		this.low = xmid + 1
	}

	// Update predictor
	this.predictor.Update(byte(bit))

	// Read from bitstream
	for ((this.low ^ this.high) & uint64(0xFF000000)) == 0 {
		err := this.Read()

		if err != nil {
			return 0, err
		}
	}

	return bit, nil
}

func (this *BinaryEntropyDecoder) Read() error {
	this.low = uint64(this.low << 8)
	this.high = uint64((this.high << 8) | 255)
	read, err := this.bitstream.ReadBits(8)

	if err != nil {
		return err
	}

	this.current = uint64((this.current << 8) | read)
	return nil
}

func (this *BinaryEntropyDecoder) Decode(block []byte) (int, error) {
	err := error(nil)

	// Deferred initialization: the bistream may not be ready at build time
	// Initialize 'current' with bytes read from the bitstream
	if this.Initialized() == false {
		if err = this.Initialize(); err != nil {
		   return 0, err
		}
	}

	for i := range block {
		if block[i], err = this.decodeByte_(); err != nil {
			return i, err
		}
	}

	return len(block), err
}

func (this *BinaryEntropyDecoder) BitStream() kanzi.InputBitStream {
	return this.bitstream
}

func (this *BinaryEntropyDecoder) Dispose() {
}
<|endoftext|>


<commit_before>// Package klient provides an instance and abstraction to a remote klient kite.
// It is used to easily call methods of a klient kite
package klient

import (
	"fmt"
	"koding/kites/klient/usage"
	"time"

	"github.com/koding/kite"
	"github.com/koding/kite/protocol"
)

// Klient represents a remote klient instance
type Klient struct {
	client   *kite.Client
	kite     *kite.Kite
	Username string
}

// New returns a new connected klient instance to the given queryString. The
// klient is ready to use. It's connected and will redial if there is any
// disconnections.
func New(k *kite.Kite, queryString string) (*Klient, error) {
	query, err := protocol.KiteFromString(queryString)
	if err != nil {
		return nil, err
	}

	k.Log.Debug("Querying for Klient: %s", queryString)

	kites, err := k.GetKites(query.Query())
	if err != nil {
		return nil, err
	}

	remoteKite := kites[0]
	if err := remoteKite.Dial(); err != nil {
		return nil, err
	}

	// klient connection is ready now
	return &Klient{
		kite:     k,
		client:   remoteKite,
		Username: remoteKite.Username,
	}, nil

}

// Klient returns a new connected klient instance to the given queryString. The
// klient is ready to use. It's tries to connect for the given timeout duration
func NewWithTimeout(k *kite.Kite, queryString string, t time.Duration) (*Klient, error) {
	timeout := time.After(t)

	k.Log.Debug("Querying for Klient: %s", queryString)
	for {
		select {
		case <-time.Tick(time.Second * 2):
			if klient, err := New(k, queryString); err == nil {
				return klient, nil
			}

		case <-timeout:
			return nil, fmt.Errorf("timeout while connection for kite")
		}
	}
}

func (k *Klient) Close() {
	k.client.Close()
}

// Usage calls the usage method of remote and get's the result back
func (k *Klient) Usage() (*usage.Usage, error) {
	resp, err := k.client.Tell(usage.MethodName)
	if err != nil {
		return nil, err
	}

	var usg *usage.Usage
	if err := resp.Unmarshal(&usg); err != nil {
		return nil, err
	}

	return usg, nil
}

// Ping checks if the given klient response with "pong" to the "ping" we send.
// A nil error means a successfull pong result.
func (k *Klient) Ping() error {
	resp, err := k.client.Tell("kite.ping")
	if err != nil {
		return err
	}

	out, err := resp.String()
	if err != nil {
		return err
	}

	if out == "pong" {
		return nil
	}

	return fmt.Errorf("wrong response %s", out)
}
<commit_msg>kloud/klient: add exec method that is needed for wall broadcasting<commit_after>// Package klient provides an instance and abstraction to a remote klient kite.
// It is used to easily call methods of a klient kite
package klient

import (
	"fmt"
	"koding/kite-handler/command"
	"koding/kites/klient/usage"
	"time"

	"github.com/koding/kite"
	"github.com/koding/kite/protocol"
)

// Klient represents a remote klient instance
type Klient struct {
	client   *kite.Client
	kite     *kite.Kite
	Username string
}

// New returns a new connected klient instance to the given queryString. The
// klient is ready to use. It's connected and will redial if there is any
// disconnections.
func New(k *kite.Kite, queryString string) (*Klient, error) {
	query, err := protocol.KiteFromString(queryString)
	if err != nil {
		return nil, err
	}

	k.Log.Debug("Querying for Klient: %s", queryString)

	kites, err := k.GetKites(query.Query())
	if err != nil {
		return nil, err
	}

	remoteKite := kites[0]
	if err := remoteKite.Dial(); err != nil {
		return nil, err
	}

	// klient connection is ready now
	return &Klient{
		kite:     k,
		client:   remoteKite,
		Username: remoteKite.Username,
	}, nil

}

// Klient returns a new connected klient instance to the given queryString. The
// klient is ready to use. It's tries to connect for the given timeout duration
func NewWithTimeout(k *kite.Kite, queryString string, t time.Duration) (*Klient, error) {
	timeout := time.After(t)

	k.Log.Debug("Querying for Klient: %s", queryString)
	for {
		select {
		case <-time.Tick(time.Second * 2):
			if klient, err := New(k, queryString); err == nil {
				return klient, nil
			}

		case <-timeout:
			return nil, fmt.Errorf("timeout while connection for kite")
		}
	}
}

func (k *Klient) Close() {
	k.client.Close()
}

// Usage calls the usage method of remote and get's the result back
func (k *Klient) Usage() (*usage.Usage, error) {
	resp, err := k.client.Tell(usage.MethodName)
	if err != nil {
		return nil, err
	}

	var usg *usage.Usage
	if err := resp.Unmarshal(&usg); err != nil {
		return nil, err
	}

	return usg, nil
}

// Exec runs a shell command on remote klient machinek
func (k *Klient) Exec(cmd string) (*command.Output, error) {
	var params = struct {
		Command string
	}{
		Command: cmd,
	}

	resp, err := k.client.Tell("exec", params)
	if err != nil {
		return nil, err
	}

	var out *command.Output
	if err := resp.Unmarshal(&out); err != nil {
		return nil, err
	}

	return out, nil
}

// Ping checks if the given klient response with "pong" to the "ping" we send.
// A nil error means a successfull pong result.
func (k *Klient) Ping() error {
	resp, err := k.client.Tell("kite.ping")
	if err != nil {
		return err
	}

	out, err := resp.String()
	if err != nil {
		return err
	}

	if out == "pong" {
		return nil
	}

	return fmt.Errorf("wrong response %s", out)
}
<|endoftext|>


<commit_before>package utils

import (
	"log"
	"net/http"

	. "github.com/julienschmidt/httprouter"
)

type HttpError struct {
	Error   error
	Message string
	Code    int
}

type ErrHandler func(http.ResponseWriter, *http.Request, Params) *HttpError

// ErrWrap processes errors in the query.
// Logging the queries.
// If the url has parameters of success or failure,
// in case of redirect performs on them
// depending on the result of the request.
// Parameters are passed in the request url, for example:
// 	http://localhost:8000/login?success=http://google.com&failure=http://ya.ru
func ErrWrap(eh ErrHandler) Handle {
	return Handle(func(writer http.ResponseWriter, request *http.Request, params Params) {
		if e := eh(writer, request, params); e != nil {
			log.Printf("\033[7m\033[1m\t ✗ Error: %v Message: %v Code: %v\033[0m",
				e.Error, e.Message, e.Code)

			if failureURL := request.URL.Query().Get("failure"); failureURL != "" {
				http.Redirect(writer, request, failureURL, 301)
			} else {
				http.Error(writer, e.Message, e.Code)
			}
		} else {
			log.Printf("\033[7m\033[1m\t ✓ Successfully.\033[0m")

			if successURL := request.URL.Query().Get("success"); successURL != "" {
				http.Redirect(writer, request, successURL, 301)
			}
		}
	})
}
<commit_msg>Refactor.<commit_after>package utils

import (
	"log"
	"net/http"

	. "github.com/julienschmidt/httprouter"
)

type HttpError struct {
	Error   error
	Message string
	Code    int
}

type ErrHandler func(http.ResponseWriter, *http.Request, Params) *HttpError

// ErrWrap processes errors in the query.
// Logging the queries.
// If the url has parameters of success or failure,
// in case of redirect performs on them
// depending on the result of the request.
// Parameters are passed in the request url, for example:
// 	http://localhost:8000/login?success=http://google.com&failure=http://ya.ru
func ErrWrap(eh ErrHandler) Handle {
	return Handle(func(writer http.ResponseWriter, request *http.Request, params Params) {
		if e := eh(writer, request, params); e != nil {
			log.Printf("\033[7m\033[1m\t ✗ Error: %v Message: %v Code: %v\033[0m",
				e.Error, e.Message, e.Code)

			if failureURL := request.URL.Query().Get("failure"); failureURL != "" {
				http.Redirect(writer, request, failureURL, 301)
			} else {
				http.Error(writer, e.Message, e.Code)
			}
		} else {
			log.Print("\033[7m\033[1m\t ✓ Successfully.\033[0m")

			if successURL := request.URL.Query().Get("success"); successURL != "" {
				http.Redirect(writer, request, successURL, 301)
			}
		}
	})
}
<|endoftext|>