<commit_before>package writer

import (
	"bufio"
	"io"
	"net"
	"net/http"
)

// NewResponseWriter wraps an http.ResponseWriter, returning a proxy that allows you to
// hook into various parts of the response process.
func NewResponseWriter(w http.ResponseWriter, protoMajor int) ResponseWriter {
	bw := BasicWriter{inner: w}
	if protoMajor == 2 {
		return &HttpTwoWriter{bw}
	}
	return &HttpOneWriter{bw}
}

func SetProxyStatusCode(w ResponseWriter, code int) {
	w.(*BasicWriter).code = code
}

// ResponseWriter is a proxy around an http.ResponseWriter that allows you to hook
// into various parts of the response process.
type ResponseWriter interface {
	http.ResponseWriter
	// Status returns the HTTP status of the request, or 0 if one has not
	// yet been sent.
	Status() int
	// BytesWritten returns the total number of bytes sent to the client.
	BytesWritten() int
	// Tee causes the response body to be written to the given io.Writer in
	// addition to proxying the writes through. Only one io.Writer can be
	// tee'd to at once: setting a second one will overwrite the first.
	// Writes will be sent to the proxy before being written to this
	// io.Writer. It is illegal for the tee'd writer to be modified
	// concurrently with writes.
	Tee(io.Writer)
	// Unwrap returns the original proxied target.
	Unwrap() http.ResponseWriter
	Flush()
}

// basicWriter wraps a http.ResponseWriter that implements the minimal
// http.ResponseWriter interface.
type BasicWriter struct {
	inner         http.ResponseWriter
	wroteHeader   bool
	headerFlushed bool
	code          int
	bytes         int
	tee           io.Writer
}

func (b *BasicWriter) Header() http.Header {
	return b.inner.Header()
}

func (b *BasicWriter) WriteHeader(code int) {
	b.code = code
	b.wroteHeader = true
}

func (b *BasicWriter) WriteHeaderImmediate(code int) {
	b.WriteHeader(code)
	if code != 0 {
		b.inner.WriteHeader(code)
		b.headerFlushed = true
	}
}

func (b *BasicWriter) FlushHeadersIfRequired() {
	if !b.headerFlushed && b.wroteHeader {
		b.WriteHeaderImmediate(b.code)
	}
}

func (b *BasicWriter) Write(buf []byte) (int, error) {
	b.FlushHeadersIfRequired()
	n, err := b.inner.Write(buf)
	if b.tee != nil {
		_, err2 := b.tee.Write(buf)
		// Prefer errors generated by the proxied writer.
		if err == nil {
			err = err2
		}
	}
	b.bytes += n
	return n, err
}

func (b *BasicWriter) Status() int {
	// if b.code == 0 {
	// 	return 200
	// }
	return b.code
}

func (b *BasicWriter) BytesWritten() int {
	return b.bytes
}

func (b *BasicWriter) Tee(w io.Writer) {
	b.tee = w
}

func (b *BasicWriter) Unwrap() http.ResponseWriter {
	return b.inner
}

func (b *BasicWriter) CloseNotify() <-chan bool {
	cn := b.inner.(http.CloseNotifier)
	return cn.CloseNotify()
}

func (b *BasicWriter) Flush() {
	b.FlushHeadersIfRequired()
	fl := b.inner.(http.Flusher)
	fl.Flush()
}

// HttpOneWriter is a HTTP writer that additionally satisfies http.Hijacker,
// and io.ReaderFrom.
type HttpOneWriter struct {
	BasicWriter
}

func (f *HttpOneWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
	hj := f.BasicWriter.inner.(http.Hijacker)
	return hj.Hijack()
}

func (f *HttpOneWriter) ReadFrom(r io.Reader) (int64, error) {
	if f.BasicWriter.tee != nil {
		return io.Copy(&f.BasicWriter, r)
	}
	f.BasicWriter.FlushHeadersIfRequired()
	rf := f.BasicWriter.inner.(io.ReaderFrom)
	n, err := rf.ReadFrom(r)
	f.BasicWriter.bytes += int(n)
	return n, err
}

// HttpTwoWriter is a HTTP2 writer that additionally satisfies
// Push
type HttpTwoWriter struct {
	BasicWriter
}

func (f *HttpTwoWriter) Push(target string, opts *http.PushOptions) error {
	return f.BasicWriter.inner.(http.Pusher).Push(target, opts)
}
<commit_msg>change: better flushing mechanism<commit_after>package writer

import (
	"bufio"
	"io"
	"net"
	"net/http"
)

// NewResponseWriter wraps an http.ResponseWriter, returning a proxy that allows you to
// hook into various parts of the response process.
func NewResponseWriter(w http.ResponseWriter, protoMajor int) ResponseWriter {
	bw := BasicWriter{inner: w}
	if protoMajor == 2 {
		return &HttpTwoWriter{bw}
	}
	return &HttpOneWriter{bw}
}

func SetProxyStatusCode(w ResponseWriter, code int) {
	w.(*BasicWriter).code = code
}

// ResponseWriter is a proxy around an http.ResponseWriter that allows you to hook
// into various parts of the response process.
type ResponseWriter interface {
	http.ResponseWriter
	// Status returns the HTTP status of the request, or 0 if one has not
	// yet been sent.
	Status() int
	// BytesWritten returns the total number of bytes sent to the client.
	BytesWritten() int
	// Tee causes the response body to be written to the given io.Writer in
	// addition to proxying the writes through. Only one io.Writer can be
	// tee'd to at once: setting a second one will overwrite the first.
	// Writes will be sent to the proxy before being written to this
	// io.Writer. It is illegal for the tee'd writer to be modified
	// concurrently with writes.
	Tee(io.Writer)
	// Unwrap returns the original proxied target.
	Unwrap() http.ResponseWriter
	Flush()
}

// basicWriter wraps a http.ResponseWriter that implements the minimal
// http.ResponseWriter interface.
type BasicWriter struct {
	inner         http.ResponseWriter
	wroteHeader   bool
	headerFlushed bool
	code          int
	bytes         int
	tee           io.Writer
}

func (b *BasicWriter) Header() http.Header {
	return b.inner.Header()
}

func (b *BasicWriter) WriteHeader(code int) {
	b.code = code
	if code != 0 {
		b.wroteHeader = true
	} else {
		b.wroteHeader = false
	}
}

func (b *BasicWriter) WriteHeaderImmediate(code int) {
	b.WriteHeader(code)
	if b.wroteHeader {
		b.inner.WriteHeader(code)
		b.headerFlushed = true
	}
}

func (b *BasicWriter) FlushHeadersIfRequired() {
	if !b.headerFlushed {
		b.WriteHeaderImmediate(b.code)
	}
}

func (b *BasicWriter) Write(buf []byte) (int, error) {
	b.FlushHeadersIfRequired()
	n, err := b.inner.Write(buf)
	if b.tee != nil {
		_, err2 := b.tee.Write(buf)
		// Prefer errors generated by the proxied writer.
		if err == nil {
			err = err2
		}
	}
	b.bytes += n
	return n, err
}

func (b *BasicWriter) Status() int {
	// if b.code == 0 {
	// 	return 200
	// }
	return b.code
}

func (b *BasicWriter) BytesWritten() int {
	return b.bytes
}

func (b *BasicWriter) Tee(w io.Writer) {
	b.tee = w
}

func (b *BasicWriter) Unwrap() http.ResponseWriter {
	return b.inner
}

func (b *BasicWriter) CloseNotify() <-chan bool {
	cn := b.inner.(http.CloseNotifier)
	return cn.CloseNotify()
}

func (b *BasicWriter) Flush() {
	b.FlushHeadersIfRequired()
	fl := b.inner.(http.Flusher)
	fl.Flush()
}

// HttpOneWriter is a HTTP writer that additionally satisfies http.Hijacker,
// and io.ReaderFrom.
type HttpOneWriter struct {
	BasicWriter
}

func (f *HttpOneWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
	hj := f.BasicWriter.inner.(http.Hijacker)
	return hj.Hijack()
}

func (f *HttpOneWriter) ReadFrom(r io.Reader) (int64, error) {
	f.BasicWriter.FlushHeadersIfRequired()
	if f.BasicWriter.tee != nil {
		return io.Copy(&f.BasicWriter, r)
	}
	rf := f.BasicWriter.inner.(io.ReaderFrom)
	n, err := rf.ReadFrom(r)
	f.BasicWriter.bytes += int(n)
	return n, err
}

// HttpTwoWriter is a HTTP2 writer that additionally satisfies
// Push
type HttpTwoWriter struct {
	BasicWriter
}

func (f *HttpTwoWriter) Push(target string, opts *http.PushOptions) error {
	return f.BasicWriter.inner.(http.Pusher).Push(target, opts)
}
<|endoftext|>


<commit_before>package main

// MPPC структура объекта для распаковки пакета данных от игрового сервера
type MPPC struct {
	packedBytes		[]byte
	unpackedBytes	[]byte
	packedOffset	byte
	code1 			int
	code2 			int
	code3 			int
	code4 			int
}<commit_msg>Добавлены методы шифровки/дешифровки и метод распаковки MPPC<commit_after>package main
import (
	"crypto/hmac"
	"crypto/md5"
	"crypto/rand"
	"fmt"
)

// MPPC структура объекта для распаковки пакета данных от игрового сервера
type MPPC struct {
	packedBytes		[]byte
	unpackedBytes	[]byte
	packedOffset	byte
	code1 			int
	code2 			int
	code3 			int
	code4 			int
}

func randomNextBytes(count int) ([]byte, error) {
	b := make([]byte, count)
	_, err := rand.Read(b)
	if err != nil {
		return nil, err
	}
	return b, nil
}

func getRC4Key(encoderhash, uid, hash []byte) []byte {
	nhash := append(hash, encoderhash...)
	hmacmd5 := hmac.New(md5.New, uid)
	hmacmd5.Write(nhash)
	data := hmacmd5.Sum(nil)
	result := make([]byte, len(data))
	copy(result, data)
	return result
}

func newMPPC() *MPPC {
	mppc := &MPPC{}
	return mppc
}

// UnpackByte расшифровывает байт из очереди по модернизированному алгоритму MPPC
func (mppc *MPPC) UnpackByte(packedByte byte) []byte {
	code1 := mppc.code1
	code2 := mppc.code2
	code3 := mppc.code3
	code4 := mppc.code4

	mppc.packedBytes = append(mppc.packedBytes, packedByte)
	var unpackedChunk []byte
	var tempbuf []byte

	if len(mppc.unpackedBytes) >= 10240 {
		// удалить первые 2048 байт
		fmt.Println("Очистить первые 2048 байт.")
		tempbuf = make([]byte, len(mppc.unpackedBytes)-2048)
		copy(tempbuf, mppc.unpackedBytes[2048:])
		mppc.unpackedBytes = tempbuf
	}

	loop:
	for {
		switch code3 {
		case 0:
			if mppc.hasbits(4) == true {
				if mppc.getpackedbits(1) == 0 {
					code1 = 1
					code3 = 1
				} else {
					if mppc.getpackedbits(1) == 0 {
						code1 = 2
						code3 = 1
					} else {
						if mppc.getpackedbits(1) == 0 {
							code1 = 3
							code3 = 1
						} else {
							if mppc.getpackedbits(1) == 0 {
								code1 = 4
								code3 = 1
							} else {
								code1 = 5
								code3 = 1
							}
						}
					}
				}
			} else {
				break loop
			}
		case 1:
			switch code1 {
			case 1:
				if mppc.hasbits(7) == true {
					outB := byte(mppc.getpackedbits(7))
					unpackedChunk = append(unpackedChunk, outB)
					mppc.unpackedBytes = append(mppc.unpackedBytes, outB)
					code3 = 0
				} else {
					break loop
				}
			case 2:
				if mppc.hasbits(7) == true {
					outB := byte(mppc.getpackedbits(7) | 0x80)
					unpackedChunk = append(unpackedChunk, outB)
					mppc.unpackedBytes = append(mppc.unpackedBytes, outB)
					code3 = 0
				} else {
					break loop
				}
			case 3:
				if mppc.hasbits(13) == true {
					code4 = int(mppc.getpackedbits(13)) + 0x140
					code3 = 2
				} else {
					break loop
				}
			case 4:
				if mppc.hasbits(8) == true {
					code4 = int(mppc.getpackedbits(8)) + 0x40
					code3 = 2
				} else {
					break loop
				}
			case 5:
				if mppc.hasbits(6) == true {
					code4 = int(mppc.getpackedbits(6))
					code3 = 2
				} else {
					break loop
				}
			}
		case 2:
			if code4 == 0 {
				if mppc.packedOffset != 0 {
					mppc.packedOffset = 0
					// удалить первый байт в mppc.packedBytes
					tempbuf = make([]byte, len(mppc.packedBytes)-1)
					copy(tempbuf, mppc.packedBytes[1:])
					mppc.packedBytes = tempbuf
				}
				code3 = 0
				continue loop
			}
			code2 = 0
			code3 = 3
		case 3:
			if mppc.hasbits(1) == true {
				if mppc.getpackedbits(1) == 0 {
					code3 = 4
				} else {
					code2++
				}
			} else {
				break loop
			}
		case 4:
			var copySize int
			if code2 == 0 {
				copySize = 3
			} else {
				size := code2 + 1
				if mppc.hasbits(size) == true {
					copySize = int(mppc.getpackedbits(size)) + (1 << uint(size))
				} else {
					break loop
				}
			}
			unpackedChunk = mppc.CopyArray(code4, copySize, unpackedChunk)
			code3 = 0
		}
	}
	mppc.code1 = code1
	mppc.code2 = code2
	mppc.code3 = code3
	mppc.code4 = code4

	return unpackedChunk
}

// Unpack метод расшифровывает массив байтов по модернизированному алгоритму MPPC
func (mppc *MPPC) Unpack(compressedBytes []byte) []byte {
	var rtnList []byte
	for _, b := range compressedBytes {
		rtnList = append(rtnList, mppc.UnpackByte(b)...)
	}
	return rtnList
}

// CopyArray метод отсекает часть массива. Остаток переносит в начало массива
func (mppc *MPPC) CopyArray(shift, size int, unpackedChunkData []byte) []byte {
	for i := 0; i < size; i++ {
		pIndex := len(mppc.unpackedBytes) - shift
		if pIndex < 0 {
			return unpackedChunkData
		}
		b := mppc.unpackedBytes[pIndex]
		mppc.unpackedBytes = append(mppc.unpackedBytes, b)
		unpackedChunkData = append(unpackedChunkData, b)
	}
	return unpackedChunkData
}

func (mppc *MPPC) getpackedbits(bitCount int) uint {
	if bitCount > 16 {
		return 0
	}

	if mppc.hasbits(bitCount) == false {
		panic(fmt.Sprintln("Unpack bit stream overflow"))
	}

	alBitCount := bitCount + int(mppc.packedOffset)
	alByteCount := (alBitCount + 7) / 8

	var v uint32
	for i := 0; i < alByteCount; i++ {
		v |= uint32(mppc.packedBytes[i]) << uint32(24-i*8)
	}

	v <<= mppc.packedOffset
	v >>= uint32(32 - bitCount)

	mppc.packedOffset += byte(bitCount)
	freeBytes := mppc.packedOffset / 8

	if freeBytes != 0 {
		// удалить первые n-байт
		//fmt.Printf("getpackedbits. Удалить первые %d байт. mppc.packedBytes = [% X]\n", freeBytes, mppc.packedBytes)
		tempbuf := make([]byte, len(mppc.packedBytes)-int(freeBytes))
		copy(tempbuf, mppc.packedBytes[int(freeBytes):])
		mppc.packedBytes = tempbuf
		//fmt.Printf("getpackedbits. Новый mppc.packedBytes = [% X]\n", mppc.packedBytes)
	}
	mppc.packedOffset %= 8
	return uint(v)
}

func (mppc *MPPC) hasbits(count int) bool {
	if len(mppc.packedBytes)*8-int(mppc.packedOffset) >= count {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package missinggo

// todo move to httptoo as ResponseRecorder

import (
	"bufio"
	"net"
	"net/http"
	"time"
)

// A http.ResponseWriter that tracks the status of the response. The status
// code, and number of bytes written for example.
type StatusResponseWriter struct {
	http.ResponseWriter
	Code         int
	BytesWritten int64
	Started      time.Time
	Ttfb         time.Duration // Time to first byte
	GotFirstByte bool
	WroteHeader  Event
	Hijacked     bool
}

var _ interface {
	http.ResponseWriter
	http.Hijacker
} = (*StatusResponseWriter)(nil)

func (me *StatusResponseWriter) Write(b []byte) (n int, err error) {
	// Exactly how it's done in the standard library. This ensures Code is
	// correct.
	if !me.WroteHeader.IsSet() {
		me.WriteHeader(http.StatusOK)
	}
	if !me.GotFirstByte && len(b) > 0 {
		if me.Started.IsZero() {
			panic("Started was not initialized")
		}
		me.Ttfb = time.Since(me.Started)
		me.GotFirstByte = true
	}
	n, err = me.ResponseWriter.Write(b)
	me.BytesWritten += int64(n)
	return
}

func (me *StatusResponseWriter) WriteHeader(code int) {
	me.ResponseWriter.WriteHeader(code)
	if !me.WroteHeader.IsSet() {
		me.Code = code
		me.WroteHeader.Set()
	}
}

func (me *StatusResponseWriter) Hijack() (c net.Conn, b *bufio.ReadWriter, err error) {
	me.Hijacked = true
	c, b, err = me.ResponseWriter.(http.Hijacker).Hijack()
	if b.Writer.Buffered() != 0 {
		panic("unexpected buffered writes")
	}
	c = responseConn{c, me}
	return
}

type responseConn struct {
	net.Conn
	s *StatusResponseWriter
}

func (me responseConn) Write(b []byte) (n int, err error) {
	n, err = me.Conn.Write(b)
	me.s.BytesWritten += int64(n)
	return
}
<commit_msg>Rename Ttfb->TimeToFirstByte and only set it after a byte is written<commit_after>package missinggo

// todo move to httptoo as ResponseRecorder

import (
	"bufio"
	"net"
	"net/http"
	"time"
)

// A http.ResponseWriter that tracks the status of the response. The status
// code, and number of bytes written for example.
type StatusResponseWriter struct {
	http.ResponseWriter
	Code            int
	BytesWritten    int64
	Started         time.Time
	TimeToFirstByte time.Duration // Time to first byte
	GotFirstByte    bool
	WroteHeader     Event
	Hijacked        bool
}

var _ interface {
	http.ResponseWriter
	http.Hijacker
} = (*StatusResponseWriter)(nil)

func (me *StatusResponseWriter) Write(b []byte) (n int, err error) {
	// Exactly how it's done in the standard library. This ensures Code is
	// correct.
	if !me.WroteHeader.IsSet() {
		me.WriteHeader(http.StatusOK)
	}
	if me.Started.IsZero() {
		panic("Started was not initialized")
	}
	timeBeforeWrite := time.Now()
	n, err = me.ResponseWriter.Write(b)
	if n > 0 && !me.GotFirstByte {
		me.TimeToFirstByte = timeBeforeWrite.Sub(me.Started)
		me.GotFirstByte = true
	}
	me.BytesWritten += int64(n)
	return
}

func (me *StatusResponseWriter) WriteHeader(code int) {
	me.ResponseWriter.WriteHeader(code)
	if !me.WroteHeader.IsSet() {
		me.Code = code
		me.WroteHeader.Set()
	}
}

func (me *StatusResponseWriter) Hijack() (c net.Conn, b *bufio.ReadWriter, err error) {
	me.Hijacked = true
	c, b, err = me.ResponseWriter.(http.Hijacker).Hijack()
	if b.Writer.Buffered() != 0 {
		panic("unexpected buffered writes")
	}
	c = responseConn{c, me}
	return
}

type responseConn struct {
	net.Conn
	s *StatusResponseWriter
}

func (me responseConn) Write(b []byte) (n int, err error) {
	n, err = me.Conn.Write(b)
	me.s.BytesWritten += int64(n)
	return
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"strings"

	golog "github.com/ipfs/go-log"
	host "github.com/libp2p/go-libp2p-host"
	inet "github.com/libp2p/go-libp2p-net"
	net "github.com/libp2p/go-libp2p-net"
	peer "github.com/libp2p/go-libp2p-peer"
	pstore "github.com/libp2p/go-libp2p-peerstore"
	swarm "github.com/libp2p/go-libp2p-swarm"
	bhost "github.com/libp2p/go-libp2p/p2p/host/basic"
	testutil "github.com/libp2p/go-testutil"
	ma "github.com/multiformats/go-multiaddr"
	gologging "github.com/whyrusleeping/go-logging"
)

// create a 'Host' with a random peer to listen on the given address
func makeBasicHost(listen string, secio bool) (host.Host, error) {
	addr, err := ma.NewMultiaddr(listen)
	if err != nil {
		return nil, err
	}

	ps := pstore.NewPeerstore()
	var pid peer.ID

	if secio {
		ident, err := testutil.RandIdentity()
		if err != nil {
			return nil, err
		}

		ident.PrivateKey()
		ps.AddPrivKey(ident.ID(), ident.PrivateKey())
		ps.AddPubKey(ident.ID(), ident.PublicKey())
		pid = ident.ID()
	} else {
		fakepid, err := testutil.RandPeerID()
		if err != nil {
			return nil, err
		}
		pid = fakepid
	}

	ctx := context.Background()

	// create a new swarm to be used by the service host
	netw, err := swarm.NewNetwork(ctx, []ma.Multiaddr{addr}, pid, ps, nil)
	if err != nil {
		return nil, err
	}

	log.Printf("I am %s/ipfs/%s\n", addr, pid.Pretty())
	return bhost.New(netw), nil
}

func main() {
	golog.SetAllLoggers(gologging.INFO) // Change to DEBUG for extra info
	listenF := flag.Int("l", 0, "wait for incoming connections")
	target := flag.String("d", "", "target peer to dial")
	secio := flag.Bool("secio", false, "enable secio")

	flag.Parse()

	if *listenF == 0 {
		log.Fatal("Please provide a port to bind on with -l")
	}

	listenaddr := fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", *listenF)

	ha, err := makeBasicHost(listenaddr, *secio)
	if err != nil {
		log.Fatal(err)
	}

	// Set a stream handler on host A
	ha.SetStreamHandler("/echo/1.0.0", func(s net.Stream) {
		log.Println("Got a new stream!")
		defer s.Close()
		doEcho(s)
	})

	if *target == "" {
		log.Println("listening for connections")
		select {} // hang forever
	}
	// This is where the listener code ends

	ipfsaddr, err := ma.NewMultiaddr(*target)
	if err != nil {
		log.Fatalln(err)
	}

	pid, err := ipfsaddr.ValueForProtocol(ma.P_IPFS)
	if err != nil {
		log.Fatalln(err)
	}

	peerid, err := peer.IDB58Decode(pid)
	if err != nil {
		log.Fatalln(err)
	}

	tptaddr := strings.Split(ipfsaddr.String(), "/ipfs/")[0]
	// This creates a MA with the "/ip4/ipaddr/tcp/port" part of the target
	tptmaddr, err := ma.NewMultiaddr(tptaddr)
	if err != nil {
		log.Fatalln(err)
	}

	// We need to add the target to our peerstore, so we know how we can
	// contact it
	ha.Peerstore().AddAddr(peerid, tptmaddr, pstore.PermanentAddrTTL)

	log.Println("opening stream")
	// make a new stream from host B to host A
	// it should be handled on host A by the handler we set above
	s, err := ha.NewStream(context.Background(), peerid, "/echo/1.0.0")
	if err != nil {
		log.Fatalln(err)
	}

	_, err = s.Write([]byte("Hello, world!"))
	if err != nil {
		log.Fatalln(err)
	}

	out, err := ioutil.ReadAll(s)
	if err != nil {
		log.Fatalln(err)
	}

	log.Printf("read reply: %q\n", out)
}

// doEcho reads some data from a stream, writes it back and closes the
// stream.
func doEcho(s inet.Stream) {
	buf := make([]byte, 1024)
	n, err := s.Read(buf)
	if err != nil {
		log.Println(err)
		return
	}

	log.Printf("read request: %q\n", buf[:n])
	_, err = s.Write(buf[:n])
	if err != nil {
		log.Println(err)
		return
	}
}
<commit_msg>seed rand for generation of peer IDs in echo example<commit_after>package main

import (
	"context"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"strings"
	"time"

	golog "github.com/ipfs/go-log"
	host "github.com/libp2p/go-libp2p-host"
	inet "github.com/libp2p/go-libp2p-net"
	net "github.com/libp2p/go-libp2p-net"
	peer "github.com/libp2p/go-libp2p-peer"
	pstore "github.com/libp2p/go-libp2p-peerstore"
	swarm "github.com/libp2p/go-libp2p-swarm"
	bhost "github.com/libp2p/go-libp2p/p2p/host/basic"
	testutil "github.com/libp2p/go-testutil"
	ma "github.com/multiformats/go-multiaddr"
	gologging "github.com/whyrusleeping/go-logging"
)

// create a 'Host' with a random peer to listen on the given address
func makeBasicHost(listen string, secio bool) (host.Host, error) {
	addr, err := ma.NewMultiaddr(listen)
	if err != nil {
		return nil, err
	}

	ps := pstore.NewPeerstore()
	var pid peer.ID

	if secio {
		ident, err := testutil.RandIdentity()
		if err != nil {
			return nil, err
		}

		ident.PrivateKey()
		ps.AddPrivKey(ident.ID(), ident.PrivateKey())
		ps.AddPubKey(ident.ID(), ident.PublicKey())
		pid = ident.ID()
	} else {
		fakepid, err := testutil.RandPeerID()
		if err != nil {
			return nil, err
		}
		pid = fakepid
	}

	ctx := context.Background()

	// create a new swarm to be used by the service host
	netw, err := swarm.NewNetwork(ctx, []ma.Multiaddr{addr}, pid, ps, nil)
	if err != nil {
		return nil, err
	}

	log.Printf("I am %s/ipfs/%s\n", addr, pid.Pretty())
	return bhost.New(netw), nil
}

func main() {
	rand.Seed(time.Now().UnixNano())
	golog.SetAllLoggers(gologging.INFO) // Change to DEBUG for extra info
	listenF := flag.Int("l", 0, "wait for incoming connections")
	target := flag.String("d", "", "target peer to dial")
	secio := flag.Bool("secio", false, "enable secio")

	flag.Parse()

	if *listenF == 0 {
		log.Fatal("Please provide a port to bind on with -l")
	}

	listenaddr := fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", *listenF)

	ha, err := makeBasicHost(listenaddr, *secio)
	if err != nil {
		log.Fatal(err)
	}

	// Set a stream handler on host A
	ha.SetStreamHandler("/echo/1.0.0", func(s net.Stream) {
		log.Println("Got a new stream!")
		defer s.Close()
		doEcho(s)
	})

	if *target == "" {
		log.Println("listening for connections")
		select {} // hang forever
	}
	// This is where the listener code ends

	ipfsaddr, err := ma.NewMultiaddr(*target)
	if err != nil {
		log.Fatalln(err)
	}

	pid, err := ipfsaddr.ValueForProtocol(ma.P_IPFS)
	if err != nil {
		log.Fatalln(err)
	}

	peerid, err := peer.IDB58Decode(pid)
	if err != nil {
		log.Fatalln(err)
	}

	tptaddr := strings.Split(ipfsaddr.String(), "/ipfs/")[0]
	// This creates a MA with the "/ip4/ipaddr/tcp/port" part of the target
	tptmaddr, err := ma.NewMultiaddr(tptaddr)
	if err != nil {
		log.Fatalln(err)
	}

	// We need to add the target to our peerstore, so we know how we can
	// contact it
	ha.Peerstore().AddAddr(peerid, tptmaddr, pstore.PermanentAddrTTL)

	log.Println("opening stream")
	// make a new stream from host B to host A
	// it should be handled on host A by the handler we set above
	s, err := ha.NewStream(context.Background(), peerid, "/echo/1.0.0")
	if err != nil {
		log.Fatalln(err)
	}

	_, err = s.Write([]byte("Hello, world!"))
	if err != nil {
		log.Fatalln(err)
	}

	out, err := ioutil.ReadAll(s)
	if err != nil {
		log.Fatalln(err)
	}

	log.Printf("read reply: %q\n", out)
}

// doEcho reads some data from a stream, writes it back and closes the
// stream.
func doEcho(s inet.Stream) {
	buf := make([]byte, 1024)
	n, err := s.Read(buf)
	if err != nil {
		log.Println(err)
		return
	}

	log.Printf("read request: %q\n", buf[:n])
	_, err = s.Write(buf[:n])
	if err != nil {
		log.Println(err)
		return
	}
}
<|endoftext|>


<commit_before>package fetch

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"

	"github.com/robertkrimen/otto"

	"fknsrs.biz/p/ottoext/loop"
)

func must(err error) {
	if err != nil {
		panic(err)
	}
}

func TestFetch(t *testing.T) {
	m := http.NewServeMux()
	m.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("hello"))
	})
	s := httptest.NewServer(m)
	defer s.Close()

	vm := otto.New()
	l := loop.New(vm)

	if err := Define(vm, l); err != nil {
		panic(err)
	}

	must(l.EvalAndRun(`fetch('http://` + s.Config.Addr + `/').then(function(r) {
    return r.text();
  }).then(function(d) {
  	if (d.indexOf('hello') === -1) {
  		throw new Error('what');
  	}
	});`))
}

func TestFetchCallback(t *testing.T) {
	m := http.NewServeMux()
	m.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("hello"))
	})
	s := httptest.NewServer(m)
	defer s.Close()

	vm := otto.New()
	l := loop.New(vm)

	if err := Define(vm, l); err != nil {
		panic(err)
	}

	if err := vm.Set("__capture", func(s string) {
		if !strings.Contains(s, "hello") {
			panic(fmt.Errorf("expected to find `hello' in response"))
		}
	}); err != nil {
		panic(err)
	}

	must(l.EvalAndRun(`fetch('` + s.Config.Addr + `').then(function(r) {
		return r.text();
	}).then(__capture)`))
}
<commit_msg>fix fetch() tests, making them fail<commit_after>package fetch

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"

	"github.com/robertkrimen/otto"

	"fknsrs.biz/p/ottoext/loop"
)

func must(err error) {
	if err != nil {
		panic(err)
	}
}

func TestFetch(t *testing.T) {
	m := http.NewServeMux()
	m.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("hello"))
	})
	s := httptest.NewServer(m)
	defer s.Close()

	vm := otto.New()
	l := loop.New(vm)

	if err := Define(vm, l); err != nil {
		panic(err)
	}

	must(l.EvalAndRun(`fetch('` + s.URL + `').then(function(r) {
    return r.text();
  }).then(function(d) {
    if (d.indexOf('hellox') === -1) {
      throw new Error('what');
    }
  });`))
}

func TestFetchCallback(t *testing.T) {
	m := http.NewServeMux()
	m.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("hello"))
	})
	s := httptest.NewServer(m)
	defer s.Close()

	vm := otto.New()
	l := loop.New(vm)

	if err := Define(vm, l); err != nil {
		panic(err)
	}

	ch := make(chan bool, 1)

	if err := vm.Set("__capture", func(s string) {
		defer func() { ch <- true }()

		if !strings.Contains(s, "hello") {
			panic(fmt.Errorf("expected to find `hello' in response"))
		}
	}); err != nil {
		panic(err)
	}

	must(l.EvalAndRun(`fetch('` + s.URL + `').then(function(r) {
    return r.text();
  }).then(__capture)`))

	<-ch
}

func TestFetchHeaders(t *testing.T) {
	m := http.NewServeMux()
	m.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("header-one", "1")
		w.Header().Add("header-two", "2a")
		w.Header().Add("header-two", "2b")

		w.Write([]byte("hello"))
	})
	s := httptest.NewServer(m)
	defer s.Close()

	vm := otto.New()
	l := loop.New(vm)

	if err := Define(vm, l); err != nil {
		panic(err)
	}

	ch := make(chan bool, 1)

	if err := vm.Set("__capture", func(s string) {
		defer func() { ch <- true }()

		if s != `{"header-one":["1"],"header-two":["2a","2b"]}` {
			panic(fmt.Errorf("expected headers to contain 1, 2a, and 2b"))
		}
	}); err != nil {
		panic(err)
	}

	must(l.EvalAndRun(`fetch('` + s.URL + `').then(function(r) {
    return __capture(JSON.stringify({
      'header-one': r.headers.getAll('header-one'),
      'header-two': r.headers.getAll('header-two'),
    }));
  })`))

	<-ch
}
<|endoftext|>


<commit_before>package zipfs

import (
	"archive/zip"
	"crypto/md5"
	"fmt"
	"io"
	"os"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestFileSystem(t *testing.T) {
	assert := assert.New(t)
	require := require.New(t)

	fs, err := New("testdata/testdata.zip")
	require.NoError(err)
	require.NotNil(fs)

	f, err := fs.Open("/xxx")
	assert.Error(err)
	assert.Nil(f)

	f, err = fs.Open("test.html")
	assert.NoError(err)
	assert.NotNil(f)

}

func TestOpen(t *testing.T) {
	assert := assert.New(t)
	require := require.New(t)
	fs, err := New("testdata/testdata.zip")
	require.NoError(err)

	testCases := []struct {
		Path  string
		Error string
	}{
		{
			Path:  "/does/not/exist",
			Error: "file does not exist",
		},
		{
			Path:  "/img",
			Error: "",
		},
		{
			Path:  "/img/circle.png",
			Error: "",
		},
	}
	for _, tc := range testCases {
		f, err := fs.Open(tc.Path)
		if tc.Error == "" {
			assert.NoError(err)
			assert.NotNil(f)
			f.Close()

			// testing error after closing
			var buf [50]byte
			_, err := f.Read(buf[:])
			assert.Error(err)
			_, err = f.Seek(20, 0)
			assert.Error(err)
		} else {
			assert.Error(err)
			assert.True(strings.Contains(err.Error(), tc.Error), err.Error())
			assert.True(strings.Contains(err.Error(), tc.Path), err.Error())
		}
	}

	err = fs.Close()
	assert.NoError(err)
	f, err := fs.Open("/img/circle.png")
	assert.Error(err)
	assert.Nil(f)
	assert.True(strings.Contains(err.Error(), "filesystem closed"), err.Error())
}

func TestReaddir(t *testing.T) {
	assert := assert.New(t)
	require := require.New(t)
	fs, err := New("testdata/testdata.zip")
	require.NoError(err)

	testCases := []struct {
		Path  string
		Count int
		Error string
		Files []string
	}{
		{
			Path:  "/img",
			Error: "",
			Files: []string{
				"another-circle.png",
				"circle.png",
			},
		},
		{
			Path:  "/",
			Error: "",
			Files: []string{
				"empty",
				"img",
				"index.html",
				"js",
				"lots-of-files",
				"not-a-zip-file.txt",
				"random.dat",
				"test.html",
			},
		},
		{
			Path:  "/lots-of-files",
			Error: "",
			Files: []string{
				"file-01",
				"file-02",
				"file-03",
				"file-04",
				"file-05",
				"file-06",
				"file-07",
				"file-08",
				"file-09",
				"file-10",
				"file-11",
				"file-12",
				"file-13",
				"file-14",
				"file-15",
				"file-16",
				"file-17",
				"file-18",
				"file-19",
				"file-20",
			},
		},
		{
			Path:  "/img/circle.png",
			Error: "not a directory",
		},
		{
			Path:  "/img/circle.png",
			Error: "not a directory",
			Count: 2,
		},
	}

	for _, tc := range testCases {
		f, err := fs.Open(tc.Path)
		require.NoError(err)
		require.NotNil(f)

		files, err := f.Readdir(tc.Count)
		if tc.Error == "" {
			assert.NoError(err)
			assert.NotNil(files)
			printError := false
			if len(files) != len(tc.Files) {
				printError = true
			} else {
				for i, file := range files {
					if file.Name() != tc.Files[i] {
						printError = true
						break
					}
				}
			}
			if printError {
				t.Log(tc.Path, "Readdir expected:")
				for i, f := range tc.Files {
					t.Logf("    %d: %s\n", i, f)
				}
				t.Log(tc.Path, "Readdir actual:")
				for i, f := range files {
					t.Logf("    %d: %s\n", i, f.Name())
				}
				t.Error("Readdir failed test")
			}
		} else {
			assert.Error(err)
			assert.Nil(files)
			assert.True(strings.Contains(err.Error(), tc.Error), err.Error())
			assert.True(strings.Contains(err.Error(), tc.Path), err.Error())
		}
	}

	file, err := fs.Open("/lots-of-files")
	require.NoError(err)
	for i := 0; i < 10; i++ {
		a, err := file.Readdir(2)
		require.NoError(err)
		assert.Equal(len(a), 2)
		assert.Equal(fmt.Sprintf("file-%02d", i*2+1), a[0].Name())
		assert.Equal(fmt.Sprintf("file-%02d", i*2+2), a[1].Name())
	}
	a, err := file.Readdir(2)
	assert.Error(err)
	assert.Equal(io.EOF, err)
	assert.Equal(0, len(a))
}

// TestFileInfo tests the os.FileInfo associated with the http.File
func TestFileInfo(t *testing.T) {
	require := require.New(t)
	assert := assert.New(t)
	fs, err := New("testdata/testdata.zip")
	require.NoError(err)

	testCases := []struct {
		Path       string
		Name       string
		Size       int64
		Mode       os.FileMode
		IsDir      bool
		HasZipFile bool
	}{
		// Don't use any text files here because the sizes
		// are different betwen Windows and Unix-like OSs.
		{
			Path:       "/img/circle.png",
			Name:       "circle.png",
			Size:       5973,
			Mode:       0444,
			IsDir:      false,
			HasZipFile: true,
		},
		{
			Path:       "/img/",
			Name:       "img",
			Size:       0,
			Mode:       os.ModeDir | 0555,
			IsDir:      true,
			HasZipFile: true,
		},
		{
			Path:       "/",
			Name:       "/",
			Size:       0,
			Mode:       os.ModeDir | 0555,
			IsDir:      true,
			HasZipFile: true,
		},
	}

	for _, tc := range testCases {
		file, err := fs.Open(tc.Path)
		require.NoError(err)
		fi, err := file.Stat()
		require.NoError(err)
		assert.Equal(tc.Name, fi.Name())
		assert.Equal(tc.Size, fi.Size())
		assert.Equal(tc.Mode, fi.Mode())
		assert.Equal(tc.IsDir, fi.IsDir())
		_, hasZipFile := fi.Sys().(*zip.File)
		assert.Equal(tc.HasZipFile, hasZipFile, fi.Name())
		assert.False(fi.ModTime().IsZero())
	}
}

// TestFile tests the file reading capabilities.
func TestFile(t *testing.T) {
	require := require.New(t)
	assert := assert.New(t)
	fs, err := New("testdata/testdata.zip")
	require.NoError(err)

	testCases := []struct {
		Path string
		Size int
		MD5  string
	}{
		{
			Path: "/random.dat",
			Size: 10000,
			MD5:  "3c9fe0521cabb2ab38484cd1c024a61d",
		},
		{
			Path: "/img/circle.png",
			Size: 5973,
			MD5:  "05e3048db45e71749e06658ccfc0753b",
		},
	}

	calcMD5 := func(r io.ReadSeeker, size int, seek bool) string {
		if seek {
			n, err := r.Seek(0, 0)
			require.NoError(err)
			require.Equal(int64(0), n)
		}
		buf := make([]byte, size)
		n, err := r.Read(buf)
		require.NoError(err)
		require.Equal(size, n)
		md5Text := fmt.Sprintf("%x", md5.Sum(buf))
		n, err = r.Read(buf)
		require.Error(err)
		require.Equal(io.EOF, err)
		require.Equal(0, n)
		return md5Text
	}

	for _, tc := range testCases {
		file, err := fs.Open(tc.Path)
		assert.NoError(err)
		assert.Equal(tc.MD5, calcMD5(file, tc.Size, false))

		// seek back to the beginning, should not have
		// to create a temporary file
		nseek, err := file.Seek(0, 0)
		assert.NoError(err)
		assert.Equal(int64(0), nseek)
		assert.Equal(tc.MD5, calcMD5(file, tc.Size, true))

		nSeek, err := file.Seek(int64(tc.Size/2), 0)
		assert.NoError(err)
		assert.Equal(int64(tc.Size/2), nSeek)
		assert.Equal(tc.MD5, calcMD5(file, tc.Size, true))

		file.Close()
	}
}
<commit_msg>Fix type found by go report card<commit_after>package zipfs

import (
	"archive/zip"
	"crypto/md5"
	"fmt"
	"io"
	"os"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestFileSystem(t *testing.T) {
	assert := assert.New(t)
	require := require.New(t)

	fs, err := New("testdata/testdata.zip")
	require.NoError(err)
	require.NotNil(fs)

	f, err := fs.Open("/xxx")
	assert.Error(err)
	assert.Nil(f)

	f, err = fs.Open("test.html")
	assert.NoError(err)
	assert.NotNil(f)

}

func TestOpen(t *testing.T) {
	assert := assert.New(t)
	require := require.New(t)
	fs, err := New("testdata/testdata.zip")
	require.NoError(err)

	testCases := []struct {
		Path  string
		Error string
	}{
		{
			Path:  "/does/not/exist",
			Error: "file does not exist",
		},
		{
			Path:  "/img",
			Error: "",
		},
		{
			Path:  "/img/circle.png",
			Error: "",
		},
	}
	for _, tc := range testCases {
		f, err := fs.Open(tc.Path)
		if tc.Error == "" {
			assert.NoError(err)
			assert.NotNil(f)
			f.Close()

			// testing error after closing
			var buf [50]byte
			_, err := f.Read(buf[:])
			assert.Error(err)
			_, err = f.Seek(20, 0)
			assert.Error(err)
		} else {
			assert.Error(err)
			assert.True(strings.Contains(err.Error(), tc.Error), err.Error())
			assert.True(strings.Contains(err.Error(), tc.Path), err.Error())
		}
	}

	err = fs.Close()
	assert.NoError(err)
	f, err := fs.Open("/img/circle.png")
	assert.Error(err)
	assert.Nil(f)
	assert.True(strings.Contains(err.Error(), "filesystem closed"), err.Error())
}

func TestReaddir(t *testing.T) {
	assert := assert.New(t)
	require := require.New(t)
	fs, err := New("testdata/testdata.zip")
	require.NoError(err)

	testCases := []struct {
		Path  string
		Count int
		Error string
		Files []string
	}{
		{
			Path:  "/img",
			Error: "",
			Files: []string{
				"another-circle.png",
				"circle.png",
			},
		},
		{
			Path:  "/",
			Error: "",
			Files: []string{
				"empty",
				"img",
				"index.html",
				"js",
				"lots-of-files",
				"not-a-zip-file.txt",
				"random.dat",
				"test.html",
			},
		},
		{
			Path:  "/lots-of-files",
			Error: "",
			Files: []string{
				"file-01",
				"file-02",
				"file-03",
				"file-04",
				"file-05",
				"file-06",
				"file-07",
				"file-08",
				"file-09",
				"file-10",
				"file-11",
				"file-12",
				"file-13",
				"file-14",
				"file-15",
				"file-16",
				"file-17",
				"file-18",
				"file-19",
				"file-20",
			},
		},
		{
			Path:  "/img/circle.png",
			Error: "not a directory",
		},
		{
			Path:  "/img/circle.png",
			Error: "not a directory",
			Count: 2,
		},
	}

	for _, tc := range testCases {
		f, err := fs.Open(tc.Path)
		require.NoError(err)
		require.NotNil(f)

		files, err := f.Readdir(tc.Count)
		if tc.Error == "" {
			assert.NoError(err)
			assert.NotNil(files)
			printError := false
			if len(files) != len(tc.Files) {
				printError = true
			} else {
				for i, file := range files {
					if file.Name() != tc.Files[i] {
						printError = true
						break
					}
				}
			}
			if printError {
				t.Log(tc.Path, "Readdir expected:")
				for i, f := range tc.Files {
					t.Logf("    %d: %s\n", i, f)
				}
				t.Log(tc.Path, "Readdir actual:")
				for i, f := range files {
					t.Logf("    %d: %s\n", i, f.Name())
				}
				t.Error("Readdir failed test")
			}
		} else {
			assert.Error(err)
			assert.Nil(files)
			assert.True(strings.Contains(err.Error(), tc.Error), err.Error())
			assert.True(strings.Contains(err.Error(), tc.Path), err.Error())
		}
	}

	file, err := fs.Open("/lots-of-files")
	require.NoError(err)
	for i := 0; i < 10; i++ {
		a, err := file.Readdir(2)
		require.NoError(err)
		assert.Equal(len(a), 2)
		assert.Equal(fmt.Sprintf("file-%02d", i*2+1), a[0].Name())
		assert.Equal(fmt.Sprintf("file-%02d", i*2+2), a[1].Name())
	}
	a, err := file.Readdir(2)
	assert.Error(err)
	assert.Equal(io.EOF, err)
	assert.Equal(0, len(a))
}

// TestFileInfo tests the os.FileInfo associated with the http.File
func TestFileInfo(t *testing.T) {
	require := require.New(t)
	assert := assert.New(t)
	fs, err := New("testdata/testdata.zip")
	require.NoError(err)

	testCases := []struct {
		Path       string
		Name       string
		Size       int64
		Mode       os.FileMode
		IsDir      bool
		HasZipFile bool
	}{
		// Don't use any text files here because the sizes
		// are different between Windows and Unix-like OSs.
		{
			Path:       "/img/circle.png",
			Name:       "circle.png",
			Size:       5973,
			Mode:       0444,
			IsDir:      false,
			HasZipFile: true,
		},
		{
			Path:       "/img/",
			Name:       "img",
			Size:       0,
			Mode:       os.ModeDir | 0555,
			IsDir:      true,
			HasZipFile: true,
		},
		{
			Path:       "/",
			Name:       "/",
			Size:       0,
			Mode:       os.ModeDir | 0555,
			IsDir:      true,
			HasZipFile: true,
		},
	}

	for _, tc := range testCases {
		file, err := fs.Open(tc.Path)
		require.NoError(err)
		fi, err := file.Stat()
		require.NoError(err)
		assert.Equal(tc.Name, fi.Name())
		assert.Equal(tc.Size, fi.Size())
		assert.Equal(tc.Mode, fi.Mode())
		assert.Equal(tc.IsDir, fi.IsDir())
		_, hasZipFile := fi.Sys().(*zip.File)
		assert.Equal(tc.HasZipFile, hasZipFile, fi.Name())
		assert.False(fi.ModTime().IsZero())
	}
}

// TestFile tests the file reading capabilities.
func TestFile(t *testing.T) {
	require := require.New(t)
	assert := assert.New(t)
	fs, err := New("testdata/testdata.zip")
	require.NoError(err)

	testCases := []struct {
		Path string
		Size int
		MD5  string
	}{
		{
			Path: "/random.dat",
			Size: 10000,
			MD5:  "3c9fe0521cabb2ab38484cd1c024a61d",
		},
		{
			Path: "/img/circle.png",
			Size: 5973,
			MD5:  "05e3048db45e71749e06658ccfc0753b",
		},
	}

	calcMD5 := func(r io.ReadSeeker, size int, seek bool) string {
		if seek {
			n, err := r.Seek(0, 0)
			require.NoError(err)
			require.Equal(int64(0), n)
		}
		buf := make([]byte, size)
		n, err := r.Read(buf)
		require.NoError(err)
		require.Equal(size, n)
		md5Text := fmt.Sprintf("%x", md5.Sum(buf))
		n, err = r.Read(buf)
		require.Error(err)
		require.Equal(io.EOF, err)
		require.Equal(0, n)
		return md5Text
	}

	for _, tc := range testCases {
		file, err := fs.Open(tc.Path)
		assert.NoError(err)
		assert.Equal(tc.MD5, calcMD5(file, tc.Size, false))

		// seek back to the beginning, should not have
		// to create a temporary file
		nseek, err := file.Seek(0, 0)
		assert.NoError(err)
		assert.Equal(int64(0), nseek)
		assert.Equal(tc.MD5, calcMD5(file, tc.Size, true))

		nSeek, err := file.Seek(int64(tc.Size/2), 0)
		assert.NoError(err)
		assert.Equal(int64(tc.Size/2), nSeek)
		assert.Equal(tc.MD5, calcMD5(file, tc.Size, true))

		file.Close()
	}
}
<|endoftext|>


<commit_before>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

// Package log provides some handy types and method to activate and deactivate specific log
// behavior within files in a transparent way.
package log

import (
	"fmt"
	"testing"
)

// Logger is a minimalist interface to represent logger
type Logger interface {
	Log(format string, a ...interface{})
}

// DebugLogger can be used in development to display loglines in the console
type DebugLogger struct {
	Tag string
}

// Log implements the Logger interface
func (l DebugLogger) Log(format string, a ...interface{}) {
	fmt.Printf("\033[33m[ %s ]\033[0m ", l.Tag) // Tag printed in yellow
	fmt.Printf(format, a...)
	fmt.Print("\n")
}

// TestLogger can be used in a test environnement to display log only on failure
type TestLogger struct {
	Tag string
	T   *testing.T
}

// Log implements the Logger interface
func (l TestLogger) Log(format string, a ...interface{}) {
	l.T.Logf("\033[33m[ %s ]\033[0m %s", l.Tag, fmt.Sprintf(format, a...)) // Tag printed in yellow
}

// MultiLogger aggregates several loggers log to each of them
type MultiLogger struct {
	loggers []Logger
}

// Log implements the Logger interface
func (l MultiLogger) Log(format string, a ...interface{}) {
	for _, logger := range l.loggers {
		logger.Log(format, a...)
	}
}
<commit_msg>[broker] Make Loggers an exported field for MultiLogger<commit_after>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

// Package log provides some handy types and method to activate and deactivate specific log
// behavior within files in a transparent way.
package log

import (
	"fmt"
	"testing"
)

// Logger is a minimalist interface to represent logger
type Logger interface {
	Log(format string, a ...interface{})
}

// DebugLogger can be used in development to display loglines in the console
type DebugLogger struct {
	Tag string
}

// Log implements the Logger interface
func (l DebugLogger) Log(format string, a ...interface{}) {
	fmt.Printf("\033[33m[ %s ]\033[0m ", l.Tag) // Tag printed in yellow
	fmt.Printf(format, a...)
	fmt.Print("\n")
}

// TestLogger can be used in a test environnement to display log only on failure
type TestLogger struct {
	Tag string
	T   *testing.T
}

// Log implements the Logger interface
func (l TestLogger) Log(format string, a ...interface{}) {
	l.T.Logf("\033[33m[ %s ]\033[0m %s", l.Tag, fmt.Sprintf(format, a...)) // Tag printed in yellow
}

// MultiLogger aggregates several loggers log to each of them
type MultiLogger struct {
	Loggers []Logger
}

// Log implements the Logger interface
func (l MultiLogger) Log(format string, a ...interface{}) {
	for _, logger := range l.Loggers {
		logger.Log(format, a...)
	}
}
<|endoftext|>


<commit_before>package client_test

import (
	"github.com/nelhage/livegrep/client"
	. "launchpad.net/gocheck"
	"net"
	"strings"
	"testing"
)

// We assume a codesearch running on localhost:9999. This could be
// improved.

func Test(t *testing.T) { TestingT(t) }

type ClientSuite struct {
	client client.Client
}

var _ = Suite(&ClientSuite{})

func (s *ClientSuite) SetUpTest(c *C) {
	var err error
	s.client, err = client.Dial("tcp", "localhost:9999")
	if err != nil {
		panic(err.Error())
	}
}

func (s *ClientSuite) TestQuery(c *C) {
	search, err := s.client.Query(&client.Query{".", "", ""})
	c.Assert(err, IsNil)
	var n int
	for r := range search.Results() {
		n++
		c.Assert(r.Line, Not(Equals), "")
	}
	c.Assert(n, Not(Equals), 0)
	st, e := search.Close()
	c.Assert(st, Not(IsNil))
	c.Assert(e, IsNil)
}

func (s *ClientSuite) TestTwoQueries(c *C) {
	search, err := s.client.Query(&client.Query{".", "", ""})
	c.Assert(err, IsNil)
	_, err = search.Close()
	c.Assert(err, IsNil)

	search, err = s.client.Query(&client.Query{".", "", ""})
	c.Assert(err, IsNil)
	n := 0
	for _ = range search.Results() {
		n++
	}
	_, err = search.Close()
	if err != nil {
		c.Fatalf("Unexpected error: %s", err.Error())
	}
	c.Assert(n, Not(Equals), 0)
}

func (s *ClientSuite) TestBadRegex(c *C) {
	search, err := s.client.Query(&client.Query{"(", "", ""})
	c.Assert(err, IsNil)
	for _ = range search.Results() {
		c.Fatal("Got back a result from an erroneous query!")
	}
	st, e := search.Close()
	c.Assert(st, IsNil)
	if e == nil {
		c.Fatal("Didn't get back an error")
	}
	if q, ok := e.(client.QueryError); ok {
		c.Assert(q.Query.Line, Equals, "(")
		if strings.HasPrefix(q.Err, "FATAL") {
			c.Errorf("Error includes FATAL prefix: %s", q.Err)
		}
	} else {
		c.Fatalf("Error %v wasn't a QueryError", e)
	}
}

func shutdownClient(addr string, ready chan<- bool) {
	ln, err := net.Listen("tcp", addr)
	if err != nil {
		panic(err.Error())
	}
	defer ln.Close()
	ready <- true
	conn, err := ln.Accept()
	if err != nil {
		panic(err.Error())
	}
	conn.Write([]byte("READY {}\n"))
	conn.Close()
}

func (s *ClientSuite) TestShutdown(c *C) {
	ready := make(chan bool, 1)
	go shutdownClient("127.0.0.1:10999", ready)
	<-ready

	cl, err := client.Dial("tcp", "127.0.0.1:10999")
	c.Assert(err, IsNil)

	search, err := cl.Query(&client.Query{Line: "l"})
	c.Assert(err, IsNil)
	c.Assert(search, Not(IsNil))

	results := search.Results()
	c.Assert(results, Not(IsNil))
	for r := range results {
		c.Errorf("Got a result back: %+v", r)
	}
	st, err := search.Close()
	c.Assert(st, IsNil)
	c.Assert(err, Not(IsNil))

	search, err = cl.Query(&client.Query{Line: "l"})
	c.Assert(err, Not(IsNil))
	c.Assert(search, IsNil)
}
<commit_msg>refactor a mock server in client_test<commit_after>package client_test

import (
	"github.com/nelhage/livegrep/client"
	. "launchpad.net/gocheck"
	"net"
	"strings"
	"testing"
)

// We assume a codesearch running on localhost:9999. This could be
// improved.

func Test(t *testing.T) { TestingT(t) }

type ClientSuite struct {
	client client.Client
}

var _ = Suite(&ClientSuite{})

func (s *ClientSuite) SetUpTest(c *C) {
	var err error
	s.client, err = client.Dial("tcp", "localhost:9999")
	if err != nil {
		panic(err.Error())
	}
}

func (s *ClientSuite) TestQuery(c *C) {
	search, err := s.client.Query(&client.Query{".", "", ""})
	c.Assert(err, IsNil)
	var n int
	for r := range search.Results() {
		n++
		c.Assert(r.Line, Not(Equals), "")
	}
	c.Assert(n, Not(Equals), 0)
	st, e := search.Close()
	c.Assert(st, Not(IsNil))
	c.Assert(e, IsNil)
}

func (s *ClientSuite) TestTwoQueries(c *C) {
	search, err := s.client.Query(&client.Query{".", "", ""})
	c.Assert(err, IsNil)
	_, err = search.Close()
	c.Assert(err, IsNil)

	search, err = s.client.Query(&client.Query{".", "", ""})
	c.Assert(err, IsNil)
	n := 0
	for _ = range search.Results() {
		n++
	}
	_, err = search.Close()
	if err != nil {
		c.Fatalf("Unexpected error: %s", err.Error())
	}
	c.Assert(n, Not(Equals), 0)
}

func (s *ClientSuite) TestBadRegex(c *C) {
	search, err := s.client.Query(&client.Query{"(", "", ""})
	c.Assert(err, IsNil)
	for _ = range search.Results() {
		c.Fatal("Got back a result from an erroneous query!")
	}
	st, e := search.Close()
	c.Assert(st, IsNil)
	if e == nil {
		c.Fatal("Didn't get back an error")
	}
	if q, ok := e.(client.QueryError); ok {
		c.Assert(q.Query.Line, Equals, "(")
		if strings.HasPrefix(q.Err, "FATAL") {
			c.Errorf("Error includes FATAL prefix: %s", q.Err)
		}
	} else {
		c.Fatalf("Error %v wasn't a QueryError", e)
	}
}

func mockServerShutdown() <-chan string {
	ready := make(chan string, 1)
	go func() {
		ln, err := net.Listen("tcp", ":0")
		if err != nil {
			panic(err.Error())
		}
		defer ln.Close()
		ready <- ln.Addr().String()
		conn, err := ln.Accept()
		if err != nil {
			panic(err.Error())
		}
		conn.Write([]byte("READY {}\n"))
		conn.Close()
	}()
	return ready
}

func (s *ClientSuite) TestShutdown(c *C) {
	ready := mockServerShutdown()
	addr := <-ready

	cl, err := client.Dial("tcp", addr)
	c.Assert(err, IsNil)

	search, err := cl.Query(&client.Query{Line: "l"})
	c.Assert(err, IsNil)
	c.Assert(search, Not(IsNil))

	results := search.Results()
	c.Assert(results, Not(IsNil))
	for r := range results {
		c.Errorf("Got a result back: %+v", r)
	}
	st, err := search.Close()
	c.Assert(st, IsNil)
	c.Assert(err, Not(IsNil))

	search, err = cl.Query(&client.Query{Line: "l"})
	c.Assert(err, Not(IsNil))
	c.Assert(search, IsNil)
}
<|endoftext|>


<commit_before>package cli

import (
  "strings"
  "fmt"

  "github.com/jbenet/go-ipfs/commands"
)

func Parse(input []string, root *commands.Command) (*commands.Request, error) {
  path, input, err := parsePath(input, root)
  if err != nil {
    return nil, err
  }

  options, err := root.GetOptions(path)
  if err != nil {
    return nil, err
  }

  opts, args, err := parseOptions(input, options)
  if err != nil {
    return nil, err
  }

  return commands.NewRequest(path, opts, args), nil
}

// parsePath gets the command path from the command line input
func parsePath(input []string, root *commands.Command) ([]string, []string, error) {
  cmd := root
  i := 0

  for _, blob := range input {
    if strings.HasPrefix(blob, "-") {
      break
    }

    cmd := cmd.Sub(blob)
    if cmd == nil {
      break
    }

    i++
  }

  return input[:i], input[i:], nil
}

// parseOptions parses the raw string values of the given options
// returns the parsed options as strings, along with the CLI args
func parseOptions(input []string, options map[string]commands.Option) (map[string]interface{}, []string, error) {
  opts := make(map[string]interface{})
  args := make([]string, 0)

  // TODO: error if one option is defined multiple times

  for i := 0; i < len(input); i++ {
    blob := input[i]

    if strings.HasPrefix(blob, "--") {
      name := blob[2:]
      value := ""

      if strings.Contains(name, "=") {
        split := strings.SplitN(name, "=", 2)
        name = split[0]
        value = split[1]
      }

      opts[name] = value

    } else if strings.HasPrefix(blob, "-") {
      blob = blob[1:]

      if strings.ContainsAny(blob, "-=\"") {
        return nil, nil, fmt.Errorf("Invalid option blob: '%s'", input[i])
      }

      nameS := ""
      for _, name := range blob {
        nameS = string(name)
        opts[nameS] = ""
      }

      if nameS != "" {
        opt, ok := options[nameS]
        if ok && opt.Type != commands.Bool {
          i++
          if i <= len(input) {
            opts[nameS] = input[i]
          }
        }
      }

    } else {
      args = append(args, blob)
    }
  }

  return opts, args, nil
}
<commit_msg>commands/cli: Refactored CLI parsing to match go tooling conventions<commit_after>package cli

import (
  "strings"

  "github.com/jbenet/go-ipfs/commands"
)

func Parse(input []string, root *commands.Command) (*commands.Request, error) {
  path, input, err := parsePath(input, root)
  if err != nil {
    return nil, err
  }

  opts, args, err := parseOptions(input)
  if err != nil {
    return nil, err
  }

  return commands.NewRequest(path, opts, args), nil
}

// parsePath gets the command path from the command line input
func parsePath(input []string, root *commands.Command) ([]string, []string, error) {
  cmd := root
  i := 0

  for _, blob := range input {
    if strings.HasPrefix(blob, "-") {
      break
    }

    cmd := cmd.Sub(blob)
    if cmd == nil {
      break
    }

    i++
  }

  return input[:i], input[i:], nil
}

// parseOptions parses the raw string values of the given options
// returns the parsed options as strings, along with the CLI args
func parseOptions(input []string) (map[string]interface{}, []string, error) {
  opts := make(map[string]interface{})
  args := make([]string, 0)

  // TODO: error if one option is defined multiple times

  for i := 0; i < len(input); i++ {
    blob := input[i]

    if strings.HasPrefix(blob, "-") {
      name := blob[1:]
      value := ""

      // support single and double dash
      if strings.HasPrefix(name, "-") {
        name = name[1:]
      }

      if strings.Contains(name, "=") {
        split := strings.SplitN(name, "=", 2)
        name = split[0]
        value = split[1]
      }

      opts[name] = value

    } else {
      args = append(args, blob)
    }
  }

  return opts, args, nil
}
<|endoftext|>


<commit_before>package commands

import (
	"bytes"
	"crypto/sha256"
	"fmt"
	"io/ioutil"
	"net/http"
	"os/exec"
	"strings"
)

func init() {
	// Nothing happens here.
}

// LengthCheck makes sure a string has at least minLength lines.
func LengthCheck(data string, minLength int) bool {
	length := LineCount(data)
	Log(fmt.Sprintf("length='%d' minLength='%d'", length, minLength), "debug")
	if length >= minLength {
		return true
	}
	return false
}

// ReadURL grabs a URL and returns the string from the body.
func ReadURL(url string, dogstatsd bool) string {
	resp, err := http.Get(url)
	if err != nil {
		Log(fmt.Sprintf("function='ReadURL' panic='true' url='%s'", url), "info")
		if dogstatsd {
			StatsdPanic(url, "read_url")
		}
	}
	defer resp.Body.Close()
	body, _ := ioutil.ReadAll(resp.Body)
	return string(body)
}

// LineCount splits a string by linebreak and returns the number of lines.
func LineCount(data string) int {
	var length int
	if strings.ContainsAny(data, "\n") {
		length = strings.Count(data, "\n")
	} else {
		length = 1
	}
	return length
}

// ComputeChecksum takes a string and computes a SHA256 checksum.
func ComputeChecksum(data string) string {
	dataBytes := []byte(data)
	computedChecksum := sha256.Sum256(dataBytes)
	finalChecksum := fmt.Sprintf("%x\n", computedChecksum)
	Log(fmt.Sprintf("computedChecksum='%s'", finalChecksum), "debug")
	return finalChecksum
}

// ChecksumCompare takes a string, generates a SHA256 checksum and compares
// against the passed checksum to see if they match.
func ChecksumCompare(data string, checksum string) bool {
	computedChecksum := ComputeChecksum(data)
	Log(fmt.Sprintf("checksum='%s' computedChecksum='%s'", checksum, computedChecksum), "debug")
	if strings.TrimSpace(computedChecksum) == strings.TrimSpace(checksum) {
		return true
	}
	return false
}

// UnixDiff runs diff to generate text for the Datadog events.
func UnixDiff(old, new string) string {
	diff, _ := exec.Command("diff", "-u", old, new).Output()
	text := string(diff)
	finalText := removeLines(text, 3)
	return finalText
}

// removeLines trims the top n number of lines from a string.
func removeLines(text string, number int) string {
	lines := strings.Split(text, "\n")
	var cleaned []string
	cleaned = append(cleaned, lines[number:]...)
	finalText := strings.Join(cleaned, "\n")
	return finalText
}

// RunCommand runs a cli command with arguments.
func RunCommand(command string) bool {
	parts := strings.Fields(command)
	cli := parts[0]
	args := parts[1:len(parts)]
	cmd := exec.Command(cli, args...)
	var out bytes.Buffer
	cmd.Stdout = &out
	err := cmd.Run()
	if err != nil {
		Log(fmt.Sprintf("exec='error' message='%v'", err), "info")
		return false
	}
	return true
}
<commit_msg>Remove extra line break we were stripping anyways.<commit_after>package commands

import (
	"bytes"
	"crypto/sha256"
	"fmt"
	"io/ioutil"
	"net/http"
	"os/exec"
	"strings"
)

func init() {
	// Nothing happens here.
}

// LengthCheck makes sure a string has at least minLength lines.
func LengthCheck(data string, minLength int) bool {
	length := LineCount(data)
	Log(fmt.Sprintf("length='%d' minLength='%d'", length, minLength), "debug")
	if length >= minLength {
		return true
	}
	return false
}

// ReadURL grabs a URL and returns the string from the body.
func ReadURL(url string, dogstatsd bool) string {
	resp, err := http.Get(url)
	if err != nil {
		Log(fmt.Sprintf("function='ReadURL' panic='true' url='%s'", url), "info")
		if dogstatsd {
			StatsdPanic(url, "read_url")
		}
	}
	defer resp.Body.Close()
	body, _ := ioutil.ReadAll(resp.Body)
	return string(body)
}

// LineCount splits a string by linebreak and returns the number of lines.
func LineCount(data string) int {
	var length int
	if strings.ContainsAny(data, "\n") {
		length = strings.Count(data, "\n")
	} else {
		length = 1
	}
	return length
}

// ComputeChecksum takes a string and computes a SHA256 checksum.
func ComputeChecksum(data string) string {
	dataBytes := []byte(data)
	computedChecksum := sha256.Sum256(dataBytes)
	finalChecksum := fmt.Sprintf("%x", computedChecksum)
	Log(fmt.Sprintf("computedChecksum='%s'", finalChecksum), "debug")
	return finalChecksum
}

// ChecksumCompare takes a string, generates a SHA256 checksum and compares
// against the passed checksum to see if they match.
func ChecksumCompare(data string, checksum string) bool {
	computedChecksum := ComputeChecksum(data)
	Log(fmt.Sprintf("checksum='%s' computedChecksum='%s'", checksum, computedChecksum), "debug")
	if strings.TrimSpace(computedChecksum) == strings.TrimSpace(checksum) {
		return true
	}
	return false
}

// UnixDiff runs diff to generate text for the Datadog events.
func UnixDiff(old, new string) string {
	diff, _ := exec.Command("diff", "-u", old, new).Output()
	text := string(diff)
	finalText := removeLines(text, 3)
	return finalText
}

// removeLines trims the top n number of lines from a string.
func removeLines(text string, number int) string {
	lines := strings.Split(text, "\n")
	var cleaned []string
	cleaned = append(cleaned, lines[number:]...)
	finalText := strings.Join(cleaned, "\n")
	return finalText
}

// RunCommand runs a cli command with arguments.
func RunCommand(command string) bool {
	parts := strings.Fields(command)
	cli := parts[0]
	args := parts[1:len(parts)]
	cmd := exec.Command(cli, args...)
	var out bytes.Buffer
	cmd.Stdout = &out
	err := cmd.Run()
	if err != nil {
		Log(fmt.Sprintf("exec='error' message='%v'", err), "info")
		return false
	}
	return true
}
<|endoftext|>


<commit_before>package utils

import (
	"bytes"
	"crypto/md5"
	"crypto/rand"
	"encoding/base64"
	"strconv"

	e "github.com/techjanitor/pram-post/errors"
)

var nRandBytes = 32

// creates a new random session id with user id
func NewSession(userid uint) (cookieToken string, err error) {

	uid := strconv.Itoa(int(userid))

	// Initialize cache handle
	cache := RedisCache

	// make slice for token, with user + semicolon
	token := make([]byte, nRandBytes+len(uid)+1)
	// copy key into token
	copy(token, []byte(uid))
	// add semicolon
	token[len(uid)] = ';'

	// read in random bytes
	_, err = rand.Read(token[len(uid)+1:])
	if err != nil {
		return
	}

	// goes in the cookie
	cookieToken = base64.URLEncoding.EncodeToString(token)

	// goes to redis
	sum := md5.Sum(token)
	storageToken := base64.StdEncoding.EncodeToString(sum[:])

	// set key in redis
	err = cache.SetEx(storageToken, 2592000, uid)
	if err != nil {
		return
	}

	return

}

// validate compares provided session id to redis
func ValidateSession(key []byte) (err error) {

	// decode key
	token, err := base64.URLEncoding.DecodeString(string(key))
	if err != nil {
		return
	}

	// get uid
	index := bytes.IndexByte(token, ';')

	// check to see if user is there
	if index < 0 {
		return e.ErrInvalidSession
	}

	// get given uid
	uid := string(token[:index])

	// hash token
	sum := md5.Sum(token)

	// base64 encode sum
	providedHash := base64.StdEncoding.EncodeToString(sum[:])

	// check for match
	result, err := cache.Get(providedHash)
	if err == u.ErrCacheMiss {
		return e.ErrInvalidSession
	}
	if err != nil {
		return
	}

	// check if uid matches
	if uid != string(result) {
		return e.ErrInvalidSession
	}

	return

}
<commit_msg>add login handler<commit_after>package utils

import (
	"bytes"
	"crypto/md5"
	"crypto/rand"
	"encoding/base64"
	"strconv"

	e "github.com/techjanitor/pram-post/errors"
)

var nRandBytes = 32

// creates a new random session id with user id
func NewSession(userid uint) (cookieToken string, err error) {

	uid := strconv.Itoa(int(userid))

	// Initialize cache handle
	cache := RedisCache

	// make slice for token, with user + semicolon
	token := make([]byte, nRandBytes+len(uid)+1)
	// copy key into token
	copy(token, []byte(uid))
	// add semicolon
	token[len(uid)] = ';'

	// read in random bytes
	_, err = rand.Read(token[len(uid)+1:])
	if err != nil {
		return
	}

	// goes in the cookie
	cookieToken = base64.URLEncoding.EncodeToString(token)

	// goes to redis
	sum := md5.Sum(token)
	storageToken := base64.StdEncoding.EncodeToString(sum[:])

	// set key in redis
	err = cache.SetEx(storageToken, 2592000, []byte(uid))
	if err != nil {
		return
	}

	return

}

// validate compares provided session id to redis
func ValidateSession(key []byte) (err error) {

	// Initialize cache handle
	cache := RedisCache

	// decode key
	token, err := base64.URLEncoding.DecodeString(string(key))
	if err != nil {
		return
	}

	// get uid
	index := bytes.IndexByte(token, ';')

	// check to see if user is there
	if index < 0 {
		return e.ErrInvalidSession
	}

	// get given uid
	uid := string(token[:index])

	// hash token
	sum := md5.Sum(token)

	// base64 encode sum
	providedHash := base64.StdEncoding.EncodeToString(sum[:])

	// check for match
	result, err := cache.Get(providedHash)
	if err == ErrCacheMiss {
		return e.ErrInvalidSession
	}
	if err != nil {
		return
	}

	// check if uid matches
	if uid != string(result) {
		return e.ErrInvalidSession
	}

	return

}
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"github.com/mitchellh/mapstructure"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"testing"
)

func TestCheckUnusedConfig(t *testing.T) {
	md := &mapstructure.Metadata{
		Unused: make([]string, 0),
	}

	err := CheckUnusedConfig(md)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	md.Unused = []string{"foo", "bar"}
	err = CheckUnusedConfig(md)
	if err == nil {
		t.Fatal("should have error")
	}
}

func TestDecodeConfig(t *testing.T) {
	type Local struct {
		Foo string
		Bar string
	}

	raws := []interface{}{
		map[string]interface{}{
			"foo": "bar",
		},
		map[string]interface{}{
			"bar": "baz",
			"baz": "what",
		},
	}

	var result Local
	md, err := DecodeConfig(&result, raws...)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if result.Foo != "bar" {
		t.Fatalf("invalid: %#v", result.Foo)
	}

	if result.Bar != "baz" {
		t.Fatalf("invalid: %#v", result.Bar)
	}

	if md == nil {
		t.Fatal("metadata should not be nil")
	}

	if !reflect.DeepEqual(md.Unused, []string{"baz"}) {
		t.Fatalf("unused: %#v", md.Unused)
	}
}

func TestDownloadableURL(t *testing.T) {
	// Invalid URL: has hex code in host
	_, err := DownloadableURL("http://what%20.com")
	if err == nil {
		t.Fatal("expected err")
	}

	// Invalid: unsupported scheme
	_, err = DownloadableURL("ftp://host.com/path")
	if err == nil {
		t.Fatal("expected err")
	}

	// Valid: http
	u, err := DownloadableURL("HTTP://packer.io/path")
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if u != "http://packer.io/path" {
		t.Fatalf("bad: %s", u)
	}

	// No path
	u, err = DownloadableURL("HTTP://packer.io")
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if u != "http://packer.io" {
		t.Fatalf("bad: %s", u)
	}
}

func TestDownloadableURL_FilePaths(t *testing.T) {
	tf, err := ioutil.TempFile("", "packer")
	if err != nil {
		t.Fatalf("tempfile err: %s", err)
	}
	defer os.Remove(tf.Name())
	tf.Close()

	tfPath, err := filepath.EvalSymlinks(tf.Name())
	if err != nil {
		t.Fatalf("tempfile err: %s", err)
	}

	tfPath = filepath.Clean(tfPath)

	// Relative filepath. We run this test in a func so that
	// the defers run right away.
	func() {
		wd, err := os.Getwd()
		if err != nil {
			t.Fatalf("getwd err: %s", err)
		}

		err = os.Chdir(filepath.Dir(tfPath))
		if err != nil {
			t.Fatalf("chdir err: %s", err)
		}
		defer os.Chdir(wd)

		filename := filepath.Base(tfPath)
		u, err := DownloadableURL(filename)
		if err != nil {
			t.Fatalf("err: %s", err)
		}

		if u != fmt.Sprintf("file://%s", tfPath) {
			t.Fatalf("unexpected: %s", u)
		}
	}()

	// Test some cases with and without a schema prefix
	for _, prefix := range []string{"", "file://"} {
		// Nonexistent file
		_, err = DownloadableURL(prefix + "i/dont/exist")
		if err == nil {
			t.Fatal("expected err")
		}

		// Good file
		u, err := DownloadableURL(prefix + tfPath)
		if err != nil {
			t.Fatalf("err: %s", err)
		}

		if u != fmt.Sprintf("file://%s", tfPath) {
			t.Fatalf("unexpected: %s", u)
		}
	}
}

func TestScrubConfig(t *testing.T) {
	type Inner struct {
		Baz string
	}
	type Local struct {
		Foo string
		Bar string
		Inner
	}
	c := Local{"foo", "bar", Inner{"bar"}}
	expect := "Config: {Foo:foo Bar:<Filtered> Inner:{Baz:<Filtered>}}"
	conf := ScrubConfig(c, c.Bar)
	if conf != expect {
		t.Fatalf("got %s, expected %s", conf, expect)
	}
}
<commit_msg>common: fix tests<commit_after>package common

import (
	"fmt"
	"github.com/mitchellh/mapstructure"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"testing"
)

func TestCheckUnusedConfig(t *testing.T) {
	md := &mapstructure.Metadata{
		Unused: make([]string, 0),
	}

	err := CheckUnusedConfig(md)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	md.Unused = []string{"foo", "bar"}
	err = CheckUnusedConfig(md)
	if err == nil {
		t.Fatal("should have error")
	}
}

func TestDecodeConfig(t *testing.T) {
	type Local struct {
		Foo string
		Bar string
	}

	raws := []interface{}{
		map[string]interface{}{
			"foo": "bar",
		},
		map[string]interface{}{
			"bar": "baz",
			"baz": "what",
		},
	}

	var result Local
	md, err := DecodeConfig(&result, raws...)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if result.Foo != "bar" {
		t.Fatalf("invalid: %#v", result.Foo)
	}

	if result.Bar != "baz" {
		t.Fatalf("invalid: %#v", result.Bar)
	}

	if md == nil {
		t.Fatal("metadata should not be nil")
	}

	if !reflect.DeepEqual(md.Unused, []string{"baz"}) {
		t.Fatalf("unused: %#v", md.Unused)
	}
}

func TestDownloadableURL(t *testing.T) {
	// Invalid URL: has hex code in host
	_, err := DownloadableURL("http://what%20.com")
	if err == nil {
		t.Fatal("expected err")
	}

	// Invalid: unsupported scheme
	_, err = DownloadableURL("ftp://host.com/path")
	if err == nil {
		t.Fatal("expected err")
	}

	// Valid: http
	u, err := DownloadableURL("HTTP://packer.io/path")
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if u != "http://packer.io/path" {
		t.Fatalf("bad: %s", u)
	}

	// No path
	u, err = DownloadableURL("HTTP://packer.io")
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if u != "http://packer.io" {
		t.Fatalf("bad: %s", u)
	}
}

func TestDownloadableURL_FilePaths(t *testing.T) {
	tf, err := ioutil.TempFile("", "packer")
	if err != nil {
		t.Fatalf("tempfile err: %s", err)
	}
	defer os.Remove(tf.Name())
	tf.Close()

	tfPath, err := filepath.EvalSymlinks(tf.Name())
	if err != nil {
		t.Fatalf("tempfile err: %s", err)
	}

	tfPath = filepath.Clean(tfPath)

	// Relative filepath. We run this test in a func so that
	// the defers run right away.
	func() {
		wd, err := os.Getwd()
		if err != nil {
			t.Fatalf("getwd err: %s", err)
		}

		err = os.Chdir(filepath.Dir(tfPath))
		if err != nil {
			t.Fatalf("chdir err: %s", err)
		}
		defer os.Chdir(wd)

		filename := filepath.Base(tfPath)
		u, err := DownloadableURL(filename)
		if err != nil {
			t.Fatalf("err: %s", err)
		}

		if u != fmt.Sprintf("file://%s", tfPath) {
			t.Fatalf("unexpected: %s", u)
		}
	}()

	// Test some cases with and without a schema prefix
	for _, prefix := range []string{"", "file://"} {
		// Nonexistent file
		_, err = DownloadableURL(prefix + "i/dont/exist")
		if err != nil {
			t.Fatalf("err: %s", err)
		}

		// Good file
		u, err := DownloadableURL(prefix + tfPath)
		if err != nil {
			t.Fatalf("err: %s", err)
		}

		if u != fmt.Sprintf("file://%s", tfPath) {
			t.Fatalf("unexpected: %s", u)
		}
	}
}

func TestScrubConfig(t *testing.T) {
	type Inner struct {
		Baz string
	}
	type Local struct {
		Foo string
		Bar string
		Inner
	}
	c := Local{"foo", "bar", Inner{"bar"}}
	expect := "Config: {Foo:foo Bar:<Filtered> Inner:{Baz:<Filtered>}}"
	conf := ScrubConfig(c, c.Bar)
	if conf != expect {
		t.Fatalf("got %s, expected %s", conf, expect)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"

	humanize "github.com/dustin/go-humanize"
	"github.com/gorilla/schema"
	"github.com/igungor/cmd/filmdizibot/bot"
	"github.com/igungor/cmd/filmdizibot/command"
	"github.com/igungor/telegram"
)

const groupWhatsup = -230439016

func main() {
	log.SetFlags(0)
	log.SetPrefix("filmdizibot: ")
	var (
		flagHost = flag.String("h", "0.0.0.0", "host to listen to")
		flagPort = flag.String("p", "1989", "port to listen to")
	)
	flag.Parse()

	ctx := context.Background()
	bot, err := bot.New(ctx)
	if err != nil {
		log.Fatalf("Error creating the bot: %v\n", err)
	}

	md := telegram.WithParseMode(telegram.ModeMarkdown)

	mux := http.NewServeMux()
	mux.HandleFunc("/", bot.Handler())
	mux.HandleFunc("/cb", func(w http.ResponseWriter, r *http.Request) {
		err := r.ParseForm()
		if err != nil {
			bot.SendMessage(groupWhatsup, fmt.Sprintf("ParseForm failed: %v", err))
			return
		}

		dec := schema.NewDecoder()
		dec.IgnoreUnknownKeys(true)
		dec.SetAliasTag("json")

		var t transfer
		err = dec.Decode(&t, r.PostForm)
		if err != nil {
			bot.SendMessage(groupWhatsup, fmt.Sprintf("Decode failed: %v", err))
			return
		}

		// ignore spam requests
		if t.Name == "" {
			return
		}

		txt := fmt.Sprintf("🗣 New file downloaded!\n\n*%v*\n\nSize: %v", t.Name, humanize.Bytes(uint64(t.Size)))
		bot.SendMessage(groupWhatsup, txt, md)
	})

	addr := net.JoinHostPort(*flagHost, *flagPort)
	go func() {
		log.Fatal(http.ListenAndServe(addr, mux))
	}()

	for msg := range bot.Messages() {
		if msg.IsService() {
			log.Printf("Skipping service message...\n")
			continue
		}

		cmdname := msg.Command()
		cmd := command.Match(cmdname)
		if cmd == nil {
			continue
		}

		log.Printf("New request: %v\n", msg.Text)
		go cmd.Run(ctx, bot, msg)
	}
}

type transfer struct {
	Name       string `json:"name"`
	Size       int    `json:"size"`
	FileID     int64  `json:"file_id"`
	DownloadID int64  `json:"download_id"`
	ParentID   int64  `json:"save_parent_id"`
}

func (t transfer) String() string {
	return fmt.Sprintf("%q\n\n indirilmeye baslandi.\n Boyut: **%v**\n", t.Name, humanize.Bytes(uint64(t.Size)))
}
<commit_msg>filmdizibot: make addr variable local<commit_after>package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"

	humanize "github.com/dustin/go-humanize"
	"github.com/gorilla/schema"
	"github.com/igungor/cmd/filmdizibot/bot"
	"github.com/igungor/cmd/filmdizibot/command"
	"github.com/igungor/telegram"
)

const groupWhatsup = -230439016

func main() {
	log.SetFlags(0)
	log.SetPrefix("filmdizibot: ")
	var (
		flagHost = flag.String("h", "0.0.0.0", "host to listen to")
		flagPort = flag.String("p", "1989", "port to listen to")
	)
	flag.Parse()

	ctx := context.Background()
	bot, err := bot.New(ctx)
	if err != nil {
		log.Fatalf("Error creating the bot: %v\n", err)
	}

	md := telegram.WithParseMode(telegram.ModeMarkdown)

	mux := http.NewServeMux()
	mux.HandleFunc("/", bot.Handler())
	mux.HandleFunc("/cb", func(w http.ResponseWriter, r *http.Request) {
		err := r.ParseForm()
		if err != nil {
			bot.SendMessage(groupWhatsup, fmt.Sprintf("ParseForm failed: %v", err))
			return
		}

		dec := schema.NewDecoder()
		dec.IgnoreUnknownKeys(true)
		dec.SetAliasTag("json")

		var t transfer
		err = dec.Decode(&t, r.PostForm)
		if err != nil {
			bot.SendMessage(groupWhatsup, fmt.Sprintf("Decode failed: %v", err))
			return
		}

		// ignore spam requests
		if t.Name == "" {
			return
		}

		txt := fmt.Sprintf("🗣 New file downloaded!\n\n*%v*\n\nSize: %v", t.Name, humanize.Bytes(uint64(t.Size)))
		bot.SendMessage(groupWhatsup, txt, md)
	})

	go func() {
		addr := net.JoinHostPort(*flagHost, *flagPort)
		log.Fatal(http.ListenAndServe(addr, mux))
	}()

	for msg := range bot.Messages() {
		if msg.IsService() {
			log.Printf("Skipping service message...\n")
			continue
		}

		cmdname := msg.Command()
		cmd := command.Match(cmdname)
		if cmd == nil {
			continue
		}

		log.Printf("New request: %v\n", msg.Text)
		go cmd.Run(ctx, bot, msg)
	}
}

type transfer struct {
	Name       string `json:"name"`
	Size       int    `json:"size"`
	FileID     int64  `json:"file_id"`
	DownloadID int64  `json:"download_id"`
	ParentID   int64  `json:"save_parent_id"`
}

func (t transfer) String() string {
	return fmt.Sprintf("%q\n\n indirilmeye baslandi.\n Boyut: **%v**\n", t.Name, humanize.Bytes(uint64(t.Size)))
}
<|endoftext|>


<commit_before>package filters

import (
	"fmt"
	base "github.com/sjwhitworth/golearn/base"
	"math"
)

// ChiMergeFilter implements supervised discretisation
// by merging successive numeric intervals if the difference
// in their class distribution is not statistically signficant.
// See Bramer, "Principles of Data Mining", 2nd Edition
//  pp 105--115
type ChiMergeFilter struct {
	Attributes   []int
	Instances    *base.Instances
	Tables       map[int][]*FrequencyTableEntry
	Significance float64
	MinRows      int
	MaxRows      int
	_Trained     bool
}

// NewChiMergeFilter creates a ChiMergeFilter with some helpful initialisations.
func NewChiMergeFilter(inst *base.Instances, significance float64) ChiMergeFilter {
	return ChiMergeFilter{
		make([]int, 0),
		inst,
		make(map[int][]*FrequencyTableEntry),
		significance,
		0,
		0,
		false,
	}
}

// Build trains a ChiMergeFilter on the ChiMergeFilter.Instances given
func (c *ChiMergeFilter) Build() {
	for _, attr := range c.Attributes {
		tab := chiMerge(c.Instances, attr, c.Significance, c.MinRows, c.MaxRows)
		c.Tables[attr] = tab
		c._Trained = true
	}
}

// AddAllNumericAttributes adds every suitable attribute
// to the ChiMergeFilter for discretisation
func (c *ChiMergeFilter) AddAllNumericAttributes() {
	for i := 0; i < c.Instances.Cols; i++ {
		if i == c.Instances.ClassIndex {
			continue
		}
		attr := c.Instances.GetAttr(i)
		if attr.GetType() != base.Float64Type {
			continue
		}
		c.Attributes = append(c.Attributes, i)
	}
}

// Run discretises the set of Instances `on'
//
// IMPORTANT: ChiMergeFilter discretises in place.
func (c *ChiMergeFilter) Run(on *base.Instances) {
	if !c._Trained {
		panic("Call Build() beforehand")
	}
	for attr := range c.Tables {
		table := c.Tables[attr]
		for i := 0; i < on.Rows; i++ {
			val := on.Get(i, attr)
			dis := 0
			for j, k := range table {
				if k.Value < val {
					dis = j
					continue
				}
				break
			}
			on.Set(i, attr, float64(dis))
		}
		newAttribute := new(base.CategoricalAttribute)
		newAttribute.SetName(on.GetAttr(attr).GetName())
		for _, k := range table {
			newAttribute.GetSysValFromString(fmt.Sprintf("%f", k.Value))
		}
		on.ReplaceAttr(attr, newAttribute)
	}
}

// AddAttribute add a given numeric Attribute `attr' to the
// filter.
//
// IMPORTANT: This function panic()s if it can't locate the
// attribute in the Instances set.
func (c *ChiMergeFilter) AddAttribute(attr base.Attribute) {
	if attr.GetType() != base.Float64Type {
		panic("ChiMerge only works on Float64Attributes")
	}
	attrIndex := c.Instances.GetAttrIndex(attr)
	if attrIndex == -1 {
		panic("Invalid attribute!")
	}
	c.Attributes = append(c.Attributes, attrIndex)
}

type FrequencyTableEntry struct {
	Value     float64
	Frequency map[string]int
}

func (t *FrequencyTableEntry) String() string {
	return fmt.Sprintf("%.2f %s", t.Value, t.Frequency)
}

func ChiMBuildFrequencyTable(attr int, inst *base.Instances) []*FrequencyTableEntry {
	ret := make([]*FrequencyTableEntry, 0)
	var attribute *base.FloatAttribute
	attribute, ok := inst.GetAttr(attr).(*base.FloatAttribute)
	if !ok {
		panic("only use Chi-M on numeric stuff")
	}
	for i := 0; i < inst.Rows; i++ {
		value := inst.Get(i, attr)
		valueConv := attribute.GetUsrVal(value)
		class := inst.GetClass(i)
		// Search the frequency table for the value
		found := false
		for _, entry := range ret {
			if entry.Value == valueConv {
				found = true
				entry.Frequency[class]++
			}
		}
		if !found {
			newEntry := &FrequencyTableEntry{
				valueConv,
				make(map[string]int),
			}
			newEntry.Frequency[class] = 1
			ret = append(ret, newEntry)
		}
	}

	return ret
}

func chiSquaredPdf(k float64, x float64) float64 {
	if x < 0 {
		return 0
	}
	top := math.Pow(x, (k/2)-1) * math.Exp(-x/2)
	bottom := math.Pow(2, k/2) * math.Gamma(k/2)
	return top / bottom
}

func chiSquaredPercentile(k int, x float64) float64 {
	// Implements Yahya et al.'s "A Numerical Procedure
	//  for Computing Chi-Square Percentage Points"
	// InterStat Journal 01/2007; April 25:page:1-8.
	steps := 32
	intervals := 4 * steps
	w := x / (4.0 * float64(steps))
	values := make([]float64, intervals+1)
	for i := 0; i < intervals+1; i++ {
		c := w * float64(i)
		v := chiSquaredPdf(float64(k), c)
		values[i] = v
	}

	ret1 := values[0] + values[len(values)-1]
	ret2 := 0.0
	ret3 := 0.0
	ret4 := 0.0

	for i := 2; i < intervals-1; i += 4 {
		ret2 += values[i]
	}

	for i := 4; i < intervals-3; i += 4 {
		ret3 += values[i]
	}

	for i := 1; i < intervals; i += 2 {
		ret4 += values[i]
	}

	return (2.0 * w / 45) * (7*ret1 + 12*ret2 + 14*ret3 + 32*ret4)
}

func chiCountClasses(entries []*FrequencyTableEntry) map[string]int {
	classCounter := make(map[string]int)
	for _, e := range entries {
		for k := range e.Frequency {
			classCounter[k] += e.Frequency[k]
		}
	}
	return classCounter
}

func chiComputeStatistic(entry1 *FrequencyTableEntry, entry2 *FrequencyTableEntry) float64 {

	// Sum the number of things observed per class
	classCounter := make(map[string]int)
	for k := range entry1.Frequency {
		classCounter[k] += entry1.Frequency[k]
	}
	for k := range entry2.Frequency {
		classCounter[k] += entry2.Frequency[k]
	}

	// Sum the number of things observed per value
	entryObservations1 := 0
	entryObservations2 := 0
	for k := range entry1.Frequency {
		entryObservations1 += entry1.Frequency[k]
	}
	for k := range entry2.Frequency {
		entryObservations2 += entry2.Frequency[k]
	}

	totalObservations := entryObservations1 + entryObservations2
	// Compute the expected values per class
	expectedClassValues1 := make(map[string]float64)
	expectedClassValues2 := make(map[string]float64)
	for k := range classCounter {
		expectedClassValues1[k] = float64(classCounter[k])
		expectedClassValues1[k] *= float64(entryObservations1)
		expectedClassValues1[k] /= float64(totalObservations)
	}
	for k := range classCounter {
		expectedClassValues2[k] = float64(classCounter[k])
		expectedClassValues2[k] *= float64(entryObservations2)
		expectedClassValues2[k] /= float64(totalObservations)
	}

	// Compute chi-squared value
	chiSum := 0.0
	for k := range expectedClassValues1 {
		numerator := float64(entry1.Frequency[k])
		numerator -= expectedClassValues1[k]
		numerator = math.Pow(numerator, 2)
		denominator := float64(expectedClassValues1[k])
		if denominator < 0.5 {
			denominator = 0.5
		}
		chiSum += numerator / denominator
	}
	for k := range expectedClassValues2 {
		numerator := float64(entry2.Frequency[k])
		numerator -= expectedClassValues2[k]
		numerator = math.Pow(numerator, 2)
		denominator := float64(expectedClassValues2[k])
		if denominator < 0.5 {
			denominator = 0.5
		}
		chiSum += numerator / denominator
	}

	return chiSum
}

func chiMergeMergeZipAdjacent(freq []*FrequencyTableEntry, minIndex int) []*FrequencyTableEntry {
	mergeEntry1 := freq[minIndex]
	mergeEntry2 := freq[minIndex+1]
	classCounter := make(map[string]int)
	for k := range mergeEntry1.Frequency {
		classCounter[k] += mergeEntry1.Frequency[k]
	}
	for k := range mergeEntry2.Frequency {
		classCounter[k] += mergeEntry2.Frequency[k]
	}
	newVal := freq[minIndex].Value
	newEntry := &FrequencyTableEntry{
		newVal,
		classCounter,
	}
	lowerSlice := freq
	upperSlice := freq
	if minIndex > 0 {
		lowerSlice = freq[0:minIndex]
		upperSlice = freq[minIndex+1:]
	} else {
		lowerSlice = make([]*FrequencyTableEntry, 0)
		upperSlice = freq[1:]
	}
	upperSlice[0] = newEntry
	freq = append(lowerSlice, upperSlice...)
	return freq
}

func chiMergePrintTable(freq []*FrequencyTableEntry) {
	classes := chiCountClasses(freq)
	fmt.Printf("Attribute value\t")
	for k := range classes {
		fmt.Printf("\t%s", k)
	}
	fmt.Printf("\tTotal\n")
	for _, f := range freq {
		fmt.Printf("%.2f\t", f.Value)
		total := 0
		for k := range classes {
			fmt.Printf("\t%d", f.Frequency[k])
			total += f.Frequency[k]
		}
		fmt.Printf("\t%d\n", total)
	}
}

// Produces a value mapping table
//   inst: The base.Instances which need discretising
//   sig:  The significance level (e.g. 0.95)
//   minrows: The minimum number of rows required in the frequency table
//   maxrows: The maximum number of rows allowed in the frequency table
//            If the number of rows is above this, statistically signficant
//            adjacent rows will be merged
//   precision: internal number of decimal places to round E value to
//              (useful for verification)
func chiMerge(inst *base.Instances, attr int, sig float64, minrows int, maxrows int) []*FrequencyTableEntry {

	// Parameter sanity checking
	if !(2 <= minrows) {
		minrows = 2
	}
	if !(minrows < maxrows) {
		maxrows = minrows + 1
	}
	if sig == 0 {
		sig = 10
	}

	// Build a frequency table
	freq := ChiMBuildFrequencyTable(attr, inst)
	// Count the number of classes
	classes := chiCountClasses(freq)
	for {
		// chiMergePrintTable(freq) DEBUG
		if len(freq) <= minrows {
			break
		}
		minChiVal := math.Inf(1)
		// There may be more than one index to merge
		minChiIndexes := make([]int, 0)
		for i := 0; i < len(freq)-1; i++ {
			chiVal := chiComputeStatistic(freq[i], freq[i+1])
			if chiVal < minChiVal {
				minChiVal = chiVal
				minChiIndexes = make([]int, 0)
			}
			if chiVal == minChiVal {
				minChiIndexes = append(minChiIndexes, i)
			}
		}
		// Only merge if:
		//  We're above the maximum number of rows
		//  OR the chiVal is significant
		//   AS LONG AS we're above the minimum row count
		merge := false
		if len(freq) > maxrows {
			merge = true
		}
		// Compute the degress of freedom |classes - 1| * |rows - 1|
		degsOfFree := len(classes) - 1
		sigVal := chiSquaredPercentile(degsOfFree, minChiVal)
		if sigVal < sig {
			merge = true
		}
		// If we don't need to merge, then break
		if !merge {
			break
		}
		// Otherwise merge the rows i, i+1 by taking
		//  The higher of the two things as the value
		//  Combining the class frequencies
		for i, v := range minChiIndexes {
			freq = chiMergeMergeZipAdjacent(freq, v-i)
		}
	}
	return freq
}
<commit_msg>arg t.Frequency for printf verb %s of wrong type: map[string]int<commit_after>package filters

import (
	"fmt"
	base "github.com/sjwhitworth/golearn/base"
	"math"
)

// ChiMergeFilter implements supervised discretisation
// by merging successive numeric intervals if the difference
// in their class distribution is not statistically signficant.
// See Bramer, "Principles of Data Mining", 2nd Edition
//  pp 105--115
type ChiMergeFilter struct {
	Attributes   []int
	Instances    *base.Instances
	Tables       map[int][]*FrequencyTableEntry
	Significance float64
	MinRows      int
	MaxRows      int
	_Trained     bool
}

// NewChiMergeFilter creates a ChiMergeFilter with some helpful initialisations.
func NewChiMergeFilter(inst *base.Instances, significance float64) ChiMergeFilter {
	return ChiMergeFilter{
		make([]int, 0),
		inst,
		make(map[int][]*FrequencyTableEntry),
		significance,
		0,
		0,
		false,
	}
}

// Build trains a ChiMergeFilter on the ChiMergeFilter.Instances given
func (c *ChiMergeFilter) Build() {
	for _, attr := range c.Attributes {
		tab := chiMerge(c.Instances, attr, c.Significance, c.MinRows, c.MaxRows)
		c.Tables[attr] = tab
		c._Trained = true
	}
}

// AddAllNumericAttributes adds every suitable attribute
// to the ChiMergeFilter for discretisation
func (c *ChiMergeFilter) AddAllNumericAttributes() {
	for i := 0; i < c.Instances.Cols; i++ {
		if i == c.Instances.ClassIndex {
			continue
		}
		attr := c.Instances.GetAttr(i)
		if attr.GetType() != base.Float64Type {
			continue
		}
		c.Attributes = append(c.Attributes, i)
	}
}

// Run discretises the set of Instances `on'
//
// IMPORTANT: ChiMergeFilter discretises in place.
func (c *ChiMergeFilter) Run(on *base.Instances) {
	if !c._Trained {
		panic("Call Build() beforehand")
	}
	for attr := range c.Tables {
		table := c.Tables[attr]
		for i := 0; i < on.Rows; i++ {
			val := on.Get(i, attr)
			dis := 0
			for j, k := range table {
				if k.Value < val {
					dis = j
					continue
				}
				break
			}
			on.Set(i, attr, float64(dis))
		}
		newAttribute := new(base.CategoricalAttribute)
		newAttribute.SetName(on.GetAttr(attr).GetName())
		for _, k := range table {
			newAttribute.GetSysValFromString(fmt.Sprintf("%f", k.Value))
		}
		on.ReplaceAttr(attr, newAttribute)
	}
}

// AddAttribute add a given numeric Attribute `attr' to the
// filter.
//
// IMPORTANT: This function panic()s if it can't locate the
// attribute in the Instances set.
func (c *ChiMergeFilter) AddAttribute(attr base.Attribute) {
	if attr.GetType() != base.Float64Type {
		panic("ChiMerge only works on Float64Attributes")
	}
	attrIndex := c.Instances.GetAttrIndex(attr)
	if attrIndex == -1 {
		panic("Invalid attribute!")
	}
	c.Attributes = append(c.Attributes, attrIndex)
}

type FrequencyTableEntry struct {
	Value     float64
	Frequency map[string]int
}

func (t *FrequencyTableEntry) String() string {
	return fmt.Sprintf("%.2f %v", t.Value, t.Frequency)
}

func ChiMBuildFrequencyTable(attr int, inst *base.Instances) []*FrequencyTableEntry {
	ret := make([]*FrequencyTableEntry, 0)
	var attribute *base.FloatAttribute
	attribute, ok := inst.GetAttr(attr).(*base.FloatAttribute)
	if !ok {
		panic("only use Chi-M on numeric stuff")
	}
	for i := 0; i < inst.Rows; i++ {
		value := inst.Get(i, attr)
		valueConv := attribute.GetUsrVal(value)
		class := inst.GetClass(i)
		// Search the frequency table for the value
		found := false
		for _, entry := range ret {
			if entry.Value == valueConv {
				found = true
				entry.Frequency[class]++
			}
		}
		if !found {
			newEntry := &FrequencyTableEntry{
				valueConv,
				make(map[string]int),
			}
			newEntry.Frequency[class] = 1
			ret = append(ret, newEntry)
		}
	}

	return ret
}

func chiSquaredPdf(k float64, x float64) float64 {
	if x < 0 {
		return 0
	}
	top := math.Pow(x, (k/2)-1) * math.Exp(-x/2)
	bottom := math.Pow(2, k/2) * math.Gamma(k/2)
	return top / bottom
}

func chiSquaredPercentile(k int, x float64) float64 {
	// Implements Yahya et al.'s "A Numerical Procedure
	//  for Computing Chi-Square Percentage Points"
	// InterStat Journal 01/2007; April 25:page:1-8.
	steps := 32
	intervals := 4 * steps
	w := x / (4.0 * float64(steps))
	values := make([]float64, intervals+1)
	for i := 0; i < intervals+1; i++ {
		c := w * float64(i)
		v := chiSquaredPdf(float64(k), c)
		values[i] = v
	}

	ret1 := values[0] + values[len(values)-1]
	ret2 := 0.0
	ret3 := 0.0
	ret4 := 0.0

	for i := 2; i < intervals-1; i += 4 {
		ret2 += values[i]
	}

	for i := 4; i < intervals-3; i += 4 {
		ret3 += values[i]
	}

	for i := 1; i < intervals; i += 2 {
		ret4 += values[i]
	}

	return (2.0 * w / 45) * (7*ret1 + 12*ret2 + 14*ret3 + 32*ret4)
}

func chiCountClasses(entries []*FrequencyTableEntry) map[string]int {
	classCounter := make(map[string]int)
	for _, e := range entries {
		for k := range e.Frequency {
			classCounter[k] += e.Frequency[k]
		}
	}
	return classCounter
}

func chiComputeStatistic(entry1 *FrequencyTableEntry, entry2 *FrequencyTableEntry) float64 {

	// Sum the number of things observed per class
	classCounter := make(map[string]int)
	for k := range entry1.Frequency {
		classCounter[k] += entry1.Frequency[k]
	}
	for k := range entry2.Frequency {
		classCounter[k] += entry2.Frequency[k]
	}

	// Sum the number of things observed per value
	entryObservations1 := 0
	entryObservations2 := 0
	for k := range entry1.Frequency {
		entryObservations1 += entry1.Frequency[k]
	}
	for k := range entry2.Frequency {
		entryObservations2 += entry2.Frequency[k]
	}

	totalObservations := entryObservations1 + entryObservations2
	// Compute the expected values per class
	expectedClassValues1 := make(map[string]float64)
	expectedClassValues2 := make(map[string]float64)
	for k := range classCounter {
		expectedClassValues1[k] = float64(classCounter[k])
		expectedClassValues1[k] *= float64(entryObservations1)
		expectedClassValues1[k] /= float64(totalObservations)
	}
	for k := range classCounter {
		expectedClassValues2[k] = float64(classCounter[k])
		expectedClassValues2[k] *= float64(entryObservations2)
		expectedClassValues2[k] /= float64(totalObservations)
	}

	// Compute chi-squared value
	chiSum := 0.0
	for k := range expectedClassValues1 {
		numerator := float64(entry1.Frequency[k])
		numerator -= expectedClassValues1[k]
		numerator = math.Pow(numerator, 2)
		denominator := float64(expectedClassValues1[k])
		if denominator < 0.5 {
			denominator = 0.5
		}
		chiSum += numerator / denominator
	}
	for k := range expectedClassValues2 {
		numerator := float64(entry2.Frequency[k])
		numerator -= expectedClassValues2[k]
		numerator = math.Pow(numerator, 2)
		denominator := float64(expectedClassValues2[k])
		if denominator < 0.5 {
			denominator = 0.5
		}
		chiSum += numerator / denominator
	}

	return chiSum
}

func chiMergeMergeZipAdjacent(freq []*FrequencyTableEntry, minIndex int) []*FrequencyTableEntry {
	mergeEntry1 := freq[minIndex]
	mergeEntry2 := freq[minIndex+1]
	classCounter := make(map[string]int)
	for k := range mergeEntry1.Frequency {
		classCounter[k] += mergeEntry1.Frequency[k]
	}
	for k := range mergeEntry2.Frequency {
		classCounter[k] += mergeEntry2.Frequency[k]
	}
	newVal := freq[minIndex].Value
	newEntry := &FrequencyTableEntry{
		newVal,
		classCounter,
	}
	lowerSlice := freq
	upperSlice := freq
	if minIndex > 0 {
		lowerSlice = freq[0:minIndex]
		upperSlice = freq[minIndex+1:]
	} else {
		lowerSlice = make([]*FrequencyTableEntry, 0)
		upperSlice = freq[1:]
	}
	upperSlice[0] = newEntry
	freq = append(lowerSlice, upperSlice...)
	return freq
}

func chiMergePrintTable(freq []*FrequencyTableEntry) {
	classes := chiCountClasses(freq)
	fmt.Printf("Attribute value\t")
	for k := range classes {
		fmt.Printf("\t%s", k)
	}
	fmt.Printf("\tTotal\n")
	for _, f := range freq {
		fmt.Printf("%.2f\t", f.Value)
		total := 0
		for k := range classes {
			fmt.Printf("\t%d", f.Frequency[k])
			total += f.Frequency[k]
		}
		fmt.Printf("\t%d\n", total)
	}
}

// Produces a value mapping table
//   inst: The base.Instances which need discretising
//   sig:  The significance level (e.g. 0.95)
//   minrows: The minimum number of rows required in the frequency table
//   maxrows: The maximum number of rows allowed in the frequency table
//            If the number of rows is above this, statistically signficant
//            adjacent rows will be merged
//   precision: internal number of decimal places to round E value to
//              (useful for verification)
func chiMerge(inst *base.Instances, attr int, sig float64, minrows int, maxrows int) []*FrequencyTableEntry {

	// Parameter sanity checking
	if !(2 <= minrows) {
		minrows = 2
	}
	if !(minrows < maxrows) {
		maxrows = minrows + 1
	}
	if sig == 0 {
		sig = 10
	}

	// Build a frequency table
	freq := ChiMBuildFrequencyTable(attr, inst)
	// Count the number of classes
	classes := chiCountClasses(freq)
	for {
		// chiMergePrintTable(freq) DEBUG
		if len(freq) <= minrows {
			break
		}
		minChiVal := math.Inf(1)
		// There may be more than one index to merge
		minChiIndexes := make([]int, 0)
		for i := 0; i < len(freq)-1; i++ {
			chiVal := chiComputeStatistic(freq[i], freq[i+1])
			if chiVal < minChiVal {
				minChiVal = chiVal
				minChiIndexes = make([]int, 0)
			}
			if chiVal == minChiVal {
				minChiIndexes = append(minChiIndexes, i)
			}
		}
		// Only merge if:
		//  We're above the maximum number of rows
		//  OR the chiVal is significant
		//   AS LONG AS we're above the minimum row count
		merge := false
		if len(freq) > maxrows {
			merge = true
		}
		// Compute the degress of freedom |classes - 1| * |rows - 1|
		degsOfFree := len(classes) - 1
		sigVal := chiSquaredPercentile(degsOfFree, minChiVal)
		if sigVal < sig {
			merge = true
		}
		// If we don't need to merge, then break
		if !merge {
			break
		}
		// Otherwise merge the rows i, i+1 by taking
		//  The higher of the two things as the value
		//  Combining the class frequencies
		for i, v := range minChiIndexes {
			freq = chiMergeMergeZipAdjacent(freq, v-i)
		}
	}
	return freq
}
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"strconv"
	"strings"
)

type CallFuncType func([]string) IResponse

type ItemsResponse struct {
	items []IItem
}

func NewItemsResponse(items []IItem) *ItemsResponse {
	return &ItemsResponse{items}
}

func (self *ItemsResponse) GetResponse() string {
	data := make([]string, 0, 3+9*len(self.items))
	data = append(data, "+DATA %")
	data = append(data, strconv.Itoa(len(self.items)))
	for _, item := range self.items {
		data = append(data, " ")
		itemId := item.GetId()
		itemData := item.GetContent()
		data = append(data, "$")
		data = append(data, strconv.Itoa(len(itemId)))
		data = append(data, " ")
		data = append(data, itemId)
		data = append(data, "$")
		data = append(data, strconv.Itoa(len(itemData)))
		data = append(data, " ")
		data = append(data, itemData)
	}
	return strings.Join(data, "")
}

func (self *ItemsResponse) IsError() bool {
	return false
}

// Error response.
type ErrorResponse struct {
	ErrorText string
	ErrorCode int64
}

func (e *ErrorResponse) Error() string {
	return e.ErrorText
}

func (e *ErrorResponse) GetResponse() string {
	return fmt.Sprintf("-ERR:%d:%s", e.ErrorCode, e.ErrorText)
}

func (e *ErrorResponse) IsError() bool {
	return true
}

// Error response.
type OkResponse struct {
	Code int64
}

func (e *OkResponse) GetResponse() string {
	return fmt.Sprintf("+OK:%d", e.Code)
}

func (self *OkResponse) IsError() bool {
	return false
}

var OK200_RESPONSE *OkResponse = &OkResponse{200}
<commit_msg>Added dictionary response builder for stats.<commit_after>package common

import (
	"fmt"
	"strconv"
	"strings"
)

type CallFuncType func([]string) IResponse

type ItemsResponse struct {
	items []IItem
}

type DictResponse struct {
	dict map[string]interface{}
}

func NewDictResponse(dict map[string]interface{}) *DictResponse {
	return &DictResponse{dict}
}

func (self *DictResponse) GetResponse() string {
	data := make([]string, 0, 3+9*len(self.dict))
	data = append(data, "+DATA %")
	data = append(data, strconv.Itoa(len(self.dict)))
	for k, v := range self.dict {
		data = append(data, "\n")
		data = append(data, k)
		data = append(data, " ")
		switch t := v.(type) {
		case string:
			data = append(data, t)
		case int:
			data = append(data, ":")
			data = append(data, strconv.Itoa(t))
		case int64:
			data = append(data, ":")
			data = append(data, strconv.Itoa(int(t)))
		}
	}
	return strings.Join(data, "")
}

func (self *DictResponse) IsError() bool {
	return false
}

func NewItemsResponse(items []IItem) *ItemsResponse {
	return &ItemsResponse{items}
}

func (self *ItemsResponse) GetResponse() string {
	data := make([]string, 0, 3+9*len(self.items))
	data = append(data, "+DATA %")
	data = append(data, strconv.Itoa(len(self.items)))
	for _, item := range self.items {
		data = append(data, " ")
		itemId := item.GetId()
		itemData := item.GetContent()
		data = append(data, "$")
		data = append(data, strconv.Itoa(len(itemId)))
		data = append(data, " ")
		data = append(data, itemId)
		data = append(data, "$")
		data = append(data, strconv.Itoa(len(itemData)))
		data = append(data, " ")
		data = append(data, itemData)
	}
	return strings.Join(data, "")
}

func (self *ItemsResponse) IsError() bool {
	return false
}

// Error response.
type ErrorResponse struct {
	ErrorText string
	ErrorCode int64
}

func (e *ErrorResponse) Error() string {
	return e.ErrorText
}

func (e *ErrorResponse) GetResponse() string {
	return fmt.Sprintf("-ERR:%d:%s", e.ErrorCode, e.ErrorText)
}

func (e *ErrorResponse) IsError() bool {
	return true
}

// Error response.
type OkResponse struct {
	Code int64
}

func (e *OkResponse) GetResponse() string {
	return fmt.Sprintf("+OK:%d", e.Code)
}

func (self *OkResponse) IsError() bool {
	return false
}

var OK200_RESPONSE *OkResponse = &OkResponse{200}
<|endoftext|>