<commit_before>// Copyright 2017 ibelie, Chen Jie, Joungtao. All rights reserved.
// Use of this source code is governed by The MIT License
// that can be found in the LICENSE file.

package rpc

import (
	"bytes"
	"fmt"
	"path"
	"sort"
	"strings"

	"io/ioutil"
)

func injectJavascript(dir string, entities []*Entity) {
	var buffer bytes.Buffer
	requireMap := make(map[string]bool)
	methodsMap := make(map[string]bool)

	var methods []string
	for _, e := range entities {
		for _, c := range e.Components {
			if c.Protocol == nil {
				continue
			}
			for _, m := range c.Protocol.Methods {
				if ok, exist := methodsMap[m.Name]; exist && ok {
					continue
				} else if len(m.Results) > 0 {
					continue
				}
				var params []string
				for i, _ := range m.Params {
					params = append(params, fmt.Sprintf("a%d", i))
				}
				localParams := append([]string{"v"}, params...)
				methods = append(methods, fmt.Sprintf(`
Entity.prototype.Deserialize%s = function(data) {
	return ibelie.rpc.%s.Deserialize%sParam(data);
};

Entity.prototype.%s = function(%s) {
	if (!this.isAwake) {
		console.warn('[Entity] Not awake:', this);
		return;
	}
	for (var k in this) {
		var v = this[k];
		v.%s && v.%s.call(%s);
	}
	var data = ibelie.rpc.%s.Serialize%sParam(%s);
	this.connection.send(this, ibelie.rpc.Symbols.%s, data);
};
`, m.Name, c.Name, m.Name, m.Name, strings.Join(params, ", "),
					m.Name, m.Name, strings.Join(localParams, ", "),
					c.Name, m.Name, strings.Join(params, ", "), m.Name))
				requireMap[fmt.Sprintf(`
goog.require('ibelie.rpc.%s');`, c.Name)] = true
				methodsMap[m.Name] = true
			}
		}
	}

	var requires []string
	for require, ok := range requireMap {
		if ok {
			requires = append(requires, require)
		}
	}
	sort.Strings(requires)

	buffer.Write([]byte(fmt.Sprintf(`// Generated by ibelie-rpc.  DO NOT EDIT!

goog.provide('Entity');

goog.require('tyts.String');
goog.require('tyts.ProtoBuf');
goog.require('tyts.SizeVarint');%s

var ZERO_RUID = 'AAAAAAAAAAA';

Entity = function() {
	this.__class__ = 'Entity';
	this.isAwake = false;
	this.RUID = ZERO_RUID;
	this.Key  = ZERO_RUID;
	this.Type = 0;
};

Entity.prototype.ByteSize = function() {
	var size = tyts.SizeVarint(this.Type << 2);
	if (this.RUID != ZERO_RUID) {
		size += 8;
	}
	if (this.Key != ZERO_RUID) {
		size += 8;
	}
	return size;
};

Entity.prototype.SerializeUnsealed = function(protobuf) {
	var t = this.Type << 2;
	if (this.RUID != ZERO_RUID) {
		t &= 1;
	}
	if (this.Key != ZERO_RUID) {
		t &= 2;
	}
	protobuf.WriteVarint(t);
	if (this.RUID != ZERO_RUID) {
		protobuf.WriteBase64(this.RUID);
	}
	if (this.Key != ZERO_RUID) {
		protobuf.WriteBase64(this.Key);
	}
};

Entity.prototype.Serialize = function() {
	var protobuf = new tyts.ProtoBuf(new Uint8Array(this.ByteSize()));
	this.SerializeUnsealed(protobuf);
	return protobuf.buffer;
};

Entity.prototype.Deserialize = function(data) {
	var protobuf = new tyts.ProtoBuf(data);
	var t = protobuf.ReadVarint();
	this.Type = t >>> 2;
	this.RUID = (t & 1) ? protobuf.ReadBase64(8) : ZERO_RUID;
	this.Key  = (t & 2) ? protobuf.ReadBase64(8) : ZERO_RUID;
};

var ibelie = {};
ibelie.rpc = {};
ibelie.rpc.Entity = Entity;

ibelie.rpc.Component = function(entity) {
	this.Entity = entity;
};

ibelie.rpc.Component.prototype.Awake = function(e) {
	if (e.isAwake) {
		console.warn('[Entity] Already awaked:', e);
		return e;
	}
	var conn = this.Entity.connection;
	var entity = conn.entities[e.RUID];
	if (entity) {
		return entity
	}
	entity = new entities[ibelie.rpc.Dictionary[e.Type]]();
	entity.RUID	= e.RUID;
	entity.Key	= e.Key;
	entity.Type	= e.Type;
	entity.connection = conn;
	conn.send(e, ibelie.rpc.Symbols.OBSERVE);
	conn.entities[entity.RUID] = entity;
	return entity;
};

ibelie.rpc.Component.prototype.Drop = function(e) {
	if (!e || !e.isAwake) {
		console.warn('[Entity] Not awaked:', e);
		return;
	}
	for (var k in e) {
		var v = e[k];
		v.onDrop && v.onDrop();
		if (v.Entity) {
			delete v.Entity;
		}
	}
	e.isAwake = false;
	var conn = this.Entity.connection;
	conn.send(e, ibelie.rpc.Symbols.IGNORE);
	delete conn.entities[e.RUID];
	var entity = new Entity();
	entity.RUID	= e.RUID;
	entity.Key	= e.Key;
	entity.Type	= e.Type;
	return entity;
};

ibelie.rpc.Connection = function(url) {
	var conn = this;
	var socket = new WebSocket(url);
	socket.onopen = function (event) {
		socket.onmessage = function(event) {
			var entity;
			var protobuf = tyts.ProtoBuf.FromBase64(event.data);
			var id = protobuf.ReadBase64(8);
			if (!ibelie.rpc.Symbols) {
				ibelie.rpc.Symbols = {};
				ibelie.rpc.Dictionary = {};
				var buffer = new tyts.ProtoBuf(protobuf.ReadBuffer());
				while (!buffer.End()) {
					var symbol = tyts.String.Deserialize(null, buffer);
					var value = buffer.ReadVarint();
					ibelie.rpc.Symbols[symbol] = value;
					ibelie.rpc.Dictionary[value] = symbol;
				}
				entity = new entities.Session();
				entity.connection = conn;
				entity.Type = ibelie.rpc.Symbols.Session;
				entity.Key  = 0;
				conn.entities[id] = entity;
			} else {
				entity = conn.entities[id];
				if (!entity) {
					console.error('[Connection] Cannot find entity:', id);
					return;
				}
			}
			while (!protobuf.End()) {
				var name = ibelie.rpc.Dictionary[protobuf.ReadVarint()];
				var data = protobuf.ReadBuffer();
				if (ibelie.rpc[name]) {
					ibelie.rpc[name].prototype.Deserialize.call(entity[name], data);
				} else if (!entity.isAwake) {
					console.error('[Connection] Entity is not awake:', id, name, entity);
					continue;
				} else if (name == 'NOTIFY') {
					var buffer = new tyts.ProtoBuf(data);
					var component = ibelie.rpc.Dictionary[buffer.ReadVarint()];
					var property = ibelie.rpc.Dictionary[buffer.ReadVarint()];
					var newValue = ibelie.rpc[component]['Deserialize' + property](buffer.Bytes())[0];
					var oldValue = entity[component][property];
					var handler = entity[component][property + 'Handler'];
					if (oldValue.concat) {
						entity[component][property] = oldValue.concat(newValue);
						handler && handler(oldValue, newValue);
					} else if ((newValue instanceof Object) && !newValue.__class__) {
						if (!entity[component][property]) {
							entity[component][property] = {};
						}
						for (var k in newValue) {
							var o = oldValue[k];
							var n = newValue[k];
							oldValue[k] = n;
							handler && handler(k, o, n);
						}
					} else {
						entity[component][property] = newValue;
						handler && handler(oldValue, newValue);
					}
				} else {
					var args = entity['Deserialize' + name](data);
					for (var k in entity) {
						var v = entity[k];
						v[name] && v[name].apply(v, args);
					}
				}
			}
			if (entity && !entity.isAwake) {
				entity.isAwake = true;
				for (var k in entity) {
					var v = entity[k];
					v.onAwake && v.onAwake();
				}
			}
		};
		socket.onclose = function(event) {
			console.warn('[Connection] Socket has been closed:', event, conn);
		};
	};
	this.socket = socket;
	this.entities = {};
};

ibelie.rpc.Connection.prototype.send = function(entity, method, data) {
	var size = entity.ByteSize() + tyts.SizeVarint(method);
	if (data) {
		size += data.length;
	}
	var protobuf = new tyts.ProtoBuf(new Uint8Array(size));
	entity.SerializeUnsealed(protobuf);
	protobuf.WriteVarint(method);
	if (data) {
		protobuf.WriteBytes(data);
	}
	this.socket.send(protobuf.ToBase64());
};

ibelie.rpc.Connection.prototype.disconnect = function() {
	this.socket.close();
};
%s`, strings.Join(requires, ""), strings.Join(methods, ""))))

	ioutil.WriteFile(path.Join(dir, "rpc.js"), buffer.Bytes(), 0666)
}
<commit_msg>fix a bug<commit_after>// Copyright 2017 ibelie, Chen Jie, Joungtao. All rights reserved.
// Use of this source code is governed by The MIT License
// that can be found in the LICENSE file.

package rpc

import (
	"bytes"
	"fmt"
	"path"
	"sort"
	"strings"

	"io/ioutil"
)

func injectJavascript(dir string, entities []*Entity) {
	var buffer bytes.Buffer
	requireMap := make(map[string]bool)
	methodsMap := make(map[string]bool)

	var methods []string
	for _, e := range entities {
		for _, c := range e.Components {
			if c.Protocol == nil {
				continue
			}
			for _, m := range c.Protocol.Methods {
				if ok, exist := methodsMap[m.Name]; exist && ok {
					continue
				} else if len(m.Results) > 0 {
					continue
				}
				var params []string
				for i, _ := range m.Params {
					params = append(params, fmt.Sprintf("a%d", i))
				}
				localParams := append([]string{"v"}, params...)
				methods = append(methods, fmt.Sprintf(`
Entity.prototype.Deserialize%s = function(data) {
	return ibelie.rpc.%s.Deserialize%sParam(data);
};

Entity.prototype.%s = function(%s) {
	if (!this.isAwake) {
		console.warn('[Entity] Not awake:', this);
		return;
	}
	for (var k in this) {
		var v = this[k];
		v.%s && v.%s.call(%s);
	}
	var data = ibelie.rpc.%s.Serialize%sParam(%s);
	this.connection.send(this, ibelie.rpc.Symbols.%s, data);
};
`, m.Name, c.Name, m.Name, m.Name, strings.Join(params, ", "),
					m.Name, m.Name, strings.Join(localParams, ", "),
					c.Name, m.Name, strings.Join(params, ", "), m.Name))
				requireMap[fmt.Sprintf(`
goog.require('ibelie.rpc.%s');`, c.Name)] = true
				methodsMap[m.Name] = true
			}
		}
	}

	var requires []string
	for require, ok := range requireMap {
		if ok {
			requires = append(requires, require)
		}
	}
	sort.Strings(requires)

	buffer.Write([]byte(fmt.Sprintf(`// Generated by ibelie-rpc.  DO NOT EDIT!

goog.provide('Entity');

goog.require('tyts.String');
goog.require('tyts.ProtoBuf');
goog.require('tyts.SizeVarint');%s

var ZERO_RUID = 'AAAAAAAAAAA';

Entity = function() {
	this.__class__ = 'Entity';
	this.isAwake = false;
	this.RUID = ZERO_RUID;
	this.Key  = ZERO_RUID;
	this.Type = 0;
};

Entity.prototype.ByteSize = function() {
	var size = tyts.SizeVarint(this.Type << 2);
	if (this.RUID != ZERO_RUID) {
		size += 8;
	}
	if (this.Key != ZERO_RUID) {
		size += 8;
	}
	return size;
};

Entity.prototype.SerializeUnsealed = function(protobuf) {
	var t = this.Type << 2;
	if (this.RUID != ZERO_RUID) {
		t &= 1;
	}
	if (this.Key != ZERO_RUID) {
		t &= 2;
	}
	protobuf.WriteVarint(t);
	if (this.RUID != ZERO_RUID) {
		protobuf.WriteBase64(this.RUID);
	}
	if (this.Key != ZERO_RUID) {
		protobuf.WriteBase64(this.Key);
	}
};

Entity.prototype.Serialize = function() {
	var protobuf = new tyts.ProtoBuf(new Uint8Array(this.ByteSize()));
	this.SerializeUnsealed(protobuf);
	return protobuf.buffer;
};

Entity.prototype.Deserialize = function(data) {
	var protobuf = new tyts.ProtoBuf(data);
	var t = protobuf.ReadVarint();
	this.Type = t >>> 2;
	this.RUID = (t & 1) ? protobuf.ReadBase64(8) : ZERO_RUID;
	this.Key  = (t & 2) ? protobuf.ReadBase64(8) : ZERO_RUID;
};

var ibelie = {};
ibelie.rpc = {};
ibelie.rpc.Entity = Entity;

ibelie.rpc.Component = function(entity) {
	this.Entity = entity;
};

ibelie.rpc.Component.prototype.Awake = function(e) {
	if (e.isAwake) {
		console.warn('[Entity] Already awaked:', e);
		return e;
	}
	var conn = this.Entity.connection;
	var entity = conn.entities[e.RUID];
	if (entity) {
		return entity
	}
	entity = new entities[ibelie.rpc.Dictionary[e.Type]]();
	entity.RUID	= e.RUID;
	entity.Key	= e.Key;
	entity.Type	= e.Type;
	entity.connection = conn;
	conn.send(e, ibelie.rpc.Symbols.OBSERVE);
	conn.entities[entity.RUID] = entity;
	return entity;
};

ibelie.rpc.Component.prototype.Drop = function(e) {
	if (!e || !e.isAwake) {
		console.warn('[Entity] Not awaked:', e);
		return;
	}
	for (var k in e) {
		var v = e[k];
		v.onDrop && v.onDrop();
		if (v.Entity) {
			delete v.Entity;
		}
	}
	e.isAwake = false;
	var conn = this.Entity.connection;
	conn.send(e, ibelie.rpc.Symbols.IGNORE);
	delete conn.entities[e.RUID];
	var entity = new Entity();
	entity.RUID	= e.RUID;
	entity.Key	= e.Key;
	entity.Type	= e.Type;
	return entity;
};

ibelie.rpc.Connection = function(url) {
	var conn = this;
	var socket = new WebSocket(url);
	socket.onopen = function (event) {
		socket.onmessage = function(event) {
			var entity;
			var protobuf = tyts.ProtoBuf.FromBase64(event.data);
			var id = protobuf.ReadBase64(8);
			if (!ibelie.rpc.Symbols) {
				ibelie.rpc.Symbols = {};
				ibelie.rpc.Dictionary = {};
				var buffer = new tyts.ProtoBuf(protobuf.ReadBuffer());
				while (!buffer.End()) {
					var symbol = tyts.String.Deserialize(null, buffer);
					var value = buffer.ReadVarint();
					ibelie.rpc.Symbols[symbol] = value;
					ibelie.rpc.Dictionary[value] = symbol;
				}
				entity = new entities.Session();
				entity.connection = conn;
				entity.Type = ibelie.rpc.Symbols.Session;
				entity.Key  = ZERO_RUID;
				conn.entities[id] = entity;
			} else {
				entity = conn.entities[id];
				if (!entity) {
					console.error('[Connection] Cannot find entity:', id);
					return;
				}
			}
			while (!protobuf.End()) {
				var name = ibelie.rpc.Dictionary[protobuf.ReadVarint()];
				var data = protobuf.ReadBuffer();
				if (ibelie.rpc[name]) {
					ibelie.rpc[name].prototype.Deserialize.call(entity[name], data);
				} else if (!entity.isAwake) {
					console.error('[Connection] Entity is not awake:', id, name, entity);
					continue;
				} else if (name == 'NOTIFY') {
					var buffer = new tyts.ProtoBuf(data);
					var component = ibelie.rpc.Dictionary[buffer.ReadVarint()];
					var property = ibelie.rpc.Dictionary[buffer.ReadVarint()];
					var newValue = ibelie.rpc[component]['Deserialize' + property](buffer.Bytes())[0];
					var oldValue = entity[component][property];
					var handler = entity[component][property + 'Handler'];
					if (oldValue.concat) {
						entity[component][property] = oldValue.concat(newValue);
						handler && handler(oldValue, newValue);
					} else if ((newValue instanceof Object) && !newValue.__class__) {
						if (!entity[component][property]) {
							entity[component][property] = {};
						}
						for (var k in newValue) {
							var o = oldValue[k];
							var n = newValue[k];
							oldValue[k] = n;
							handler && handler(k, o, n);
						}
					} else {
						entity[component][property] = newValue;
						handler && handler(oldValue, newValue);
					}
				} else {
					var args = entity['Deserialize' + name](data);
					for (var k in entity) {
						var v = entity[k];
						v[name] && v[name].apply(v, args);
					}
				}
			}
			if (entity && !entity.isAwake) {
				entity.isAwake = true;
				for (var k in entity) {
					var v = entity[k];
					v.onAwake && v.onAwake();
				}
			}
		};
		socket.onclose = function(event) {
			console.warn('[Connection] Socket has been closed:', event, conn);
		};
	};
	this.socket = socket;
	this.entities = {};
};

ibelie.rpc.Connection.prototype.send = function(entity, method, data) {
	var size = entity.ByteSize() + tyts.SizeVarint(method);
	if (data) {
		size += data.length;
	}
	var protobuf = new tyts.ProtoBuf(new Uint8Array(size));
	entity.SerializeUnsealed(protobuf);
	protobuf.WriteVarint(method);
	if (data) {
		protobuf.WriteBytes(data);
	}
	this.socket.send(protobuf.ToBase64());
};

ibelie.rpc.Connection.prototype.disconnect = function() {
	this.socket.close();
};
%s`, strings.Join(requires, ""), strings.Join(methods, ""))))

	ioutil.WriteFile(path.Join(dir, "rpc.js"), buffer.Bytes(), 0666)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"log"
	"net"
)

// Can be tested using nc tool:
//    echo "asdad" | nc 127.0.0.1 27017
//
type TCPInput struct {
	data     chan []byte
	address  string
	listener net.Listener
}

func NewTCPInput(address string) (i *TCPInput) {
	i = new(TCPInput)
	i.data = make(chan []byte)
	i.address = address

	i.listen(address)

	return
}

func (i *TCPInput) Read(data []byte) (int, error) {
	buf := <-i.data
	copy(data, buf)

	return len(buf), nil
}

func (i *TCPInput) listen(address string) {
	listener, err := net.Listen("tcp", address)
	i.listener = listener

	if err != nil {
		log.Fatal("Can't start:", err)
	}

	go func() {
		for {
			conn, err := listener.Accept()

			if err != nil {
				log.Println("Error while Accept()", err)
				continue
			}

			go i.handleConnection(conn)
		}
	}()
}

func scanBytes(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}

	// Search for ¶ symbol
	if i := bytes.IndexByte(data, 194); i >= 0 {
		if len(data) > i+1 && data[i+1] == 182 {
			// We have a full newline-terminated line.
			return i + 2, data[0:i], nil
		}
	}
	// If we're at EOF, we have a final, non-terminated line. Return it.
	if atEOF {
		return len(data), data, nil
	}
	// Request more data.
	return 0, nil, nil
}

func (i *TCPInput) handleConnection(conn net.Conn) {
	defer conn.Close()

	scanner := bufio.NewScanner(conn)

	scanner.Split(scanBytes)

	for scanner.Scan() {
		i.data <- scanner.Bytes()
	}
}

func (i *TCPInput) String() string {
	return "TCP input: " + i.address
}
<commit_msg>make tcpinput use readbytes and create new slice prior to copying to the buffer.<commit_after>package main

import (
	"bufio"
	"io"
	"log"
	"net"
)

// Can be tested using nc tool:
//    echo "asdad" | nc 127.0.0.1 27017
//
type TCPInput struct {
	data     chan []byte
	address  string
	listener net.Listener
}

func NewTCPInput(address string) (i *TCPInput) {
	i = new(TCPInput)
	i.data = make(chan []byte)
	i.address = address

	i.listen(address)

	return
}

func (i *TCPInput) Read(data []byte) (int, error) {
	buf := <-i.data
	copy(data, buf)

	return len(buf), nil
}

func (i *TCPInput) listen(address string) {
	listener, err := net.Listen("tcp", address)
	i.listener = listener

	if err != nil {
		log.Fatal("Can't start:", err)
	}

	go func() {
		for {
			conn, err := listener.Accept()

			if err != nil {
				log.Println("Error while Accept()", err)
				continue
			}

			go i.handleConnection(conn)
		}
	}()
}

func (i *TCPInput) handleConnection(conn net.Conn) {
	defer conn.Close()

	reader := bufio.NewReader(conn)

	for {
		buf,err := reader.ReadBytes('¶')
		new_buf_len := len(buf) - 2
		new_buf := make([]byte, new_buf_len)
		copy(new_buf, buf[:new_buf_len])
		i.data <- new_buf
		if err != nil {
			if err != io.EOF {
				log.Printf("error: %s\n", err)
			}
		}
	}
}

func (i *TCPInput) String() string {
	return "TCP input: " + i.address
}
<|endoftext|>


<commit_before>// Copyright 2015 trivago GmbH
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tgo

import (
	"github.com/trivago/tgo/ttesting"
	"sync"
	"testing"
)

func getMockMetric() metrics {
	return metrics{
		new(sync.Mutex),
		make(map[string]*int64),
	}
}

func TestMetricsSet(t *testing.T) {
	expect := ttesting.NewExpect(t)
	mockMetric := getMockMetric()

	// test for initialization to zero
	mockMetric.New("MockMetric")
	count, err := mockMetric.Get("MockMetric")
	expect.Nil(err)
	expect.Equal(int64(0), count)

	// test for setting to a particular value
	mockMetric.Set("MockMetric", int64(5))
	count, err = mockMetric.Get("MockMetric")
	expect.Nil(err)
	expect.Equal(int64(5), count)

	// test for setting to a particular int
	mockMetric.SetI("MockMetric", 5)
	count, err = mockMetric.Get("MockMetric")
	expect.Nil(err)
	expect.Equal(int64(5), count)

	// test for setting to a particular float
	mockMetric.SetF("MockMetric", 4.3)
	count, err = mockMetric.Get("MockMetric")
	expect.Nil(err)
	expect.Equal(int64(4), count)
}

func TestMetricsAddSub(t *testing.T) {
	expect := ttesting.NewExpect(t)
	mockMetric := getMockMetric()

	mockMetric.New("MockMetric")
	mockMetric.Add("MockMetric", int64(1))
	count, err := mockMetric.Get("MockMetric")
	expect.Nil(err)
	expect.Equal(int64(1), count)

	mockMetric.AddI("MockMetric", 1)
	count, err = mockMetric.Get("MockMetric")
	expect.Nil(err)
	expect.Equal(int64(2), count)

	mockMetric.AddF("MockMetric", 2.4)
	count, err = mockMetric.Get("MockMetric")
	expect.Nil(err)
	expect.Equal(int64(4), count)

	mockMetric.Sub("MockMetric", int64(1))
	count, err = mockMetric.Get("MockMetric")
	expect.Nil(err)
	expect.Equal(int64(3), count)

	mockMetric.SubF("MockMetric", 1.6)
	count, err = mockMetric.Get("MockMetric")
	expect.Nil(err)
	expect.Equal(int64(1), count)

	mockMetric.SubI("MockMetric", 1)
	count, err = mockMetric.Get("MockMetric")
	expect.Nil(err)
	expect.Equal(int64(0), count)
}

func TestMetricsIncDec(t *testing.T) {
	expect := ttesting.NewExpect(t)
	mockMetric := getMockMetric()
	mockMetric.New("MockMetric")

	mockMetric.Inc("MockMetric")
	count, err := mockMetric.Get("MockMetric")
	expect.Nil(err)
	expect.Equal(int64(1), count)

	mockMetric.Dec("MockMetric")
	count, err = mockMetric.Get("MockMetric")
	expect.Nil(err)
	expect.Equal(int64(0), count)

}
<commit_msg>fixed mutex type<commit_after>// Copyright 2015 trivago GmbH
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tgo

import (
	"github.com/trivago/tgo/ttesting"
	"sync"
	"testing"
)

func getMockMetric() metrics {
	return metrics{
		new(sync.RWMutex),
		make(map[string]*int64),
	}
}

func TestMetricsSet(t *testing.T) {
	expect := ttesting.NewExpect(t)
	mockMetric := getMockMetric()

	// test for initialization to zero
	mockMetric.New("MockMetric")
	count, err := mockMetric.Get("MockMetric")
	expect.Nil(err)
	expect.Equal(int64(0), count)

	// test for setting to a particular value
	mockMetric.Set("MockMetric", int64(5))
	count, err = mockMetric.Get("MockMetric")
	expect.Nil(err)
	expect.Equal(int64(5), count)

	// test for setting to a particular int
	mockMetric.SetI("MockMetric", 5)
	count, err = mockMetric.Get("MockMetric")
	expect.Nil(err)
	expect.Equal(int64(5), count)

	// test for setting to a particular float
	mockMetric.SetF("MockMetric", 4.3)
	count, err = mockMetric.Get("MockMetric")
	expect.Nil(err)
	expect.Equal(int64(4), count)
}

func TestMetricsAddSub(t *testing.T) {
	expect := ttesting.NewExpect(t)
	mockMetric := getMockMetric()

	mockMetric.New("MockMetric")
	mockMetric.Add("MockMetric", int64(1))
	count, err := mockMetric.Get("MockMetric")
	expect.Nil(err)
	expect.Equal(int64(1), count)

	mockMetric.AddI("MockMetric", 1)
	count, err = mockMetric.Get("MockMetric")
	expect.Nil(err)
	expect.Equal(int64(2), count)

	mockMetric.AddF("MockMetric", 2.4)
	count, err = mockMetric.Get("MockMetric")
	expect.Nil(err)
	expect.Equal(int64(4), count)

	mockMetric.Sub("MockMetric", int64(1))
	count, err = mockMetric.Get("MockMetric")
	expect.Nil(err)
	expect.Equal(int64(3), count)

	mockMetric.SubF("MockMetric", 1.6)
	count, err = mockMetric.Get("MockMetric")
	expect.Nil(err)
	expect.Equal(int64(1), count)

	mockMetric.SubI("MockMetric", 1)
	count, err = mockMetric.Get("MockMetric")
	expect.Nil(err)
	expect.Equal(int64(0), count)
}

func TestMetricsIncDec(t *testing.T) {
	expect := ttesting.NewExpect(t)
	mockMetric := getMockMetric()
	mockMetric.New("MockMetric")

	mockMetric.Inc("MockMetric")
	count, err := mockMetric.Get("MockMetric")
	expect.Nil(err)
	expect.Equal(int64(1), count)

	mockMetric.Dec("MockMetric")
	count, err = mockMetric.Get("MockMetric")
	expect.Nil(err)
	expect.Equal(int64(0), count)

}
<|endoftext|>


<commit_before>// Package metrics provides a framework for application instrumentation. It's
// primarily designed to help you get started with good and robust
// instrumentation, and to help you migrate from a less-capable system like
// Graphite to a more-capable system like Prometheus. If your organization has
// already standardized on an instrumentation system like Prometheus, and has no
// plans to change, it may make sense to use that system's instrumentation
// library directly.
//
// This package provides three core metric abstractions (Counter, Gauge, and
// Histogram) and implementations for almost all common instrumentation
// backends. Each metric has an observation method (Add, Set, or Observe,
// respectively) used to record values, and a With method to "scope" the
// observation by various parameters. For example, you might have a Histogram to
// record request durations, parameterized by the method that's being called.
//
//    var requestDuration metrics.Histogram
//    // ...
//    requestDuration.With("method", "MyMethod").Observe(time.Since(begin))
//
// This allows a single high-level metrics object (requestDuration) to work with
// many code paths somewhat dynamically. The concept of With is fully supported
// in some backends like Prometheus, and not supported in other backends like
// Graphite. So, With may be a no-op, depending on the concrete implementation
// you choose.
//
// Usage
//
// Metrics are dependencies, and should be passed to the components that need
// them in the same way you'd construct and pass a database handle, or reference
// to another component. Metrics should *not* be created in the global scope.
// Instead, instantiate metrics in your func main, using whichever concrete
// implementation is appropriate for your organization.
//
//    latency := prometheus.NewSummaryFrom(stdprometheus.SummaryOpts{
//        Namespace: "myteam",
//        Subsystem: "foosvc",
//        Name:      "request_latency_seconds",
//        Help:      "Incoming request latency in seconds."
//    }, []string{"method", "status_code"})
//
// Write your components to take the metrics they will use as parameters to
// their constructors. Use the interface types, not the concrete types. That is,
//
//    // NewAPI takes metrics.Histogram, not *prometheus.Summary
//    func NewAPI(s Store, logger log.Logger, latency metrics.Histogram) *API {
//        // ...
//    }
//
//    func (a *API) ServeFoo(w http.ResponseWriter, r *http.Request) {
//        begin := time.Now()
//        // ...
//        a.latency.Observe(time.Since(begin).Seconds())
//    }
//
// Finally, pass the metrics as dependencies when building your object graph.
// This should happen in func main, not in the global scope.
//
//    api := NewAPI(store, logger, latency)
//    http.ListenAndServe("/", api)
//
// Note that metrics are "write-only" interfaces.
//
// Implementation details
//
// All metrics are safe for concurrent use. Considerable design influence has
// been taken from https://github.com/codahale/metrics and
// https://prometheus.io.
//
// Each telemetry system has different semantics for label values, push vs.
// pull, support for histograms, etc. These properties influence the design of
// their respective packages. This table attempts to summarize the key points of
// distinction.
//
//    SYSTEM      DIM  COUNTERS               GAUGES                 HISTOGRAMS
//    dogstatsd   n    batch, push-aggregate  batch, push-aggregate  native, batch, push-each
//    statsd      1    batch, push-aggregate  batch, push-aggregate  native, batch, push-each
//    graphite    1    batch, push-aggregate  batch, push-aggregate  synthetic, batch, push-aggregate
//    expvar      1    atomic                 atomic                 synthetic, batch, in-place expose
//    influx      n    custom                 custom                 custom
//    prometheus  n    native                 native                 native
//    circonus    1    native                 native                 native
//    pcp         1    native                 native                 native
//
package metrics
<commit_msg>metrics: further clarification to docs<commit_after>// Package metrics provides a framework for application instrumentation. It's
// primarily designed to help you get started with good and robust
// instrumentation, and to help you migrate from a less-capable system like
// Graphite to a more-capable system like Prometheus. If your organization has
// already standardized on an instrumentation system like Prometheus, and has no
// plans to change, it may make sense to use that system's instrumentation
// library directly.
//
// This package provides three core metric abstractions (Counter, Gauge, and
// Histogram) and implementations for almost all common instrumentation
// backends. Each metric has an observation method (Add, Set, or Observe,
// respectively) used to record values, and a With method to "scope" the
// observation by various parameters. For example, you might have a Histogram to
// record request durations, parameterized by the method that's being called.
//
//    var requestDuration metrics.Histogram
//    // ...
//    requestDuration.With("method", "MyMethod").Observe(time.Since(begin))
//
// This allows a single high-level metrics object (requestDuration) to work with
// many code paths somewhat dynamically. The concept of With is fully supported
// in some backends like Prometheus, and not supported in other backends like
// Graphite. So, With may be a no-op, depending on the concrete implementation
// you choose. Please check the implementation to know for sure. For
// implementations that don't provide With, it's necessary to fully parameterize
// each metric in the metric name, e.g.
//
//    // Statsd
//    c := statsd.NewCounter("request_duration_MyMethod_200")
//    c.Add(1)
//
//    // Prometheus
//    c := prometheus.NewCounter(stdprometheus.CounterOpts{
//        Name: "request_duration",
//        ...
//    }, []string{"method", "status_code"})
//    c.With("method", "MyMethod", "status_code", strconv.Itoa(code)).Add(1)
//
// Usage
//
// Metrics are dependencies, and should be passed to the components that need
// them in the same way you'd construct and pass a database handle, or reference
// to another component. Metrics should *not* be created in the global scope.
// Instead, instantiate metrics in your func main, using whichever concrete
// implementation is appropriate for your organization.
//
//    latency := prometheus.NewSummaryFrom(stdprometheus.SummaryOpts{
//        Namespace: "myteam",
//        Subsystem: "foosvc",
//        Name:      "request_latency_seconds",
//        Help:      "Incoming request latency in seconds."
//    }, []string{"method", "status_code"})
//
// Write your components to take the metrics they will use as parameters to
// their constructors. Use the interface types, not the concrete types. That is,
//
//    // NewAPI takes metrics.Histogram, not *prometheus.Summary
//    func NewAPI(s Store, logger log.Logger, latency metrics.Histogram) *API {
//        // ...
//    }
//
//    func (a *API) ServeFoo(w http.ResponseWriter, r *http.Request) {
//        begin := time.Now()
//        // ...
//        a.latency.Observe(time.Since(begin).Seconds())
//    }
//
// Finally, pass the metrics as dependencies when building your object graph.
// This should happen in func main, not in the global scope.
//
//    api := NewAPI(store, logger, latency)
//    http.ListenAndServe("/", api)
//
// Note that metrics are "write-only" interfaces.
//
// Implementation details
//
// All metrics are safe for concurrent use. Considerable design influence has
// been taken from https://github.com/codahale/metrics and
// https://prometheus.io.
//
// Each telemetry system has different semantics for label values, push vs.
// pull, support for histograms, etc. These properties influence the design of
// their respective packages. This table attempts to summarize the key points of
// distinction.
//
//    SYSTEM      DIM  COUNTERS               GAUGES                 HISTOGRAMS
//    dogstatsd   n    batch, push-aggregate  batch, push-aggregate  native, batch, push-each
//    statsd      1    batch, push-aggregate  batch, push-aggregate  native, batch, push-each
//    graphite    1    batch, push-aggregate  batch, push-aggregate  synthetic, batch, push-aggregate
//    expvar      1    atomic                 atomic                 synthetic, batch, in-place expose
//    influx      n    custom                 custom                 custom
//    prometheus  n    native                 native                 native
//    circonus    1    native                 native                 native
//    pcp         1    native                 native                 native
//
package metrics
<|endoftext|>


<commit_before>package gojison

import (
	"net/http"
	"strings"

	"encoding/json"

	"github.com/ndyakov/whatever"
	"github.com/zenazn/goji/web"
)

func Response(h http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
	}
	return http.HandlerFunc(fn)
}

func Request(c *web.C, h http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {
		contentTypeSlice := strings.Split(r.Header.Get("Content-Type"), ";")
		if contentTypeSlice[0] == "application/json" {
			var params whatever.Params
			c.Env["GojisonDecodeError"] = json.NewDecoder(r.Body).Decode(&params)
			c.Env["Params"] = params
		}
		h.ServeHTTP(w, r)
	}
	return http.HandlerFunc(fn)
}
<commit_msg>Add comments for the middlewares.<commit_after>package gojison

import (
	"net/http"
	"strings"

	"encoding/json"

	"github.com/ndyakov/whatever"
	"github.com/zenazn/goji/web"
)

// Response will set the Content-Type of the http response to:
//     "application/json"
func Response(h http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
	}
	return http.HandlerFunc(fn)
}

// Request will parse the request body to a whatever.Params structure and
// then add this structure to the goji context map with the key "Params".
// The error (or nil) of the decoding will be available in the context with
// the key "GojisonDecodeError".
//
// The parsing of the body will happen only if the Content-Type of the request
// is application/json.
//
// For more information about how to work with the whatever.Params type, please refer to:
// http://godoc.org/github.com/ndyakov/whatever
func Request(c *web.C, h http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {
		contentTypeSlice := strings.Split(r.Header.Get("Content-Type"), ";")
		if contentTypeSlice[0] == "application/json" {
			var params whatever.Params
			c.Env["GojisonDecodeError"] = json.NewDecoder(r.Body).Decode(&params)
			c.Env["Params"] = params
		}
		h.ServeHTTP(w, r)
	}
	return http.HandlerFunc(fn)
}
<|endoftext|>


<commit_before>package jail

import (
	"bufio"
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"regexp"
	"strings"
	"time"

	"github.com/naoina/toml"
)

type configJail struct {
	Name        string
	LogFile     string
	TimeFormat  string
	Regexp      []string
	MaxFail     int
	BanTime     int
	FindTime    int
	ActionBan   string
	ActionUnBan string
	ActionSetup string
	Enabled     bool
	Testing     bool
}

type Jail struct {
	name        string
	logFile     string
	timeFormat  string
	regexp      []*regexp.Regexp
	maxFail     int
	banTime     int
	findTime    int
	actionBan   string
	actionUnBan string
	actionSetup string
	enabled     bool
	testing     bool
	logreader   *logReader
	jailees     []*jailee
	Cells       map[string]time.Time
}

type jailee struct {
	q         map[string]string
	ip        string
	failcount int
}

func NewJail(jailfile string) *Jail {
	f, err := os.Open(jailfile)
	if err != nil {
		panic(err)
	}
	defer f.Close()
	buf, err := ioutil.ReadAll(f)
	if err != nil {
		panic(err)
	}
	var config configJail
	if err := toml.Unmarshal(buf, &config); err != nil {
		panic(err)
	}

	rg := make([]*regexp.Regexp, 0)
	for _, v := range config.Regexp {
		rr := regexp.MustCompile(v)
		rg = append(rg, rr)

	}
	j := Jail{
		name:        basename(jailfile),
		logreader:   newLogReader(config.LogFile),
		logFile:     config.LogFile,
		timeFormat:  config.TimeFormat,
		regexp:      rg,
		maxFail:     config.MaxFail,
		banTime:     config.BanTime,
		findTime:    config.FindTime,
		actionBan:   config.ActionBan,
		actionUnBan: config.ActionUnBan,
		actionSetup: config.ActionSetup,
		enabled:     config.Enabled,
		testing:     config.Testing,
		jailees:     make([]*jailee, 0),
		Cells:       make(map[string]time.Time),
	}
	j.executeSetup()
	return &j
}

func (j *Jail) getJailee(ip string) (int, *jailee, bool) {
	for i, ja := range j.jailees {
		if ja.ip == ip {
			return i, ja, true
		}
	}
	return -1, nil, false
}

func (j *Jail) add(q map[string]string) {
	ip := q["HOST"]
	if _, jj, ok := j.getJailee(ip); !ok {
		j.jinit(q)
	} else {
		jj.failcount++
	}
}

func (j *Jail) jinit(q map[string]string) {
	ja := jailee{failcount: 1, ip: q["HOST"], q: q}
	j.jailees = append(j.jailees, &ja)
	j.Cells[q["HOST"]] = time.Now()
}

func (j *Jail) check(ip string) {
	if _, jj, ok := j.getJailee(ip); ok {
		if jj.failcount == j.maxFail {
			go j.executeBan(jj)
		}
	}
}

func (j *Jail) remove(jj *jailee) bool {
	// we have a slice and concurent access
	// so we cannot remove it hence we do a soft delete
	delete(j.Cells, jj.ip)
	jj.failcount = 0
	if jj.failcount == 0 {
		return true
	}
	return false
}

func (j *Jail) checkFind(toCheck string) bool {
	to, _ := time.Parse(j.timeFormat, toCheck)
	if to.Year() == 0 {
		nowY := time.Now().Year()
		to = to.AddDate(nowY, 0, 0)
	}
	if time.Since(to) > time.Duration(j.findTime)*time.Minute && !j.testing {
		return false
	}
	return true
}

func (j *Jail) executeSetup() {
	cmd := j.parseCommand(j.actionSetup, nil)

	if j.testing {
		fmt.Println(cmd)
	} else {

		err := cmd.Start()
		if err != nil {
			fmt.Println(err)
		}
		err = cmd.Wait()
	}
}

func (j *Jail) executeBan(jj *jailee) {
	cmd := j.parseCommand(j.actionBan, jj)

	if j.testing {
		fmt.Println(cmd)
	} else {
		err := cmd.Start()
		if err != nil {
			fmt.Println(err)
		}
		err = cmd.Wait()
	}

	timer := time.NewTimer(time.Duration(j.banTime) * time.Minute)
	<-timer.C
	j.executeUnBan(jj)
}

func (j *Jail) executeUnBan(jj *jailee) {
	cmd := j.parseCommand(j.actionUnBan, jj)

	if j.testing {
		fmt.Println(cmd)
	} else {
		err := cmd.Start()
		if err != nil {
			fmt.Println(err)
		}
		err = cmd.Wait()
	}

	if ok := j.remove(jj); !ok {
		fmt.Println("cannot remove jailee")
	}
}

func (j *Jail) parseCommand(cmd string, jj *jailee) *exec.Cmd {
	bin := strings.Fields(cmd)[0]
	args := strings.Fields(cmd)[1:]
	if jj != nil {
		for i, k := range args {
			if strings.HasPrefix(k, "<") && strings.HasSuffix(k, ">") {
				s := strings.TrimPrefix(strings.TrimSuffix(k, ">"), "<")
				args[i] = jj.q[s]
			}
		}
	}
	c := exec.Command(bin, strings.Join(args, " "))
	c.Stdout = os.Stdout
	return c
}

func (j *Jail) Run() {
	if j.enabled {
		for {
			j.logreader.readLine()
			select {
			case <-j.logreader.errors:
				j.logreader.reset()
			case z := <-j.logreader.lines:
				if q, ok := j.matchLine(z); ok {
					if j.checkFind(q["DATETIME"]) {
						j.add(q)
						j.check(q["HOST"])
					}
				}
			}
		}
	}
}

func (j *Jail) matchLine(line string) (map[string]string, bool) {
	result := make(map[string]string)
	for _, z := range j.regexp {
		match := z.FindStringSubmatch(line)
		if match != nil {
			for i, name := range z.SubexpNames() {
				if i == 0 || name == "" {
					continue
				}
				result[name] = match[i]
			}
			return result, true
		}
	}
	return result, false
}

func sameLog(file string, sum string) bool {
	f, err := os.Open(file)
	if err != nil {
		fmt.Println(err)
		return false
	}

	r := bufio.NewReader(f)
	line, _, er := r.ReadLine()
	if er != nil {
		fmt.Println(er)
		return false
	}

	hash := md5.Sum(line)
	strHash := hex.EncodeToString(hash[:])

	if strHash == sum {
		return true
	}
	return false
}

func basename(s string) string {
	base := path.Base(s)
	n := strings.LastIndexByte(base, '.')
	if n >= 0 {
		return base[:n]
	}
	return base
}
<commit_msg>go2ban: implemented prettyprint for Jail actions.<commit_after>package jail

import (
	"bufio"
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"regexp"
	"strings"
	"time"

	"github.com/naoina/toml"
)

type configJail struct {
	Name        string
	LogFile     string
	TimeFormat  string
	Regexp      []string
	MaxFail     int
	BanTime     int
	FindTime    int
	ActionBan   string
	ActionUnBan string
	ActionSetup string
	Enabled     bool
	Testing     bool
}

type Jail struct {
	name        string
	logFile     string
	timeFormat  string
	regexp      []*regexp.Regexp
	maxFail     int
	banTime     int
	findTime    int
	actionBan   string
	actionUnBan string
	actionSetup string
	enabled     bool
	testing     bool
	logreader   *logReader
	jailees     []*jailee
	Cells       map[string]time.Time
}

type jailee struct {
	q         map[string]string
	ip        string
	failcount int
}

func NewJail(jailfile string) *Jail {
	f, err := os.Open(jailfile)
	if err != nil {
		panic(err)
	}
	defer f.Close()
	buf, err := ioutil.ReadAll(f)
	if err != nil {
		panic(err)
	}
	var config configJail
	if err := toml.Unmarshal(buf, &config); err != nil {
		panic(err)
	}

	rg := make([]*regexp.Regexp, 0)
	for _, v := range config.Regexp {
		rr := regexp.MustCompile(v)
		rg = append(rg, rr)

	}
	j := Jail{
		name:        basename(jailfile),
		logreader:   newLogReader(config.LogFile),
		logFile:     config.LogFile,
		timeFormat:  config.TimeFormat,
		regexp:      rg,
		maxFail:     config.MaxFail,
		banTime:     config.BanTime,
		findTime:    config.FindTime,
		actionBan:   config.ActionBan,
		actionUnBan: config.ActionUnBan,
		actionSetup: config.ActionSetup,
		enabled:     config.Enabled,
		testing:     config.Testing,
		jailees:     make([]*jailee, 0),
		Cells:       make(map[string]time.Time),
	}
	j.executeSetup()
	return &j
}

func (j *Jail) getJailee(ip string) (int, *jailee, bool) {
	for i, ja := range j.jailees {
		if ja.ip == ip {
			return i, ja, true
		}
	}
	return -1, nil, false
}

func (j *Jail) add(q map[string]string) {
	ip := q["HOST"]
	if _, jj, ok := j.getJailee(ip); !ok {
		j.jinit(q)
	} else {
		jj.failcount++
	}
}

func (j *Jail) jinit(q map[string]string) {
	ja := jailee{failcount: 1, ip: q["HOST"], q: q}
	j.jailees = append(j.jailees, &ja)
	j.Cells[q["HOST"]] = time.Now()
}

func (j *Jail) check(ip string) {
	if _, jj, ok := j.getJailee(ip); ok {
		if jj.failcount == j.maxFail {
			go j.executeBan(jj)
		}
	}
}

func (j *Jail) remove(jj *jailee) bool {
	// we have a slice and concurent access
	// so we cannot remove it hence we do a soft delete
	delete(j.Cells, jj.ip)
	jj.failcount = 0
	if jj.failcount == 0 {
		return true
	}
	return false
}

func (j *Jail) checkFind(toCheck string) bool {
	to, _ := time.Parse(j.timeFormat, toCheck)
	if to.Year() == 0 {
		nowY := time.Now().Year()
		to = to.AddDate(nowY, 0, 0)
	}
	if time.Since(to) > time.Duration(j.findTime)*time.Minute && !j.testing {
		return false
	}
	return true
}

func (j *Jail) executeSetup() {
	cmd := j.parseCommand(j.actionSetup, nil)

	if j.testing {
		prettyprint(cmd)
	} else {

		err := cmd.Start()
		if err != nil {
			fmt.Println(err)
		}
		err = cmd.Wait()
	}
}

func (j *Jail) executeBan(jj *jailee) {
	cmd := j.parseCommand(j.actionBan, jj)

	if j.testing {
		prettyprint(cmd)
	} else {
		err := cmd.Start()
		if err != nil {
			fmt.Println(err)
		}
		err = cmd.Wait()
	}

	timer := time.NewTimer(time.Duration(j.banTime) * time.Minute)
	<-timer.C
	j.executeUnBan(jj)
}

func (j *Jail) executeUnBan(jj *jailee) {
	cmd := j.parseCommand(j.actionUnBan, jj)

	if j.testing {
		prettyprint(cmd)
	} else {
		err := cmd.Start()
		if err != nil {
			fmt.Println(err)
		}
		err = cmd.Wait()
	}

	if ok := j.remove(jj); !ok {
		fmt.Println("cannot remove jailee")
	}
}

func (j *Jail) parseCommand(cmd string, jj *jailee) *exec.Cmd {
	bin := strings.Fields(cmd)[0]
	args := strings.Fields(cmd)[1:]
	if jj != nil {
		for i, k := range args {
			if strings.HasPrefix(k, "<") && strings.HasSuffix(k, ">") {
				s := strings.TrimPrefix(strings.TrimSuffix(k, ">"), "<")
				args[i] = jj.q[s]
			}
		}
	}
	c := exec.Command(bin, strings.Join(args, " "))
	c.Stdout = os.Stdout
	return c
}

func (j *Jail) Run() {
	if j.enabled {
		for {
			j.logreader.readLine()
			select {
			case <-j.logreader.errors:
				j.logreader.reset()
			case z := <-j.logreader.lines:
				if q, ok := j.matchLine(z); ok {
					if j.checkFind(q["DATETIME"]) {
						j.add(q)
						j.check(q["HOST"])
					}
				}
			}
		}
	}
}

func (j *Jail) matchLine(line string) (map[string]string, bool) {
	result := make(map[string]string)
	for _, z := range j.regexp {
		match := z.FindStringSubmatch(line)
		if match != nil {
			for i, name := range z.SubexpNames() {
				if i == 0 || name == "" {
					continue
				}
				result[name] = match[i]
			}
			return result, true
		}
	}
	return result, false
}

func sameLog(file string, sum string) bool {
	f, err := os.Open(file)
	if err != nil {
		fmt.Println(err)
		return false
	}

	r := bufio.NewReader(f)
	line, _, er := r.ReadLine()
	if er != nil {
		fmt.Println(er)
		return false
	}

	hash := md5.Sum(line)
	strHash := hex.EncodeToString(hash[:])

	if strHash == sum {
		return true
	}
	return false
}

func basename(s string) string {
	base := path.Base(s)
	n := strings.LastIndexByte(base, '.')
	if n >= 0 {
		return base[:n]
	}
	return base
}

func prettyprint(c *exec.Cmd) {
	var s string = "In testing mode. If not, would have executed:\n"
	s += c.Path
	s += " "
	for i, w := range c.Args {
		if i != 0 {
			s += w
			s += " "
		}
	}
	fmt.Println(s)
}
<|endoftext|>


<commit_before>// pkgc is a tool for generating wrappers for Go packages imported by Grumpy
// programs.
//
// usage: pkgc PACKAGE
//
// Where PACKAGE is the full Go package name. Generated code is dumped to
// stdout. Packages generated in this way can be imported by Grumpy programs
// using string literal import syntax, e.g.:
//
// import "__go__/encoding/json"
//
// Or:
//
// from "__go__/time" import Duration

package main

import (
	"bytes"
	"fmt"
	"go/constant"
	"go/importer"
	"go/types"
	"math"
	"os"
	"path"
)

const packageTemplate = `package %[1]s
import (
	"grumpy"
	"reflect"
	mod %[2]q
)
func fun(f *grumpy.Frame, _ []*grumpy.Object) (*grumpy.Object, *grumpy.BaseException) {
%[3]s
	return nil, nil
}
var Code = grumpy.NewCode("<module>", %[2]q, nil, 0, fun)
func init() {
	grumpy.RegisterModule("__go__/%[2]s", Code)
}
`

const typeTemplate = `	if true {
		var x mod.%[1]s
		if o, raised := grumpy.WrapNative(f, reflect.ValueOf(x)); raised != nil {
			return nil, raised
		} else if raised = f.Globals().SetItemString(f, %[1]q, o.Type().ToObject()); raised != nil {
			return nil, raised
		}
	}
`

const varTemplate = `	if o, raised := grumpy.WrapNative(f, reflect.ValueOf(%[1]s)); raised != nil {
		return nil, raised
	} else if raised = f.Globals().SetItemString(f, %[2]q, o); raised != nil {
		return nil, raised
	}
`

func getConst(name string, v constant.Value) string {
	format := "%s"
	switch v.Kind() {
	case constant.Int:
		if constant.Sign(v) >= 0 {
			if i, exact := constant.Uint64Val(v); exact {
				if i < math.MaxInt8 {
					format = "uint(%s)"
				} else if i < math.MaxInt32 {
					format = "uint32(%s)"
				} else {
					format = "uint64(%s)"
				}
			} else {
				format = "float64(%s)"
			}
		}
	case constant.Float:
		format = "float64(%s)"
	}
	return fmt.Sprintf(format, name)
}

func main() {
	if len(os.Args) != 2 {
		fmt.Fprint(os.Stderr, "usage: pkgc PACKAGE")
		os.Exit(1)
	}
	pkgPath := os.Args[1]
	pkg, err := importer.Default().Import(pkgPath)
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to import: %q: %v\n", pkgPath, err)
		os.Exit(2)
	}
	var buf bytes.Buffer
	scope := pkg.Scope()
	for _, name := range scope.Names() {
		o := scope.Lookup(name)
		if !o.Exported() {
			continue
		}
		switch x := o.(type) {
		case *types.TypeName:
			if types.IsInterface(x.Type()) {
				continue
			}
			buf.WriteString(fmt.Sprintf(typeTemplate, name))
		case *types.Const:
			expr := getConst("mod." + name, x.Val())
			buf.WriteString(fmt.Sprintf(varTemplate, expr, name))
		default:
			expr := "mod." + name
			buf.WriteString(fmt.Sprintf(varTemplate, expr, name))
		}
	}
	fmt.Printf(packageTemplate, path.Base(pkgPath), pkgPath, buf.Bytes())
}
<commit_msg>Fix int underflow on 32bit CPUs<commit_after>// pkgc is a tool for generating wrappers for Go packages imported by Grumpy
// programs.
//
// usage: pkgc PACKAGE
//
// Where PACKAGE is the full Go package name. Generated code is dumped to
// stdout. Packages generated in this way can be imported by Grumpy programs
// using string literal import syntax, e.g.:
//
// import "__go__/encoding/json"
//
// Or:
//
// from "__go__/time" import Duration

package main

import (
	"bytes"
	"fmt"
	"go/constant"
	"go/importer"
	"go/types"
	"math"
	"os"
	"path"
)

const packageTemplate = `package %[1]s
import (
	"grumpy"
	"reflect"
	mod %[2]q
)
func fun(f *grumpy.Frame, _ []*grumpy.Object) (*grumpy.Object, *grumpy.BaseException) {
%[3]s
	return nil, nil
}
var Code = grumpy.NewCode("<module>", %[2]q, nil, 0, fun)
func init() {
	grumpy.RegisterModule("__go__/%[2]s", Code)
}
`

const typeTemplate = `	if true {
		var x mod.%[1]s
		if o, raised := grumpy.WrapNative(f, reflect.ValueOf(x)); raised != nil {
			return nil, raised
		} else if raised = f.Globals().SetItemString(f, %[1]q, o.Type().ToObject()); raised != nil {
			return nil, raised
		}
	}
`

const varTemplate = `	if o, raised := grumpy.WrapNative(f, reflect.ValueOf(%[1]s)); raised != nil {
		return nil, raised
	} else if raised = f.Globals().SetItemString(f, %[2]q, o); raised != nil {
		return nil, raised
	}
`

func getConst(name string, v constant.Value) string {
	format := "%s"
	switch v.Kind() {
	case constant.Int:
		if constant.Sign(v) >= 0 {
			if i, exact := constant.Uint64Val(v); exact {
				if i < math.MaxInt8 {
					format = "uint(%s)"
				} else if i < math.MaxInt32 {
					format = "uint32(%s)"
				} else {
					format = "uint64(%s)"
				}
			} else {
				format = "float64(%s)"
			}
		} else {
			if i, exact := constant.Int64Val(v); exact {
				if i > math.MinInt8 {
					format = "int(%s)"
				} else if i > math.MinInt32 {
					format = "int32(%s)"
				} else {
					format = "int64(%s)"
				}
			}
		}
	case constant.Float:
		format = "float64(%s)"
	}
	return fmt.Sprintf(format, name)
}

func main() {
	if len(os.Args) != 2 {
		fmt.Fprint(os.Stderr, "usage: pkgc PACKAGE")
		os.Exit(1)
	}
	pkgPath := os.Args[1]
	pkg, err := importer.Default().Import(pkgPath)
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to import: %q: %v\n", pkgPath, err)
		os.Exit(2)
	}
	var buf bytes.Buffer
	scope := pkg.Scope()
	for _, name := range scope.Names() {
		o := scope.Lookup(name)
		if !o.Exported() {
			continue
		}
		switch x := o.(type) {
		case *types.TypeName:
			if types.IsInterface(x.Type()) {
				continue
			}
			buf.WriteString(fmt.Sprintf(typeTemplate, name))
		case *types.Const:
			expr := getConst("mod." + name, x.Val())
			buf.WriteString(fmt.Sprintf(varTemplate, expr, name))
		default:
			expr := "mod." + name
			buf.WriteString(fmt.Sprintf(varTemplate, expr, name))
		}
	}
	fmt.Printf(packageTemplate, path.Base(pkgPath), pkgPath, buf.Bytes())
}
<|endoftext|>


<commit_before>// pkgc is a tool for generating wrappers for Go packages imported by Grumpy
// programs.
//
// usage: pkgc PACKAGE
//
// Where PACKAGE is the full Go package name. Generated code is dumped to
// stdout. Packages generated in this way can be imported by Grumpy programs
// using string literal import syntax, e.g.:
//
// import "__go__/encoding/json"
//
// Or:
//
// from "__go__/time" import Duration

package main

import (
	"bytes"
	"fmt"
	"go/constant"
	"go/importer"
	"go/types"
	"math"
	"os"
	"path"
)

const packageTemplate = `package %[1]s
import (
	"grumpy"
	"reflect"
	mod %[2]q
)
func fun(f *grumpy.Frame, _ []*grumpy.Object) (*grumpy.Object, *grumpy.BaseException) {
%[3]s
	return nil, nil
}
var Code = grumpy.NewCode("<module>", %[2]q, nil, 0, fun)
func init() {
	grumpy.RegisterModule("__go__/%[2]s", Code)
}
`

const typeTemplate = `	if true {
		var x mod.%[1]s
		if o, raised := grumpy.WrapNative(f, reflect.ValueOf(x)); raised != nil {
			return nil, raised
		} else if raised = f.Globals().SetItemString(f, %[1]q, o.Type().ToObject()); raised != nil {
			return nil, raised
		}
	}
`

const varTemplate = `	if o, raised := grumpy.WrapNative(f, reflect.ValueOf(%[1]s)); raised != nil {
		return nil, raised
	} else if raised = f.Globals().SetItemString(f, %[2]q, o); raised != nil {
		return nil, raised
	}
`

func getConst(name string, v constant.Value) string {
	format := "%s"
	switch v.Kind() {
	case constant.Int:
		if constant.Sign(v) >= 0 {
			if i, exact := constant.Uint64Val(v); exact {
				if i > math.MaxInt64 {
					format = "uint64(%s)"
				}
			} else {
				format = "float64(%s)"
			}
		}
	case constant.Float:
		format = "float64(%s)"
	}
	return fmt.Sprintf(format, name)
}

func main() {
	if len(os.Args) != 2 {
		fmt.Fprint(os.Stderr, "usage: pkgc PACKAGE")
		os.Exit(1)
	}
	pkgPath := os.Args[1]
	pkg, err := importer.Default().Import(pkgPath)
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to import: %q: %v\n", pkgPath, err)
		os.Exit(2)
	}
	var buf bytes.Buffer
	scope := pkg.Scope()
	for _, name := range scope.Names() {
		o := scope.Lookup(name)
		if !o.Exported() {
			continue
		}
		switch x := o.(type) {
		case *types.TypeName:
			if types.IsInterface(x.Type()) {
				continue
			}
			buf.WriteString(fmt.Sprintf(typeTemplate, name))
		case *types.Const:
			expr := getConst("mod." + name, x.Val())
			buf.WriteString(fmt.Sprintf(varTemplate, expr, name))
		default:
			expr := "mod." + name
			buf.WriteString(fmt.Sprintf(varTemplate, expr, name))
		}
	}
	fmt.Printf(packageTemplate, path.Base(pkgPath), pkgPath, buf.Bytes())
}
<commit_msg>Fix overflow on math.MaxInt64<commit_after>// pkgc is a tool for generating wrappers for Go packages imported by Grumpy
// programs.
//
// usage: pkgc PACKAGE
//
// Where PACKAGE is the full Go package name. Generated code is dumped to
// stdout. Packages generated in this way can be imported by Grumpy programs
// using string literal import syntax, e.g.:
//
// import "__go__/encoding/json"
//
// Or:
//
// from "__go__/time" import Duration

package main

import (
	"bytes"
	"fmt"
	"go/constant"
	"go/importer"
	"go/types"
	"math"
	"os"
	"path"
)

const packageTemplate = `package %[1]s
import (
	"grumpy"
	"reflect"
	mod %[2]q
)
func fun(f *grumpy.Frame, _ []*grumpy.Object) (*grumpy.Object, *grumpy.BaseException) {
%[3]s
	return nil, nil
}
var Code = grumpy.NewCode("<module>", %[2]q, nil, 0, fun)
func init() {
	grumpy.RegisterModule("__go__/%[2]s", Code)
}
`

const typeTemplate = `	if true {
		var x mod.%[1]s
		if o, raised := grumpy.WrapNative(f, reflect.ValueOf(x)); raised != nil {
			return nil, raised
		} else if raised = f.Globals().SetItemString(f, %[1]q, o.Type().ToObject()); raised != nil {
			return nil, raised
		}
	}
`

const varTemplate = `	if o, raised := grumpy.WrapNative(f, reflect.ValueOf(%[1]s)); raised != nil {
		return nil, raised
	} else if raised = f.Globals().SetItemString(f, %[2]q, o); raised != nil {
		return nil, raised
	}
`

func getConst(name string, v constant.Value) string {
	format := "%s"
	switch v.Kind() {
	case constant.Int:
		if constant.Sign(v) >= 0 {
			if i, exact := constant.Uint64Val(v); exact {
				if i < math.MaxInt8 {
					format = "uint(%s)"
				} else if i < math.MaxInt32 {
					format = "uint32(%s)"
				} else {
					format = "uint64(%s)"
				}
			} else {
				format = "float64(%s)"
			}
		}
	case constant.Float:
		format = "float64(%s)"
	}
	return fmt.Sprintf(format, name)
}

func main() {
	if len(os.Args) != 2 {
		fmt.Fprint(os.Stderr, "usage: pkgc PACKAGE")
		os.Exit(1)
	}
	pkgPath := os.Args[1]
	pkg, err := importer.Default().Import(pkgPath)
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to import: %q: %v\n", pkgPath, err)
		os.Exit(2)
	}
	var buf bytes.Buffer
	scope := pkg.Scope()
	for _, name := range scope.Names() {
		o := scope.Lookup(name)
		if !o.Exported() {
			continue
		}
		switch x := o.(type) {
		case *types.TypeName:
			if types.IsInterface(x.Type()) {
				continue
			}
			buf.WriteString(fmt.Sprintf(typeTemplate, name))
		case *types.Const:
			expr := getConst("mod." + name, x.Val())
			buf.WriteString(fmt.Sprintf(varTemplate, expr, name))
		default:
			expr := "mod." + name
			buf.WriteString(fmt.Sprintf(varTemplate, expr, name))
		}
	}
	fmt.Printf(packageTemplate, path.Base(pkgPath), pkgPath, buf.Bytes())
}
<|endoftext|>


<commit_before>package ipns

import (
	"bytes"
	"crypto/rand"
	"io/ioutil"
	"os"
	"runtime"
	"strings"
	"testing"
	"time"

	fstest "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil"
	core "github.com/jbenet/go-ipfs/core"
	u "github.com/jbenet/go-ipfs/util"
)

func maybeSkipFuseTests(t *testing.T) bool {
	v := "TEST_NO_FUSE"
	n := strings.ToLower(os.Getenv(v))
	skip := n != "" && n != "false" && n != "f"

	if skip {
		t.Skipf("Skipping FUSE tests (%s=%s)", v, n)
	}
	return skip
}

func randBytes(size int) []byte {
	b := make([]byte, size)
	rand.Read(b)
	return b
}

func writeFile(t *testing.T, size int, path string) []byte {
	return writeFileData(t, randBytes(size), path)
}

func writeFileData(t *testing.T, data []byte, path string) []byte {
	fi, err := os.Create(path)
	if err != nil {
		t.Fatal(err)
	}

	n, err := fi.Write(data)
	if err != nil {
		t.Fatal(err)
	}

	if n != len(data) {
		t.Fatal("Didnt write proper amount!")
	}

	err = fi.Close()
	if err != nil {
		t.Fatal(err)
	}

	return data
}

func setupIpnsTest(t *testing.T, node *core.IpfsNode) (*core.IpfsNode, *fstest.Mount) {
	maybeSkipFuseTests(t)

	var err error
	if node == nil {
		node, err = core.NewMockNode()
		if err != nil {
			t.Fatal(err)
		}
	}

	fs, err := NewIpns(node, "")
	if err != nil {
		t.Fatal(err)
	}
	mnt, err := fstest.MountedT(t, fs)
	if err != nil {
		t.Fatal(err)
	}

	return node, mnt
}

// Test writing a file and reading it back
func TestIpnsBasicIO(t *testing.T) {
	_, mnt := setupIpnsTest(t, nil)
	defer mnt.Close()

	fname := mnt.Dir + "/local/testfile"
	data := writeFile(t, 12345, fname)

	rbuf, err := ioutil.ReadFile(fname)
	if err != nil {
		t.Fatal(err)
	}

	if !bytes.Equal(rbuf, data) {
		t.Fatal("Incorrect Read!")
	}
}

// Test to make sure file changes persist over mounts of ipns
func TestFilePersistence(t *testing.T) {
	node, mnt := setupIpnsTest(t, nil)

	fname := "/local/atestfile"
	data := writeFile(t, 127, mnt.Dir+fname)

	// Wait for publish: TODO: make publish happen faster in tests
	time.Sleep(time.Millisecond * 40)

	mnt.Close()

	node, mnt = setupIpnsTest(t, node)
	defer mnt.Close()

	rbuf, err := ioutil.ReadFile(mnt.Dir + fname)
	if err != nil {
		t.Fatal(err)
	}

	if !bytes.Equal(rbuf, data) {
		t.Fatalf("File data changed between mounts! sizes differ: %d != %d", len(data), len(rbuf))
	}
}

// Test to make sure the filesystem reports file sizes correctly
func TestFileSizeReporting(t *testing.T) {
	_, mnt := setupIpnsTest(t, nil)
	defer mnt.Close()

	fname := mnt.Dir + "/local/sizecheck"
	data := writeFile(t, 5555, fname)

	finfo, err := os.Stat(fname)
	if err != nil {
		t.Fatal(err)
	}

	if finfo.Size() != int64(len(data)) {
		t.Fatal("Read incorrect size from stat!")
	}
}

// Test to make sure you cant create multiple entries with the same name
func TestDoubleEntryFailure(t *testing.T) {
	_, mnt := setupIpnsTest(t, nil)
	defer mnt.Close()

	dname := mnt.Dir + "/local/thisisadir"
	err := os.Mkdir(dname, 0777)
	if err != nil {
		t.Fatal(err)
	}

	err = os.Mkdir(dname, 0777)
	if err == nil {
		t.Fatal("Should have gotten error one creating new directory.")
	}
}

func TestAppendFile(t *testing.T) {
	_, mnt := setupIpnsTest(t, nil)
	defer mnt.Close()

	fname := mnt.Dir + "/local/file"
	data := writeFile(t, 1300, fname)

	fi, err := os.OpenFile(fname, os.O_RDWR|os.O_APPEND, 0666)
	if err != nil {
		t.Fatal(err)
	}

	nudata := randBytes(500)

	n, err := fi.Write(nudata)
	if err != nil {
		t.Fatal(err)
	}
	err = fi.Close()
	if err != nil {
		t.Fatal(err)
	}

	if n != len(nudata) {
		t.Fatal("Failed to write enough bytes.")
	}

	data = append(data, nudata...)

	rbuf, err := ioutil.ReadFile(fname)
	if err != nil {
		t.Fatal(err)
	}
	if !bytes.Equal(rbuf, data) {
		t.Fatal("Data inconsistent!")
	}
}

func TestFastRepublish(t *testing.T) {

	// make timeout noticeable.
	osrt := shortRepublishTimeout
	shortRepublishTimeout = time.Millisecond * 100

	olrt := longRepublishTimeout
	longRepublishTimeout = time.Second

	node, mnt := setupIpnsTest(t, nil)

	h, err := node.Identity.PrivKey().GetPublic().Hash()
	if err != nil {
		t.Fatal(err)
	}
	pubkeyHash := u.Key(h).Pretty()

	// set them back
	defer func() {
		shortRepublishTimeout = osrt
		longRepublishTimeout = olrt
		mnt.Close()
	}()

	closed := make(chan struct{})
	dataA := []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
	dataB := []byte("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")

	fname := mnt.Dir + "/local/file"

	// get first resolved hash
	log.Debug("publishing first hash")
	writeFileData(t, dataA, fname) // random
	<-time.After(shortRepublishTimeout * 11 / 10)
	log.Debug("resolving first hash")
	resolvedHash, err := node.Namesys.Resolve(pubkeyHash)
	if err != nil {
		t.Fatal("resolve err:", pubkeyHash, err)
	}

	// constantly keep writing to the file
	go func(timeout time.Duration) {
		for {
			select {
			case <-closed:
				return

			case <-time.After(timeout * 8 / 10):
				writeFileData(t, dataB, fname)
			}
		}
	}(shortRepublishTimeout)

	hasPublished := func() bool {
		res, err := node.Namesys.Resolve(pubkeyHash)
		if err != nil {
			t.Fatalf("resolve err: %v", err)
		}
		return res != resolvedHash
	}

	// test things

	// at this point, should not have written dataA and not have written dataB
	rbuf, err := ioutil.ReadFile(fname)
	if err != nil || !bytes.Equal(rbuf, dataA) {
		t.Fatalf("Data inconsistent! %v %v", err, string(rbuf))
	}

	if hasPublished() {
		t.Fatal("published (wrote)")
	}

	<-time.After(shortRepublishTimeout * 11 / 10)

	// at this point, should have written written dataB, but not published it
	rbuf, err = ioutil.ReadFile(fname)
	if err != nil || !bytes.Equal(rbuf, dataB) {
		t.Fatalf("Data inconsistent! %v %v", err, string(rbuf))
	}

	if hasPublished() {
		t.Fatal("published (wrote)")
	}

	<-time.After(longRepublishTimeout * 11 / 10)

	// at this point, should have written written dataB, and published it
	rbuf, err = ioutil.ReadFile(fname)
	if err != nil || !bytes.Equal(rbuf, dataB) {
		t.Fatalf("Data inconsistent! %v %v", err, string(rbuf))
	}

	if !hasPublished() {
		t.Fatal("not published")
	}

	close(closed)
}

// Test writing a medium sized file one byte at a time
func TestMultiWrite(t *testing.T) {
	/*
		if runtime.GOOS == "darwin" {
			link := "https://github.com/jbenet/go-ipfs/issues/147"
			t.Skipf("Skipping as is broken in OSX. See %s", link)
		}
	*/

	_, mnt := setupIpnsTest(t, nil)
	defer mnt.Close()

	fpath := mnt.Dir + "/local/file"
	fi, err := os.Create(fpath)
	if err != nil {
		t.Fatal(err)
	}

	data := randBytes(1001)
	for i := 0; i < len(data); i++ {
		n, err := fi.Write(data[i : i+1])
		if err != nil {
			t.Fatal(err)
		}
		if n != 1 {
			t.Fatal("Somehow wrote the wrong number of bytes! (n != 1)")
		}
	}
	fi.Close()

	rbuf, err := ioutil.ReadFile(fpath)
	if err != nil {
		t.Fatal(err)
	}

	if !bytes.Equal(rbuf, data) {
		t.Fatal("File on disk did not match bytes written")
	}
}
<commit_msg>ipns test: unused import + rmv old comment<commit_after>package ipns

import (
	"bytes"
	"crypto/rand"
	"io/ioutil"
	"os"
	"strings"
	"testing"
	"time"

	fstest "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil"
	core "github.com/jbenet/go-ipfs/core"
	u "github.com/jbenet/go-ipfs/util"
)

func maybeSkipFuseTests(t *testing.T) bool {
	v := "TEST_NO_FUSE"
	n := strings.ToLower(os.Getenv(v))
	skip := n != "" && n != "false" && n != "f"

	if skip {
		t.Skipf("Skipping FUSE tests (%s=%s)", v, n)
	}
	return skip
}

func randBytes(size int) []byte {
	b := make([]byte, size)
	rand.Read(b)
	return b
}

func writeFile(t *testing.T, size int, path string) []byte {
	return writeFileData(t, randBytes(size), path)
}

func writeFileData(t *testing.T, data []byte, path string) []byte {
	fi, err := os.Create(path)
	if err != nil {
		t.Fatal(err)
	}

	n, err := fi.Write(data)
	if err != nil {
		t.Fatal(err)
	}

	if n != len(data) {
		t.Fatal("Didnt write proper amount!")
	}

	err = fi.Close()
	if err != nil {
		t.Fatal(err)
	}

	return data
}

func setupIpnsTest(t *testing.T, node *core.IpfsNode) (*core.IpfsNode, *fstest.Mount) {
	maybeSkipFuseTests(t)

	var err error
	if node == nil {
		node, err = core.NewMockNode()
		if err != nil {
			t.Fatal(err)
		}
	}

	fs, err := NewIpns(node, "")
	if err != nil {
		t.Fatal(err)
	}
	mnt, err := fstest.MountedT(t, fs)
	if err != nil {
		t.Fatal(err)
	}

	return node, mnt
}

// Test writing a file and reading it back
func TestIpnsBasicIO(t *testing.T) {
	_, mnt := setupIpnsTest(t, nil)
	defer mnt.Close()

	fname := mnt.Dir + "/local/testfile"
	data := writeFile(t, 12345, fname)

	rbuf, err := ioutil.ReadFile(fname)
	if err != nil {
		t.Fatal(err)
	}

	if !bytes.Equal(rbuf, data) {
		t.Fatal("Incorrect Read!")
	}
}

// Test to make sure file changes persist over mounts of ipns
func TestFilePersistence(t *testing.T) {
	node, mnt := setupIpnsTest(t, nil)

	fname := "/local/atestfile"
	data := writeFile(t, 127, mnt.Dir+fname)

	// Wait for publish: TODO: make publish happen faster in tests
	time.Sleep(time.Millisecond * 40)

	mnt.Close()

	node, mnt = setupIpnsTest(t, node)
	defer mnt.Close()

	rbuf, err := ioutil.ReadFile(mnt.Dir + fname)
	if err != nil {
		t.Fatal(err)
	}

	if !bytes.Equal(rbuf, data) {
		t.Fatalf("File data changed between mounts! sizes differ: %d != %d", len(data), len(rbuf))
	}
}

// Test to make sure the filesystem reports file sizes correctly
func TestFileSizeReporting(t *testing.T) {
	_, mnt := setupIpnsTest(t, nil)
	defer mnt.Close()

	fname := mnt.Dir + "/local/sizecheck"
	data := writeFile(t, 5555, fname)

	finfo, err := os.Stat(fname)
	if err != nil {
		t.Fatal(err)
	}

	if finfo.Size() != int64(len(data)) {
		t.Fatal("Read incorrect size from stat!")
	}
}

// Test to make sure you cant create multiple entries with the same name
func TestDoubleEntryFailure(t *testing.T) {
	_, mnt := setupIpnsTest(t, nil)
	defer mnt.Close()

	dname := mnt.Dir + "/local/thisisadir"
	err := os.Mkdir(dname, 0777)
	if err != nil {
		t.Fatal(err)
	}

	err = os.Mkdir(dname, 0777)
	if err == nil {
		t.Fatal("Should have gotten error one creating new directory.")
	}
}

func TestAppendFile(t *testing.T) {
	_, mnt := setupIpnsTest(t, nil)
	defer mnt.Close()

	fname := mnt.Dir + "/local/file"
	data := writeFile(t, 1300, fname)

	fi, err := os.OpenFile(fname, os.O_RDWR|os.O_APPEND, 0666)
	if err != nil {
		t.Fatal(err)
	}

	nudata := randBytes(500)

	n, err := fi.Write(nudata)
	if err != nil {
		t.Fatal(err)
	}
	err = fi.Close()
	if err != nil {
		t.Fatal(err)
	}

	if n != len(nudata) {
		t.Fatal("Failed to write enough bytes.")
	}

	data = append(data, nudata...)

	rbuf, err := ioutil.ReadFile(fname)
	if err != nil {
		t.Fatal(err)
	}
	if !bytes.Equal(rbuf, data) {
		t.Fatal("Data inconsistent!")
	}
}

func TestFastRepublish(t *testing.T) {

	// make timeout noticeable.
	osrt := shortRepublishTimeout
	shortRepublishTimeout = time.Millisecond * 100

	olrt := longRepublishTimeout
	longRepublishTimeout = time.Second

	node, mnt := setupIpnsTest(t, nil)

	h, err := node.Identity.PrivKey().GetPublic().Hash()
	if err != nil {
		t.Fatal(err)
	}
	pubkeyHash := u.Key(h).Pretty()

	// set them back
	defer func() {
		shortRepublishTimeout = osrt
		longRepublishTimeout = olrt
		mnt.Close()
	}()

	closed := make(chan struct{})
	dataA := []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
	dataB := []byte("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")

	fname := mnt.Dir + "/local/file"

	// get first resolved hash
	log.Debug("publishing first hash")
	writeFileData(t, dataA, fname) // random
	<-time.After(shortRepublishTimeout * 11 / 10)
	log.Debug("resolving first hash")
	resolvedHash, err := node.Namesys.Resolve(pubkeyHash)
	if err != nil {
		t.Fatal("resolve err:", pubkeyHash, err)
	}

	// constantly keep writing to the file
	go func(timeout time.Duration) {
		for {
			select {
			case <-closed:
				return

			case <-time.After(timeout * 8 / 10):
				writeFileData(t, dataB, fname)
			}
		}
	}(shortRepublishTimeout)

	hasPublished := func() bool {
		res, err := node.Namesys.Resolve(pubkeyHash)
		if err != nil {
			t.Fatalf("resolve err: %v", err)
		}
		return res != resolvedHash
	}

	// test things

	// at this point, should not have written dataA and not have written dataB
	rbuf, err := ioutil.ReadFile(fname)
	if err != nil || !bytes.Equal(rbuf, dataA) {
		t.Fatalf("Data inconsistent! %v %v", err, string(rbuf))
	}

	if hasPublished() {
		t.Fatal("published (wrote)")
	}

	<-time.After(shortRepublishTimeout * 11 / 10)

	// at this point, should have written written dataB, but not published it
	rbuf, err = ioutil.ReadFile(fname)
	if err != nil || !bytes.Equal(rbuf, dataB) {
		t.Fatalf("Data inconsistent! %v %v", err, string(rbuf))
	}

	if hasPublished() {
		t.Fatal("published (wrote)")
	}

	<-time.After(longRepublishTimeout * 11 / 10)

	// at this point, should have written written dataB, and published it
	rbuf, err = ioutil.ReadFile(fname)
	if err != nil || !bytes.Equal(rbuf, dataB) {
		t.Fatalf("Data inconsistent! %v %v", err, string(rbuf))
	}

	if !hasPublished() {
		t.Fatal("not published")
	}

	close(closed)
}

// Test writing a medium sized file one byte at a time
func TestMultiWrite(t *testing.T) {

	_, mnt := setupIpnsTest(t, nil)
	defer mnt.Close()

	fpath := mnt.Dir + "/local/file"
	fi, err := os.Create(fpath)
	if err != nil {
		t.Fatal(err)
	}

	data := randBytes(1001)
	for i := 0; i < len(data); i++ {
		n, err := fi.Write(data[i : i+1])
		if err != nil {
			t.Fatal(err)
		}
		if n != 1 {
			t.Fatal("Somehow wrote the wrong number of bytes! (n != 1)")
		}
	}
	fi.Close()

	rbuf, err := ioutil.ReadFile(fpath)
	if err != nil {
		t.Fatal(err)
	}

	if !bytes.Equal(rbuf, data) {
		t.Fatal("File on disk did not match bytes written")
	}
}
<|endoftext|>


<commit_before>/* Index management handlers. */
package v1

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
)

func Index(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Cache-Control", "must-revalidate")
	var col, path string
	if !Require(w, r, "col", &col) {
		return
	}
	if !Require(w, r, "path", &path) {
		return
	}
	dbcol := V1DB.Use(col)
	if dbcol == nil {
		http.Error(w, fmt.Sprintf("Collection '%s' does not exist.", col), 400)
		return
	}
	if err := dbcol.Index(strings.Split(path, ",")); err != nil {
		http.Error(w, fmt.Sprint(err), 400)
		return
	}
}

func Indexes(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Cache-Control", "must-revalidate")
	var col string
	if !Require(w, r, "col", &col) {
		return
	}
	dbcol := V1DB.Use(col)
	if dbcol == nil {
		http.Error(w, fmt.Sprintf("Collection '%s' does not exist.", col), 400)
		return
	}
	indexes := make([]string, 0)
	for path := range dbcol.StrHT {
		indexes = append(indexes, path)
	}
	resp, err := json.Marshal(indexes)
	if err != nil {
		http.Error(w, fmt.Sprint("Server error."), 500)
		return
	}
	w.Write(resp)
}

func Unindex(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Cache-Control", "must-revalidate")
	var col, path string
	if !Require(w, r, "col", &col) {
		return
	}
	if !Require(w, r, "path", &path) {
		return
	}
	dbcol := V1DB.Use(col)
	if dbcol == nil {
		http.Error(w, fmt.Sprintf("Collection '%s' does not exist.", col), 400)
		return
	}
	if err := dbcol.Unindex(strings.Split(path, ",")); err != nil {
		http.Error(w, fmt.Sprint(err), 400)
		return
	}
}
<commit_msg>return http 201 when creating index<commit_after>/* Index management handlers. */
package v1

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
)

func Index(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Cache-Control", "must-revalidate")
	var col, path string
	if !Require(w, r, "col", &col) {
		return
	}
	if !Require(w, r, "path", &path) {
		return
	}
	dbcol := V1DB.Use(col)
	if dbcol == nil {
		http.Error(w, fmt.Sprintf("Collection '%s' does not exist.", col), 400)
		return
	}
	if err := dbcol.Index(strings.Split(path, ",")); err != nil {
		http.Error(w, fmt.Sprint(err), 400)
		return
	}
	w.WriteHeader(201)
}

func Indexes(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Cache-Control", "must-revalidate")
	var col string
	if !Require(w, r, "col", &col) {
		return
	}
	dbcol := V1DB.Use(col)
	if dbcol == nil {
		http.Error(w, fmt.Sprintf("Collection '%s' does not exist.", col), 400)
		return
	}
	indexes := make([]string, 0)
	for path := range dbcol.StrHT {
		indexes = append(indexes, path)
	}
	resp, err := json.Marshal(indexes)
	if err != nil {
		http.Error(w, fmt.Sprint("Server error."), 500)
		return
	}
	w.Write(resp)
}

func Unindex(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Cache-Control", "must-revalidate")
	var col, path string
	if !Require(w, r, "col", &col) {
		return
	}
	if !Require(w, r, "path", &path) {
		return
	}
	dbcol := V1DB.Use(col)
	if dbcol == nil {
		http.Error(w, fmt.Sprintf("Collection '%s' does not exist.", col), 400)
		return
	}
	if err := dbcol.Unindex(strings.Split(path, ",")); err != nil {
		http.Error(w, fmt.Sprint(err), 400)
		return
	}
}
<|endoftext|>


<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package auth contains authorization check wrapper for handlers.
// Example:
// h, err := auth.WithAuth(handler, checker, Requirement{ClientID: true, ClientSecret: true, Role: Admin}
// if err != nil { ... }
// r.HandleFunc("/path", h)
package auth

import (
	"context"
	"fmt"
	"net/http"
	"strings"

	"github.com/golang/glog" /* copybara-comment */
	"github.com/gorilla/mux" /* copybara-comment */
	"google.golang.org/grpc/codes" /* copybara-comment */
	"google.golang.org/grpc/status" /* copybara-comment */
	"github.com/GoogleCloudPlatform/healthcare-federated-access-services/lib/common" /* copybara-comment: common */
	"github.com/GoogleCloudPlatform/healthcare-federated-access-services/lib/ga4gh" /* copybara-comment: ga4gh */
	"github.com/GoogleCloudPlatform/healthcare-federated-access-services/lib/httputil" /* copybara-comment: httputil */
	"github.com/GoogleCloudPlatform/healthcare-federated-access-services/lib/oathclients" /* copybara-comment: oathclients */
)

const (
	// maxHTTPBody = 2M
	maxHTTPBody = 2 * 1000 * 1000

	// None -> no bearer token required
	None Role = ""
	// User -> requires any valid bearer token, need to match {user} in path
	User Role = "user"
	// Admin -> requires bearer token with admin permission
	Admin Role = "admin"
)

var (
	// RequireNone -> requires nothing for authorization
	RequireNone = Require{ClientID: false, ClientSecret: false, Role: None}
	// RequireClientID -> only require client id
	RequireClientID = Require{ClientID: true, ClientSecret: false, Role: None}
	// RequireClientIDAndSecret -> require client id and matched secret
	RequireClientIDAndSecret = Require{ClientID: true, ClientSecret: true, Role: None}
	// RequireAdminToken -> require an admin token, also the client id and secret
	RequireAdminToken = Require{ClientID: true, ClientSecret: true, Role: Admin}
	// RequireUserToken -> require an user token, also the client id and secret
	RequireUserToken = Require{ClientID: true, ClientSecret: true, Role: User}
)

// Role requirement of access.
type Role string

// Checker stores information and functions for authorization check.
type Checker struct {
	// Accepted oidc Issuer url.
	Issuer string
	// FetchClientSecrets fetchs client id and client secret.
	FetchClientSecrets func() (map[string]string, error)
	// TransformIdentity transform as needed, will run just after token convert to identity.
	// eg. hydra stores custom claims in "ext" fields for access token. need to move to top
	// level field.
	TransformIdentity func(*ga4gh.Identity) *ga4gh.Identity
	// IsAdmin checks if the given identity has admin permission.
	IsAdmin func(*ga4gh.Identity) error
}

// Require defines the Authorization Requirement.
type Require struct {
	ClientID     bool
	ClientSecret bool
	// Roles current supports "user" and "admin". Check will check the role inside the bearer token.
	// not requirement bearer token if "Role" is empty.
	Role Role
}

// MustWithAuth wraps the handler func with authorization check includes client credentials, bearer token validation and role in token.
// function will cause fatal if passed in invalid requirement. This is cleaner when calling in main.
func MustWithAuth(handler func(http.ResponseWriter, *http.Request), checker *Checker, require Require) func(http.ResponseWriter, *http.Request) {
	h, err := WithAuth(handler, checker, require)
	if err != nil {
		log.Fatalf("WithAuth(): %v", err)
	}
	return h
}

// WithAuth wraps the handler func with authorization check includes client credentials, bearer token validation and role in token.
// function will return error if passed in invalid requirement.
func WithAuth(handler func(http.ResponseWriter, *http.Request), checker *Checker, require Require) (func(http.ResponseWriter, *http.Request), error) {
	if !require.ClientID && (require.ClientSecret || len(require.Role) != 0) {
		return nil, fmt.Errorf("must require client_id when require client_secret or bearer token")
	}

	if require.Role != None && require.Role != User && require.Role != Admin {
		return nil, fmt.Errorf("undefined role: %s", require.Role)
	}

	return func(w http.ResponseWriter, r *http.Request) {
		if err := checker.check(r, require); err != nil {
			httputil.WriteRPCResp(w, nil, err)
			return
		}

		handler(w, r)
	}, nil
}

// checkRequest need to validate the request before actually read data from it.
func checkRequest(r *http.Request) error {
	// TODO: maybe should also cover content-length = -1
	if r.ContentLength > maxHTTPBody {
		return status.Error(codes.FailedPrecondition, "body too large")
	}

	return nil
}

// Check checks request meet all authorization requirements.
func (s *Checker) check(r *http.Request, require Require) error {
	if err := checkRequest(r); err != nil {
		return err
	}

	if !require.ClientID {
		return nil
	}

	r.ParseForm()

	cID := oathclients.ExtractClientID(r)
	cSec := oathclients.ExtractClientSecret(r)

	if err := s.verifyClientCredentials(cID, cSec, require); err != nil {
		return err
	}

	// Not require bearer token.
	if require.Role == None {
		return nil
	}

	tok := extractBearerToken(r)

	if err := verifyToken(r.Context(), tok, s.Issuer, cID); err != nil {
		return err
	}

	id, err := s.tokenToIdentityWithoutVerification(tok)
	if err != nil {
		return err
	}

	if err := verifyIdentity(id, s.Issuer, cID); err != nil {
		return err
	}

	err = s.IsAdmin(id)

	switch require.Role {
	case Admin:
		if err != nil {
			// TODO: token maybe leaked at this point, consider auto revoke or contact user/admin.
			return status.Errorf(codes.Unauthenticated, "requires admin permission %v", err)
		}
		return nil

	case User:
		// Token is for an administrator, who is able to act on behalf of any user, so short-circuit remaining checks.
		if err == nil {
			return nil
		}
		if user := mux.Vars(r)["user"]; len(user) != 0 && user != id.Subject {
			// TODO: token maybe leaked at this point, consider auto revoke or contact user/admin.
			return status.Errorf(codes.Unauthenticated, "user in path does not match token")
		}
		return nil

	default:
		return status.Errorf(codes.Unauthenticated, "unknown role %q", require.Role)
	}
}

// verifyClientCredentials based on the provided requirement, the function
// checks if the client is known and the provided secret matches the secret
// for that client.
func (s *Checker) verifyClientCredentials(client, secret string, require Require) error {
	secrets, err := s.FetchClientSecrets()
	if err != nil {
		return err
	}

	// Check that the client ID exists and it is a known.
	if len(client) == 0 {
		return status.Error(codes.Unauthenticated, "requires a valid client ID")
	}

	want, ok := secrets[client]
	if !ok {
		return status.Errorf(codes.Unauthenticated, "client ID %q is unrecognized", client)
	}

	if !require.ClientSecret {
		return nil
	}

	// Check that the client secret match the client ID.
	if want != secret {
		return status.Error(codes.Unauthenticated, "requires a valid client secret")
	}

	return nil
}

// extractBearerToken from Authorization Header.
func extractBearerToken(r *http.Request) string {
	parts := strings.SplitN(r.Header.Get("Authorization"), " ", 2)
	if len(parts) == 2 && strings.ToLower(parts[0]) == "bearer" {
		return parts[1]
	}
	return ""
}

// tokenToIdentityWithoutVerification parse the token to Identity struct.
// Also normalize the issuer string inside Identity and apply the transform needed in Checker.
func (s *Checker) tokenToIdentityWithoutVerification(tok string) (*ga4gh.Identity, error) {
	id, err := common.ConvertTokenToIdentityUnsafe(tok)
	if err != nil {
		return nil, status.Errorf(codes.Unauthenticated, "invalid token format: %v", err)
	}

	id.Issuer = normalize(id.Issuer)

	id = s.TransformIdentity(id)

	return id, nil
}

// verifyIdentity verifies:
// - token issuer
// - subject is not empty
// - aud and azp allow given clientID
// - id.Valid(): expire, notBefore, issueAt
func verifyIdentity(id *ga4gh.Identity, issuer, clientID string) error {
	iss := normalize(issuer)
	if id.Issuer != iss {
		// TODO: token maybe leaked at this point, consider auto revoke or contact user/admin.
		return status.Errorf(codes.Unauthenticated, "token unauthorized: for issuer %s", id.Issuer)
	}

	if len(id.Subject) == 0 {
		return status.Error(codes.Unauthenticated, "token unauthorized: no subject")
	}

	if !common.IsAudience(id, clientID, iss) {
		// TODO: token maybe leaked at this point, consider auto revoke or contact user/admin.
		return status.Errorf(codes.Unauthenticated, "token unauthorized: unauthorized party")
	}

	if err := id.Valid(); err != nil {
		return status.Errorf(codes.Unauthenticated, "token unauthorized: %v", err)
	}

	return nil
}

// verifyToken oidc spec verfiy token.
func verifyToken(ctx context.Context, tok, iss, clientID string) error {
	v, err := common.GetOIDCTokenVerifier(ctx, clientID, iss)
	if err != nil {
		return status.Errorf(codes.Unauthenticated, "GetOIDCTokenVerifier failed: %v", err)
	}

	if _, err = v.Verify(ctx, tok); err != nil {
		return status.Errorf(codes.Unauthenticated, "token verify failed: %v", err)
	}

	return nil
}

// normalize ensure the issuer string and tailling slash.
func normalize(issuer string) string {
	return strings.TrimSuffix(issuer, "/")
}
<commit_msg>use glog instead of log<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package auth contains authorization check wrapper for handlers.
// Example:
// h, err := auth.WithAuth(handler, checker, Requirement{ClientID: true, ClientSecret: true, Role: Admin}
// if err != nil { ... }
// r.HandleFunc("/path", h)
package auth

import (
	"context"
	"fmt"
	"net/http"
	"strings"

	glog "github.com/golang/glog" /* copybara-comment */
	"github.com/gorilla/mux" /* copybara-comment */
	"google.golang.org/grpc/codes" /* copybara-comment */
	"google.golang.org/grpc/status" /* copybara-comment */
	"github.com/GoogleCloudPlatform/healthcare-federated-access-services/lib/common" /* copybara-comment: common */
	"github.com/GoogleCloudPlatform/healthcare-federated-access-services/lib/ga4gh" /* copybara-comment: ga4gh */
	"github.com/GoogleCloudPlatform/healthcare-federated-access-services/lib/httputil" /* copybara-comment: httputil */
	"github.com/GoogleCloudPlatform/healthcare-federated-access-services/lib/oathclients" /* copybara-comment: oathclients */
)

const (
	// maxHTTPBody = 2M
	maxHTTPBody = 2 * 1000 * 1000

	// None -> no bearer token required
	None Role = ""
	// User -> requires any valid bearer token, need to match {user} in path
	User Role = "user"
	// Admin -> requires bearer token with admin permission
	Admin Role = "admin"
)

var (
	// RequireNone -> requires nothing for authorization
	RequireNone = Require{ClientID: false, ClientSecret: false, Role: None}
	// RequireClientID -> only require client id
	RequireClientID = Require{ClientID: true, ClientSecret: false, Role: None}
	// RequireClientIDAndSecret -> require client id and matched secret
	RequireClientIDAndSecret = Require{ClientID: true, ClientSecret: true, Role: None}
	// RequireAdminToken -> require an admin token, also the client id and secret
	RequireAdminToken = Require{ClientID: true, ClientSecret: true, Role: Admin}
	// RequireUserToken -> require an user token, also the client id and secret
	RequireUserToken = Require{ClientID: true, ClientSecret: true, Role: User}
)

// Role requirement of access.
type Role string

// Checker stores information and functions for authorization check.
type Checker struct {
	// Accepted oidc Issuer url.
	Issuer string
	// FetchClientSecrets fetchs client id and client secret.
	FetchClientSecrets func() (map[string]string, error)
	// TransformIdentity transform as needed, will run just after token convert to identity.
	// eg. hydra stores custom claims in "ext" fields for access token. need to move to top
	// level field.
	TransformIdentity func(*ga4gh.Identity) *ga4gh.Identity
	// IsAdmin checks if the given identity has admin permission.
	IsAdmin func(*ga4gh.Identity) error
}

// Require defines the Authorization Requirement.
type Require struct {
	ClientID     bool
	ClientSecret bool
	// Roles current supports "user" and "admin". Check will check the role inside the bearer token.
	// not requirement bearer token if "Role" is empty.
	Role Role
}

// MustWithAuth wraps the handler func with authorization check includes client credentials, bearer token validation and role in token.
// function will cause fatal if passed in invalid requirement. This is cleaner when calling in main.
func MustWithAuth(handler func(http.ResponseWriter, *http.Request), checker *Checker, require Require) func(http.ResponseWriter, *http.Request) {
	h, err := WithAuth(handler, checker, require)
	if err != nil {
		glog.Fatalf("WithAuth(): %v", err)
	}
	return h
}

// WithAuth wraps the handler func with authorization check includes client credentials, bearer token validation and role in token.
// function will return error if passed in invalid requirement.
func WithAuth(handler func(http.ResponseWriter, *http.Request), checker *Checker, require Require) (func(http.ResponseWriter, *http.Request), error) {
	if !require.ClientID && (require.ClientSecret || len(require.Role) != 0) {
		return nil, fmt.Errorf("must require client_id when require client_secret or bearer token")
	}

	if require.Role != None && require.Role != User && require.Role != Admin {
		return nil, fmt.Errorf("undefined role: %s", require.Role)
	}

	return func(w http.ResponseWriter, r *http.Request) {
		if err := checker.check(r, require); err != nil {
			httputil.WriteRPCResp(w, nil, err)
			return
		}

		handler(w, r)
	}, nil
}

// checkRequest need to validate the request before actually read data from it.
func checkRequest(r *http.Request) error {
	// TODO: maybe should also cover content-length = -1
	if r.ContentLength > maxHTTPBody {
		return status.Error(codes.FailedPrecondition, "body too large")
	}

	return nil
}

// Check checks request meet all authorization requirements.
func (s *Checker) check(r *http.Request, require Require) error {
	if err := checkRequest(r); err != nil {
		return err
	}

	if !require.ClientID {
		return nil
	}

	r.ParseForm()

	cID := oathclients.ExtractClientID(r)
	cSec := oathclients.ExtractClientSecret(r)

	if err := s.verifyClientCredentials(cID, cSec, require); err != nil {
		return err
	}

	// Not require bearer token.
	if require.Role == None {
		return nil
	}

	tok := extractBearerToken(r)

	if err := verifyToken(r.Context(), tok, s.Issuer, cID); err != nil {
		return err
	}

	id, err := s.tokenToIdentityWithoutVerification(tok)
	if err != nil {
		return err
	}

	if err := verifyIdentity(id, s.Issuer, cID); err != nil {
		return err
	}

	err = s.IsAdmin(id)

	switch require.Role {
	case Admin:
		if err != nil {
			// TODO: token maybe leaked at this point, consider auto revoke or contact user/admin.
			return status.Errorf(codes.Unauthenticated, "requires admin permission %v", err)
		}
		return nil

	case User:
		// Token is for an administrator, who is able to act on behalf of any user, so short-circuit remaining checks.
		if err == nil {
			return nil
		}
		if user := mux.Vars(r)["user"]; len(user) != 0 && user != id.Subject {
			// TODO: token maybe leaked at this point, consider auto revoke or contact user/admin.
			return status.Errorf(codes.Unauthenticated, "user in path does not match token")
		}
		return nil

	default:
		return status.Errorf(codes.Unauthenticated, "unknown role %q", require.Role)
	}
}

// verifyClientCredentials based on the provided requirement, the function
// checks if the client is known and the provided secret matches the secret
// for that client.
func (s *Checker) verifyClientCredentials(client, secret string, require Require) error {
	secrets, err := s.FetchClientSecrets()
	if err != nil {
		return err
	}

	// Check that the client ID exists and it is a known.
	if len(client) == 0 {
		return status.Error(codes.Unauthenticated, "requires a valid client ID")
	}

	want, ok := secrets[client]
	if !ok {
		return status.Errorf(codes.Unauthenticated, "client ID %q is unrecognized", client)
	}

	if !require.ClientSecret {
		return nil
	}

	// Check that the client secret match the client ID.
	if want != secret {
		return status.Error(codes.Unauthenticated, "requires a valid client secret")
	}

	return nil
}

// extractBearerToken from Authorization Header.
func extractBearerToken(r *http.Request) string {
	parts := strings.SplitN(r.Header.Get("Authorization"), " ", 2)
	if len(parts) == 2 && strings.ToLower(parts[0]) == "bearer" {
		return parts[1]
	}
	return ""
}

// tokenToIdentityWithoutVerification parse the token to Identity struct.
// Also normalize the issuer string inside Identity and apply the transform needed in Checker.
func (s *Checker) tokenToIdentityWithoutVerification(tok string) (*ga4gh.Identity, error) {
	id, err := common.ConvertTokenToIdentityUnsafe(tok)
	if err != nil {
		return nil, status.Errorf(codes.Unauthenticated, "invalid token format: %v", err)
	}

	id.Issuer = normalize(id.Issuer)

	id = s.TransformIdentity(id)

	return id, nil
}

// verifyIdentity verifies:
// - token issuer
// - subject is not empty
// - aud and azp allow given clientID
// - id.Valid(): expire, notBefore, issueAt
func verifyIdentity(id *ga4gh.Identity, issuer, clientID string) error {
	iss := normalize(issuer)
	if id.Issuer != iss {
		// TODO: token maybe leaked at this point, consider auto revoke or contact user/admin.
		return status.Errorf(codes.Unauthenticated, "token unauthorized: for issuer %s", id.Issuer)
	}

	if len(id.Subject) == 0 {
		return status.Error(codes.Unauthenticated, "token unauthorized: no subject")
	}

	if !common.IsAudience(id, clientID, iss) {
		// TODO: token maybe leaked at this point, consider auto revoke or contact user/admin.
		return status.Errorf(codes.Unauthenticated, "token unauthorized: unauthorized party")
	}

	if err := id.Valid(); err != nil {
		return status.Errorf(codes.Unauthenticated, "token unauthorized: %v", err)
	}

	return nil
}

// verifyToken oidc spec verfiy token.
func verifyToken(ctx context.Context, tok, iss, clientID string) error {
	v, err := common.GetOIDCTokenVerifier(ctx, clientID, iss)
	if err != nil {
		return status.Errorf(codes.Unauthenticated, "GetOIDCTokenVerifier failed: %v", err)
	}

	if _, err = v.Verify(ctx, tok); err != nil {
		return status.Errorf(codes.Unauthenticated, "token verify failed: %v", err)
	}

	return nil
}

// normalize ensure the issuer string and tailling slash.
func normalize(issuer string) string {
	return strings.TrimSuffix(issuer, "/")
}
<|endoftext|>