<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package rpc

import (
	"fmt"
	"http/httptest"
	"log"
	"net"
	"os"
	"runtime"
	"strings"
	"sync"
	"testing"
	"time"
)

var (
	serverAddr, newServerAddr string
	httpServerAddr            string
	once, newOnce, httpOnce   sync.Once
)

const (
	second      = 1e9
	newHttpPath = "/foo"
)

type Args struct {
	A, B int
}

type Reply struct {
	C int
}

type Arith int

func (t *Arith) Add(args *Args, reply *Reply) os.Error {
	reply.C = args.A + args.B
	return nil
}

func (t *Arith) Mul(args *Args, reply *Reply) os.Error {
	reply.C = args.A * args.B
	return nil
}

func (t *Arith) Div(args *Args, reply *Reply) os.Error {
	if args.B == 0 {
		return os.ErrorString("divide by zero")
	}
	reply.C = args.A / args.B
	return nil
}

func (t *Arith) String(args *Args, reply *string) os.Error {
	*reply = fmt.Sprintf("%d+%d=%d", args.A, args.B, args.A+args.B)
	return nil
}

func (t *Arith) Scan(args *string, reply *Reply) (err os.Error) {
	_, err = fmt.Sscan(*args, &reply.C)
	return
}

func (t *Arith) Error(args *Args, reply *Reply) os.Error {
	panic("ERROR")
}

func listenTCP() (net.Listener, string) {
	l, e := net.Listen("tcp", "127.0.0.1:0") // any available address
	if e != nil {
		log.Fatalf("net.Listen tcp :0: %v", e)
	}
	return l, l.Addr().String()
}

func startServer() {
	Register(new(Arith))

	var l net.Listener
	l, serverAddr = listenTCP()
	log.Println("Test RPC server listening on", serverAddr)
	go Accept(l)

	HandleHTTP()
	httpOnce.Do(startHttpServer)
}

func startNewServer() {
	s := NewServer()
	s.Register(new(Arith))

	var l net.Listener
	l, newServerAddr = listenTCP()
	log.Println("NewServer test RPC server listening on", newServerAddr)
	go Accept(l)

	s.HandleHTTP(newHttpPath, "/bar")
	httpOnce.Do(startHttpServer)
}

func startHttpServer() {
	server := httptest.NewServer(nil)
	httpServerAddr = server.Listener.Addr().String()
	log.Println("Test HTTP RPC server listening on", httpServerAddr)
}

func TestRPC(t *testing.T) {
	once.Do(startServer)
	testRPC(t, serverAddr)
	newOnce.Do(startNewServer)
	testRPC(t, newServerAddr)
}

func testRPC(t *testing.T, addr string) {
	client, err := Dial("tcp", addr)
	if err != nil {
		t.Fatal("dialing", err)
	}

	// Synchronous calls
	args := &Args{7, 8}
	reply := new(Reply)
	err = client.Call("Arith.Add", args, reply)
	if err != nil {
		t.Errorf("Add: expected no error but got string %q", err.String())
	}
	if reply.C != args.A+args.B {
		t.Errorf("Add: expected %d got %d", reply.C, args.A+args.B)
	}

	// Nonexistent method
	args = &Args{7, 0}
	reply = new(Reply)
	err = client.Call("Arith.BadOperation", args, reply)
	// expect an error
	if err == nil {
		t.Error("BadOperation: expected error")
	} else if !strings.HasPrefix(err.String(), "rpc: can't find method ") {
		t.Errorf("BadOperation: expected can't find method error; got %q", err)
	}

	// Unknown service
	args = &Args{7, 8}
	reply = new(Reply)
	err = client.Call("Arith.Unknown", args, reply)
	if err == nil {
		t.Error("expected error calling unknown service")
	} else if strings.Index(err.String(), "method") < 0 {
		t.Error("expected error about method; got", err)
	}

	// Out of order.
	args = &Args{7, 8}
	mulReply := new(Reply)
	mulCall := client.Go("Arith.Mul", args, mulReply, nil)
	addReply := new(Reply)
	addCall := client.Go("Arith.Add", args, addReply, nil)

	addCall = <-addCall.Done
	if addCall.Error != nil {
		t.Errorf("Add: expected no error but got string %q", addCall.Error.String())
	}
	if addReply.C != args.A+args.B {
		t.Errorf("Add: expected %d got %d", addReply.C, args.A+args.B)
	}

	mulCall = <-mulCall.Done
	if mulCall.Error != nil {
		t.Errorf("Mul: expected no error but got string %q", mulCall.Error.String())
	}
	if mulReply.C != args.A*args.B {
		t.Errorf("Mul: expected %d got %d", mulReply.C, args.A*args.B)
	}

	// Error test
	args = &Args{7, 0}
	reply = new(Reply)
	err = client.Call("Arith.Div", args, reply)
	// expect an error: zero divide
	if err == nil {
		t.Error("Div: expected error")
	} else if err.String() != "divide by zero" {
		t.Error("Div: expected divide by zero error; got", err)
	}

	// Bad type.
	reply = new(Reply)
	err = client.Call("Arith.Add", reply, reply) // args, reply would be the correct thing to use
	if err == nil {
		t.Error("expected error calling Arith.Add with wrong arg type")
	} else if strings.Index(err.String(), "type") < 0 {
		t.Error("expected error about type; got", err)
	}

	// Non-struct argument
	const Val = 12345
	str := fmt.Sprint(Val)
	reply = new(Reply)
	err = client.Call("Arith.Scan", &str, reply)
	if err != nil {
		t.Errorf("Scan: expected no error but got string %q", err.String())
	} else if reply.C != Val {
		t.Errorf("Scan: expected %d got %d", Val, reply.C)
	}

	// Non-struct reply
	args = &Args{27, 35}
	str = ""
	err = client.Call("Arith.String", args, &str)
	if err != nil {
		t.Errorf("String: expected no error but got string %q", err.String())
	}
	expect := fmt.Sprintf("%d+%d=%d", args.A, args.B, args.A+args.B)
	if str != expect {
		t.Errorf("String: expected %s got %s", expect, str)
	}

	args = &Args{7, 8}
	reply = new(Reply)
	err = client.Call("Arith.Mul", args, reply)
	if err != nil {
		t.Errorf("Mul: expected no error but got string %q", err.String())
	}
	if reply.C != args.A*args.B {
		t.Errorf("Mul: expected %d got %d", reply.C, args.A*args.B)
	}
}

func TestHTTP(t *testing.T) {
	once.Do(startServer)
	testHTTPRPC(t, "")
	newOnce.Do(startNewServer)
	testHTTPRPC(t, newHttpPath)
}

func testHTTPRPC(t *testing.T, path string) {
	var client *Client
	var err os.Error
	if path == "" {
		client, err = DialHTTP("tcp", httpServerAddr)
	} else {
		client, err = DialHTTPPath("tcp", httpServerAddr, path)
	}
	if err != nil {
		t.Fatal("dialing", err)
	}

	// Synchronous calls
	args := &Args{7, 8}
	reply := new(Reply)
	err = client.Call("Arith.Add", args, reply)
	if err != nil {
		t.Errorf("Add: expected no error but got string %q", err.String())
	}
	if reply.C != args.A+args.B {
		t.Errorf("Add: expected %d got %d", reply.C, args.A+args.B)
	}
}

type ArgNotPointer int
type ReplyNotPointer int
type ArgNotPublic int
type ReplyNotPublic int
type local struct{}

func (t *ArgNotPointer) ArgNotPointer(args Args, reply *Reply) os.Error {
	return nil
}

func (t *ReplyNotPointer) ReplyNotPointer(args *Args, reply Reply) os.Error {
	return nil
}

func (t *ArgNotPublic) ArgNotPublic(args *local, reply *Reply) os.Error {
	return nil
}

func (t *ReplyNotPublic) ReplyNotPublic(args *Args, reply *local) os.Error {
	return nil
}

// Check that registration handles lots of bad methods and a type with no suitable methods.
func TestRegistrationError(t *testing.T) {
	err := Register(new(ArgNotPointer))
	if err == nil {
		t.Errorf("expected error registering ArgNotPointer")
	}
	err = Register(new(ReplyNotPointer))
	if err == nil {
		t.Errorf("expected error registering ReplyNotPointer")
	}
	err = Register(new(ArgNotPublic))
	if err == nil {
		t.Errorf("expected error registering ArgNotPublic")
	}
	err = Register(new(ReplyNotPublic))
	if err == nil {
		t.Errorf("expected error registering ReplyNotPublic")
	}
}

type WriteFailCodec int

func (WriteFailCodec) WriteRequest(*Request, interface{}) os.Error {
	// the panic caused by this error used to not unlock a lock.
	return os.NewError("fail")
}

func (WriteFailCodec) ReadResponseHeader(*Response) os.Error {
	time.Sleep(60e9)
	panic("unreachable")
}

func (WriteFailCodec) ReadResponseBody(interface{}) os.Error {
	time.Sleep(60e9)
	panic("unreachable")
}

func (WriteFailCodec) Close() os.Error {
	return nil
}

func TestSendDeadlock(t *testing.T) {
	client := NewClientWithCodec(WriteFailCodec(0))

	done := make(chan bool)
	go func() {
		testSendDeadlock(client)
		testSendDeadlock(client)
		done <- true
	}()
	select {
	case <-done:
		return
	case <-time.After(5e9):
		t.Fatal("deadlock")
	}
}

func testSendDeadlock(client *Client) {
	defer func() {
		recover()
	}()
	args := &Args{7, 8}
	reply := new(Reply)
	client.Call("Arith.Add", args, reply)
}

func TestCountMallocs(t *testing.T) {
	once.Do(startServer)
	client, err := Dial("tcp", serverAddr)
	if err != nil {
		t.Error("error dialing", err)
	}
	args := &Args{7, 8}
	reply := new(Reply)
	mallocs := 0 - runtime.MemStats.Mallocs
	const count = 100
	for i := 0; i < count; i++ {
		err = client.Call("Arith.Add", args, reply)
		if err != nil {
			t.Errorf("Add: expected no error but got string %q", err.String())
		}
		if reply.C != args.A+args.B {
			t.Errorf("Add: expected %d got %d", reply.C, args.A+args.B)
		}
	}
	mallocs += runtime.MemStats.Mallocs
	fmt.Printf("mallocs per rpc round trip: %d\n", mallocs/count)
}

func BenchmarkEndToEnd(b *testing.B) {
	b.StopTimer()
	once.Do(startServer)
	client, err := Dial("tcp", serverAddr)
	if err != nil {
		fmt.Println("error dialing", err)
		return
	}

	// Synchronous calls
	args := &Args{7, 8}
	reply := new(Reply)
	b.StartTimer()
	for i := 0; i < b.N; i++ {
		err = client.Call("Arith.Add", args, reply)
		if err != nil {
			fmt.Printf("Add: expected no error but got string %q", err.String())
			break
		}
		if reply.C != args.A+args.B {
			fmt.Printf("Add: expected %d got %d", reply.C, args.A+args.B)
			break
		}
	}
}
<commit_msg>rpc: increase server_test timeout<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package rpc

import (
	"fmt"
	"http/httptest"
	"log"
	"net"
	"os"
	"runtime"
	"strings"
	"sync"
	"testing"
	"time"
)

var (
	serverAddr, newServerAddr string
	httpServerAddr            string
	once, newOnce, httpOnce   sync.Once
)

const (
	second      = 1e9
	newHttpPath = "/foo"
)

type Args struct {
	A, B int
}

type Reply struct {
	C int
}

type Arith int

func (t *Arith) Add(args *Args, reply *Reply) os.Error {
	reply.C = args.A + args.B
	return nil
}

func (t *Arith) Mul(args *Args, reply *Reply) os.Error {
	reply.C = args.A * args.B
	return nil
}

func (t *Arith) Div(args *Args, reply *Reply) os.Error {
	if args.B == 0 {
		return os.ErrorString("divide by zero")
	}
	reply.C = args.A / args.B
	return nil
}

func (t *Arith) String(args *Args, reply *string) os.Error {
	*reply = fmt.Sprintf("%d+%d=%d", args.A, args.B, args.A+args.B)
	return nil
}

func (t *Arith) Scan(args *string, reply *Reply) (err os.Error) {
	_, err = fmt.Sscan(*args, &reply.C)
	return
}

func (t *Arith) Error(args *Args, reply *Reply) os.Error {
	panic("ERROR")
}

func listenTCP() (net.Listener, string) {
	l, e := net.Listen("tcp", "127.0.0.1:0") // any available address
	if e != nil {
		log.Fatalf("net.Listen tcp :0: %v", e)
	}
	return l, l.Addr().String()
}

func startServer() {
	Register(new(Arith))

	var l net.Listener
	l, serverAddr = listenTCP()
	log.Println("Test RPC server listening on", serverAddr)
	go Accept(l)

	HandleHTTP()
	httpOnce.Do(startHttpServer)
}

func startNewServer() {
	s := NewServer()
	s.Register(new(Arith))

	var l net.Listener
	l, newServerAddr = listenTCP()
	log.Println("NewServer test RPC server listening on", newServerAddr)
	go Accept(l)

	s.HandleHTTP(newHttpPath, "/bar")
	httpOnce.Do(startHttpServer)
}

func startHttpServer() {
	server := httptest.NewServer(nil)
	httpServerAddr = server.Listener.Addr().String()
	log.Println("Test HTTP RPC server listening on", httpServerAddr)
}

func TestRPC(t *testing.T) {
	once.Do(startServer)
	testRPC(t, serverAddr)
	newOnce.Do(startNewServer)
	testRPC(t, newServerAddr)
}

func testRPC(t *testing.T, addr string) {
	client, err := Dial("tcp", addr)
	if err != nil {
		t.Fatal("dialing", err)
	}

	// Synchronous calls
	args := &Args{7, 8}
	reply := new(Reply)
	err = client.Call("Arith.Add", args, reply)
	if err != nil {
		t.Errorf("Add: expected no error but got string %q", err.String())
	}
	if reply.C != args.A+args.B {
		t.Errorf("Add: expected %d got %d", reply.C, args.A+args.B)
	}

	// Nonexistent method
	args = &Args{7, 0}
	reply = new(Reply)
	err = client.Call("Arith.BadOperation", args, reply)
	// expect an error
	if err == nil {
		t.Error("BadOperation: expected error")
	} else if !strings.HasPrefix(err.String(), "rpc: can't find method ") {
		t.Errorf("BadOperation: expected can't find method error; got %q", err)
	}

	// Unknown service
	args = &Args{7, 8}
	reply = new(Reply)
	err = client.Call("Arith.Unknown", args, reply)
	if err == nil {
		t.Error("expected error calling unknown service")
	} else if strings.Index(err.String(), "method") < 0 {
		t.Error("expected error about method; got", err)
	}

	// Out of order.
	args = &Args{7, 8}
	mulReply := new(Reply)
	mulCall := client.Go("Arith.Mul", args, mulReply, nil)
	addReply := new(Reply)
	addCall := client.Go("Arith.Add", args, addReply, nil)

	addCall = <-addCall.Done
	if addCall.Error != nil {
		t.Errorf("Add: expected no error but got string %q", addCall.Error.String())
	}
	if addReply.C != args.A+args.B {
		t.Errorf("Add: expected %d got %d", addReply.C, args.A+args.B)
	}

	mulCall = <-mulCall.Done
	if mulCall.Error != nil {
		t.Errorf("Mul: expected no error but got string %q", mulCall.Error.String())
	}
	if mulReply.C != args.A*args.B {
		t.Errorf("Mul: expected %d got %d", mulReply.C, args.A*args.B)
	}

	// Error test
	args = &Args{7, 0}
	reply = new(Reply)
	err = client.Call("Arith.Div", args, reply)
	// expect an error: zero divide
	if err == nil {
		t.Error("Div: expected error")
	} else if err.String() != "divide by zero" {
		t.Error("Div: expected divide by zero error; got", err)
	}

	// Bad type.
	reply = new(Reply)
	err = client.Call("Arith.Add", reply, reply) // args, reply would be the correct thing to use
	if err == nil {
		t.Error("expected error calling Arith.Add with wrong arg type")
	} else if strings.Index(err.String(), "type") < 0 {
		t.Error("expected error about type; got", err)
	}

	// Non-struct argument
	const Val = 12345
	str := fmt.Sprint(Val)
	reply = new(Reply)
	err = client.Call("Arith.Scan", &str, reply)
	if err != nil {
		t.Errorf("Scan: expected no error but got string %q", err.String())
	} else if reply.C != Val {
		t.Errorf("Scan: expected %d got %d", Val, reply.C)
	}

	// Non-struct reply
	args = &Args{27, 35}
	str = ""
	err = client.Call("Arith.String", args, &str)
	if err != nil {
		t.Errorf("String: expected no error but got string %q", err.String())
	}
	expect := fmt.Sprintf("%d+%d=%d", args.A, args.B, args.A+args.B)
	if str != expect {
		t.Errorf("String: expected %s got %s", expect, str)
	}

	args = &Args{7, 8}
	reply = new(Reply)
	err = client.Call("Arith.Mul", args, reply)
	if err != nil {
		t.Errorf("Mul: expected no error but got string %q", err.String())
	}
	if reply.C != args.A*args.B {
		t.Errorf("Mul: expected %d got %d", reply.C, args.A*args.B)
	}
}

func TestHTTP(t *testing.T) {
	once.Do(startServer)
	testHTTPRPC(t, "")
	newOnce.Do(startNewServer)
	testHTTPRPC(t, newHttpPath)
}

func testHTTPRPC(t *testing.T, path string) {
	var client *Client
	var err os.Error
	if path == "" {
		client, err = DialHTTP("tcp", httpServerAddr)
	} else {
		client, err = DialHTTPPath("tcp", httpServerAddr, path)
	}
	if err != nil {
		t.Fatal("dialing", err)
	}

	// Synchronous calls
	args := &Args{7, 8}
	reply := new(Reply)
	err = client.Call("Arith.Add", args, reply)
	if err != nil {
		t.Errorf("Add: expected no error but got string %q", err.String())
	}
	if reply.C != args.A+args.B {
		t.Errorf("Add: expected %d got %d", reply.C, args.A+args.B)
	}
}

type ArgNotPointer int
type ReplyNotPointer int
type ArgNotPublic int
type ReplyNotPublic int
type local struct{}

func (t *ArgNotPointer) ArgNotPointer(args Args, reply *Reply) os.Error {
	return nil
}

func (t *ReplyNotPointer) ReplyNotPointer(args *Args, reply Reply) os.Error {
	return nil
}

func (t *ArgNotPublic) ArgNotPublic(args *local, reply *Reply) os.Error {
	return nil
}

func (t *ReplyNotPublic) ReplyNotPublic(args *Args, reply *local) os.Error {
	return nil
}

// Check that registration handles lots of bad methods and a type with no suitable methods.
func TestRegistrationError(t *testing.T) {
	err := Register(new(ArgNotPointer))
	if err == nil {
		t.Errorf("expected error registering ArgNotPointer")
	}
	err = Register(new(ReplyNotPointer))
	if err == nil {
		t.Errorf("expected error registering ReplyNotPointer")
	}
	err = Register(new(ArgNotPublic))
	if err == nil {
		t.Errorf("expected error registering ArgNotPublic")
	}
	err = Register(new(ReplyNotPublic))
	if err == nil {
		t.Errorf("expected error registering ReplyNotPublic")
	}
}

type WriteFailCodec int

func (WriteFailCodec) WriteRequest(*Request, interface{}) os.Error {
	// the panic caused by this error used to not unlock a lock.
	return os.NewError("fail")
}

func (WriteFailCodec) ReadResponseHeader(*Response) os.Error {
	time.Sleep(120e9)
	panic("unreachable")
}

func (WriteFailCodec) ReadResponseBody(interface{}) os.Error {
	time.Sleep(120e9)
	panic("unreachable")
}

func (WriteFailCodec) Close() os.Error {
	return nil
}

func TestSendDeadlock(t *testing.T) {
	client := NewClientWithCodec(WriteFailCodec(0))

	done := make(chan bool)
	go func() {
		testSendDeadlock(client)
		testSendDeadlock(client)
		done <- true
	}()
	select {
	case <-done:
		return
	case <-time.After(5e9):
		t.Fatal("deadlock")
	}
}

func testSendDeadlock(client *Client) {
	defer func() {
		recover()
	}()
	args := &Args{7, 8}
	reply := new(Reply)
	client.Call("Arith.Add", args, reply)
}

func TestCountMallocs(t *testing.T) {
	once.Do(startServer)
	client, err := Dial("tcp", serverAddr)
	if err != nil {
		t.Error("error dialing", err)
	}
	args := &Args{7, 8}
	reply := new(Reply)
	mallocs := 0 - runtime.MemStats.Mallocs
	const count = 100
	for i := 0; i < count; i++ {
		err = client.Call("Arith.Add", args, reply)
		if err != nil {
			t.Errorf("Add: expected no error but got string %q", err.String())
		}
		if reply.C != args.A+args.B {
			t.Errorf("Add: expected %d got %d", reply.C, args.A+args.B)
		}
	}
	mallocs += runtime.MemStats.Mallocs
	fmt.Printf("mallocs per rpc round trip: %d\n", mallocs/count)
}

func BenchmarkEndToEnd(b *testing.B) {
	b.StopTimer()
	once.Do(startServer)
	client, err := Dial("tcp", serverAddr)
	if err != nil {
		fmt.Println("error dialing", err)
		return
	}

	// Synchronous calls
	args := &Args{7, 8}
	reply := new(Reply)
	b.StartTimer()
	for i := 0; i < b.N; i++ {
		err = client.Call("Arith.Add", args, reply)
		if err != nil {
			fmt.Printf("Add: expected no error but got string %q", err.String())
			break
		}
		if reply.C != args.A+args.B {
			fmt.Printf("Add: expected %d got %d", reply.C, args.A+args.B)
			break
		}
	}
}
<|endoftext|>


<commit_before>package main

//go:generate goembed static

import (
	"flag"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"github.com/jeanfric/embedfs"
	"github.com/jeanfric/liftca"
	"github.com/jeanfric/liftca/cmd/liftca/handlers"
	"github.com/jeanfric/liftca/ht"
)

func main() {
	quit := make(chan bool)

	var addressArg string
	var storeFileArg string

	flag.StringVar(&addressArg, "a", ":8080", "listen address")
	flag.StringVar(&storeFileArg, "s", "store.gob", "path to state storage file")
	flag.Parse()

	storeFile := filepath.Clean(storeFileArg)
	backingFile, err := os.OpenFile(storeFile, os.O_CREATE|os.O_RDWR, 0666)
	if err != nil {
		log.Fatal(err)
	}
	defer backingFile.Close()
	store := liftca.LoadStore(backingFile)
	storeChanged := make(chan struct{})
	store.Updates(storeChanged)
	go func(c <-chan struct{}) {
		for {
			<-c
			store.DumpStore(backingFile)
		}
	}(storeChanged)

	assets, err := loadAssets() // method generated by goembed
	if err != nil {
		log.Fatal(err)
	}

	r := ht.NewRouter()
	fileServer := ht.NewLoggingFileServer(embedfs.New(assets))

	r.Handle("GET", "/ca", ht.NewHandler(store, handlers.GetCAs))
	r.Handle("POST", "/ca", ht.NewHandler(store, handlers.PostCA))
	r.Handle("GET", "/ca/{ca_id}-certificate.cer", ht.NewHandler(store, handlers.GetCACertificateCER))
	r.Handle("GET", "/ca/{ca_id}-crl.crl", ht.NewHandler(store, handlers.GetCACRLCER))
	r.Handle("GET", "/ca/{ca_id}-private-key.cer", ht.NewHandler(store, handlers.GetCAPrivateKeyCER))
	r.Handle("GET", "/ca/{ca_id}-certificate.pem", ht.NewHandler(store, handlers.GetCACertificatePEM))
	r.Handle("GET", "/ca/{ca_id}-certificate.pem.txt", ht.NewHandler(store, handlers.GetCACertificatePEMTXT))
	r.Handle("GET", "/ca/{ca_id}-private-key.pem", ht.NewHandler(store, handlers.GetCAPrivateKeyPEM))
	r.Handle("GET", "/ca/{ca_id}-private-key.pem.txt", ht.NewHandler(store, handlers.GetCAPrivateKeyPEMTXT))
	r.Handle("GET", "/ca/{ca_id}-crl.pem", ht.NewHandler(store, handlers.GetCACRLPEM))
	r.Handle("GET", "/ca/{ca_id}-crl.pem.txt", ht.NewHandler(store, handlers.GetCACRLPEMTXT))
	r.Handle("GET", "/ca/{ca_id}", ht.NewHandler(store, handlers.GetCA))
	r.Handle("GET", "/ca/{ca_id}/cert", ht.NewHandler(store, handlers.GetCerts))
	r.Handle("POST", "/ca/{ca_id}/cert", ht.NewHandler(store, handlers.PostCert))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-certificate.pem", ht.NewHandler(store, handlers.GetCertificatePEM))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-certificate.pem.txt", ht.NewHandler(store, handlers.GetCertificatePEMTXT))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-private-key.pem", ht.NewHandler(store, handlers.GetCertificatePrivateKeyPEM))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-private-key.pem.txt", ht.NewHandler(store, handlers.GetCertificatePrivateKeyPEMTXT))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-private-key.cer", ht.NewHandler(store, handlers.GetCertificatePrivateKeyCER))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-certificate.cer", ht.NewHandler(store, handlers.GetCertificateCER))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}", ht.NewHandler(store, handlers.GetCert))
	r.Handle("POST", "/ca/{ca_id}/crl", ht.NewHandler(store, handlers.PostCRL))
	r.Handle("GET", "/ca/{ca_id}/crl", ht.NewHandler(store, handlers.GetCRL))
	r.Handle("DELETE", "/ca/{ca_id}/crl/{cert_id}", ht.NewHandler(store, handlers.DeleteCRL))
	r.Handle("GET", "/", fileServer)
	r.Handle("GET", "/{f}", fileServer)
	r.Handle("GET", "/js/{f}", fileServer)
	r.Handle("GET", "/fonts/{f}", fileServer)
	r.Handle("GET", "/css/{f}", fileServer)
	r.Handle("GET", "/img/{f}", fileServer)
	r.Handle("GET", "/partials/{f}", fileServer)

	s := &http.Server{
		Addr:           addressArg,
		ReadTimeout:    10 * time.Second,
		WriteTimeout:   10 * time.Second,
		MaxHeaderBytes: 1 << 20,
		Handler:        r,
	}
	go s.ListenAndServe()

	log.Printf("liftCA engaged at '%v', data file '%v'", addressArg, storeFile)
	<-quit
}
<commit_msg>Allow serving static assets from filesystem<commit_after>package main

//go:generate goembed static

import (
	"flag"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"github.com/jeanfric/embedfs"
	"github.com/jeanfric/liftca"
	"github.com/jeanfric/liftca/cmd/liftca/handlers"
	"github.com/jeanfric/liftca/ht"
)

func main() {
	quit := make(chan bool)

	var addressArg string
	var storeFileArg string
	var serveDir string

	flag.StringVar(&addressArg, "a", ":8080", "listen address")
	flag.StringVar(&storeFileArg, "s", "store.gob", "path to state storage file")
	flag.StringVar(&serveDir, "d", "", "if set, directory to serve static assets from; else use embedded assets")
	flag.Parse()

	storeFile := filepath.Clean(storeFileArg)
	backingFile, err := os.OpenFile(storeFile, os.O_CREATE|os.O_RDWR, 0666)
	if err != nil {
		log.Fatal(err)
	}
	defer backingFile.Close()
	store := liftca.LoadStore(backingFile)
	storeChanged := make(chan struct{})
	store.Updates(storeChanged)
	go func(c <-chan struct{}) {
		for {
			<-c
			store.DumpStore(backingFile)
		}
	}(storeChanged)

	var fileServer http.Handler

	if serveDir != "" {
		log.Printf("Serving from '%s'", serveDir)
		fileServer = ht.NewLoggingFileServer(http.Dir(serveDir))
	} else {
		assets, err := loadAssets() // method generated by goembed
		if err != nil {
			log.Fatal(err)
		}
		fileServer = ht.NewLoggingFileServer(embedfs.New(assets))
	}

	r := ht.NewRouter()

	r.Handle("GET", "/ca", ht.NewHandler(store, handlers.GetCAs))
	r.Handle("POST", "/ca", ht.NewHandler(store, handlers.PostCA))
	r.Handle("GET", "/ca/{ca_id}-certificate.cer", ht.NewHandler(store, handlers.GetCACertificateCER))
	r.Handle("GET", "/ca/{ca_id}-crl.crl", ht.NewHandler(store, handlers.GetCACRLCER))
	r.Handle("GET", "/ca/{ca_id}-private-key.cer", ht.NewHandler(store, handlers.GetCAPrivateKeyCER))
	r.Handle("GET", "/ca/{ca_id}-certificate.pem", ht.NewHandler(store, handlers.GetCACertificatePEM))
	r.Handle("GET", "/ca/{ca_id}-certificate.pem.txt", ht.NewHandler(store, handlers.GetCACertificatePEMTXT))
	r.Handle("GET", "/ca/{ca_id}-private-key.pem", ht.NewHandler(store, handlers.GetCAPrivateKeyPEM))
	r.Handle("GET", "/ca/{ca_id}-private-key.pem.txt", ht.NewHandler(store, handlers.GetCAPrivateKeyPEMTXT))
	r.Handle("GET", "/ca/{ca_id}-crl.pem", ht.NewHandler(store, handlers.GetCACRLPEM))
	r.Handle("GET", "/ca/{ca_id}-crl.pem.txt", ht.NewHandler(store, handlers.GetCACRLPEMTXT))
	r.Handle("GET", "/ca/{ca_id}", ht.NewHandler(store, handlers.GetCA))
	r.Handle("GET", "/ca/{ca_id}/cert", ht.NewHandler(store, handlers.GetCerts))
	r.Handle("POST", "/ca/{ca_id}/cert", ht.NewHandler(store, handlers.PostCert))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-certificate.pem", ht.NewHandler(store, handlers.GetCertificatePEM))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-certificate.pem.txt", ht.NewHandler(store, handlers.GetCertificatePEMTXT))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-private-key.pem", ht.NewHandler(store, handlers.GetCertificatePrivateKeyPEM))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-private-key.pem.txt", ht.NewHandler(store, handlers.GetCertificatePrivateKeyPEMTXT))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-private-key.cer", ht.NewHandler(store, handlers.GetCertificatePrivateKeyCER))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}-certificate.cer", ht.NewHandler(store, handlers.GetCertificateCER))
	r.Handle("GET", "/ca/{ca_id}/cert/{cert_id}", ht.NewHandler(store, handlers.GetCert))
	r.Handle("POST", "/ca/{ca_id}/crl", ht.NewHandler(store, handlers.PostCRL))
	r.Handle("GET", "/ca/{ca_id}/crl", ht.NewHandler(store, handlers.GetCRL))
	r.Handle("DELETE", "/ca/{ca_id}/crl/{cert_id}", ht.NewHandler(store, handlers.DeleteCRL))
	r.Handle("GET", "/", fileServer)
	r.Handle("GET", "/{f}", fileServer)
	r.Handle("GET", "/js/{f}", fileServer)
	r.Handle("GET", "/fonts/{f}", fileServer)
	r.Handle("GET", "/css/{f}", fileServer)
	r.Handle("GET", "/img/{f}", fileServer)
	r.Handle("GET", "/partials/{f}", fileServer)

	s := &http.Server{
		Addr:           addressArg,
		ReadTimeout:    10 * time.Second,
		WriteTimeout:   10 * time.Second,
		MaxHeaderBytes: 1 << 20,
		Handler:        r,
	}
	go s.ListenAndServe()

	log.Printf("liftCA engaged at '%v', data file '%v'", addressArg, storeFile)
	<-quit
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"regexp"
	"strings"

	"github.com/github/hub/git"
	"github.com/github/hub/github"
	"github.com/github/hub/ui"
	"github.com/github/hub/utils"
)

var cmdCreate = &Command{
	Run:   create,
	Usage: "create [-poc] [-d <DESCRIPTION>] [-h <HOMEPAGE>] [[<ORGANIZATION>/]<NAME>]",
	Long: `Create a new repository on GitHub and add a git remote for it.

## Options:
	-p, --private
		Create a private repository.

	-d, --description=<DESCRIPTION>
		Use this text as the description of the GitHub repository.

	-h, --homepage=<HOMEPAGE>
		Use this text as the URL of the GitHub repository.

	-o, --browse
		Open the new repository in a web browser.

	-c, --copy
		Put the URL of the new repository to clipboard instead of printing it.

	[<ORGANIZATION>/]<NAME>
		The name for the repository on GitHub (default: name of the current working
		directory).

		Optionally, create the repository within <ORGANIZATION>.

## Examples:
		$ hub create
		[ repo created on GitHub ]
		> git remote add -f origin git@github.com:USER/REPO.git

		$ hub create sinatra/recipes
		[ repo created in GitHub organization ]
		> git remote add -f origin git@github.com:sinatra/recipes.git

## See also:

hub-init(1), hub(1)
`,
}

var (
	flagCreatePrivate,
	flagCreateBrowse,
	flagCreateCopy bool

	flagCreateDescription,
	flagCreateHomepage string
)

func init() {
	cmdCreate.Flag.BoolVarP(&flagCreatePrivate, "private", "p", false, "PRIVATE")
	cmdCreate.Flag.BoolVarP(&flagCreateBrowse, "browse", "o", false, "BROWSE")
	cmdCreate.Flag.BoolVarP(&flagCreateCopy, "copy", "c", false, "COPY")
	cmdCreate.Flag.StringVarP(&flagCreateDescription, "description", "d", "", "DESCRIPTION")
	cmdCreate.Flag.StringVarP(&flagCreateHomepage, "homepage", "h", "", "HOMEPAGE")

	CmdRunner.Use(cmdCreate)
}

func create(command *Command, args *Args) {
	_, err := git.Dir()
	if err != nil {
		err = fmt.Errorf("'create' must be run from inside a git repository")
		utils.Check(err)
	}

	var newRepoName string
	if args.IsParamsEmpty() {
		dirName, err := git.WorkdirName()
		utils.Check(err)
		newRepoName = github.SanitizeProjectName(dirName)
	} else {
		reg := regexp.MustCompile("^[^-]")
		if !reg.MatchString(args.FirstParam()) {
			err = fmt.Errorf("invalid argument: %s", args.FirstParam())
			utils.Check(err)
		}
		newRepoName = args.FirstParam()
	}

	config := github.CurrentConfig()
	host, err := config.DefaultHost()
	if err != nil {
		utils.Check(github.FormatError("creating repository", err))
	}

	owner := host.User
	if strings.Contains(newRepoName, "/") {
		split := strings.SplitN(newRepoName, "/", 2)
		owner = split[0]
		newRepoName = split[1]
	}

	project := github.NewProject(owner, newRepoName, host.Host)
	gh := github.NewClient(project.Host)

	repo, err := gh.Repository(project)
	if err == nil {
		foundProject := github.NewProject(repo.FullName, "", project.Host)
		if foundProject.SameAs(project) {
			if !repo.Private && flagCreatePrivate {
				err = fmt.Errorf("Repository '%s' already exists and is public", repo.FullName)
				utils.Check(err)
			} else {
				ui.Errorln("Existing repository detected")
				project = foundProject
			}
		} else {
			repo = nil
		}
	} else {
		repo = nil
	}

	if repo == nil {
		if !args.Noop {
			repo, err := gh.CreateRepository(project, flagCreateDescription, flagCreateHomepage, flagCreatePrivate)
			utils.Check(err)
			project = github.NewProject(repo.FullName, "", project.Host)
		}
	}

	localRepo, err := github.LocalRepo()
	utils.Check(err)

	originName := "origin"
	if originRemote, err := localRepo.RemoteByName(originName); err == nil {
		originProject, err := originRemote.Project()
		if err != nil || !originProject.SameAs(project) {
			ui.Errorf(`A git remote named "%s" already exists and is set to push to '%s'.\n`, originRemote.Name, originRemote.PushURL)
		}
	} else {
		url := project.GitURL("", "", true)
		args.Before("git", "remote", "add", "-f", originName, url)
	}

	webUrl := project.WebURL("", "", "")
	args.NoForward()
	printBrowseOrCopy(args, webUrl, flagCreateBrowse, flagCreateCopy)
}
<commit_msg>Switch `create` to new args parser<commit_after>package commands

import (
	"fmt"
	"regexp"
	"strings"

	"github.com/github/hub/git"
	"github.com/github/hub/github"
	"github.com/github/hub/ui"
	"github.com/github/hub/utils"
)

var cmdCreate = &Command{
	Run:   create,
	Usage: "create [-poc] [-d <DESCRIPTION>] [-h <HOMEPAGE>] [[<ORGANIZATION>/]<NAME>]",
	Long: `Create a new repository on GitHub and add a git remote for it.

## Options:
	-p, --private
		Create a private repository.

	-d, --description=<DESCRIPTION>
		Use this text as the description of the GitHub repository.

	-h, --homepage=<HOMEPAGE>
		Use this text as the URL of the GitHub repository.

	-o, --browse
		Open the new repository in a web browser.

	-c, --copy
		Put the URL of the new repository to clipboard instead of printing it.

	[<ORGANIZATION>/]<NAME>
		The name for the repository on GitHub (default: name of the current working
		directory).

		Optionally, create the repository within <ORGANIZATION>.

## Examples:
		$ hub create
		[ repo created on GitHub ]
		> git remote add -f origin git@github.com:USER/REPO.git

		$ hub create sinatra/recipes
		[ repo created in GitHub organization ]
		> git remote add -f origin git@github.com:sinatra/recipes.git

## See also:

hub-init(1), hub(1)
`,
}

func init() {
	CmdRunner.Use(cmdCreate)
}

func create(command *Command, args *Args) {
	_, err := git.Dir()
	if err != nil {
		err = fmt.Errorf("'create' must be run from inside a git repository")
		utils.Check(err)
	}

	var newRepoName string
	if args.IsParamsEmpty() {
		dirName, err := git.WorkdirName()
		utils.Check(err)
		newRepoName = github.SanitizeProjectName(dirName)
	} else {
		reg := regexp.MustCompile("^[^-]")
		if !reg.MatchString(args.FirstParam()) {
			err = fmt.Errorf("invalid argument: %s", args.FirstParam())
			utils.Check(err)
		}
		newRepoName = args.FirstParam()
	}

	config := github.CurrentConfig()
	host, err := config.DefaultHost()
	if err != nil {
		utils.Check(github.FormatError("creating repository", err))
	}

	owner := host.User
	if strings.Contains(newRepoName, "/") {
		split := strings.SplitN(newRepoName, "/", 2)
		owner = split[0]
		newRepoName = split[1]
	}

	project := github.NewProject(owner, newRepoName, host.Host)
	gh := github.NewClient(project.Host)

	flagCreatePrivate := args.Flag.Bool("--private")

	repo, err := gh.Repository(project)
	if err == nil {
		foundProject := github.NewProject(repo.FullName, "", project.Host)
		if foundProject.SameAs(project) {
			if !repo.Private && flagCreatePrivate {
				err = fmt.Errorf("Repository '%s' already exists and is public", repo.FullName)
				utils.Check(err)
			} else {
				ui.Errorln("Existing repository detected")
				project = foundProject
			}
		} else {
			repo = nil
		}
	} else {
		repo = nil
	}

	if repo == nil {
		if !args.Noop {
			flagCreateDescription := args.Flag.Value("--description")
			flagCreateHomepage := args.Flag.Value("--homepage")
			repo, err := gh.CreateRepository(project, flagCreateDescription, flagCreateHomepage, flagCreatePrivate)
			utils.Check(err)
			project = github.NewProject(repo.FullName, "", project.Host)
		}
	}

	localRepo, err := github.LocalRepo()
	utils.Check(err)

	originName := "origin"
	if originRemote, err := localRepo.RemoteByName(originName); err == nil {
		originProject, err := originRemote.Project()
		if err != nil || !originProject.SameAs(project) {
			ui.Errorf(`A git remote named "%s" already exists and is set to push to '%s'.\n`, originRemote.Name, originRemote.PushURL)
		}
	} else {
		url := project.GitURL("", "", true)
		args.Before("git", "remote", "add", "-f", originName, url)
	}

	webUrl := project.WebURL("", "", "")
	args.NoForward()
	flagCreateBrowse := args.Flag.Bool("--browse")
	flagCreateCopy := args.Flag.Bool("--copy")
	printBrowseOrCopy(args, webUrl, flagCreateBrowse, flagCreateCopy)
}
<|endoftext|>


<commit_before>package apptail

import (
	"encoding/json"
	"fmt"
	"github.com/ActiveState/log"
	"github.com/ActiveState/tail"
	"logyard"
	"logyard/util/pubsub"
	"path/filepath"
	"unicode/utf8"
)

// AppInstance is the NATS message sent by dea/stager to notify of new
// instances.
type AppInstance struct {
	AppID    int
	AppName  string
	Type     string
	Index    int
	LogFiles []string
}

// AppLogMessage is a struct corresponding to an entry in the app log stream.
type AppLogMessage struct {
	Text          string
	LogFilename   string
	UnixTime      int64
	HumanTime     string
	Source        string // example: app, staging, stackato.dea, stackato.stager
	InstanceIndex int
	AppID         int
	AppName       string
	NodeID        string // Host (DEA,stager) IP of this app instance
}

// Publish publishes an AppLogMessage to logyard after sanity checks.
func (line *AppLogMessage) Publish(pub *pubsub.Publisher, allowInvalidJson bool) error {
	// JSON must be a UTF-8 encoded string.
	if !utf8.ValidString(line.Text) {
		line.Text = string([]rune(line.Text))
	}

	data, err := json.Marshal(line)
	if err != nil {
		if allowInvalidJson {
			log.Errorf("Cannot encode %+v into JSON -- %s. Skipping this message", line, err)
		} else {
			return fmt.Errorf("Failed to encode app log record to JSON: ", err)
		}
	}
	key := fmt.Sprintf("apptail.%d", line.AppID)
	pub.MustPublish(key, string(data))
	return nil
}

// AppInstanceStarted is a function to be invoked when dea/stager
// starts an application instance.
func AppInstanceStarted(instance *AppInstance, nodeid string) {
	log.Infof("New app instance was started: %+v", instance)

	for _, filename := range instance.LogFiles {
		go func(filename string) {
			pub := logyard.Broker.NewPublisherMust()
			defer pub.Stop()

			tail, err := tail.TailFile(filename, tail.Config{
				MaxLineSize: Config.MaxRecordSize,
				MustExist:   true,
				Follow:      true,
				Location:    -1,
				ReOpen:      false,
				Poll:        true})
			if err != nil {
				log.Errorf("Cannot tail file (%s); %s", filename, err)
				return
			}
			for line := range tail.Lines {
				// JSON must be a valid UTF-8 string
				if !utf8.ValidString(line.Text) {
					line.Text = string([]rune(line.Text))
				}
				err := (&AppLogMessage{
					Text:          line.Text,
					LogFilename:   filepath.Base(filename),
					UnixTime:      line.Time.Unix(),
					HumanTime:     line.Time.Format("2006-01-02T15:04:05-07:00"), // heroku-format
					Source:        instance.Type,
					InstanceIndex: instance.Index,
					AppID:         instance.AppID,
					AppName:       instance.AppName,
					NodeID:        nodeid,
				}).Publish(pub, false)
				if err != nil {
					log.Fatal(err)
				}
			}
			err = tail.Wait()
			if err != nil {
				log.Error(err)
			}
		}(filename)
	}
}
<commit_msg>Bug #98687 - add AppGroup field to app log stream<commit_after>package apptail

import (
	"encoding/json"
	"fmt"
	"github.com/ActiveState/log"
	"github.com/ActiveState/tail"
	"logyard"
	"logyard/util/pubsub"
	"path/filepath"
	"unicode/utf8"
)

// AppInstance is the NATS message sent by dea/stager to notify of new
// instances.
type AppInstance struct {
	AppID    int
	AppName  string
	AppGroup string `json:"group"`
	Type     string
	Index    int
	LogFiles []string
}

// AppLogMessage is a struct corresponding to an entry in the app log stream.
type AppLogMessage struct {
	Text          string
	LogFilename   string
	UnixTime      int64
	HumanTime     string
	Source        string // example: app, staging, stackato.dea, stackato.stager
	InstanceIndex int
	AppID         int
	AppName       string
	AppGroup      string
	NodeID        string // Host (DEA,stager) IP of this app instance
}

// Publish publishes an AppLogMessage to logyard after sanity checks.
func (line *AppLogMessage) Publish(pub *pubsub.Publisher, allowInvalidJson bool) error {
	// JSON must be a UTF-8 encoded string.
	if !utf8.ValidString(line.Text) {
		line.Text = string([]rune(line.Text))
	}

	data, err := json.Marshal(line)
	if err != nil {
		if allowInvalidJson {
			log.Errorf("Cannot encode %+v into JSON -- %s. Skipping this message", line, err)
		} else {
			return fmt.Errorf("Failed to encode app log record to JSON: ", err)
		}
	}
	key := fmt.Sprintf("apptail.%d", line.AppID)
	pub.MustPublish(key, string(data))
	return nil
}

// AppInstanceStarted is a function to be invoked when dea/stager
// starts an application instance.
func AppInstanceStarted(instance *AppInstance, nodeid string) {
	log.Infof("New app instance was started: %+v", instance)

	for _, filename := range instance.LogFiles {
		go func(filename string) {
			pub := logyard.Broker.NewPublisherMust()
			defer pub.Stop()

			tail, err := tail.TailFile(filename, tail.Config{
				MaxLineSize: Config.MaxRecordSize,
				MustExist:   true,
				Follow:      true,
				Location:    -1,
				ReOpen:      false,
				Poll:        true})
			if err != nil {
				log.Errorf("Cannot tail file (%s); %s", filename, err)
				return
			}
			for line := range tail.Lines {
				// JSON must be a valid UTF-8 string
				if !utf8.ValidString(line.Text) {
					line.Text = string([]rune(line.Text))
				}
				err := (&AppLogMessage{
					Text:          line.Text,
					LogFilename:   filepath.Base(filename),
					UnixTime:      line.Time.Unix(),
					HumanTime:     line.Time.Format("2006-01-02T15:04:05-07:00"), // heroku-format
					Source:        instance.Type,
					InstanceIndex: instance.Index,
					AppID:         instance.AppID,
					AppName:       instance.AppName,
					AppGroup:      instance.AppGroup,
					NodeID:        nodeid,
				}).Publish(pub, false)
				if err != nil {
					log.Fatal(err)
				}
			}
			err = tail.Wait()
			if err != nil {
				log.Error(err)
			}
		}(filename)
	}
}
<|endoftext|>


<commit_before>package socket

import (
	"time"
	"github.com/neliseev/logger"
)

// Defaults vars
var msgSep = byte(":")

// Constants
const maxTCPQueries  int           = 256
const tcpIdleTimeout time.Duration = 60 * time.Second
const rtimeout       time.Duration = 2 * time.Second // Socket read timeout
const udpMsgSize     int           = 508             // RFC 791 (Min IP Size - Max IP Header Size - UDP Header Size)
const maxMsgSize     int           = 128             // ToDo Set configurable?

// Init logger subsystem
var log logger.Log

func init() {
	log.New()
}<commit_msg>Fix errors with types in conditions<commit_after>package socket

import (
	"time"
	"github.com/neliseev/logger"
)

// Defaults vars
var msgSep = byte(':')

// Constants
const maxTCPQueries  int           = 256
const tcpIdleTimeout time.Duration = 60 * time.Second
const rtimeout       time.Duration = 2 * time.Second // Socket read timeout
const udpMsgSize     int           = 508             // RFC 791 (Min IP Size - Max IP Header Size - UDP Header Size)
const maxMsgSize     int           = 128             // ToDo Set configurable?

// Init logger subsystem
var log logger.Log

func init() {
	log.New()
}<|endoftext|>


<commit_before>package runner
import (
	"net/http"
	"fmt"
	"sync"
	"github.com/dudang/golt/parser"
)

func ExecuteJsonGolt(testPlan parser.GoltJsons) {
	for _, element := range testPlan.Golt {
		executeElement(element)
	}
}

func executeElement(testElement parser.GoltJson) {
	waitGroup := sync.WaitGroup
	waitGroup.Add(testElement.Threads)
	for i:= 0; i < testElement.Threads; i++ {
		go spawnRoutine(testElement)
	}
	waitGroup.Wait()
}

func spawnRoutine(testElement parser.GoltJson) {
	switch testElement.Method {
		case "GET":
			getRequest(testElement.URL)
		default:
			return
	}
}

func getRequest(url string) {
	resp, err := http.Get(url)
	resp.Body.Close()
	if err != nil {
		fmt.Printf("%v\n", err)
	}
	fmt.Println(resp.StatusCode)
}<commit_msg>Fix variable initialization which failed travis build<commit_after>package runner
import (
	"net/http"
	"fmt"
	"sync"
	"github.com/dudang/golt/parser"
)

func ExecuteJsonGolt(testPlan parser.GoltJsons) {
	for _, element := range testPlan.Golt {
		executeElement(element)
	}
}

func executeElement(testElement parser.GoltJson) {
	var wg sync.WaitGroup
	wg.Add(testElement.Threads)
	for i:= 0; i < testElement.Threads; i++ {
		go spawnRoutine(testElement)
	}
	wg.Wait()
}

func spawnRoutine(testElement parser.GoltJson) {
	switch testElement.Method {
		case "GET":
			getRequest(testElement.URL)
		default:
			return
	}
}

func getRequest(url string) {
	resp, err := http.Get(url)
	resp.Body.Close()
	if err != nil {
		fmt.Printf("%v\n", err)
	}
	fmt.Println(resp.StatusCode)
}<|endoftext|>


<commit_before>package runner

import (
	"golang.org/x/net/context"
	"sync"
	"time"
)

type Runner interface {
	Run(ctx context.Context) <-chan Result
}

type Result struct {
	Text  string
	Time  time.Duration
	Error error
}

type VU struct {
	Cancel context.CancelFunc
}

func Run(ctx context.Context, r Runner, scale <-chan int) <-chan Result {
	ch := make(chan Result)

	go func() {
		wg := sync.WaitGroup{}
		defer func() {
			wg.Wait()
			close(ch)
		}()

		currentVUs := []VU{}
		for {
			select {
			case vus := <-scale:
				for vus > len(currentVUs) {
					wg.Add(1)
					c, cancel := context.WithCancel(ctx)
					currentVUs = append(currentVUs, VU{Cancel: cancel})
					go func() {
						defer wg.Done()
						for res := range r.Run(c) {
							ch <- res
						}
					}()
				}
				for vus < len(currentVUs) {
					currentVUs[len(currentVUs)-1].Cancel()
					currentVUs = currentVUs[:len(currentVUs)-1]
				}
			case <-ctx.Done():
				return
			}
		}
	}()

	return ch
}
<commit_msg>Is this noticeably faster<commit_after>package runner

import (
	"golang.org/x/net/context"
	"sync"
	"time"
)

type Runner interface {
	Run(ctx context.Context) <-chan Result
}

type Result struct {
	Text  string
	Time  time.Duration
	Error error
}

type VU struct {
	Cancel context.CancelFunc
}

func Run(ctx context.Context, r Runner, scale <-chan int) <-chan Result {
	ch := make(chan Result)

	go func() {
		wg := sync.WaitGroup{}
		defer func() {
			wg.Wait()
			close(ch)
		}()

		currentVUs := make([]VU, 0, 100)
		for {
			select {
			case vus := <-scale:
				for vus > len(currentVUs) {
					wg.Add(1)
					c, cancel := context.WithCancel(ctx)
					currentVUs = append(currentVUs, VU{Cancel: cancel})
					go func() {
						defer wg.Done()
						for res := range r.Run(c) {
							ch <- res
						}
					}()
				}
				for vus < len(currentVUs) {
					currentVUs[len(currentVUs)-1].Cancel()
					currentVUs = currentVUs[:len(currentVUs)-1]
				}
			case <-ctx.Done():
				return
			}
		}
	}()

	return ch
}
<|endoftext|>


<commit_before>// Skeleton to part 9 of the Whispering Gophers code lab.
//
// This program extends part 8.
//
// It connects to the peer specified by -peer.
// It accepts connections from peers and receives messages from them.
// When it sees a peer with an address it hasn't seen before, it makes a
// connection to that peer.
// It adds an ID field containing a random string to each outgoing message.
// When it recevies a message with an ID it hasn't seen before, it broadcasts
// that message to all connected peers.
//
package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"sync"

	"code.google.com/p/whispering-gophers/util"
)

var (
	peerAddr = flag.String("peer", "", "peer host:port")
	self     string
)

type Message struct {
	ID   string
	Addr string
	Body string
}

func main() {
	flag.Parse()

	l, err := util.Listen()
	if err != nil {
		log.Fatal(err)
	}
	self = l.Addr().String()
	log.Println("Listening on", self)

	go dial(*peerAddr)
	go readInput()

	for {
		c, err := l.Accept()
		if err != nil {
			log.Fatal(err)
		}
		go serve(c)
	}
}

var peers = &Peers{m: make(map[string]chan<- Message)}

type Peers struct {
	m  map[string]chan<- Message
	mu sync.RWMutex
}

// Add creates and returns a new channel for the given peer address.
// If an address already exists in the registry, it returns nil.
func (p *Peers) Add(addr string) <-chan Message {
	p.mu.Lock()
	defer p.mu.Unlock()
	if _, ok := p.m[addr]; ok {
		return nil
	}
	ch := make(chan Message)
	p.m[addr] = ch
	return ch
}

// Remove deletes the specified peer from the registry.
func (p *Peers) Remove(addr string) {
	p.mu.Lock()
	defer p.mu.Unlock()
	delete(p.m, addr)
}

// List returns a slice of all active peer channels.
func (p *Peers) List() []chan<- Message {
	p.mu.RLock()
	defer p.mu.RUnlock()
	l := make([]chan<- Message, 0, len(p.m))
	for _, ch := range p.m {
		l = append(l, ch)
	}
	return l
}

func broadcast(m Message) {
	for _, ch := range peers.List() {
		select {
		case ch <- m:
		default:
			// Okay to drop messages sometimes.
		}
	}
}

func serve(c net.Conn) {
	defer c.Close()
	d := json.NewDecoder(c)
	for {
		var m Message
		err := d.Decode(&m)
		if err != nil {
			log.Println(err)
			return
		}

		// TODO: If this message has seen before, ignore it.

		fmt.Printf("%#v\n", m)
		broadcast(m)
		go dial(m.Addr)
	}
}

func readInput() {
	s := bufio.NewScanner(os.Stdin)
	for s.Scan() {
		m := Message{
			ID:   util.RandomID(),
			Addr: self,
			Body: s.Text(),
		}
		// TODO: Mark the message ID as seen.
		broadcast(m)
	}
	if err := s.Err(); err != nil {
		log.Fatal(err)
	}
}

func dial(addr string) {
	if addr == self {
		return // Don't try to dial self.
	}

	ch := peers.Add(addr)
	if ch == nil {
		return // Peer already connected.
	}
	defer peers.Remove(addr)

	c, err := net.Dial("tcp", addr)
	if err != nil {
		log.Println(addr, err)
		return
	}
	defer c.Close()

	e := json.NewEncoder(c)
	for m := range ch {
		err := e.Encode(m)
		if err != nil {
			log.Println(addr, err)
			return
		}
	}
}

// TODO: Create a new map of seen message IDs and a mutex to protect it.

// Seen returns true if the specified id has been seen before.
// If not, it returns false and marks the given id as "seen".
func Seen(id string) bool {
	// TODO: Get a write lock on the seen message IDs map and unlock it at before returning.
	// TODO: Check if the id has been seen before and return that later.
	// TODO: Mark the ID as seen in the map.
}
<commit_msg>whispering-gophers: add more TODOs to part 9<commit_after>// Skeleton to part 9 of the Whispering Gophers code lab.
//
// This program extends part 8.
//
// It connects to the peer specified by -peer.
// It accepts connections from peers and receives messages from them.
// When it sees a peer with an address it hasn't seen before, it makes a
// connection to that peer.
// It adds an ID field containing a random string to each outgoing message.
// When it recevies a message with an ID it hasn't seen before, it broadcasts
// that message to all connected peers.
//
package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"sync"

	"code.google.com/p/whispering-gophers/util"
)

var (
	peerAddr = flag.String("peer", "", "peer host:port")
	self     string
)

type Message struct {
	// TODO: add ID field
	Addr string
	Body string
}

func main() {
	flag.Parse()

	l, err := util.Listen()
	if err != nil {
		log.Fatal(err)
	}
	self = l.Addr().String()
	log.Println("Listening on", self)

	go dial(*peerAddr)
	go readInput()

	for {
		c, err := l.Accept()
		if err != nil {
			log.Fatal(err)
		}
		go serve(c)
	}
}

var peers = &Peers{m: make(map[string]chan<- Message)}

type Peers struct {
	m  map[string]chan<- Message
	mu sync.RWMutex
}

// Add creates and returns a new channel for the given peer address.
// If an address already exists in the registry, it returns nil.
func (p *Peers) Add(addr string) <-chan Message {
	p.mu.Lock()
	defer p.mu.Unlock()
	if _, ok := p.m[addr]; ok {
		return nil
	}
	ch := make(chan Message)
	p.m[addr] = ch
	return ch
}

// Remove deletes the specified peer from the registry.
func (p *Peers) Remove(addr string) {
	p.mu.Lock()
	defer p.mu.Unlock()
	delete(p.m, addr)
}

// List returns a slice of all active peer channels.
func (p *Peers) List() []chan<- Message {
	p.mu.RLock()
	defer p.mu.RUnlock()
	l := make([]chan<- Message, 0, len(p.m))
	for _, ch := range p.m {
		l = append(l, ch)
	}
	return l
}

func broadcast(m Message) {
	for _, ch := range peers.List() {
		select {
		case ch <- m:
		default:
			// Okay to drop messages sometimes.
		}
	}
}

func serve(c net.Conn) {
	defer c.Close()
	d := json.NewDecoder(c)
	for {
		var m Message
		err := d.Decode(&m)
		if err != nil {
			log.Println(err)
			return
		}

		// TODO: If this message has seen before, ignore it.

		fmt.Printf("%#v\n", m)
		broadcast(m)
		go dial(m.Addr)
	}
}

func readInput() {
	s := bufio.NewScanner(os.Stdin)
	for s.Scan() {
		m := Message{
			// TODO: use util.RandomID to populate the ID field.
			Addr: self,
			Body: s.Text(),
		}
		// TODO: Mark the message ID as seen.
		broadcast(m)
	}
	if err := s.Err(); err != nil {
		log.Fatal(err)
	}
}

func dial(addr string) {
	if addr == self {
		return // Don't try to dial self.
	}

	ch := peers.Add(addr)
	if ch == nil {
		return // Peer already connected.
	}
	defer peers.Remove(addr)

	c, err := net.Dial("tcp", addr)
	if err != nil {
		log.Println(addr, err)
		return
	}
	defer c.Close()

	e := json.NewEncoder(c)
	for m := range ch {
		err := e.Encode(m)
		if err != nil {
			log.Println(addr, err)
			return
		}
	}
}

// TODO: Create a new map of seen message IDs and a mutex to protect it.

// Seen returns true if the specified id has been seen before.
// If not, it returns false and marks the given id as "seen".
func Seen(id string) bool {
	// TODO: Get a write lock on the seen message IDs map and unlock it at before returning.
	// TODO: Check if the id has been seen before and return that later.
	// TODO: Mark the ID as seen in the map.
}
<|endoftext|>


<commit_before>package websocket

import (
	"encoding/json"
	"net/http"
	"sync"
	"time"

	log "github.com/cihub/seelog"
	"github.com/gin-gonic/gin"
	"github.com/gorilla/websocket"
	"github.com/pborman/uuid"
)

const (
	// Time allowed to write the file to the client.
	writeWait = 10 * time.Second

	// Time allowed to read the next pong message from the client.
	pongWait = 60 * time.Second

	// Send pings to client with this period. Must be less than pongWait.
	pingPeriod = (pongWait * 9) / 10
)

type Message struct {
	Type string          `json:"type"`
	Data json.RawMessage `json:"data"`
	To   string          `json:"to"`
}
type Clients struct {
	sync.Mutex
	clients []*Client
	Router  *Router `inject:""`
}
type Client struct {
	out        chan<- *Message
	done       <-chan bool
	err        <-chan error
	Id         string
	disconnect chan int
}

// Add a client to a room
func (r *Clients) appendClient(client *Client) {
	r.Lock()
	r.clients = append(r.clients, client)
	r.Unlock()

	msgs := r.Router.RunOnClientConnectHandlers()
	for _, msg := range msgs {
		client.out <- msg
	}
}

// Message all the other clients
func (r *Clients) SendToAll(t string, data interface{}) {

	out, err := json.Marshal(data)
	if err != nil {
		log.Error(err)
		return
	}
	msg := &Message{Type: t, Data: out}

	r.Lock()
	clientsToRemove := make([]*Client, 0)

	for _, c := range r.clients {
		select {
		case c.out <- msg:
			// Everything went well :)
		case <-time.After(time.Second):
			log.Warn("Failed writing to websocket: timeout (", c.Id, ")")
			clientsToRemove = append(clientsToRemove, c)
		}
	}

	r.Unlock()
	go func() {
		for _, c := range clientsToRemove {
			r.removeClient(c)
		}
	}()
}

// Remove a client
func (r *Clients) removeClient(client *Client) {
	r.Lock()
	defer r.Unlock()

	for index, c := range r.clients {
		if c == client {
			c.disconnect <- websocket.CloseInternalServerErr
			r.clients = append(r.clients[:index], r.clients[(index+1):]...)
		}
	}
}

// Disconnect all clients
func (r *Clients) disconnectAll() {
	r.Lock()
	defer r.Unlock()

	for _, c := range r.clients {
		c.disconnect <- websocket.CloseGoingAway
	}
}

func newClients() *Clients {
	return &Clients{sync.Mutex{}, make([]*Client, 0), nil}
}

func (clients *Clients) WebsocketRoute(c *gin.Context) {
	conn, err := websocket.Upgrade(c.Writer, c.Request, nil, 1024, 1024)
	if err != nil {
		http.Error(c.Writer, "Websocket error", 400)
		log.Error(err)
		return
	}

	out := make(chan *Message)
	done := make(chan bool)
	wsErr := make(chan error)
	disconnect := make(chan int)
	client := &Client{out, done, wsErr, uuid.New(), disconnect}

	go senderWorker(conn, out)
	go disconnectWorker(conn, client)

	clients.appendClient(client)

	conn.SetReadDeadline(time.Now().Add(pongWait))
	conn.SetPongHandler(func(string) error {
		//log.Debug("Got pong response from browser")
		conn.SetReadDeadline(time.Now().Add(pongWait))
		return nil
	})

	//Listen to websocket
	for {
		msg := &Message{}
		err := conn.ReadJSON(msg)
		if err != nil {
			log.Info(conn.LocalAddr(), " Disconnected")
			close(done)
			close(out)
			clients.removeClient(client)
			break
		}
		go clients.Router.Run(msg)
	}
}

func disconnectWorker(conn *websocket.Conn, c *Client) {
	for code := range c.disconnect {
		log.Debug("Closing websocket")
		conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(code, ""), time.Now().Add(writeWait))
		//We can only disconnect once so we can close this channel here
		close(c.disconnect)
		if err := conn.Close(); err != nil {
			log.Error("Connection could not be closed: %s", err)
		}
		return
	}
}
func senderWorker(conn *websocket.Conn, out chan *Message) {
	pingTicker := time.NewTicker(pingPeriod)
	for {
		select {
		case msg, opened := <-out:
			if !opened {
				log.Debug("websocket: Sendchannel closed stopping pingTicket and senderWorker")
				pingTicker.Stop()
				return
			}
			if err := conn.WriteJSON(msg); err != nil {
				log.Error(err)
			}
		case <-pingTicker.C:
			if err := conn.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(writeWait)); err != nil {
				log.Error(err)
				return
			}

		}
	}
}
<commit_msg>use defer for safer disconnect<commit_after>package websocket

import (
	"encoding/json"
	"net/http"
	"sync"
	"time"

	log "github.com/cihub/seelog"
	"github.com/gin-gonic/gin"
	"github.com/gorilla/websocket"
	"github.com/pborman/uuid"
)

const (
	// Time allowed to write the file to the client.
	writeWait = 10 * time.Second

	// Time allowed to read the next pong message from the client.
	pongWait = 60 * time.Second

	// Send pings to client with this period. Must be less than pongWait.
	pingPeriod = (pongWait * 9) / 10
)

type Message struct {
	Type string          `json:"type"`
	Data json.RawMessage `json:"data"`
	To   string          `json:"to"`
}
type Clients struct {
	sync.Mutex
	clients []*Client
	Router  *Router `inject:""`
}
type Client struct {
	out        chan<- *Message
	done       <-chan bool
	err        <-chan error
	Id         string
	disconnect chan int
}

// Add a client to a room
func (r *Clients) appendClient(client *Client) {
	r.Lock()
	r.clients = append(r.clients, client)
	r.Unlock()

	msgs := r.Router.RunOnClientConnectHandlers()
	for _, msg := range msgs {
		client.out <- msg
	}
}

// Message all the other clients
func (r *Clients) SendToAll(t string, data interface{}) {

	out, err := json.Marshal(data)
	if err != nil {
		log.Error(err)
		return
	}
	msg := &Message{Type: t, Data: out}

	r.Lock()
	clientsToRemove := make([]*Client, 0)

	for _, c := range r.clients {
		select {
		case c.out <- msg:
			// Everything went well :)
		case <-time.After(time.Second):
			log.Warn("Failed writing to websocket: timeout (", c.Id, ")")
			clientsToRemove = append(clientsToRemove, c)
		}
	}

	r.Unlock()
	go func() {
		for _, c := range clientsToRemove {
			r.removeClient(c)
		}
	}()
}

// Remove a client
func (r *Clients) removeClient(client *Client) {
	r.Lock()
	defer r.Unlock()

	for index, c := range r.clients {
		if c == client {
			c.disconnect <- websocket.CloseInternalServerErr
			r.clients = append(r.clients[:index], r.clients[(index+1):]...)
		}
	}
}

// Disconnect all clients
func (r *Clients) disconnectAll() {
	r.Lock()
	defer r.Unlock()

	for _, c := range r.clients {
		c.disconnect <- websocket.CloseGoingAway
	}
}

func newClients() *Clients {
	return &Clients{sync.Mutex{}, make([]*Client, 0), nil}
}

func (clients *Clients) WebsocketRoute(c *gin.Context) {
	conn, err := websocket.Upgrade(c.Writer, c.Request, nil, 1024, 1024)
	if err != nil {
		http.Error(c.Writer, "Websocket error", 400)
		log.Error(err)
		return
	}

	out := make(chan *Message)
	done := make(chan bool)
	wsErr := make(chan error)
	disconnect := make(chan int)

	defer func() {
		close(out)
		close(done)
		close(wsErr)
	}()

	client := &Client{out, done, wsErr, uuid.New(), disconnect}

	go senderWorker(conn, out)
	go disconnectWorker(conn, client)

	clients.appendClient(client)

	conn.SetReadDeadline(time.Now().Add(pongWait))
	conn.SetPongHandler(func(string) error {
		//log.Debug("Got pong response from browser")
		conn.SetReadDeadline(time.Now().Add(pongWait))
		return nil
	})

	//Listen to websocket
	for {
		msg := &Message{}
		err := conn.ReadJSON(msg)
		if err != nil {
			log.Info(conn.LocalAddr(), " Disconnected")
			clients.removeClient(client)
			return
		}
		go clients.Router.Run(msg)
	}
}

func disconnectWorker(conn *websocket.Conn, c *Client) {
	defer close(c.disconnect)
	for code := range c.disconnect {
		log.Debug("Closing websocket")
		conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(code, ""), time.Now().Add(writeWait))
		if err := conn.Close(); err != nil {
			log.Error("Connection could not be closed: %s", err)
		}
		return
	}
}
func senderWorker(conn *websocket.Conn, out chan *Message) {
	pingTicker := time.NewTicker(pingPeriod)
	defer func() {
		pingTicker.Stop()
	}()
	for {
		select {
		case msg, opened := <-out:
			if !opened {
				log.Debug("websocket: Sendchannel closed stopping pingTicket and senderWorker")
				return
			}
			if err := conn.WriteJSON(msg); err != nil {
				log.Error(err)
			}
		case <-pingTicker.C:
			if err := conn.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(writeWait)); err != nil {
				log.Error(err)
				return
			}

		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/go-martini/martini"
	"github.com/vanng822/uploader"
	"github.com/vanng822/uploader/storage/mongodb"
	"net/http"
	"os"
)

const (
	STORAGE_TYPE_FILE    = "file"
	STORAGE_TYPE_MONGODB = "mongodb"
)

type StorageConfig struct {
	Type           string
	Configurations map[string]string
}

func GetStorage(config *uploader.StorageConfig) uploader.ImageStorage {
	var storage uploader.ImageStorage
	switch config.Type {
	case STORAGE_TYPE_FILE:
		storage = uploader.NewImageStorageFile(config.Configurations)
	case STORAGE_TYPE_MONGODB:
		storage = storage_mongodb.New(config.Configurations)
	default:
		panic(fmt.Sprintf("Unsupported storage type %s", config.Type))
	}
	return storage
}

type EndpointConfig struct {
	Endpoint  string
	FileField string
	Storage   *StorageConfig
}

type Config struct {
	Host      string
	Port      int
	Endpoints []*EndpointConfig
}

func LoadConfig(filename string) *Config {
	file, err := os.Open(filename)
	if err != nil {
		panic(err)
	}
	defer file.Close()
	decoder := json.NewDecoder(file)
	conf := Config{}
	err = decoder.Decode(&conf)
	if err != nil {
		panic(err)
	}
	return &conf
}

func main() {

	var (
		config string
		host   string
		port   int
	)

	flag.StringVar(&host, "h", "", "Host to bind to")
	flag.IntVar(&port, "p", 0, "Port number to listen on")
	flag.StringVar(&config, "c", "./config/app.json", "Path to configurations")
	flag.Parse()

	conf := LoadConfig(config)

	if host != "" {
		conf.Host = host
	}

	if port != 0 {
		conf.Port = port
	}

	if len(conf.Endpoints) == 0 {
		panic("There is no endpoint configured")
	}

	m := martini.Classic()

	for _, endpoint := range conf.Endpoints {

		go func(endpoint *EndpointConfig) {
			u := uploader.NewUploader(GetStorage(endpoint.Storage))
			handler := uploader.NewHandler(u)

			m.Group(endpoint.Endpoint, func(r martini.Router) {
				r.Get("/:filename", func(res http.ResponseWriter, req *http.Request, params martini.Params) {
					handler.HandleGet(res, params["filename"])
				})
				r.Post("/", func(res http.ResponseWriter, req *http.Request, params martini.Params) {
					file, _, err := req.FormFile(endpoint.FileField)
					if err != nil {
						res.WriteHeader(http.StatusBadRequest)
						return
					}
					handler.HandlePost(res, file)
				})
				r.Put("/:filename", func(res http.ResponseWriter, req *http.Request, params martini.Params) {
					file, _, err := req.FormFile(endpoint.FileField)
					if err != nil {
						res.WriteHeader(http.StatusBadRequest)
						return
					}
					handler.HandlePut(res, file, params["filename"])
				})
				r.Delete("/:filename", func(res http.ResponseWriter, req *http.Request, params martini.Params) {
					handler.HandleDelete(res, params["filename"])
				})
			})
		}(endpoint)
	}

	http.ListenAndServe(fmt.Sprintf("%s:%d", conf.Host, conf.Port), m)
}
<commit_msg>Update since the move of StorageConfig<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/go-martini/martini"
	"github.com/vanng822/uploader"
	"github.com/vanng822/uploader/storage/mongodb"
	"net/http"
	"os"
)

const (
	STORAGE_TYPE_FILE    = "file"
	STORAGE_TYPE_MONGODB = "mongodb"
)

type StorageConfig struct {
	Type           string
	Configurations map[string]string
}

func GetStorage(config *StorageConfig) uploader.ImageStorage {
	var storage uploader.ImageStorage
	switch config.Type {
	case STORAGE_TYPE_FILE:
		storage = uploader.NewImageStorageFile(config.Configurations)
	case STORAGE_TYPE_MONGODB:
		storage = storage_mongodb.New(config.Configurations)
	default:
		panic(fmt.Sprintf("Unsupported storage type %s", config.Type))
	}
	return storage
}

type EndpointConfig struct {
	Endpoint  string
	FileField string
	Storage   *StorageConfig
}

type Config struct {
	Host      string
	Port      int
	Endpoints []*EndpointConfig
}

func LoadConfig(filename string) *Config {
	file, err := os.Open(filename)
	if err != nil {
		panic(err)
	}
	defer file.Close()
	decoder := json.NewDecoder(file)
	conf := Config{}
	err = decoder.Decode(&conf)
	if err != nil {
		panic(err)
	}
	return &conf
}

func main() {

	var (
		config string
		host   string
		port   int
	)

	flag.StringVar(&host, "h", "", "Host to bind to")
	flag.IntVar(&port, "p", 0, "Port number to listen on")
	flag.StringVar(&config, "c", "./config/app.json", "Path to configurations")
	flag.Parse()

	conf := LoadConfig(config)

	if host != "" {
		conf.Host = host
	}

	if port != 0 {
		conf.Port = port
	}

	if len(conf.Endpoints) == 0 {
		panic("There is no endpoint configured")
	}

	m := martini.Classic()

	for _, endpoint := range conf.Endpoints {

		go func(endpoint *EndpointConfig) {
			u := uploader.NewUploader(GetStorage(endpoint.Storage))
			handler := uploader.NewHandler(u)

			m.Group(endpoint.Endpoint, func(r martini.Router) {
				r.Get("/:filename", func(res http.ResponseWriter, req *http.Request, params martini.Params) {
					handler.HandleGet(res, params["filename"])
				})
				r.Post("/", func(res http.ResponseWriter, req *http.Request, params martini.Params) {
					file, _, err := req.FormFile(endpoint.FileField)
					if err != nil {
						res.WriteHeader(http.StatusBadRequest)
						return
					}
					handler.HandlePost(res, file)
				})
				r.Put("/:filename", func(res http.ResponseWriter, req *http.Request, params martini.Params) {
					file, _, err := req.FormFile(endpoint.FileField)
					if err != nil {
						res.WriteHeader(http.StatusBadRequest)
						return
					}
					handler.HandlePut(res, file, params["filename"])
				})
				r.Delete("/:filename", func(res http.ResponseWriter, req *http.Request, params martini.Params) {
					handler.HandleDelete(res, params["filename"])
				})
			})
		}(endpoint)
	}

	http.ListenAndServe(fmt.Sprintf("%s:%d", conf.Host, conf.Port), m)
}
<|endoftext|>


<commit_before>package version

// Version of Functions
var Version = "0.3.444"
<commit_msg>fnserver: 0.3.445 release [skip ci]<commit_after>package version

// Version of Functions
var Version = "0.3.445"
<|endoftext|>


<commit_before>package version

// Version of Functions
var Version = "0.3.492"
<commit_msg>fnserver: 0.3.493 release [skip ci]<commit_after>package version

// Version of Functions
var Version = "0.3.493"
<|endoftext|>


<commit_before>package version

// Version of Functions
var Version = "0.3.570"
<commit_msg>fnserver: 0.3.571 release [skip ci]<commit_after>package version

// Version of Functions
var Version = "0.3.571"
<|endoftext|>


<commit_before>// Copyright (c) 2014-2017 Ludovic Fauvet
// Licensed under the MIT license

package database

import (
	"errors"
	"fmt"
	"strconv"
	"strings"
	"sync"
	"time"

	. "github.com/etix/mirrorbits/config"
	"github.com/etix/mirrorbits/core"
	"github.com/gomodule/redigo/redis"
)

const (
	redisConnectionTimeout = 200 * time.Millisecond
	redisReadWriteTimeout  = 300 * time.Second
	// RedisMinimumVersion contains the minimum redis version required to run the application
	RedisMinimumVersion = "3.2.0"
)

var (
	// ErrUnreachable is returned when the endpoint is not reachable
	ErrUnreachable = errors.New("redis endpoint unreachable")
	// ErrRedisUpgradeRequired is returned when the redis server is running an unsupported version
	ErrRedisUpgradeRequired = errors.New("unsupported Redis version")
)

type redisPool interface {
	Get() redis.Conn
	Close() error
}

// Redis is the instance object of the redis database
type Redis struct {
	pool            redisPool
	Pubsub          *Pubsub
	failure         bool
	failureState    sync.RWMutex
	knownMaster     string
	knownMasterLock sync.Mutex
	stop            chan bool
	ready           chan struct{}
}

// NewRedis returns a new instance of the redis database
func NewRedis() *Redis {
	r := NewRedisCustomPool(nil)
	go func() {
	again:
		up, err := r.UpgradeNeeded()
		if err != nil {
			time.Sleep(50 * time.Millisecond)
			goto again
		}
		if up {
			t := time.Now()
			err = r.Upgrade()
			if err != nil {
				log.Fatalf("Upgrade failed: %+v", err)
			}
			log.Infof("Database upgrade successful (took %s), starting normally", time.Since(t).Round(time.Millisecond))
		}
		close(r.ready)
	}()

	return r
}

// NewRedisCli returns a new instance of the redis database for the CLI.
// Unlike NewRedis(), this function doesn't handle the upgrade. It just
// quit with an error if the db format is invalid.
// TODO the CLI should not use Redis directly but move to RPC
func NewRedisCli() *Redis {
	r := NewRedisCustomPool(nil)

	up, err := r.UpgradeNeeded()
	if err != nil {
		time.Sleep(50 * time.Millisecond)
		return r
	}
	if up {
		log.Fatalf("Database upgrade required")
	}
	close(r.ready)

	return r
}

// NewRedisCustomPool returns a new instance of the redis database
// using a custom pool
func NewRedisCustomPool(pool redisPool) *Redis {
	r := &Redis{
		stop:  make(chan bool),
		ready: make(chan struct{}),
	}

	if pool != nil {
		r.pool = pool
	} else {
		r.pool = &redis.Pool{
			MaxIdle:     10,
			IdleTimeout: 240 * time.Second,
			Dial: func() (redis.Conn, error) {
				conn, err := r.Connect()

				switch err {
				case nil:
					r.setFailureState(false)
				default:
					r.setFailureState(true)
				}

				if r.checkVersion(conn) == ErrRedisUpgradeRequired {
					log.Fatalf("Unsupported Redis version, please upgrade to Redis >= %s", RedisMinimumVersion)
				}

				return conn, err
			},
			TestOnBorrow: func(c redis.Conn, t time.Time) error {
				_, err := c.Do("PING")
				if RedisIsLoading(err) {
					return nil
				}
				return err
			},
		}
	}

	go r.connRecover()

	return r
}

// Get returns a redis connection from the pool
func (r *Redis) Get() redis.Conn {
	select {
	case <-r.ready:
	default:
		return &NotReadyError{}
	}
	return r.pool.Get()
}

// UnblockedGet returns a redis connection from the pool even
// if the database checks and/or upgrade are not finished.
func (r *Redis) UnblockedGet() redis.Conn {
	return r.pool.Get()
}

// Close closes all connections to the redis database
func (r *Redis) Close() {
	select {
	case _, _ = <-r.stop:
		return
	default:
		log.Debug("Closing databases connections")
		r.Pubsub.Close()
		r.pool.Close()
		close(r.stop)
	}
}

// ConnectPubsub initiates the connection to the pubsub
func (r *Redis) ConnectPubsub() {
	if r.Pubsub == nil {
		r.Pubsub = NewPubsub(r)
	}
}

// CheckVersion checks if the redis server version is supported
func (r *Redis) CheckVersion() error {
	c := r.UnblockedGet()
	defer c.Close()
	return r.checkVersion(c)
}

func (r *Redis) checkVersion(conn redis.Conn) error {
	if conn == nil {
		return ErrUnreachable
	}
	info, err := parseInfo(conn.Do("INFO", "server"))
	if err == nil {
		if parseVersion(info["redis_version"]) < parseVersion(RedisMinimumVersion) {
			return ErrRedisUpgradeRequired
		}
	}
	return err
}

// Connect initiates a new connection to the redis server
func (r *Redis) Connect() (redis.Conn, error) {
	sentinels := GetConfig().RedisSentinels

	if len(sentinels) > 0 {

		if len(GetConfig().RedisSentinelMasterName) == 0 {
			r.logError("Config: RedisSentinelMasterName cannot be empty!")
			goto single
		}

		for _, s := range sentinels {
			log.Debugf("Connecting to redis sentinel %s", s.Host)
			var master []string
			var masterhost string
			var cm redis.Conn

			c, err := r.connectTo(s.Host)
			if err != nil {
				r.logError("Sentinel: %s", err.Error())
				continue
			}
			//AUTH?
			role, err := r.askRole(c)
			if err != nil {
				r.logError("Sentinel: %s", err.Error())
				goto closeSentinel
			}
			if role != "sentinel" {
				r.logError("Sentinel: %s is not a sentinel but a %s", s.Host, role)
				goto closeSentinel
			}

			master, err = redis.Strings(c.Do("SENTINEL", "get-master-addr-by-name", GetConfig().RedisSentinelMasterName))
			if err == redis.ErrNil {
				r.logError("Sentinel: %s doesn't know the master-name %s", s.Host, GetConfig().RedisSentinelMasterName)
				goto closeSentinel
			} else if err != nil {
				r.logError("Sentinel: %s", err.Error())
				goto closeSentinel
			}

			masterhost = fmt.Sprintf("%s:%s", master[0], master[1])

			cm, err = r.connectTo(masterhost)
			if err != nil {
				r.logError("Redis master: %s", err.Error())
				goto closeSentinel
			}

			if r.auth(cm) != nil {
				r.logError("Redis master: auth failed")
				goto closeMaster
			}
			if err = r.selectDB(cm); err != nil {
				c.Close()
				return nil, err
			}

			role, err = r.askRole(cm)
			if err != nil {
				r.logError("Redis master: %s", err.Error())
				goto closeMaster
			}
			if role != "master" {
				r.logError("Redis master: %s is not a master but a %s", masterhost, role)
				goto closeMaster
			}

			// Close the connection to the sentinel
			c.Close()

			r.printConnectedMaster(masterhost)
			return cm, nil

		closeMaster:
			cm.Close()

		closeSentinel:
			c.Close()
		}
	}

single:

	if len(GetConfig().RedisAddress) == 0 {
		if len(sentinels) == 0 {
			log.Error("No redis master available")
		}
		return nil, ErrUnreachable
	}

	if len(sentinels) > 0 && r.Failure() == false {
		log.Warning("No redis master available, trying using the configured RedisAddress as fallback")
	}

	c, err := r.connectTo(GetConfig().RedisAddress)
	if err != nil {
		return nil, err
	}
	if err = r.auth(c); err != nil {
		c.Close()
		return nil, err
	}
	if err = r.selectDB(c); err != nil {
		c.Close()
		return nil, err
	}
	role, err := r.askRole(c)
	if err != nil {
		r.logError("Redis master: %s", err.Error())
		return nil, ErrUnreachable
	}
	if role != "master" {
		r.logError("Redis master: %s is not a master but a %s", GetConfig().RedisAddress, role)
		return nil, ErrUnreachable
	}
	r.printConnectedMaster(GetConfig().RedisAddress)
	return c, err

}

func (r *Redis) connectTo(address string) (redis.Conn, error) {
	return redis.Dial("tcp", address,
		redis.DialConnectTimeout(redisConnectionTimeout),
		redis.DialReadTimeout(redisReadWriteTimeout),
		redis.DialWriteTimeout(redisReadWriteTimeout))
}

func (r *Redis) askRole(c redis.Conn) (string, error) {
	roleReply, err := redis.Values(c.Do("ROLE"))
	if err != nil {
		return "", err
	}
	role, err := redis.String(roleReply[0], err)
	return role, err
}

func (r *Redis) auth(c redis.Conn) (err error) {
	if GetConfig().RedisPassword != "" {
		_, err = c.Do("AUTH", GetConfig().RedisPassword)
	}
	return
}

func (r *Redis) selectDB(c redis.Conn) (err error) {
	_, err = c.Do("SELECT", GetConfig().RedisDB)
	return
}

func (r *Redis) logError(format string, args ...interface{}) {
	if r.Failure() {
		log.Debugf(format, args...)
	} else {
		log.Errorf(format, args...)
	}
}

func (r *Redis) printConnectedMaster(address string) {
	r.knownMasterLock.Lock()
	defer r.knownMasterLock.Unlock()
	if address != r.knownMaster && core.Daemon {
		r.knownMaster = address
		log.Infof("Connected to redis master %s", address)
	} else {
		log.Debugf("Connected to redis master %s", address)
	}
}

func (r *Redis) setFailureState(failure bool) {
	r.failureState.Lock()
	r.failure = failure
	r.failureState.Unlock()
}

// Failure returns true if the connection is in a failure state
func (r *Redis) Failure() bool {
	r.failureState.RLock()
	defer r.failureState.RUnlock()
	return r.failure
}

func (r *Redis) connRecover() {
	ticker := time.NewTicker(1 * time.Second)

	for {
		select {
		case <-r.stop:
			return
		case <-ticker.C:
			if r.Failure() {
				if conn := r.Get(); conn != nil {
					// A successful Get() request will automatically unlock
					// other services waiting for a working connection.
					// This is only a way to ensure they wont wait forever.
					if conn.Err() != nil {
						log.Warningf("Database is down: %s", conn.Err().Error())
					}
					conn.Close()
				}
			}
		}
	}
}

// RedisIsLoading returns true if the error is of type LOADING
func RedisIsLoading(err error) bool {
	// PARSING: "LOADING Redis is loading the dataset in memory"
	if err != nil && strings.HasPrefix(err.Error(), "LOADING") {
		return true
	}
	return false
}

func parseVersion(version string) int64 {
	s := strings.Split(version, ".")
	format := fmt.Sprintf("%%s%%0%ds", 2)

	var v string
	for _, value := range s {
		v = fmt.Sprintf(format, v, value)
	}

	var result int64
	var err error
	if result, err = strconv.ParseInt(v, 10, 64); err != nil {
		return -1
	}
	return result
}

func parseInfo(i interface{}, err error) (map[string]string, error) {
	v, err := redis.String(i, err)
	if err != nil {
		return nil, err
	}

	m := make(map[string]string)
	lines := strings.Split(v, "\r\n")

	for _, l := range lines {
		if strings.HasPrefix(l, "#") {
			continue
		}

		kv := strings.SplitN(l, ":", 2)
		if len(kv) < 2 {
			continue
		}

		m[kv[0]] = kv[1]
	}

	return m, nil
}
<commit_msg>redis: fix tests<commit_after>// Copyright (c) 2014-2017 Ludovic Fauvet
// Licensed under the MIT license

package database

import (
	"errors"
	"fmt"
	"strconv"
	"strings"
	"sync"
	"time"

	. "github.com/etix/mirrorbits/config"
	"github.com/etix/mirrorbits/core"
	"github.com/gomodule/redigo/redis"
	"github.com/rafaeljusto/redigomock"
)

const (
	redisConnectionTimeout = 200 * time.Millisecond
	redisReadWriteTimeout  = 300 * time.Second
	// RedisMinimumVersion contains the minimum redis version required to run the application
	RedisMinimumVersion = "3.2.0"
)

var (
	// ErrUnreachable is returned when the endpoint is not reachable
	ErrUnreachable = errors.New("redis endpoint unreachable")
	// ErrRedisUpgradeRequired is returned when the redis server is running an unsupported version
	ErrRedisUpgradeRequired = errors.New("unsupported Redis version")
)

type redisPool interface {
	Get() redis.Conn
	Close() error
}

// Redis is the instance object of the redis database
type Redis struct {
	pool            redisPool
	Pubsub          *Pubsub
	failure         bool
	failureState    sync.RWMutex
	knownMaster     string
	knownMasterLock sync.Mutex
	stop            chan bool
	ready           chan struct{}
}

// NewRedis returns a new instance of the redis database
func NewRedis() *Redis {
	r := NewRedisCustomPool(nil)
	go func() {
	again:
		up, err := r.UpgradeNeeded()
		if err != nil {
			time.Sleep(50 * time.Millisecond)
			goto again
		}
		if up {
			t := time.Now()
			err = r.Upgrade()
			if err != nil {
				log.Fatalf("Upgrade failed: %+v", err)
			}
			log.Infof("Database upgrade successful (took %s), starting normally", time.Since(t).Round(time.Millisecond))
		}
		close(r.ready)
	}()

	return r
}

// NewRedisCli returns a new instance of the redis database for the CLI.
// Unlike NewRedis(), this function doesn't handle the upgrade. It just
// quit with an error if the db format is invalid.
// TODO the CLI should not use Redis directly but move to RPC
func NewRedisCli() *Redis {
	r := NewRedisCustomPool(nil)

	up, err := r.UpgradeNeeded()
	if err != nil {
		time.Sleep(50 * time.Millisecond)
		return r
	}
	if up {
		log.Fatalf("Database upgrade required")
	}
	close(r.ready)

	return r
}

// NewRedisCustomPool returns a new instance of the redis database
// using a custom pool
func NewRedisCustomPool(pool redisPool) *Redis {
	r := &Redis{
		stop:  make(chan bool),
		ready: make(chan struct{}),
	}

	if pool != nil {
		// Check if we are running inside `go test`
		if _, ok := pool.Get().(*redigomock.Conn); ok {
			// Close ready since we are running a mock
			close(r.ready)
		}
		r.pool = pool
	} else {
		r.pool = &redis.Pool{
			MaxIdle:     10,
			IdleTimeout: 240 * time.Second,
			Dial: func() (redis.Conn, error) {
				conn, err := r.Connect()

				switch err {
				case nil:
					r.setFailureState(false)
				default:
					r.setFailureState(true)
				}

				if r.checkVersion(conn) == ErrRedisUpgradeRequired {
					log.Fatalf("Unsupported Redis version, please upgrade to Redis >= %s", RedisMinimumVersion)
				}

				return conn, err
			},
			TestOnBorrow: func(c redis.Conn, t time.Time) error {
				_, err := c.Do("PING")
				if RedisIsLoading(err) {
					return nil
				}
				return err
			},
		}
	}

	go r.connRecover()

	return r
}

// Get returns a redis connection from the pool
func (r *Redis) Get() redis.Conn {
	select {
	case <-r.ready:
	default:
		return &NotReadyError{}
	}
	return r.pool.Get()
}

// UnblockedGet returns a redis connection from the pool even
// if the database checks and/or upgrade are not finished.
func (r *Redis) UnblockedGet() redis.Conn {
	return r.pool.Get()
}

// Close closes all connections to the redis database
func (r *Redis) Close() {
	select {
	case _, _ = <-r.stop:
		return
	default:
		log.Debug("Closing databases connections")
		r.Pubsub.Close()
		r.pool.Close()
		close(r.stop)
	}
}

// ConnectPubsub initiates the connection to the pubsub
func (r *Redis) ConnectPubsub() {
	if r.Pubsub == nil {
		r.Pubsub = NewPubsub(r)
	}
}

// CheckVersion checks if the redis server version is supported
func (r *Redis) CheckVersion() error {
	c := r.UnblockedGet()
	defer c.Close()
	return r.checkVersion(c)
}

func (r *Redis) checkVersion(conn redis.Conn) error {
	if conn == nil {
		return ErrUnreachable
	}
	info, err := parseInfo(conn.Do("INFO", "server"))
	if err == nil {
		if parseVersion(info["redis_version"]) < parseVersion(RedisMinimumVersion) {
			return ErrRedisUpgradeRequired
		}
	}
	return err
}

// Connect initiates a new connection to the redis server
func (r *Redis) Connect() (redis.Conn, error) {
	sentinels := GetConfig().RedisSentinels

	if len(sentinels) > 0 {

		if len(GetConfig().RedisSentinelMasterName) == 0 {
			r.logError("Config: RedisSentinelMasterName cannot be empty!")
			goto single
		}

		for _, s := range sentinels {
			log.Debugf("Connecting to redis sentinel %s", s.Host)
			var master []string
			var masterhost string
			var cm redis.Conn

			c, err := r.connectTo(s.Host)
			if err != nil {
				r.logError("Sentinel: %s", err.Error())
				continue
			}
			//AUTH?
			role, err := r.askRole(c)
			if err != nil {
				r.logError("Sentinel: %s", err.Error())
				goto closeSentinel
			}
			if role != "sentinel" {
				r.logError("Sentinel: %s is not a sentinel but a %s", s.Host, role)
				goto closeSentinel
			}

			master, err = redis.Strings(c.Do("SENTINEL", "get-master-addr-by-name", GetConfig().RedisSentinelMasterName))
			if err == redis.ErrNil {
				r.logError("Sentinel: %s doesn't know the master-name %s", s.Host, GetConfig().RedisSentinelMasterName)
				goto closeSentinel
			} else if err != nil {
				r.logError("Sentinel: %s", err.Error())
				goto closeSentinel
			}

			masterhost = fmt.Sprintf("%s:%s", master[0], master[1])

			cm, err = r.connectTo(masterhost)
			if err != nil {
				r.logError("Redis master: %s", err.Error())
				goto closeSentinel
			}

			if r.auth(cm) != nil {
				r.logError("Redis master: auth failed")
				goto closeMaster
			}
			if err = r.selectDB(cm); err != nil {
				c.Close()
				return nil, err
			}

			role, err = r.askRole(cm)
			if err != nil {
				r.logError("Redis master: %s", err.Error())
				goto closeMaster
			}
			if role != "master" {
				r.logError("Redis master: %s is not a master but a %s", masterhost, role)
				goto closeMaster
			}

			// Close the connection to the sentinel
			c.Close()

			r.printConnectedMaster(masterhost)
			return cm, nil

		closeMaster:
			cm.Close()

		closeSentinel:
			c.Close()
		}
	}

single:

	if len(GetConfig().RedisAddress) == 0 {
		if len(sentinels) == 0 {
			log.Error("No redis master available")
		}
		return nil, ErrUnreachable
	}

	if len(sentinels) > 0 && r.Failure() == false {
		log.Warning("No redis master available, trying using the configured RedisAddress as fallback")
	}

	c, err := r.connectTo(GetConfig().RedisAddress)
	if err != nil {
		return nil, err
	}
	if err = r.auth(c); err != nil {
		c.Close()
		return nil, err
	}
	if err = r.selectDB(c); err != nil {
		c.Close()
		return nil, err
	}
	role, err := r.askRole(c)
	if err != nil {
		r.logError("Redis master: %s", err.Error())
		return nil, ErrUnreachable
	}
	if role != "master" {
		r.logError("Redis master: %s is not a master but a %s", GetConfig().RedisAddress, role)
		return nil, ErrUnreachable
	}
	r.printConnectedMaster(GetConfig().RedisAddress)
	return c, err

}

func (r *Redis) connectTo(address string) (redis.Conn, error) {
	return redis.Dial("tcp", address,
		redis.DialConnectTimeout(redisConnectionTimeout),
		redis.DialReadTimeout(redisReadWriteTimeout),
		redis.DialWriteTimeout(redisReadWriteTimeout))
}

func (r *Redis) askRole(c redis.Conn) (string, error) {
	roleReply, err := redis.Values(c.Do("ROLE"))
	if err != nil {
		return "", err
	}
	role, err := redis.String(roleReply[0], err)
	return role, err
}

func (r *Redis) auth(c redis.Conn) (err error) {
	if GetConfig().RedisPassword != "" {
		_, err = c.Do("AUTH", GetConfig().RedisPassword)
	}
	return
}

func (r *Redis) selectDB(c redis.Conn) (err error) {
	_, err = c.Do("SELECT", GetConfig().RedisDB)
	return
}

func (r *Redis) logError(format string, args ...interface{}) {
	if r.Failure() {
		log.Debugf(format, args...)
	} else {
		log.Errorf(format, args...)
	}
}

func (r *Redis) printConnectedMaster(address string) {
	r.knownMasterLock.Lock()
	defer r.knownMasterLock.Unlock()
	if address != r.knownMaster && core.Daemon {
		r.knownMaster = address
		log.Infof("Connected to redis master %s", address)
	} else {
		log.Debugf("Connected to redis master %s", address)
	}
}

func (r *Redis) setFailureState(failure bool) {
	r.failureState.Lock()
	r.failure = failure
	r.failureState.Unlock()
}

// Failure returns true if the connection is in a failure state
func (r *Redis) Failure() bool {
	r.failureState.RLock()
	defer r.failureState.RUnlock()
	return r.failure
}

func (r *Redis) connRecover() {
	ticker := time.NewTicker(1 * time.Second)

	for {
		select {
		case <-r.stop:
			return
		case <-ticker.C:
			if r.Failure() {
				if conn := r.Get(); conn != nil {
					// A successful Get() request will automatically unlock
					// other services waiting for a working connection.
					// This is only a way to ensure they wont wait forever.
					if conn.Err() != nil {
						log.Warningf("Database is down: %s", conn.Err().Error())
					}
					conn.Close()
				}
			}
		}
	}
}

// RedisIsLoading returns true if the error is of type LOADING
func RedisIsLoading(err error) bool {
	// PARSING: "LOADING Redis is loading the dataset in memory"
	if err != nil && strings.HasPrefix(err.Error(), "LOADING") {
		return true
	}
	return false
}

func parseVersion(version string) int64 {
	s := strings.Split(version, ".")
	format := fmt.Sprintf("%%s%%0%ds", 2)

	var v string
	for _, value := range s {
		v = fmt.Sprintf(format, v, value)
	}

	var result int64
	var err error
	if result, err = strconv.ParseInt(v, 10, 64); err != nil {
		return -1
	}
	return result
}

func parseInfo(i interface{}, err error) (map[string]string, error) {
	v, err := redis.String(i, err)
	if err != nil {
		return nil, err
	}

	m := make(map[string]string)
	lines := strings.Split(v, "\r\n")

	for _, l := range lines {
		if strings.HasPrefix(l, "#") {
			continue
		}

		kv := strings.SplitN(l, ":", 2)
		if len(kv) < 2 {
			continue
		}

		m[kv[0]] = kv[1]
	}

	return m, nil
}
<|endoftext|>


<commit_before>// Utility functions for working with text
package sanitize

import (
	"testing"
)

var Format string = "\ninput:    %q\nexpected: %q\noutput:   %q"

type Test struct {
	input    string
	expected string
}

var urls = []Test{
	{"ReAd ME.md", "read-me.md"},
	{"E88E08A7-279C-4CC1-8B90-86DE0D70443C.html", "e88e08a7-279c-4cc1-8b90-86de0d70443c.html"},
	{"/user/test/I am a long url's_-?ASDF@£$%£%^testé.html", "/user/test/i-am-a-long-urls-asdfteste.html"},
	{"/../../4-icon.jpg", "/4-icon.jpg"},
	{"/Images/../4-icon.jpg", "/images/4-icon.jpg"},
	{"../4 icon.*", "/4-icon."},
	{"Spac ey/Name/test før url", "spac-ey/name/test-for-url"},
	{"../*", "/"},
}

func TestPath(t *testing.T) {
	for _, test := range urls {
		output := Path(test.input)
		if output != test.expected {
			t.Fatalf(Format, test.input, test.expected, output)
		}
	}
}

var fileNames = []Test{
	{"ReAd ME.md", "read-me.md"},
	{"/var/etc/jobs/go/go/src/pkg/foo/bar.go", "bar.go"},
	{"I am a long url's_-?ASDF@£$%£%^é.html", "i-am-a-long-urls-asdf.html"},
	{"/../../4-icon.jpg", "4-icon.jpg"},
	{"/Images/../4-icon.jpg", "4-icon.jpg"},
	{"../4 icon.jpg", "4-icon.jpg"},
}

func TestName(t *testing.T) {
	for _, test := range fileNames {
		output := Name(test.input)
		if output != test.expected {
			t.Fatalf(Format, test.input, test.expected, output)
		}
	}
}

// Test with some malformed or malicious html
// NB because we remove all tokens after a < until the next >
// and do not attempt to parse, we should be safe from invalid html,
// but will sometimes completely empty the string if we have invalid input
var html = []Test{
	{`&#x000D;`, `&amp;#x000D;`},
	{`<invalid attr="invalid"<,<p><p><p><p><p>`, ""},
	{"
Bold  Not bold
\nAlso not bold.", "Bold  Not bold\nAlso not bold."},
	{"`FOO
ZOO", "`FOO&#x000D;ZOO"},
	{`<script><!--<script </s`, ""},
	{`<a href="/" alt="Fab.com | Aqua Paper Map 22"" title="Fab.com | Aqua Paper Map 22" - fab.com">test</a>`, "test"},
	{" or <p id=0
 or <<</>><ASDF><@$!@£M<<>>>>>>>>>>>>>><>***************aaaaaaaaaaaaaaaaaaaaaaaaaa>", " or ***************aaaaaaaaaaaaaaaaaaaaaaaaaa"},
	{"
Some text
", "Some text\n"},
	{"Something
Some more", "Something\nSome more"},
	{`<a href="http://www.example.com"?>This is a 'test' of <b>bold</b> & <i>italic</i></a> </br> invalid markup.<//data>><alert><script CDATA[:Asdfjk2354115nkjafdgs]>. <div src=">">><><img src="">`, "This is a 'test' of bold & italic \n invalid markup.. \""},
	{"<![CDATA[<sender>John Smith</sender>]]>", "John Smith]]"},
	{"


<commit_before>package mongo

import (
	"errors"

	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	mgo "gopkg.in/mgo.v2"
)

var (
	MDb      *mgo.Database
	Error    error
	MongoUrl string
	Msession *mgo.Session
)

func init() {
	cobra.OnInitialize(connect)
}

func connect() {
	MongoUrl = viper.GetString("mongo_url")
	if len(MongoUrl) == 0 {
		Error = errors.New("Missing mongo_url")
	} else {
		if Msession, Error = mgo.Dial(MongoUrl); Error == nil {
			info, _ := mgo.ParseURL(MongoUrl)
			Msession.SetMode(mgo.Monotonic, true)
			MDb = Msession.DB(info.Database)
		}
	}
}
<commit_msg>added InCollection & UniqueInCollection funcs<commit_after>package mongo

import (
	"errors"

	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	mgo "gopkg.in/mgo.v2"
)

var (
	MDb      *mgo.Database
	Error    error
	MongoUrl string
	Msession *mgo.Session
)

func init() {
	cobra.OnInitialize(connect)
}

func connect() {
	MongoUrl = viper.GetString("mongo_url")
	if len(MongoUrl) == 0 {
		Error = errors.New("Missing mongo_url")
	} else {
		if Msession, Error = mgo.Dial(MongoUrl); Error == nil {
			info, _ := mgo.ParseURL(MongoUrl)
			Msession.SetMode(mgo.Monotonic, true)
			MDb = Msession.DB(info.Database)
		}
	}
}

//InCollection returns whether document(s) matching the query the specified collection exist
func InCollection(collection *mgo.Collection, selector interface{}) bool {
	n, _ := collection.Find(selector).Count()
	return n > 0
}

//UniqueInCollection returns whether one and only one document matching the query in the specified collection exists.
func UniqueInCollection(collection *mgo.Collection, selector interface{}) bool {
	n, _ := collection.Find(selector).Count()
	return n == 1
}
<|endoftext|>