<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The testing package provides support for automated testing of Go packages.
// It is intended to be used in concert with the ``gotest'' utility, which automates
// execution of any function of the form
//     func TestXxx(*testing.T)
// where Xxx can by any alphanumeric string (but the first letter must not be in
// [a-z]) and serves to identify the test routine.
// These TestXxx routines should be declared within the package they are testing.
package testing

import (
	"flag";
	"fmt";
	"os";
	"runtime";
)

// Report as tests are run; default is silent for success.
var chatty = flag.Bool("chatty", false, "chatty")

// Insert final newline if needed and tabs after internal newlines.
func tabify(s string) string {
	n := len(s);
	if n > 0 && s[n-1] != '\n' {
		s += "\n";
		n++;
	}
	for i := 0; i < n - 1; i++ {	// -1 to avoid final newline
		if s[i] == '\n' {
			return s[0:i+1] + "\t" + tabify(s[i+1:n]);
		}
	}
	return s
}

// T is a type passed to Test functions to manage test state and support formatted test logs.
// Logs are accumulated during execution and dumped to standard error when done.
type T struct {
	errors	string;
	failed	bool;
	ch	chan *T;
}

// Fail marks the Test function as having failed but continues execution.
func (t *T) Fail() {
	t.failed = true
}

// Failed returns whether the Test function has failed.
func (t *T) Failed() bool {
	return t.failed
}

// FailNow marks the Test function as having failed and stops its execution.
// Execution will continue at the next Test.
func (t *T) FailNow() {
	t.Fail();
	t.ch <- t;
	runtime.Goexit();
}

// Log formats its arguments using default formatting, analogous to Print(),
// and records the text in the error log.
func (t *T) Log(args ...) {
	t.errors += "\t" + tabify(fmt.Sprintln(args));
}

// Log formats its arguments according to the format, analogous to Printf(),
// and records the text in the error log.
func (t *T) Logf(format string, args ...) {
	t.errors += "\t" + tabify(fmt.Sprintf(format, args));
}

// Error is equivalent to Log() followed by Fail().
func (t *T) Error(args ...) {
	t.Log(args);
	t.Fail();
}

// Errorf is equivalent to Logf() followed by Fail().
func (t *T) Errorf(format string, args ...) {
	t.Logf(format, args);
	t.Fail();
}

// Fatal is equivalent to Log() followed by FailNow().
func (t *T) Fatal(args ...) {
	t.Log(args);
	t.FailNow();
}

// Fatalf is equivalent to Logf() followed by FailNow().
func (t *T) Fatalf(format string, args ...) {
	t.Logf(format, args);
	t.FailNow();
}

// An internal type but exported because it is cross-package; part of the implementation
// of gotest.
type Test struct {
	Name string;
	F func(*T);
}

func tRunner(t *T, test *Test) {
	test.F(t);
	t.ch <- t;
}

// An internal function but exported because it is cross-package; part of the implementation
// of gotest.
func Main(tests []Test) {
	flag.Parse();
	ok := true;
	if len(tests) == 0 {
		println("testing: warning: no tests to run");
	}
	for i := 0; i < len(tests); i++ {
		if *chatty {
			println("=== RUN ", tests[i].Name);
		}
		t := new(T);
		t.ch = make(chan *T);
		go tRunner(t, &tests[i]);
		<-t.ch;
		if t.failed {
			println("--- FAIL:", tests[i].Name);
			print(t.errors);
			ok = false;
		} else if *chatty {
			println("--- PASS:", tests[i].Name);
			print(t.errors);
		}
	}
	if !ok {
		println("FAIL");
		os.Exit(1);
	}
	println("PASS");
}
<commit_msg>rename -chatty to more conventional -v. add -match flag to select tests.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The testing package provides support for automated testing of Go packages.
// It is intended to be used in concert with the ``gotest'' utility, which automates
// execution of any function of the form
//     func TestXxx(*testing.T)
// where Xxx can by any alphanumeric string (but the first letter must not be in
// [a-z]) and serves to identify the test routine.
// These TestXxx routines should be declared within the package they are testing.
package testing

import (
	"flag";
	"fmt";
	"os";
	"regexp";
	"runtime";
)

// Report as tests are run; default is silent for success.
var chatty = flag.Bool("v", false, "verbose: print additional output")
var match = flag.String("match", "", "regular expression to select tests to run")

// Insert final newline if needed and tabs after internal newlines.
func tabify(s string) string {
	n := len(s);
	if n > 0 && s[n-1] != '\n' {
		s += "\n";
		n++;
	}
	for i := 0; i < n - 1; i++ {	// -1 to avoid final newline
		if s[i] == '\n' {
			return s[0:i+1] + "\t" + tabify(s[i+1:n]);
		}
	}
	return s
}

// T is a type passed to Test functions to manage test state and support formatted test logs.
// Logs are accumulated during execution and dumped to standard error when done.
type T struct {
	errors	string;
	failed	bool;
	ch	chan *T;
}

// Fail marks the Test function as having failed but continues execution.
func (t *T) Fail() {
	t.failed = true
}

// Failed returns whether the Test function has failed.
func (t *T) Failed() bool {
	return t.failed
}

// FailNow marks the Test function as having failed and stops its execution.
// Execution will continue at the next Test.
func (t *T) FailNow() {
	t.Fail();
	t.ch <- t;
	runtime.Goexit();
}

// Log formats its arguments using default formatting, analogous to Print(),
// and records the text in the error log.
func (t *T) Log(args ...) {
	t.errors += "\t" + tabify(fmt.Sprintln(args));
}

// Log formats its arguments according to the format, analogous to Printf(),
// and records the text in the error log.
func (t *T) Logf(format string, args ...) {
	t.errors += "\t" + tabify(fmt.Sprintf(format, args));
}

// Error is equivalent to Log() followed by Fail().
func (t *T) Error(args ...) {
	t.Log(args);
	t.Fail();
}

// Errorf is equivalent to Logf() followed by Fail().
func (t *T) Errorf(format string, args ...) {
	t.Logf(format, args);
	t.Fail();
}

// Fatal is equivalent to Log() followed by FailNow().
func (t *T) Fatal(args ...) {
	t.Log(args);
	t.FailNow();
}

// Fatalf is equivalent to Logf() followed by FailNow().
func (t *T) Fatalf(format string, args ...) {
	t.Logf(format, args);
	t.FailNow();
}

// An internal type but exported because it is cross-package; part of the implementation
// of gotest.
type Test struct {
	Name string;
	F func(*T);
}

func tRunner(t *T, test *Test) {
	test.F(t);
	t.ch <- t;
}

// An internal function but exported because it is cross-package; part of the implementation
// of gotest.
func Main(tests []Test) {
	flag.Parse();
	args := flag.Args();
	ok := true;
	if len(tests) == 0 {
		println("testing: warning: no tests to run");
	}
	re, err := regexp.Compile(*match);
	if err != nil {
		println("invalid regexp for -match:", err.String());
		os.Exit(1);
	}
	for i := 0; i < len(tests); i++ {
		if !re.Match(tests[i].Name) {
			continue;
		}
		if *chatty {
			println("=== RUN ", tests[i].Name);
		}
		t := new(T);
		t.ch = make(chan *T);
		go tRunner(t, &tests[i]);
		<-t.ch;
		if t.failed {
			println("--- FAIL:", tests[i].Name);
			print(t.errors);
			ok = false;
		} else if *chatty {
			println("--- PASS:", tests[i].Name);
			print(t.errors);
		}
	}
	if !ok {
		println("FAIL");
		os.Exit(1);
	}
	println("PASS");
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
	Package builtin provides documentation for Go's predeclared identifiers.
	The items documented here are not actually in package builtin
	but their descriptions here allow godoc to present documentation
	for the language's special identifiers.
*/
package builtin

// bool is the set of boolean values, true and false.
type bool bool

// true and false are the two untyped boolean values.
const (
	true  = 0 == 0 // Untyped bool.
	false = 0 != 0 // Untyped bool.
)

// uint8 is the set of all unsigned 8-bit integers.
// Range: 0 through 255.
type uint8 uint8

// uint16 is the set of all unsigned 16-bit integers.
// Range: 0 through 65535.
type uint16 uint16

// uint32 is the set of all unsigned 32-bit integers.
// Range: 0 through 4294967295.
type uint32 uint32

// uint64 is the set of all unsigned 64-bit integers.
// Range: 0 through 18446744073709551615.
type uint64 uint64

// int8 is the set of all signed 8-bit integers.
// Range: -128 through 127.
type int8 int8

// int16 is the set of all signed 16-bit integers.
// Range: -32768 through 32767.
type int16 int16

// int32 is the set of all signed 32-bit integers.
// Range: -2147483648 through 2147483647.
type int32 int32

// int64 is the set of all signed 64-bit integers.
// Range: -9223372036854775808 through 9223372036854775807.
type int64 int64

// float32 is the set of all IEEE-754 32-bit floating-point numbers.
type float32 float32

// float64 is the set of all IEEE-754 64-bit floating-point numbers.
type float64 float64

// complex64 is the set of all complex numbers with float32 real and
// imaginary parts.
type complex64 complex64

// complex128 is the set of all complex numbers with float64 real and
// imaginary parts.
type complex128 complex128

// string is the set of all strings of 8-bit bytes, conventionally but not
// necessarily representing UTF-8-encoded text. A string may be empty, but
// not nil. Values of string type are immutable.
type string string

// int is a signed integer type that is at least 32 bits in size. It is a
// distinct type, however, and not an alias for, say, int32.
type int int

// uint is an unsigned integer type that is at least 32 bits in size. It is a
// distinct type, however, and not an alias for, say, uint32.
type uint uint

// uintptr is an integer type that is large enough to hold the bit pattern of
// any pointer.
type uintptr uintptr

// byte is an alias for uint8 and is equivalent to uint8 in all ways. It is
// used, by convention, to distinguish byte values from 8-bit unsigned
// integer values.
type byte byte

// rune is an alias for int32 and is equivalent to int32 in all ways. It is
// used, by convention, to distinguish character values from integer values.
type rune rune

// iota is a predeclared identifier representing the untyped integer ordinal
// number of the current const specification in a (usually parenthesized)
// const declaration. It is zero-indexed.
const iota = 0 // Untyped int.

// nil is a predeclared identifier representing the zero value for a
// pointer, channel, func, interface, map, or slice type.
var nil Type // Type must be a pointer, channel, func, interface, map, or slice type

// Type is here for the purposes of documentation only. It is a stand-in
// for any Go type, but represents the same type for any given function
// invocation.
type Type int

// Type1 is here for the purposes of documentation only. It is a stand-in
// for any Go type, but represents the same type for any given function
// invocation.
type Type1 int

// IntegerType is here for the purposes of documentation only. It is a stand-in
// for any integer type: int, uint, int8 etc.
type IntegerType int

// FloatType is here for the purposes of documentation only. It is a stand-in
// for either float type: float32 or float64.
type FloatType float32

// ComplexType is here for the purposes of documentation only. It is a
// stand-in for either complex type: complex64 or complex128.
type ComplexType complex64

// The append built-in function appends elements to the end of a slice. If
// it has sufficient capacity, the destination is resliced to accommodate the
// new elements. If it does not, a new underlying array will be allocated.
// Append returns the updated slice. It is therefore necessary to store the
// result of append, often in the variable holding the slice itself:
//	slice = append(slice, elem1, elem2)
//	slice = append(slice, anotherSlice...)
// As a special case, it is legal to append a string to a byte slice, like this:
//	slice = append([]byte("hello "), "world"...)
func append(slice []Type, elems ...Type) []Type

// The copy built-in function copies elements from a source slice into a
// destination slice. (As a special case, it also will copy bytes from a
// string to a slice of bytes.) The source and destination may overlap. Copy
// returns the number of elements copied, which will be the minimum of
// len(src) and len(dst).
func copy(dst, src []Type) int

// The delete built-in function deletes the element with the specified key
// (m[key]) from the map. If m is nil or there is no such element, delete
// is a no-op.
func delete(m map[Type]Type1, key Type)

// The len built-in function returns the length of v, according to its type:
//	Array: the number of elements in v.
//	Pointer to array: the number of elements in *v (even if v is nil).
//	Slice, or map: the number of elements in v; if v is nil, len(v) is zero.
//	String: the number of bytes in v.
//	Channel: the number of elements queued (unread) in the channel buffer;
//	if v is nil, len(v) is zero.
func len(v Type) int

// The cap built-in function returns the capacity of v, according to its type:
//	Array: the number of elements in v (same as len(v)).
//	Pointer to array: the number of elements in *v (same as len(v)).
//	Slice: the maximum length the slice can reach when resliced;
//	if v is nil, cap(v) is zero.
//	Channel: the channel buffer capacity, in units of elements;
//	if v is nil, cap(v) is zero.
func cap(v Type) int

// The make built-in function allocates and initializes an object of type
// slice, map, or chan (only). Like new, the first argument is a type, not a
// value. Unlike new, make's return type is the same as the type of its
// argument, not a pointer to it. The specification of the result depends on
// the type:
//	Slice: The size specifies the length. The capacity of the slice is
//	equal to its length. A second integer argument may be provided to
//	specify a different capacity; it must be no smaller than the
//	length, so make([]int, 0, 10) allocates a slice of length 0 and
//	capacity 10.
//	Map: An initial allocation is made according to the size but the
//	resulting map has length 0. The size may be omitted, in which case
//	a small starting size is allocated.
//	Channel: The channel's buffer is initialized with the specified
//	buffer capacity. If zero, or the size is omitted, the channel is
//	unbuffered.
func make(Type, size IntegerType) Type

// The new built-in function allocates memory. The first argument is a type,
// not a value, and the value returned is a pointer to a newly
// allocated zero value of that type.
func new(Type) *Type

// The complex built-in function constructs a complex value from two
// floating-point values. The real and imaginary parts must be of the same
// size, either float32 or float64 (or assignable to them), and the return
// value will be the corresponding complex type (complex64 for float32,
// complex128 for float64).
func complex(r, i FloatType) ComplexType

// The real built-in function returns the real part of the complex number c.
// The return value will be floating point type corresponding to the type of c.
func real(c ComplexType) FloatType

// The imag built-in function returns the imaginary part of the complex
// number c. The return value will be floating point type corresponding to
// the type of c.
func imag(c ComplexType) FloatType

// The close built-in function closes a channel, which must be either
// bidirectional or send-only. It should be executed only by the sender,
// never the receiver, and has the effect of shutting down the channel after
// the last sent value is received. After the last value has been received
// from a closed channel c, any receive from c will succeed without
// blocking, returning the zero value for the channel element. The form
//	x, ok := <-c
// will also set ok to false for a closed channel.
func close(c chan<- Type)

// The panic built-in function stops normal execution of the current
// goroutine. When a function F calls panic, normal execution of F stops
// immediately. Any functions whose execution was deferred by F are run in
// the usual way, and then F returns to its caller. To the caller G, the
// invocation of F then behaves like a call to panic, terminating G's
// execution and running any deferred functions. This continues until all
// functions in the executing goroutine have stopped, in reverse order. At
// that point, the program is terminated and the error condition is reported,
// including the value of the argument to panic. This termination sequence
// is called panicking and can be controlled by the built-in function
// recover.
func panic(v interface{})

// The recover built-in function allows a program to manage behavior of a
// panicking goroutine. Executing a call to recover inside a deferred
// function (but not any function called by it) stops the panicking sequence
// by restoring normal execution and retrieves the error value passed to the
// call of panic. If recover is called outside the deferred function it will
// not stop a panicking sequence. In this case, or when the goroutine is not
// panicking, or if the argument supplied to panic was nil, recover returns
// nil. Thus the return value from recover reports whether the goroutine is
// panicking.
func recover() interface{}

// The print built-in function formats its arguments in an
// implementation-specific way and writes the result to standard error.
// Print is useful for bootstrapping and debugging; it is not guaranteed
// to stay in the language.
func print(args ...Type)

// The println built-in function formats its arguments in an
// implementation-specific way and writes the result to standard error.
// Spaces are always added between arguments and a newline is appended.
// Println is useful for bootstrapping and debugging; it is not guaranteed
// to stay in the language.
func println(args ...Type)

// The error built-in interface type is the conventional interface for
// representing an error condition, with the nil value representing no error.
type error interface {
	Error() string
}
<commit_msg>builtin: clarify that make(map[K]V, n) allocates space for n elements<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
	Package builtin provides documentation for Go's predeclared identifiers.
	The items documented here are not actually in package builtin
	but their descriptions here allow godoc to present documentation
	for the language's special identifiers.
*/
package builtin

// bool is the set of boolean values, true and false.
type bool bool

// true and false are the two untyped boolean values.
const (
	true  = 0 == 0 // Untyped bool.
	false = 0 != 0 // Untyped bool.
)

// uint8 is the set of all unsigned 8-bit integers.
// Range: 0 through 255.
type uint8 uint8

// uint16 is the set of all unsigned 16-bit integers.
// Range: 0 through 65535.
type uint16 uint16

// uint32 is the set of all unsigned 32-bit integers.
// Range: 0 through 4294967295.
type uint32 uint32

// uint64 is the set of all unsigned 64-bit integers.
// Range: 0 through 18446744073709551615.
type uint64 uint64

// int8 is the set of all signed 8-bit integers.
// Range: -128 through 127.
type int8 int8

// int16 is the set of all signed 16-bit integers.
// Range: -32768 through 32767.
type int16 int16

// int32 is the set of all signed 32-bit integers.
// Range: -2147483648 through 2147483647.
type int32 int32

// int64 is the set of all signed 64-bit integers.
// Range: -9223372036854775808 through 9223372036854775807.
type int64 int64

// float32 is the set of all IEEE-754 32-bit floating-point numbers.
type float32 float32

// float64 is the set of all IEEE-754 64-bit floating-point numbers.
type float64 float64

// complex64 is the set of all complex numbers with float32 real and
// imaginary parts.
type complex64 complex64

// complex128 is the set of all complex numbers with float64 real and
// imaginary parts.
type complex128 complex128

// string is the set of all strings of 8-bit bytes, conventionally but not
// necessarily representing UTF-8-encoded text. A string may be empty, but
// not nil. Values of string type are immutable.
type string string

// int is a signed integer type that is at least 32 bits in size. It is a
// distinct type, however, and not an alias for, say, int32.
type int int

// uint is an unsigned integer type that is at least 32 bits in size. It is a
// distinct type, however, and not an alias for, say, uint32.
type uint uint

// uintptr is an integer type that is large enough to hold the bit pattern of
// any pointer.
type uintptr uintptr

// byte is an alias for uint8 and is equivalent to uint8 in all ways. It is
// used, by convention, to distinguish byte values from 8-bit unsigned
// integer values.
type byte byte

// rune is an alias for int32 and is equivalent to int32 in all ways. It is
// used, by convention, to distinguish character values from integer values.
type rune rune

// iota is a predeclared identifier representing the untyped integer ordinal
// number of the current const specification in a (usually parenthesized)
// const declaration. It is zero-indexed.
const iota = 0 // Untyped int.

// nil is a predeclared identifier representing the zero value for a
// pointer, channel, func, interface, map, or slice type.
var nil Type // Type must be a pointer, channel, func, interface, map, or slice type

// Type is here for the purposes of documentation only. It is a stand-in
// for any Go type, but represents the same type for any given function
// invocation.
type Type int

// Type1 is here for the purposes of documentation only. It is a stand-in
// for any Go type, but represents the same type for any given function
// invocation.
type Type1 int

// IntegerType is here for the purposes of documentation only. It is a stand-in
// for any integer type: int, uint, int8 etc.
type IntegerType int

// FloatType is here for the purposes of documentation only. It is a stand-in
// for either float type: float32 or float64.
type FloatType float32

// ComplexType is here for the purposes of documentation only. It is a
// stand-in for either complex type: complex64 or complex128.
type ComplexType complex64

// The append built-in function appends elements to the end of a slice. If
// it has sufficient capacity, the destination is resliced to accommodate the
// new elements. If it does not, a new underlying array will be allocated.
// Append returns the updated slice. It is therefore necessary to store the
// result of append, often in the variable holding the slice itself:
//	slice = append(slice, elem1, elem2)
//	slice = append(slice, anotherSlice...)
// As a special case, it is legal to append a string to a byte slice, like this:
//	slice = append([]byte("hello "), "world"...)
func append(slice []Type, elems ...Type) []Type

// The copy built-in function copies elements from a source slice into a
// destination slice. (As a special case, it also will copy bytes from a
// string to a slice of bytes.) The source and destination may overlap. Copy
// returns the number of elements copied, which will be the minimum of
// len(src) and len(dst).
func copy(dst, src []Type) int

// The delete built-in function deletes the element with the specified key
// (m[key]) from the map. If m is nil or there is no such element, delete
// is a no-op.
func delete(m map[Type]Type1, key Type)

// The len built-in function returns the length of v, according to its type:
//	Array: the number of elements in v.
//	Pointer to array: the number of elements in *v (even if v is nil).
//	Slice, or map: the number of elements in v; if v is nil, len(v) is zero.
//	String: the number of bytes in v.
//	Channel: the number of elements queued (unread) in the channel buffer;
//	if v is nil, len(v) is zero.
func len(v Type) int

// The cap built-in function returns the capacity of v, according to its type:
//	Array: the number of elements in v (same as len(v)).
//	Pointer to array: the number of elements in *v (same as len(v)).
//	Slice: the maximum length the slice can reach when resliced;
//	if v is nil, cap(v) is zero.
//	Channel: the channel buffer capacity, in units of elements;
//	if v is nil, cap(v) is zero.
func cap(v Type) int

// The make built-in function allocates and initializes an object of type
// slice, map, or chan (only). Like new, the first argument is a type, not a
// value. Unlike new, make's return type is the same as the type of its
// argument, not a pointer to it. The specification of the result depends on
// the type:
//	Slice: The size specifies the length. The capacity of the slice is
//	equal to its length. A second integer argument may be provided to
//	specify a different capacity; it must be no smaller than the
//	length, so make([]int, 0, 10) allocates a slice of length 0 and
//	capacity 10.
//	Map: An empty map is allocated with enough space to hold the
//	specified number of elements. The size may be omitted, in which case
//	a small starting size is allocated.
//	Channel: The channel's buffer is initialized with the specified
//	buffer capacity. If zero, or the size is omitted, the channel is
//	unbuffered.
func make(Type, size IntegerType) Type

// The new built-in function allocates memory. The first argument is a type,
// not a value, and the value returned is a pointer to a newly
// allocated zero value of that type.
func new(Type) *Type

// The complex built-in function constructs a complex value from two
// floating-point values. The real and imaginary parts must be of the same
// size, either float32 or float64 (or assignable to them), and the return
// value will be the corresponding complex type (complex64 for float32,
// complex128 for float64).
func complex(r, i FloatType) ComplexType

// The real built-in function returns the real part of the complex number c.
// The return value will be floating point type corresponding to the type of c.
func real(c ComplexType) FloatType

// The imag built-in function returns the imaginary part of the complex
// number c. The return value will be floating point type corresponding to
// the type of c.
func imag(c ComplexType) FloatType

// The close built-in function closes a channel, which must be either
// bidirectional or send-only. It should be executed only by the sender,
// never the receiver, and has the effect of shutting down the channel after
// the last sent value is received. After the last value has been received
// from a closed channel c, any receive from c will succeed without
// blocking, returning the zero value for the channel element. The form
//	x, ok := <-c
// will also set ok to false for a closed channel.
func close(c chan<- Type)

// The panic built-in function stops normal execution of the current
// goroutine. When a function F calls panic, normal execution of F stops
// immediately. Any functions whose execution was deferred by F are run in
// the usual way, and then F returns to its caller. To the caller G, the
// invocation of F then behaves like a call to panic, terminating G's
// execution and running any deferred functions. This continues until all
// functions in the executing goroutine have stopped, in reverse order. At
// that point, the program is terminated and the error condition is reported,
// including the value of the argument to panic. This termination sequence
// is called panicking and can be controlled by the built-in function
// recover.
func panic(v interface{})

// The recover built-in function allows a program to manage behavior of a
// panicking goroutine. Executing a call to recover inside a deferred
// function (but not any function called by it) stops the panicking sequence
// by restoring normal execution and retrieves the error value passed to the
// call of panic. If recover is called outside the deferred function it will
// not stop a panicking sequence. In this case, or when the goroutine is not
// panicking, or if the argument supplied to panic was nil, recover returns
// nil. Thus the return value from recover reports whether the goroutine is
// panicking.
func recover() interface{}

// The print built-in function formats its arguments in an
// implementation-specific way and writes the result to standard error.
// Print is useful for bootstrapping and debugging; it is not guaranteed
// to stay in the language.
func print(args ...Type)

// The println built-in function formats its arguments in an
// implementation-specific way and writes the result to standard error.
// Spaces are always added between arguments and a newline is appended.
// Println is useful for bootstrapping and debugging; it is not guaranteed
// to stay in the language.
func println(args ...Type)

// The error built-in interface type is the conventional interface for
// representing an error condition, with the nil value representing no error.
type error interface {
	Error() string
}
<|endoftext|>


<commit_before>// Copyright 2012 Google, Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style license
// that can be found in the LICENSE file in the root of the source
// tree.

package gopacket

import ()

// SerializableLayer allows its implementations to be written out as a set of bytes,
// so those bytes may be sent on the wire or otherwise used by the caller.
// SerializableLayer is implemented by certain Layer types, and can be encoded to
// bytes using the LayerWriter object.
type SerializableLayer interface {
	// SerializeTo writes this layer to a slice, growing that slice if necessary
	// to make it fit the layer's data.
	//  Args:
	//   b:  SerializeBuffer to write this layer on to.  When called, b.Bytes()
	//     is the payload this layer should wrap, if any.  Note that this
	//     layer can either prepend itself (common), append itself
	//     (uncommon), or both (sometimes padding or footers are required at
	//     the end of packet data). It's also possible (though probably very
	//     rarely needed) to overwrite any bytes in the current payload.
	//     After this call, b.Bytes() should return the byte encoding of
	//     this layer wrapping the original b.Bytes() payload.
	//   opts:  options to use while writing out data.
	//  Returns:
	//   error if a problem was encountered during encoding.  If an error is
	//   returned, the bytes in data should be considered invalidated, and
	//   not used.
	//
	// SerializeTo calls SHOULD entirely ignore LayerContents and
	// LayerPayload.  It just serializes based on struct fields, neither
	// modifying nor using contents/payload.
	SerializeTo(b SerializeBuffer, opts SerializeOptions) error
}

// SerializeOptions provides options for behaviors that SerializableLayers may want to
// implement.
type SerializeOptions struct {
	// FixLengths determines whether, during serialization, layers should fix
	// the values for any length field that depends on the payload.
	FixLengths bool
	// ComputeChecksums determines whether, during serialization, layers
	// should recompute checksums based on their payloads.
	ComputeChecksums bool
}

// SerializeBuffer is a helper used by gopacket for writing out packet layers.
// SerializeBuffer starts off as an empty []byte.  Subsequent calls to PrependBytes
// return byte slices before the current Bytes(), AppendBytes returns byte
// slices after.
//
// Byte slices returned by PrependBytes/AppendBytes are NOT zero'd out, so if
// you want to make sure they're all zeros, set them as such.
//
// SerializeBuffer is specifically designed to handle packet writing, where unlike
// with normal writes it's easier to start writing at the inner-most layer and
// work out, meaning that we often need to prepend bytes.  This runs counter to
// typical writes to byte slices using append(), where we only write at the end
// of the buffer.
//
// It can be reused via Clear.  Note, however, that a Clear call will invalidate the
// byte slices returned by any previous Bytes() call (the same buffer is
// reused).
//
//  1) Reusing a write buffer is generally much faster than creating a new one,
//     and with the default implementation it avoids additional memory allocations.
//  2) If a byte slice from a previous Bytes() call will continue to be used,
//     it's better to create a new SerializeBuffer.
//
// The Clear method is specifically designed to minimize memory allocations for
// similar later workloads on the SerializeBuffer.  IE: if you make a set of
// Prepend/Append calls, then clear, then make the same calls with the same
// sizes, the second round (and all future similar rounds) shouldn't allocate
// any new memory.
type SerializeBuffer interface {
	// Bytes returns the contiguous set of bytes collected so far by Prepend/Append
	// calls.  The slice returned by Bytes will be modified by future Clear calls,
	// so if you're planning on clearing this SerializeBuffer, you may want to copy
	// Bytes somewhere safe first.
	Bytes() []byte
	// PrependBytes returns a set of bytes which prepends the current bytes in this
	// buffer.  These bytes start in an indeterminate state, so they should be
	// overwritten by the caller.  The caller must only call PrependBytes if they
	// know they're going to immediately overwrite all bytes returned.
	PrependBytes(num int) ([]byte, error)
	// AppendBytes returns a set of bytes which prepends the current bytes in this
	// buffer.  These bytes start in an indeterminate state, so they should be
	// overwritten by the caller.  The caller must only call AppendBytes if they
	// know they're going to immediately overwrite all bytes returned.
	AppendBytes(num int) ([]byte, error)
	// Clear resets the SerializeBuffer to a new, empty buffer.  After a call to clear,
	// the byte slice returned by any previous call to Bytes() for this buffer
	// should be considered invalidated.
	Clear() error
}

type serializeBuffer struct {
	data                []byte
	start               int
	prepended, appended int
}

// NewSerializeBuffer creates a new instance of the default implementation of
// the SerializeBuffer interface.
func NewSerializeBuffer() SerializeBuffer {
	return &serializeBuffer{}
}

// NewSerializeBufferExpectedSize creates a new buffer for serialization, optimized for an
// expected number of bytes prepended/appended.  This tends to decrease the
// number of memory allocations made by the buffer during writes.
func NewSerializeBufferExpectedSize(expectedPrependLength, expectedAppendLength int) SerializeBuffer {
	return &serializeBuffer{
		data:      make([]byte, expectedPrependLength, expectedPrependLength+expectedAppendLength),
		start:     expectedPrependLength,
		prepended: expectedPrependLength,
		appended:  expectedAppendLength,
	}
}

func (w *serializeBuffer) Bytes() []byte {
	return w.data[w.start:]
}

func (w *serializeBuffer) PrependBytes(num int) ([]byte, error) {
	if num < 0 {
		panic("num < 0")
	}
	if w.start < num {
		toPrepend := w.prepended
		if toPrepend < num {
			toPrepend = num
		}
		w.prepended += toPrepend
		length := cap(w.data) + toPrepend
		newData := make([]byte, length)
		newStart := w.start + toPrepend
		copy(newData[newStart:], w.data[w.start:])
		w.start = newStart
		w.data = newData[:toPrepend+len(w.data)]
	}
	w.start -= num
	return w.data[w.start : w.start+num], nil
}

func (w *serializeBuffer) AppendBytes(num int) ([]byte, error) {
	if num < 0 {
		panic("num < 0")
	}
	initialLength := len(w.data)
	if cap(w.data)-initialLength < num {
		toAppend := w.appended
		if toAppend < num {
			toAppend = num
		}
		w.appended += toAppend
		newData := make([]byte, cap(w.data)+toAppend)
		copy(newData[w.start:], w.data[w.start:])
		w.data = newData[:initialLength]
	}
	// Grow the buffer.  We know it'll be under capacity given above.
	w.data = w.data[:initialLength+num]
	return w.data[initialLength:], nil
}

func (w *serializeBuffer) Clear() error {
	w.start = w.prepended
	w.data = w.data[:w.start]
	return nil
}

// SerializeLayers clears the given write buffer, then writes all layers into it so
// they correctly wrap each other.  Note that by clearing the buffer, it
// invalidates all slices previously returned by w.Bytes()
//
// Example:
//   buf := gopacket.NewSerializeBuffer()
//   opts := gopacket.SerializeOptions{}
//   gopacket.SerializeLayers(buf, opts, a, b, c)
//   firstPayload := buf.Bytes()  // contains byte representation of a(b(c))
//   gopacket.SerializeLayers(buf, opts, d, e, f)
//   secondPayload := buf.Bytes()  // contains byte representation of d(e(f)). firstPayload is now invalidated, since the SerializeLayers call Clears buf.
func SerializeLayers(w SerializeBuffer, opts SerializeOptions, layers ...SerializableLayer) error {
	w.Clear()
	for i := len(layers) - 1; i >= 0; i-- {
		layer := layers[i]
		err := layer.SerializeTo(w, opts)
		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Implement SerializePacket<commit_after>// Copyright 2012 Google, Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style license
// that can be found in the LICENSE file in the root of the source
// tree.

package gopacket

import (
	"fmt"
)

// SerializableLayer allows its implementations to be written out as a set of bytes,
// so those bytes may be sent on the wire or otherwise used by the caller.
// SerializableLayer is implemented by certain Layer types, and can be encoded to
// bytes using the LayerWriter object.
type SerializableLayer interface {
	// SerializeTo writes this layer to a slice, growing that slice if necessary
	// to make it fit the layer's data.
	//  Args:
	//   b:  SerializeBuffer to write this layer on to.  When called, b.Bytes()
	//     is the payload this layer should wrap, if any.  Note that this
	//     layer can either prepend itself (common), append itself
	//     (uncommon), or both (sometimes padding or footers are required at
	//     the end of packet data). It's also possible (though probably very
	//     rarely needed) to overwrite any bytes in the current payload.
	//     After this call, b.Bytes() should return the byte encoding of
	//     this layer wrapping the original b.Bytes() payload.
	//   opts:  options to use while writing out data.
	//  Returns:
	//   error if a problem was encountered during encoding.  If an error is
	//   returned, the bytes in data should be considered invalidated, and
	//   not used.
	//
	// SerializeTo calls SHOULD entirely ignore LayerContents and
	// LayerPayload.  It just serializes based on struct fields, neither
	// modifying nor using contents/payload.
	SerializeTo(b SerializeBuffer, opts SerializeOptions) error
}

// SerializeOptions provides options for behaviors that SerializableLayers may want to
// implement.
type SerializeOptions struct {
	// FixLengths determines whether, during serialization, layers should fix
	// the values for any length field that depends on the payload.
	FixLengths bool
	// ComputeChecksums determines whether, during serialization, layers
	// should recompute checksums based on their payloads.
	ComputeChecksums bool
}

// SerializeBuffer is a helper used by gopacket for writing out packet layers.
// SerializeBuffer starts off as an empty []byte.  Subsequent calls to PrependBytes
// return byte slices before the current Bytes(), AppendBytes returns byte
// slices after.
//
// Byte slices returned by PrependBytes/AppendBytes are NOT zero'd out, so if
// you want to make sure they're all zeros, set them as such.
//
// SerializeBuffer is specifically designed to handle packet writing, where unlike
// with normal writes it's easier to start writing at the inner-most layer and
// work out, meaning that we often need to prepend bytes.  This runs counter to
// typical writes to byte slices using append(), where we only write at the end
// of the buffer.
//
// It can be reused via Clear.  Note, however, that a Clear call will invalidate the
// byte slices returned by any previous Bytes() call (the same buffer is
// reused).
//
//  1) Reusing a write buffer is generally much faster than creating a new one,
//     and with the default implementation it avoids additional memory allocations.
//  2) If a byte slice from a previous Bytes() call will continue to be used,
//     it's better to create a new SerializeBuffer.
//
// The Clear method is specifically designed to minimize memory allocations for
// similar later workloads on the SerializeBuffer.  IE: if you make a set of
// Prepend/Append calls, then clear, then make the same calls with the same
// sizes, the second round (and all future similar rounds) shouldn't allocate
// any new memory.
type SerializeBuffer interface {
	// Bytes returns the contiguous set of bytes collected so far by Prepend/Append
	// calls.  The slice returned by Bytes will be modified by future Clear calls,
	// so if you're planning on clearing this SerializeBuffer, you may want to copy
	// Bytes somewhere safe first.
	Bytes() []byte
	// PrependBytes returns a set of bytes which prepends the current bytes in this
	// buffer.  These bytes start in an indeterminate state, so they should be
	// overwritten by the caller.  The caller must only call PrependBytes if they
	// know they're going to immediately overwrite all bytes returned.
	PrependBytes(num int) ([]byte, error)
	// AppendBytes returns a set of bytes which prepends the current bytes in this
	// buffer.  These bytes start in an indeterminate state, so they should be
	// overwritten by the caller.  The caller must only call AppendBytes if they
	// know they're going to immediately overwrite all bytes returned.
	AppendBytes(num int) ([]byte, error)
	// Clear resets the SerializeBuffer to a new, empty buffer.  After a call to clear,
	// the byte slice returned by any previous call to Bytes() for this buffer
	// should be considered invalidated.
	Clear() error
}

type serializeBuffer struct {
	data                []byte
	start               int
	prepended, appended int
}

// NewSerializeBuffer creates a new instance of the default implementation of
// the SerializeBuffer interface.
func NewSerializeBuffer() SerializeBuffer {
	return &serializeBuffer{}
}

// NewSerializeBufferExpectedSize creates a new buffer for serialization, optimized for an
// expected number of bytes prepended/appended.  This tends to decrease the
// number of memory allocations made by the buffer during writes.
func NewSerializeBufferExpectedSize(expectedPrependLength, expectedAppendLength int) SerializeBuffer {
	return &serializeBuffer{
		data:      make([]byte, expectedPrependLength, expectedPrependLength+expectedAppendLength),
		start:     expectedPrependLength,
		prepended: expectedPrependLength,
		appended:  expectedAppendLength,
	}
}

func (w *serializeBuffer) Bytes() []byte {
	return w.data[w.start:]
}

func (w *serializeBuffer) PrependBytes(num int) ([]byte, error) {
	if num < 0 {
		panic("num < 0")
	}
	if w.start < num {
		toPrepend := w.prepended
		if toPrepend < num {
			toPrepend = num
		}
		w.prepended += toPrepend
		length := cap(w.data) + toPrepend
		newData := make([]byte, length)
		newStart := w.start + toPrepend
		copy(newData[newStart:], w.data[w.start:])
		w.start = newStart
		w.data = newData[:toPrepend+len(w.data)]
	}
	w.start -= num
	return w.data[w.start : w.start+num], nil
}

func (w *serializeBuffer) AppendBytes(num int) ([]byte, error) {
	if num < 0 {
		panic("num < 0")
	}
	initialLength := len(w.data)
	if cap(w.data)-initialLength < num {
		toAppend := w.appended
		if toAppend < num {
			toAppend = num
		}
		w.appended += toAppend
		newData := make([]byte, cap(w.data)+toAppend)
		copy(newData[w.start:], w.data[w.start:])
		w.data = newData[:initialLength]
	}
	// Grow the buffer.  We know it'll be under capacity given above.
	w.data = w.data[:initialLength+num]
	return w.data[initialLength:], nil
}

func (w *serializeBuffer) Clear() error {
	w.start = w.prepended
	w.data = w.data[:w.start]
	return nil
}

// SerializeLayers clears the given write buffer, then writes all layers into it so
// they correctly wrap each other.  Note that by clearing the buffer, it
// invalidates all slices previously returned by w.Bytes()
//
// Example:
//   buf := gopacket.NewSerializeBuffer()
//   opts := gopacket.SerializeOptions{}
//   gopacket.SerializeLayers(buf, opts, a, b, c)
//   firstPayload := buf.Bytes()  // contains byte representation of a(b(c))
//   gopacket.SerializeLayers(buf, opts, d, e, f)
//   secondPayload := buf.Bytes()  // contains byte representation of d(e(f)). firstPayload is now invalidated, since the SerializeLayers call Clears buf.
func SerializeLayers(w SerializeBuffer, opts SerializeOptions, layers ...SerializableLayer) error {
	w.Clear()
	for i := len(layers) - 1; i >= 0; i-- {
		layer := layers[i]
		err := layer.SerializeTo(w, opts)
		if err != nil {
			return err
		}
	}
	return nil
}

// SerializePacket is a convenience function that calls SerializeLayers
// on packet's Layers().
// It returns an error if one of the packet layers is not a SerializebleLayer.
func SerializePacket(buf SerializeBuffer, opts SerializeOptions, packet Packet) error {
	sls := []SerializableLayer{}
	for _, layer := range packet.Layers() {
		sl, ok := layer.(SerializableLayer)
		if !ok {
			return fmt.Errorf("layer %s is not serializable", layer.LayerType().String())
		}
		sls = append(sls, sl)
	}
	return SerializeLayers(buf, opts, sls...)
}
<|endoftext|>


<commit_before>/*                          _       _
 *__      _____  __ ___   ___  __ _| |_ ___
 *\ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
 * \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
 *  \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
 *
 * Copyright © 2016 - 2019 Weaviate. All rights reserved.
 * LICENSE: https://github.com/creativesoftwarefdn/weaviate/blob/develop/LICENSE.md
 * DESIGN & CONCEPT: Bob van Luijt (@bobvanluijt)
 * CONTACT: hello@creativesoftwarefdn.org
 */package restapi

import (
	"context"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"time"

	"github.com/coreos/etcd/clientv3"
	"github.com/coreos/etcd/clientv3/concurrency"
	"github.com/creativesoftwarefdn/weaviate/auth/authentication/oidc"
	weaviateBroker "github.com/creativesoftwarefdn/weaviate/broker"
	"github.com/creativesoftwarefdn/weaviate/config"
	libcontextionary "github.com/creativesoftwarefdn/weaviate/contextionary"
	"github.com/creativesoftwarefdn/weaviate/database"
	dblisting "github.com/creativesoftwarefdn/weaviate/database/listing"
	"github.com/creativesoftwarefdn/weaviate/database/schema"
	databaseSchema "github.com/creativesoftwarefdn/weaviate/database/schema_contextionary"
	schemaContextionary "github.com/creativesoftwarefdn/weaviate/database/schema_contextionary"
	etcdSchemaManager "github.com/creativesoftwarefdn/weaviate/database/schema_manager/etcd"
	"github.com/creativesoftwarefdn/weaviate/graphqlapi"
	"github.com/creativesoftwarefdn/weaviate/graphqlapi/local/fetch"
	graphqlnetwork "github.com/creativesoftwarefdn/weaviate/graphqlapi/network"
	"github.com/creativesoftwarefdn/weaviate/messages"
	libnetwork "github.com/creativesoftwarefdn/weaviate/network"
	"github.com/creativesoftwarefdn/weaviate/network/common/peers"
	libnetworkFake "github.com/creativesoftwarefdn/weaviate/network/fake"
	libnetworkP2P "github.com/creativesoftwarefdn/weaviate/network/p2p"
	"github.com/creativesoftwarefdn/weaviate/restapi/state"
	"github.com/creativesoftwarefdn/weaviate/telemetry"
	"github.com/go-openapi/strfmt"
)

// As soon as server is initialized but not run yet, this function will be called.
// If you need to modify a config, store server instance to stop it individually later, this is the place.
// This function can be called multiple times, depending on the number of serving schemes.
// scheme value will be set accordingly: "http", "https" or "unix"
func configureServer(s *http.Server, scheme, addr string) {
	// context for the startup procedure. (So far the only subcommand respecting
	// the context is the schema initialization, as this uses the etcd client
	// requiring context. Nevertheless it would make sense to have everything
	// that goes on in here pay attention to the context, so we can have a
	// "startup in x seconds or fail")
	ctx := context.Background()
	// The timeout is arbitrary we have to adjust it as we go along, if we
	// realize it is to big/small
	ctx, cancel := context.WithTimeout(ctx, 120*time.Second)
	defer cancel()

	// Create message service
	messaging = &messages.Messaging{}
	appState.Messaging = messaging

	messaging.InfoMessage(fmt.Sprintf("created the context, nothing done yet, time left is: %s", timeTillDeadline(ctx)))

	// Load the config using the flags
	serverConfig = &config.WeaviateConfig{}
	appState.ServerConfig = serverConfig
	err := serverConfig.LoadConfig(connectorOptionGroup, messaging)
	messaging.InfoMessage(fmt.Sprintf("loaded the config, time left is: %s", timeTillDeadline(ctx)))

	appState.OIDC = configureOIDC(appState)
	messaging.InfoMessage(fmt.Sprintf("configured OIDC client, time left is: %s", timeTillDeadline(ctx)))

	// Extract environment variables needed for logging
	loggingInterval := appState.ServerConfig.Environment.Logging.Interval
	loggingUrl := appState.ServerConfig.Environment.Logging.Url
	loggingEnabled := appState.ServerConfig.Environment.Logging.Enabled
	loggingDebug := appState.ServerConfig.Environment.Debug

	if loggingEnabled != true && loggingEnabled != false {
		loggingEnabled = true
	}

	// Propagate the peer name (if any), debug toggle and the enabled toggle to the requestsLog
	mainLog.PeerName = appState.ServerConfig.Environment.Network.PeerName
	mainLog.Debug = loggingDebug
	mainLog.Enabled = loggingEnabled

	// Add properties to the config
	serverConfig.Hostname = addr
	serverConfig.Scheme = scheme

	// Fatal error loading config file
	if err != nil {
		messaging.ExitError(78, err.Error())
	}

	loadContextionary()
	messaging.InfoMessage(fmt.Sprintf("loaded the contextionary, time left is: %s", timeTillDeadline(ctx)))

	connectToNetwork()
	messaging.InfoMessage(fmt.Sprintf("connected to network, time left is: %s", timeTillDeadline(ctx)))

	// Connect to MQTT via Broker
	weaviateBroker.ConnectToMqtt(serverConfig.Environment.Broker.Host, serverConfig.Environment.Broker.Port)
	messaging.InfoMessage(fmt.Sprintf("connected to broker, time left is: %s", timeTillDeadline(ctx)))

	// Create the database connector using the config
	err, dbConnector := dblisting.NewConnector(serverConfig.Environment.Database.Name, serverConfig.Environment.Database.DatabaseConfig, serverConfig.Environment)
	// Could not find, or configure connector.
	if err != nil {
		messaging.ExitError(78, err.Error())
	}

	messaging.InfoMessage(fmt.Sprintf("created db connector, time left is: %s", timeTillDeadline(ctx)))

	// parse config store URL
	configURL := serverConfig.Environment.ConfigurationStorage.URL
	configStore, err := url.Parse(configURL)
	if err != nil || configURL == "" {
		messaging.ExitError(78, fmt.Sprintf("cannot parse config store URL: %s", err))
	}

	// Construct a distributed lock
	etcdClient, err := clientv3.New(clientv3.Config{Endpoints: []string{configStore.String()}})
	if err != nil {
		log.Fatal(err)
	}

	messaging.InfoMessage(fmt.Sprintf("created an etcd client, time left is: %s", timeTillDeadline(ctx)))

	s1, err := concurrency.NewSession(etcdClient)
	if err != nil {
		log.Fatal(err)
	}
	messaging.InfoMessage(fmt.Sprintf("created an etcd session, time left is: %s", timeTillDeadline(ctx)))

	manager, err := etcdSchemaManager.New(ctx, etcdClient, dbConnector, network)
	if err != nil {
		messaging.ExitError(78, fmt.Sprintf("Could not initialize local database state: %v", err))
	}

	messaging.InfoMessage(fmt.Sprintf("initialized the schema, time left is: %s", timeTillDeadline(ctx)))

	manager.RegisterSchemaUpdateCallback(updateSchemaCallback)

	// Initialize the reporter
	reporter = telemetry.NewReporter(ctx, mainLog, loggingInterval, loggingUrl, loggingEnabled, loggingDebug, etcdClient)

	// Start reporting
	go func() {
		reporter.Start()
	}()

	// initialize the contextinoary with the rawContextionary, it will get updated on each schema update
	contextionary = rawContextionary

	// Now instantiate a database, with the configured lock, manager and connector.
	dbParams := &database.Params{
		LockerKey:     "/weaviate/schema-connector-rw-lock",
		LockerSession: s1,
		SchemaManager: manager,
		Connector:     dbConnector,
		Contextionary: contextionary,
		Messaging:     messaging,
	}
	db, err = database.New(ctx, dbParams)
	if err != nil {
		messaging.ExitError(1, fmt.Sprintf("Could not initialize the database: %s", err.Error()))
	}
	appState.Database = db

	manager.TriggerSchemaUpdateCallbacks()
	network.RegisterUpdatePeerCallback(func(peers peers.Peers) {
		manager.TriggerSchemaUpdateCallbacks()
	})

	network.RegisterSchemaGetter(&schemaGetter{db: db})
}

func updateSchemaCallback(updatedSchema schema.Schema) {
	// Note that this is thread safe; we're running in a single go-routine, because the event
	// handlers are called when the SchemaLock is still held.
	rebuildContextionary(updatedSchema)
	rebuildGraphQL(updatedSchema)
}

func rebuildContextionary(updatedSchema schema.Schema) {
	// build new contextionary extended by the local schema
	schemaContextionary, err := databaseSchema.BuildInMemoryContextionaryFromSchema(updatedSchema, &rawContextionary)
	if err != nil {
		messaging.ExitError(78, fmt.Sprintf("Could not build in-memory contextionary from schema; %+v", err))
	}

	// Combine contextionaries
	contextionaries := []libcontextionary.Contextionary{rawContextionary, *schemaContextionary}
	combined, err := libcontextionary.CombineVectorIndices(contextionaries)

	if err != nil {
		messaging.ExitError(78, fmt.Sprintf("Could not combine the contextionary database with the in-memory generated contextionary; %+v", err))
	}

	messaging.InfoMessage("Contextionary extended with names in the schema")

	contextionary = libcontextionary.Contextionary(combined)
}

func rebuildGraphQL(updatedSchema schema.Schema) {
	peers, err := network.ListPeers()
	if err != nil {
		graphQL = nil
		messaging.ErrorMessage(fmt.Sprintf("could not list network peers to regenerate schema:\n%#v\n", err))
		return
	}

	c11y := schemaContextionary.New(contextionary)
	root := graphQLRoot{Database: db, Network: network, contextionary: c11y, log: mainLog}
	updatedGraphQL, err := graphqlapi.Build(&updatedSchema, peers, root, messaging, serverConfig.Environment)
	if err != nil {
		// TODO: turn on safe mode gh-520
		graphQL = nil
		messaging.ErrorMessage(fmt.Sprintf("Could not re-generate GraphQL schema, because:\n%#v\n", err))
	} else {
		messaging.InfoMessage("Updated GraphQL schema")
		graphQL = updatedGraphQL
	}
}

type schemaGetter struct {
	db database.Database
}

func (s *schemaGetter) Schema() (schema.Schema, error) {
	dbLock, err := s.db.ConnectorLock()
	if err != nil {
		return schema.Schema{}, err
	}

	defer dbLock.Unlock()
	return dbLock.GetSchema(), nil
}

type graphQLRoot struct {
	database.Database
	libnetwork.Network
	contextionary *schemaContextionary.Contextionary
	log           *telemetry.RequestsLog
}

func (r graphQLRoot) GetNetworkResolver() graphqlnetwork.Resolver {
	return r.Network
}

func (r graphQLRoot) GetContextionary() fetch.Contextionary {
	return r.contextionary
}

func (r graphQLRoot) GetRequestsLog() fetch.RequestsLog {
	return r.log
}

// configureOIDC will always be called, even if OIDC is disabled, this way the
// middleware will still be able to provide the user with a valuable error
// message, even when OIDC is globally disabled.
func configureOIDC(appState *state.State) *oidc.Client {
	c, err := oidc.New(appState.ServerConfig.Environment)
	if err != nil {
		appState.Messaging.ExitError(1, fmt.Sprintf("oidc client couldn't start up: %v", err))
	}

	return c
}

func timeTillDeadline(ctx context.Context) time.Duration {
	dl, _ := ctx.Deadline()
	return time.Until(dl)
}

// This function loads the Contextionary database, and creates
// an in-memory database for the centroids of the classes / properties in the Schema.
func loadContextionary() {
	// First load the file backed contextionary
	if serverConfig.Environment.Contextionary.KNNFile == "" {
		messaging.ExitError(78, "Contextionary KNN file not specified")
	}

	if serverConfig.Environment.Contextionary.IDXFile == "" {
		messaging.ExitError(78, "Contextionary IDX file not specified")
	}

	mmapedContextionary, err := libcontextionary.LoadVectorFromDisk(serverConfig.Environment.Contextionary.KNNFile, serverConfig.Environment.Contextionary.IDXFile)

	if err != nil {
		messaging.ExitError(78, fmt.Sprintf("Could not load Contextionary; %+v", err))
	}

	messaging.InfoMessage("Contextionary loaded from disk")

	rawContextionary = mmapedContextionary
}

func connectToNetwork() {
	if serverConfig.Environment.Network == nil {
		messaging.InfoMessage(fmt.Sprintf("No network configured, not joining one"))
		network = libnetworkFake.FakeNetwork{}
		appState.Network = network
	} else {
		genesis_url := strfmt.URI(serverConfig.Environment.Network.GenesisURL)
		public_url := strfmt.URI(serverConfig.Environment.Network.PublicURL)
		peer_name := serverConfig.Environment.Network.PeerName

		messaging.InfoMessage(fmt.Sprintf("Network configured, connecting to Genesis '%v'", genesis_url))
		new_net, err := libnetworkP2P.BootstrapNetwork(messaging, genesis_url, public_url, peer_name)
		if err != nil {
			messaging.ExitError(78, fmt.Sprintf("Could not connect to network! Reason: %+v", err))
		} else {
			network = *new_net
			appState.Network = *new_net
		}
	}
}
<commit_msg>gh-699: reverting breaking changes<commit_after>/*                          _       _
 *__      _____  __ ___   ___  __ _| |_ ___
 *\ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
 * \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
 *  \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
 *
 * Copyright © 2016 - 2019 Weaviate. All rights reserved.
 * LICENSE: https://github.com/creativesoftwarefdn/weaviate/blob/develop/LICENSE.md
 * DESIGN & CONCEPT: Bob van Luijt (@bobvanluijt)
 * CONTACT: hello@creativesoftwarefdn.org
 */package restapi

import (
	"context"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"time"

	"github.com/coreos/etcd/clientv3"
	"github.com/coreos/etcd/clientv3/concurrency"
	"github.com/creativesoftwarefdn/weaviate/auth/authentication/oidc"
	weaviateBroker "github.com/creativesoftwarefdn/weaviate/broker"
	"github.com/creativesoftwarefdn/weaviate/config"
	libcontextionary "github.com/creativesoftwarefdn/weaviate/contextionary"
	"github.com/creativesoftwarefdn/weaviate/database"
	dblisting "github.com/creativesoftwarefdn/weaviate/database/listing"
	"github.com/creativesoftwarefdn/weaviate/database/schema"
	databaseSchema "github.com/creativesoftwarefdn/weaviate/database/schema_contextionary"
	schemaContextionary "github.com/creativesoftwarefdn/weaviate/database/schema_contextionary"
	etcdSchemaManager "github.com/creativesoftwarefdn/weaviate/database/schema_manager/etcd"
	"github.com/creativesoftwarefdn/weaviate/graphqlapi"
	"github.com/creativesoftwarefdn/weaviate/graphqlapi/local/fetch"
	graphqlnetwork "github.com/creativesoftwarefdn/weaviate/graphqlapi/network"
	"github.com/creativesoftwarefdn/weaviate/messages"
	libnetwork "github.com/creativesoftwarefdn/weaviate/network"
	"github.com/creativesoftwarefdn/weaviate/network/common/peers"
	libnetworkFake "github.com/creativesoftwarefdn/weaviate/network/fake"
	libnetworkP2P "github.com/creativesoftwarefdn/weaviate/network/p2p"
	"github.com/creativesoftwarefdn/weaviate/restapi/state"
	"github.com/creativesoftwarefdn/weaviate/telemetry"
	"github.com/go-openapi/strfmt"
)

// As soon as server is initialized but not run yet, this function will be called.
// If you need to modify a config, store server instance to stop it individually later, this is the place.
// This function can be called multiple times, depending on the number of serving schemes.
// scheme value will be set accordingly: "http", "https" or "unix"
func configureServer(s *http.Server, scheme, addr string) {
	// context for the startup procedure. (So far the only subcommand respecting
	// the context is the schema initialization, as this uses the etcd client
	// requiring context. Nevertheless it would make sense to have everything
	// that goes on in here pay attention to the context, so we can have a
	// "startup in x seconds or fail")
	ctx := context.Background()
	// The timeout is arbitrary we have to adjust it as we go along, if we
	// realize it is to big/small
	ctx, cancel := context.WithTimeout(ctx, 120*time.Second)
	defer cancel()

	// Create message service
	messaging = &messages.Messaging{}
	appState.Messaging = messaging

	messaging.InfoMessage(fmt.Sprintf("created the context, nothing done yet, time left is: %s", timeTillDeadline(ctx)))

	// Load the config using the flags
	serverConfig = &config.WeaviateConfig{}
	appState.ServerConfig = serverConfig
	err := serverConfig.LoadConfig(connectorOptionGroup, messaging)
	messaging.InfoMessage(fmt.Sprintf("loaded the config, time left is: %s", timeTillDeadline(ctx)))

	appState.OIDC = configureOIDC(appState)
	messaging.InfoMessage(fmt.Sprintf("configured OIDC client, time left is: %s", timeTillDeadline(ctx)))

	// Extract environment variables needed for logging
	loggingInterval := appState.ServerConfig.Environment.Logging.Interval
	loggingUrl := appState.ServerConfig.Environment.Logging.Url
	loggingEnabled := appState.ServerConfig.Environment.Logging.Enabled
	loggingDebug := appState.ServerConfig.Environment.Debug

	if loggingEnabled != true && loggingEnabled != false {
		loggingEnabled = true
	}

	// Propagate the peer name (if any), debug toggle and the enabled toggle to the requestsLog
	mainLog.PeerName = appState.ServerConfig.Environment.Network.PeerName
	mainLog.Debug = loggingDebug
	mainLog.Enabled = loggingEnabled

	// Add properties to the config
	serverConfig.Hostname = addr
	serverConfig.Scheme = scheme

	// Fatal error loading config file
	if err != nil {
		messaging.ExitError(78, err.Error())
	}

	loadContextionary()
	messaging.InfoMessage(fmt.Sprintf("loaded the contextionary, time left is: %s", timeTillDeadline(ctx)))

	connectToNetwork()
	messaging.InfoMessage(fmt.Sprintf("connected to network, time left is: %s", timeTillDeadline(ctx)))

	// Connect to MQTT via Broker
	weaviateBroker.ConnectToMqtt(serverConfig.Environment.Broker.Host, serverConfig.Environment.Broker.Port)
	messaging.InfoMessage(fmt.Sprintf("connected to broker, time left is: %s", timeTillDeadline(ctx)))

	// Create the database connector using the config
	err, dbConnector := dblisting.NewConnector(serverConfig.Environment.Database.Name, serverConfig.Environment.Database.DatabaseConfig, serverConfig.Environment)
	// Could not find, or configure connector.
	if err != nil {
		messaging.ExitError(78, err.Error())
	}

	messaging.InfoMessage(fmt.Sprintf("created db connector, time left is: %s", timeTillDeadline(ctx)))

	// parse config store URL
	configURL := serverConfig.Environment.ConfigurationStorage.URL
	configStore, err := url.Parse(configURL)
	if err != nil || configURL == "" {
		messaging.ExitError(78, fmt.Sprintf("cannot parse config store URL: %s", err))
	}

	// Construct a distributed lock
	etcdClient, err := clientv3.New(clientv3.Config{Endpoints: []string{configStore.String()}})
	if err != nil {
		log.Fatal(err)
	}

	messaging.InfoMessage(fmt.Sprintf("created an etcd client, time left is: %s", timeTillDeadline(ctx)))

	s1, err := concurrency.NewSession(etcdClient)
	if err != nil {
		log.Fatal(err)
	}
	messaging.InfoMessage(fmt.Sprintf("created an etcd session, time left is: %s", timeTillDeadline(ctx)))

	manager, err := etcdSchemaManager.New(ctx, etcdClient, dbConnector, network)
	if err != nil {
		messaging.ExitError(78, fmt.Sprintf("Could not initialize local database state: %v", err))
	}

	messaging.InfoMessage(fmt.Sprintf("initialized the schema, time left is: %s", timeTillDeadline(ctx)))

	manager.RegisterSchemaUpdateCallback(updateSchemaCallback)

	// Initialize the reporter
	reporter = telemetry.NewReporter(ctx, mainLog, loggingInterval, loggingUrl, loggingEnabled, loggingDebug, etcdClient, messaging)

	// Start reporting
	go func() {
		reporter.Start()
	}()

	// initialize the contextinoary with the rawContextionary, it will get updated on each schema update
	contextionary = rawContextionary

	// Now instantiate a database, with the configured lock, manager and connector.
	dbParams := &database.Params{
		LockerKey:     "/weaviate/schema-connector-rw-lock",
		LockerSession: s1,
		SchemaManager: manager,
		Connector:     dbConnector,
		Contextionary: contextionary,
		Messaging:     messaging,
	}
	db, err = database.New(ctx, dbParams)
	if err != nil {
		messaging.ExitError(1, fmt.Sprintf("Could not initialize the database: %s", err.Error()))
	}
	appState.Database = db

	manager.TriggerSchemaUpdateCallbacks()
	network.RegisterUpdatePeerCallback(func(peers peers.Peers) {
		manager.TriggerSchemaUpdateCallbacks()
	})

	network.RegisterSchemaGetter(&schemaGetter{db: db})
}

func updateSchemaCallback(updatedSchema schema.Schema) {
	// Note that this is thread safe; we're running in a single go-routine, because the event
	// handlers are called when the SchemaLock is still held.
	rebuildContextionary(updatedSchema)
	rebuildGraphQL(updatedSchema)
}

func rebuildContextionary(updatedSchema schema.Schema) {
	// build new contextionary extended by the local schema
	schemaContextionary, err := databaseSchema.BuildInMemoryContextionaryFromSchema(updatedSchema, &rawContextionary)
	if err != nil {
		messaging.ExitError(78, fmt.Sprintf("Could not build in-memory contextionary from schema; %+v", err))
	}

	// Combine contextionaries
	contextionaries := []libcontextionary.Contextionary{rawContextionary, *schemaContextionary}
	combined, err := libcontextionary.CombineVectorIndices(contextionaries)

	if err != nil {
		messaging.ExitError(78, fmt.Sprintf("Could not combine the contextionary database with the in-memory generated contextionary; %+v", err))
	}

	messaging.InfoMessage("Contextionary extended with names in the schema")

	contextionary = libcontextionary.Contextionary(combined)
}

func rebuildGraphQL(updatedSchema schema.Schema) {
	peers, err := network.ListPeers()
	if err != nil {
		graphQL = nil
		messaging.ErrorMessage(fmt.Sprintf("could not list network peers to regenerate schema:\n%#v\n", err))
		return
	}

	c11y := schemaContextionary.New(contextionary)
	root := graphQLRoot{Database: db, Network: network, contextionary: c11y, log: mainLog}
	updatedGraphQL, err := graphqlapi.Build(&updatedSchema, peers, root, messaging, serverConfig.Environment)
	if err != nil {
		// TODO: turn on safe mode gh-520
		graphQL = nil
		messaging.ErrorMessage(fmt.Sprintf("Could not re-generate GraphQL schema, because:\n%#v\n", err))
	} else {
		messaging.InfoMessage("Updated GraphQL schema")
		graphQL = updatedGraphQL
	}
}

type schemaGetter struct {
	db database.Database
}

func (s *schemaGetter) Schema() (schema.Schema, error) {
	dbLock, err := s.db.ConnectorLock()
	if err != nil {
		return schema.Schema{}, err
	}

	defer dbLock.Unlock()
	return dbLock.GetSchema(), nil
}

type graphQLRoot struct {
	database.Database
	libnetwork.Network
	contextionary *schemaContextionary.Contextionary
	log           *telemetry.RequestsLog
}

func (r graphQLRoot) GetNetworkResolver() graphqlnetwork.Resolver {
	return r.Network
}

func (r graphQLRoot) GetContextionary() fetch.Contextionary {
	return r.contextionary
}

func (r graphQLRoot) GetRequestsLog() fetch.RequestsLog {
	return r.log
}

// configureOIDC will always be called, even if OIDC is disabled, this way the
// middleware will still be able to provide the user with a valuable error
// message, even when OIDC is globally disabled.
func configureOIDC(appState *state.State) *oidc.Client {
	c, err := oidc.New(appState.ServerConfig.Environment)
	if err != nil {
		appState.Messaging.ExitError(1, fmt.Sprintf("oidc client couldn't start up: %v", err))
	}

	return c
}

func timeTillDeadline(ctx context.Context) time.Duration {
	dl, _ := ctx.Deadline()
	return time.Until(dl)
}

// This function loads the Contextionary database, and creates
// an in-memory database for the centroids of the classes / properties in the Schema.
func loadContextionary() {
	// First load the file backed contextionary
	if serverConfig.Environment.Contextionary.KNNFile == "" {
		messaging.ExitError(78, "Contextionary KNN file not specified")
	}

	if serverConfig.Environment.Contextionary.IDXFile == "" {
		messaging.ExitError(78, "Contextionary IDX file not specified")
	}

	mmapedContextionary, err := libcontextionary.LoadVectorFromDisk(serverConfig.Environment.Contextionary.KNNFile, serverConfig.Environment.Contextionary.IDXFile)

	if err != nil {
		messaging.ExitError(78, fmt.Sprintf("Could not load Contextionary; %+v", err))
	}

	messaging.InfoMessage("Contextionary loaded from disk")

	rawContextionary = mmapedContextionary
}

func connectToNetwork() {
	if serverConfig.Environment.Network == nil {
		messaging.InfoMessage(fmt.Sprintf("No network configured, not joining one"))
		network = libnetworkFake.FakeNetwork{}
		appState.Network = network
	} else {
		genesis_url := strfmt.URI(serverConfig.Environment.Network.GenesisURL)
		public_url := strfmt.URI(serverConfig.Environment.Network.PublicURL)
		peer_name := serverConfig.Environment.Network.PeerName

		messaging.InfoMessage(fmt.Sprintf("Network configured, connecting to Genesis '%v'", genesis_url))
		new_net, err := libnetworkP2P.BootstrapNetwork(messaging, genesis_url, public_url, peer_name)
		if err != nil {
			messaging.ExitError(78, fmt.Sprintf("Could not connect to network! Reason: %+v", err))
		} else {
			network = *new_net
			appState.Network = *new_net
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime

import "unsafe"

//go:linkname memmove_bytes runtime.memmove
//go:nosplit
func memmove_bytes(dst, src unsafe.Pointer, n uintptr) {
	if uintptr(dst) == uintptr(src) {
		return
	}
	if uintptr(src) < uintptr(dst) {
		for i := n - 1; i >= 0; i-- {
			b := *(*byte)(add(src, i))
			*(*byte)(add(dst, i)) = b
		}
		return
	}
	for i := uintptr(0); i <= n; i++ {
		b := *(*byte)(add(src, i))
		*(*byte)(add(dst, i)) = b
	}
	return
}

//go:linkname memclr_bytes runtime.memclr
//go:nosplit
func memclr_bytes(ptr unsafe.Pointer, n uintptr) {
	for i := uintptr(0); i < n; i++ {
		*(*byte)(add(ptr, i)) = 0
	}
}
<commit_msg>runtime: memmove should ignore 0-byte moves<commit_after>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime

import "unsafe"

//go:linkname memmove_bytes runtime.memmove
//go:nosplit
func memmove_bytes(dst, src unsafe.Pointer, n uintptr) {
	if n == 0 {
		return
	}
	if uintptr(dst) == uintptr(src) {
		return
	}
	if uintptr(src) < uintptr(dst) {
		for i := n - 1; i >= 0; i-- {
			b := *(*byte)(add(src, i))
			*(*byte)(add(dst, i)) = b
		}
		return
	}
	for i := uintptr(0); i <= n; i++ {
		b := *(*byte)(add(src, i))
		*(*byte)(add(dst, i)) = b
	}
	return
}

//go:linkname memclr_bytes runtime.memclr
//go:nosplit
func memclr_bytes(ptr unsafe.Pointer, n uintptr) {
	for i := uintptr(0); i < n; i++ {
		*(*byte)(add(ptr, i)) = 0
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"github.com/russross/blackfriday"
	"io/ioutil"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"
	"sync"
	"log"
)

type FileRepository struct {
	directory string
	posts BlogPosts
	tags []string
	mutex sync.RWMutex
}

func NewFileRepository(directory string) *FileRepository {

	f := new(FileRepository)
	f.directory = directory

	go f.update()

	return f
}

func (f *FileRepository) AllTags() []string {
	return f.tags
}

func (f *FileRepository) AllPosts() BlogPosts {
	return f.posts
}

func (f *FileRepository) PostWithUrl(url string) (*BlogPost, error) {

	f.mutex.RLock()
	defer f.mutex.RUnlock()

	for i := range f.posts {
		if f.posts[i].Url() == url {
			return f.posts[i], nil
		}
	}

	err := errors.New("Could not find post")

	return nil, err
}

func (f *FileRepository) PostsWithTag(tag string, start int, count int) (BlogPosts, int) {

	f.mutex.RLock()
	defer f.mutex.RUnlock()

	filteredPosts := BlogPosts{}

	for i := range f.posts {
		if f.posts[i].ContainsTag(tag) {
			filteredPosts = append(filteredPosts, f.posts[i])
		}
	}

	if start > len(filteredPosts) {
		return BlogPosts{}, 0
	}

	if start + count > len(filteredPosts) {
		count = len(filteredPosts) - start
	}

	return filteredPosts[start:start + count], len(filteredPosts)
}

func (f *FileRepository) SearchPosts(term string, start int, count int) (BlogPosts, int) {

	f.mutex.RLock()
	defer f.mutex.RUnlock()

	filteredPosts := BlogPosts{}

	if len(term) > 0 {
		for i := range f.posts {
			if f.posts[i].ContainsTerm(term) {
				filteredPosts = append(filteredPosts, f.posts[i])
			}
		}
	} else {
		filteredPosts = f.posts
	}

	if start > len(filteredPosts) {
		return BlogPosts{}, 0
	}

	if start + count > len(filteredPosts) {
		count = len(filteredPosts) - start
	}

	return filteredPosts[start:start + count], len(filteredPosts)
}

func (f *FileRepository) update() {

	for {

		start := time.Now()

		f.fetchAllPosts()
		f.fetchAllTags()

		end := time.Now()
		log.Printf("Cached %v posts in %v", len(f.posts), end.Sub(start))

		time.Sleep(10 * time.Minute)
	}
}

func (f *FileRepository) fetchAllPosts() error {

	f.mutex.Lock()
	defer f.mutex.Unlock()

	dirname := f.directory + string(filepath.Separator)

	files, err := ioutil.ReadDir(dirname)

	f.posts = BlogPosts{}

	for i := range files {

		if files[i].IsDir() {
			continue
		}

		if filepath.Ext(files[i].Name()) != ".md" {
			continue
		}

		post, err := f.fetchPost(dirname + files[i].Name())

		if err != nil {
			return err
		}

		f.posts = append(f.posts, post)
	}

	sort.Sort(f.posts)

	return err
}

func (f *FileRepository) fetchAllTags() {

	f.mutex.Lock()
	defer f.mutex.Unlock()

	// We're using a map to simulate a set
	tagMap := make(map[string]bool)

	for i := range f.posts {
		for j := range f.posts[i].Tags() {
			tagMap[strings.ToLower(f.posts[i].Tags()[j])] = true
		}
	}

	f.tags = []string{}

	for key := range tagMap {
		f.tags = append(f.tags, key)
	}

	sort.Strings(f.tags)
}

func (f *FileRepository) fetchPost(filename string) (*BlogPost, error) {

	post := new(BlogPost)

	file, err := ioutil.ReadFile(filename)

	if err != nil {
		return post, err
	}

	file = []byte(extractPostHeader(string(file), post))

	htmlFlags := blackfriday.HTML_USE_SMARTYPANTS
	extensions := blackfriday.EXTENSION_AUTOLINK | blackfriday.EXTENSION_HARD_LINE_BREAK | blackfriday.EXTENSION_FENCED_CODE | blackfriday.EXTENSION_NO_INTRA_EMPHASIS

	renderer := blackfriday.HtmlRenderer(htmlFlags, post.Title(), "")

	output := blackfriday.Markdown(file, renderer, extensions)

	post.SetBody(string(output))

	f.fetchCommentsForPost(post, filename)

	return post, nil
}

func (f *FileRepository) fetchCommentsForPost(post *BlogPost, filename string) {
	dirname := filename[:len(filename) - 3] + string(filepath.Separator) + "comments" + string(filepath.Separator)

	files, err := ioutil.ReadDir(dirname)

	if err != nil {
		return
	}

	post.comments = Comments{}

	for i := range files {

		if files[i].IsDir() {
			continue
		}

		if filepath.Ext(files[i].Name()) != ".md" {
			continue
		}

		comment, err := f.fetchComment(dirname + files[i].Name())

		if err != nil {
			log.Fatal(err)
			return
		}

		post.comments = append(post.comments, comment)
	}
}

func (f *FileRepository) fetchComment(filename string) (*Comment, error) {
	comment := new(Comment)

	file, err := ioutil.ReadFile(filename)

	if err != nil {
		return comment, err
	}

	file = []byte(extractCommentHeader(string(file), comment))

	htmlFlags := blackfriday.HTML_USE_SMARTYPANTS
	extensions := blackfriday.EXTENSION_AUTOLINK | blackfriday.EXTENSION_HARD_LINE_BREAK | blackfriday.EXTENSION_FENCED_CODE | blackfriday.EXTENSION_NO_INTRA_EMPHASIS

	renderer := blackfriday.HtmlRenderer(htmlFlags, "", "")

	output := blackfriday.Markdown(file, renderer, extensions)

	comment.SetBody(string(output))

	return comment, nil
}

func extractCommentHeader(text string, comment *Comment) string {

	lines := strings.Split(text, "\n")

	headerSize := 0

	for i := range lines {
		if strings.Contains(lines[i], ":") {
			components := strings.Split(lines[i], ":")

			header := strings.ToLower(strings.Trim(components[0], " "))
			separatorIndex := strings.Index(lines[i], ":") + 1
			data := strings.Trim(lines[i][separatorIndex:], " ")

			switch header {
			case "author":
				comment.SetAuthor(data)
			case "email":
				comment.SetEmail(data)
			case "date":
				comment.SetDate(stringToTime(data))
			default:
				continue
			}

			headerSize += len(lines[i]) + 1
		} else {
			break
		}
	}

	return text[headerSize:]
}

func extractPostHeader(text string, post *BlogPost) string {

	lines := strings.Split(text, "\n")

	headerSize := 0

	for i := range lines {
		if strings.Contains(lines[i], ":") {
			components := strings.Split(lines[i], ":")

			header := strings.ToLower(strings.Trim(components[0], " "))
			separatorIndex := strings.Index(lines[i], ":") + 1
			data := strings.Trim(lines[i][separatorIndex:], " ")

			switch header {
			case "title":
				post.SetTitle(data)
			case "tags":

				tags := strings.Split(data, ",")

				formattedTags := []string{}

				for j := range tags {
					tags[j] = strings.Trim(tags[j], " ")
					tags[j] = strings.Replace(tags[j], " ", "-", -1)
					tags[j] = strings.Replace(tags[j], "/", "-", -1)
					tags[j] = strings.ToLower(tags[j])

					if tags[j] != "" {
						formattedTags = append(formattedTags, tags[j])
					}
				}

				post.SetTags(formattedTags)
			case "date":
				post.SetPublishDate(stringToTime(data))
			default:
				continue
			}

			headerSize += len(lines[i]) + 1
		} else {
			break
		}
	}

	return text[headerSize:]
}

func stringToTime(s string) time.Time {

	year, err := strconv.Atoi(s[:4])

	if err != nil {
		return time.Unix(0, 0)
	}

	month, err := strconv.Atoi(s[5:7])

	if err != nil {
		return time.Unix(0, 0)
	}

	day, err := strconv.Atoi(s[8:10])

	if err != nil {
		return time.Unix(0, 0)
	}

	hour, err := strconv.Atoi(s[11:13])

	if err != nil {
		return time.Unix(0, 0)
	}

	minute, err := strconv.Atoi(s[14:16])

	if err != nil {
		return time.Unix(0, 0)
	}

	seconds, err := strconv.Atoi(s[17:19])

	if err != nil {
		return time.Unix(0, 0)
	}

	location, err := time.LoadLocation("UTC")

	return time.Date(year, time.Month(month), day, hour, minute, seconds, 0, location)
}
<commit_msg>Reduced lock time when fetching data.<commit_after>package main

import (
	"errors"
	"github.com/russross/blackfriday"
	"io/ioutil"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"
	"sync"
	"log"
)

type FileRepository struct {
	directory string
	posts BlogPosts
	tags []string
	mutex sync.RWMutex
}

func NewFileRepository(directory string) *FileRepository {

	f := new(FileRepository)
	f.directory = directory

	go f.update()

	return f
}

func (f *FileRepository) AllTags() []string {
	return f.tags
}

func (f *FileRepository) AllPosts() BlogPosts {
	return f.posts
}

func (f *FileRepository) PostWithUrl(url string) (*BlogPost, error) {

	f.mutex.RLock()
	defer f.mutex.RUnlock()

	for i := range f.posts {
		if f.posts[i].Url() == url {
			return f.posts[i], nil
		}
	}

	err := errors.New("Could not find post")

	return nil, err
}

func (f *FileRepository) PostsWithTag(tag string, start int, count int) (BlogPosts, int) {

	f.mutex.RLock()
	defer f.mutex.RUnlock()

	filteredPosts := BlogPosts{}

	for i := range f.posts {
		if f.posts[i].ContainsTag(tag) {
			filteredPosts = append(filteredPosts, f.posts[i])
		}
	}

	if start > len(filteredPosts) {
		return BlogPosts{}, 0
	}

	if start + count > len(filteredPosts) {
		count = len(filteredPosts) - start
	}

	return filteredPosts[start:start + count], len(filteredPosts)
}

func (f *FileRepository) SearchPosts(term string, start int, count int) (BlogPosts, int) {

	f.mutex.RLock()
	defer f.mutex.RUnlock()

	filteredPosts := BlogPosts{}

	if len(term) > 0 {
		for i := range f.posts {
			if f.posts[i].ContainsTerm(term) {
				filteredPosts = append(filteredPosts, f.posts[i])
			}
		}
	} else {
		filteredPosts = f.posts
	}

	if start > len(filteredPosts) {
		return BlogPosts{}, 0
	}

	if start + count > len(filteredPosts) {
		count = len(filteredPosts) - start
	}

	return filteredPosts[start:start + count], len(filteredPosts)
}

func (f *FileRepository) update() {

	for {

		start := time.Now()

		f.fetchAllPosts()
		f.fetchAllTags()

		end := time.Now()
		log.Printf("Cached %v posts in %v", len(f.posts), end.Sub(start))

		time.Sleep(10 * time.Minute)
	}
}

func (f *FileRepository) fetchAllPosts() error {

	dirname := f.directory + string(filepath.Separator)

	files, err := ioutil.ReadDir(dirname)

	posts := BlogPosts{}

	for i := range files {

		if files[i].IsDir() {
			continue
		}

		if filepath.Ext(files[i].Name()) != ".md" {
			continue
		}

		post, err := f.fetchPost(dirname + files[i].Name())

		if err != nil {
			return err
		}

		posts = append(posts, post)
	}

	sort.Sort(posts)

	f.mutex.Lock()
	f.posts = posts
	f.mutex.Unlock()

	return err
}

func (f *FileRepository) fetchAllTags() {

	// We're using a map to simulate a set
	tagMap := make(map[string]bool)

	for i := range f.posts {
		for j := range f.posts[i].Tags() {
			tagMap[strings.ToLower(f.posts[i].Tags()[j])] = true
		}
	}

	tags := []string{}

	for key := range tagMap {
		tags = append(f.tags, key)
	}

	sort.Strings(tags)

	f.mutex.Lock()
	f.tags = tags
	f.mutex.Unlock()
}

func (f *FileRepository) fetchPost(filename string) (*BlogPost, error) {

	post := new(BlogPost)

	file, err := ioutil.ReadFile(filename)

	if err != nil {
		return post, err
	}

	file = []byte(extractPostHeader(string(file), post))

	htmlFlags := blackfriday.HTML_USE_SMARTYPANTS
	extensions := blackfriday.EXTENSION_AUTOLINK | blackfriday.EXTENSION_HARD_LINE_BREAK | blackfriday.EXTENSION_FENCED_CODE | blackfriday.EXTENSION_NO_INTRA_EMPHASIS

	renderer := blackfriday.HtmlRenderer(htmlFlags, post.Title(), "")

	output := blackfriday.Markdown(file, renderer, extensions)

	post.SetBody(string(output))

	f.fetchCommentsForPost(post, filename)

	return post, nil
}

func (f *FileRepository) fetchCommentsForPost(post *BlogPost, filename string) {
	dirname := filename[:len(filename) - 3] + string(filepath.Separator) + "comments" + string(filepath.Separator)

	files, err := ioutil.ReadDir(dirname)

	if err != nil {
		return
	}

	post.comments = Comments{}

	for i := range files {

		if files[i].IsDir() {
			continue
		}

		if filepath.Ext(files[i].Name()) != ".md" {
			continue
		}

		comment, err := f.fetchComment(dirname + files[i].Name())

		if err != nil {
			log.Fatal(err)
			return
		}

		post.comments = append(post.comments, comment)
	}
}

func (f *FileRepository) fetchComment(filename string) (*Comment, error) {
	comment := new(Comment)

	file, err := ioutil.ReadFile(filename)

	if err != nil {
		return comment, err
	}

	file = []byte(extractCommentHeader(string(file), comment))

	htmlFlags := blackfriday.HTML_USE_SMARTYPANTS
	extensions := blackfriday.EXTENSION_AUTOLINK | blackfriday.EXTENSION_HARD_LINE_BREAK | blackfriday.EXTENSION_FENCED_CODE | blackfriday.EXTENSION_NO_INTRA_EMPHASIS

	renderer := blackfriday.HtmlRenderer(htmlFlags, "", "")

	output := blackfriday.Markdown(file, renderer, extensions)

	comment.SetBody(string(output))

	return comment, nil
}

func extractCommentHeader(text string, comment *Comment) string {

	lines := strings.Split(text, "\n")

	headerSize := 0

	for i := range lines {
		if strings.Contains(lines[i], ":") {
			components := strings.Split(lines[i], ":")

			header := strings.ToLower(strings.Trim(components[0], " "))
			separatorIndex := strings.Index(lines[i], ":") + 1
			data := strings.Trim(lines[i][separatorIndex:], " ")

			switch header {
			case "author":
				comment.SetAuthor(data)
			case "email":
				comment.SetEmail(data)
			case "date":
				comment.SetDate(stringToTime(data))
			default:
				continue
			}

			headerSize += len(lines[i]) + 1
		} else {
			break
		}
	}

	return text[headerSize:]
}

func extractPostHeader(text string, post *BlogPost) string {

	lines := strings.Split(text, "\n")

	headerSize := 0

	for i := range lines {
		if strings.Contains(lines[i], ":") {
			components := strings.Split(lines[i], ":")

			header := strings.ToLower(strings.Trim(components[0], " "))
			separatorIndex := strings.Index(lines[i], ":") + 1
			data := strings.Trim(lines[i][separatorIndex:], " ")

			switch header {
			case "title":
				post.SetTitle(data)
			case "tags":

				tags := strings.Split(data, ",")

				formattedTags := []string{}

				for j := range tags {
					tags[j] = strings.Trim(tags[j], " ")
					tags[j] = strings.Replace(tags[j], " ", "-", -1)
					tags[j] = strings.Replace(tags[j], "/", "-", -1)
					tags[j] = strings.ToLower(tags[j])

					if tags[j] != "" {
						formattedTags = append(formattedTags, tags[j])
					}
				}

				post.SetTags(formattedTags)
			case "date":
				post.SetPublishDate(stringToTime(data))
			default:
				continue
			}

			headerSize += len(lines[i]) + 1
		} else {
			break
		}
	}

	return text[headerSize:]
}

func stringToTime(s string) time.Time {

	year, err := strconv.Atoi(s[:4])

	if err != nil {
		return time.Unix(0, 0)
	}

	month, err := strconv.Atoi(s[5:7])

	if err != nil {
		return time.Unix(0, 0)
	}

	day, err := strconv.Atoi(s[8:10])

	if err != nil {
		return time.Unix(0, 0)
	}

	hour, err := strconv.Atoi(s[11:13])

	if err != nil {
		return time.Unix(0, 0)
	}

	minute, err := strconv.Atoi(s[14:16])

	if err != nil {
		return time.Unix(0, 0)
	}

	seconds, err := strconv.Atoi(s[17:19])

	if err != nil {
		return time.Unix(0, 0)
	}

	location, err := time.LoadLocation("UTC")

	return time.Date(year, time.Month(month), day, hour, minute, seconds, 0, location)
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/codegangsta/cli"
	_ "github.com/go-sql-driver/mysql"
	"github.com/karamani/iostreams"
)

var (
	debugMode           bool
	queryArg            string
	connectionStringArg string
	formatArg           string
	fakeMode            bool
)

func main() {
	app := cli.NewApp()
	app.Name = "massquery"
	app.Usage = "massquery"
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:        "debug",
			Usage:       "debug mode",
			EnvVar:      "MASSQUERY_DEBUG",
			Destination: &debugMode,
		},
		cli.StringFlag{
			Name:        "query",
			Usage:       "sql-query",
			Destination: &queryArg,
		},
		cli.StringFlag{
			Name:        "cnn",
			Usage:       "db connection string",
			EnvVar:      "MASSQUERY_CNN",
			Destination: &connectionStringArg,
		},
		cli.StringFlag{
			Name:        "format",
			Usage:       "output format",
			Destination: &formatArg,
		},
		cli.BoolFlag{
			Name:        "fake",
			Usage:       "fake mode",
			Destination: &fakeMode,
		},
	}

	app.Action = func(c *cli.Context) {

		connectionString := connectionStringArg
		queryArg := c.String("query")

		if !iostreams.StdinReady() {
			res, err := runQuery(connectionString, queryArg)
			if err != nil {
				log.Println(err.Error())
				printRes(formatArg, "", "", connectionString, "error", "")
			} else {
				debug("%#v", res)
				for _, resrow := range res {
					s := strings.Join(resrow, "\t")
					printRes(formatArg, "", "", connectionString, "success", s)
				}
			}
			return
		}

		// this func's called for each stdin's row
		process := func(row []byte) error {

			debug(string(row))

			params := strings.Split(string(row), "\t")

			if len(connectionStringArg) == 0 && len(params) < 2 {
				log.Printf("[ERROR] Не хватает параметров в stdin. Нужно минимум 2. Получено %d\n", len(params))
				return nil
			}

			if len(connectionStringArg) == 0 {
				connectionString = params[1]
			}

			query := queryArg
			for i, param := range params {
				paramTpl := fmt.Sprintf("{%d}", i)
				query = strings.Replace(query, paramTpl, param, -1)
			}

			debug(query)

			status := "success"
			res, err := runQuery(connectionString, query)
			if err != nil {
				status = "error"
				log.Println(err.Error())
				printRes(formatArg, string(row), params[0], connectionString, status, "")
			}

			for _, resrow := range res {
				s := strings.Join(resrow, "\t")
				printRes(formatArg, string(row), params[0], connectionString, status, s)
			}

			return nil
		}

		if err := iostreams.ProcessStdin(process); err != nil {
			log.Panicln(err.Error())
		}
	}

	app.Run(os.Args)
}

func printRes(format, input, id, cnn, status, res string) {
	if len(format) > 0 {
		s := format
		s = strings.Replace(s, "\\t", "\t", -1) // лишнее экранирование при получении аргумента программы
		s = strings.Replace(s, "\\n", "\n", -1) // лишнее экранирование при получении аргумента программы
		s = strings.Replace(s, "{input}", input, -1)
		s = strings.Replace(s, "{res}", res, -1)
		s = strings.Replace(s, "{id}", id, -1)
		s = strings.Replace(s, "{cnn}", cnn, -1)
		s = strings.Replace(s, "{status}", status, -1)
		fmt.Println(s)
	} else {
		fmt.Printf("%s\t%s\t%s\n", id, status, res)
	}
}

func runQuery(connectionString, query string) ([][]string, error) {

	var (
		res       [][]string
		container []string
		pointers  []interface{}
	)

	db, err := sql.Open("mysql", connectionString)
	if err != nil {
		log.Println(err.Error())
		return nil, err
	}
	defer db.Close()

	err = db.Ping()
	if err != nil {
		return nil, err
	}

	if fakeMode {
		return nil, nil
	}

	rows, err := db.Query(query)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	cols, err := rows.Columns()
	if err != nil {
		return nil, err
	}

	colsCount := len(cols)

	for rows.Next() {

		pointers = make([]interface{}, colsCount)
		container = make([]string, colsCount)
		for i, _ := range pointers {
			pointers[i] = &container[i]
		}

		if err := rows.Scan(pointers...); err != nil {
			return nil, err
		}
		res = append(res, container)
	}

	if err := rows.Err(); err != nil {
		return nil, err
	}

	return res, nil
}

func debug(format string, args ...interface{}) {
	if debugMode {
		log.Printf("[DEBUG] "+format+"\n", args...)
	}
}
<commit_msg>exec arg added<commit_after>package main

import (
	"database/sql"
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/codegangsta/cli"
	_ "github.com/go-sql-driver/mysql"
	"github.com/karamani/iostreams"
)

var (
	debugMode           bool
	queryArg            string
	execArg             string
	connectionStringArg string
	formatArg           string
	fakeMode            bool
	isExec              bool
)

func main() {
	app := cli.NewApp()
	app.Name = "massquery"
	app.Usage = "massquery"
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:        "debug",
			Usage:       "debug mode",
			EnvVar:      "MASSQUERY_DEBUG",
			Destination: &debugMode,
		},
		cli.StringFlag{
			Name:        "query",
			Usage:       "sql-query",
			Destination: &queryArg,
		},
		cli.StringFlag{
			Name:        "exec",
			Usage:       "exec-string (insert, update or delete)",
			Destination: &execArg,
		},
		cli.StringFlag{
			Name:        "cnn",
			Usage:       "db connection string",
			EnvVar:      "MASSQUERY_CNN",
			Destination: &connectionStringArg,
		},
		cli.StringFlag{
			Name:        "format",
			Usage:       "output format",
			Destination: &formatArg,
		},
		cli.BoolFlag{
			Name:        "fake",
			Usage:       "fake mode",
			Destination: &fakeMode,
		},
	}

	app.Action = func(c *cli.Context) {

		connectionString := connectionStringArg

		query := queryArg
		isExec = false
		if len(query) == 0 {
			query = execArg
			isExec = true
		}

		debug("%#v", isExec)

		if !iostreams.StdinReady() {
			res, err := runQuery(connectionString, queryArg)
			if err != nil {
				log.Println(err.Error())
				printRes(formatArg, "", "", connectionString, "error", "")
			} else {
				debug("%#v", res)
				for _, resrow := range res {
					s := strings.Join(resrow, "\t")
					printRes(formatArg, "", "", connectionString, "success", s)
				}
			}
			return
		}

		// this func's called for each stdin's row
		process := func(row []byte) error {

			debug(string(row))

			params := strings.Split(string(row), "\t")

			if len(connectionStringArg) == 0 && len(params) < 2 {
				log.Printf("[ERROR] Не хватает параметров в stdin. Нужно минимум 2. Получено %d\n", len(params))
				return nil
			}

			if len(connectionStringArg) == 0 {
				connectionString = params[1]
			}

			rowQuery := query
			for i, param := range params {
				paramTpl := fmt.Sprintf("{%d}", i)
				rowQuery = strings.Replace(rowQuery, paramTpl, param, -1)
			}

			debug(rowQuery)

			status := "success"
			res, err := runQuery(connectionString, rowQuery)
			if err != nil {
				status = "error"
				log.Println(err.Error())
				printRes(formatArg, string(row), params[0], connectionString, status, "")
			}

			for _, resrow := range res {
				s := strings.Join(resrow, "\t")
				printRes(formatArg, string(row), params[0], connectionString, status, s)
			}

			return nil
		}

		if err := iostreams.ProcessStdin(process); err != nil {
			log.Panicln(err.Error())
		}
	}

	app.Run(os.Args)
}

func printRes(format, input, id, cnn, status, res string) {
	if len(format) > 0 {
		s := format
		s = strings.Replace(s, "\\t", "\t", -1) // лишнее экранирование при получении аргумента программы
		s = strings.Replace(s, "\\n", "\n", -1) // лишнее экранирование при получении аргумента программы
		s = strings.Replace(s, "{input}", input, -1)
		s = strings.Replace(s, "{res}", res, -1)
		s = strings.Replace(s, "{id}", id, -1)
		s = strings.Replace(s, "{cnn}", cnn, -1)
		s = strings.Replace(s, "{status}", status, -1)
		fmt.Println(s)
	} else {
		fmt.Printf("%s\t%s\t%s\n", id, status, res)
	}
}

func runQuery(connectionString, query string) ([][]string, error) {

	var (
		res       [][]string
		container []string
		pointers  []interface{}
	)

	db, err := sql.Open("mysql", connectionString)
	if err != nil {
		log.Println(err.Error())
		return nil, err
	}
	defer db.Close()

	err = db.Ping()
	if err != nil {
		return nil, err
	}

	if fakeMode {
		return nil, nil
	}

	rows, err := db.Query(query)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	cols, err := rows.Columns()
	if err != nil {
		return nil, err
	}

	colsCount := len(cols)

	for rows.Next() {

		pointers = make([]interface{}, colsCount)
		container = make([]string, colsCount)
		for i, _ := range pointers {
			pointers[i] = &container[i]
		}

		if err := rows.Scan(pointers...); err != nil {
			return nil, err
		}
		res = append(res, container)
	}

	if err := rows.Err(); err != nil {
		return nil, err
	}

	return res, nil
}

func debug(format string, args ...interface{}) {
	if debugMode {
		log.Printf("[DEBUG] "+format+"\n", args...)
	}
}
<|endoftext|>