<commit_before>package xmmsclient

import (
	"bytes"
	"encoding/hex"
	"reflect"
	"testing"
)

func checkBuffer(t *testing.T, expected []byte, actual []byte) {
	if !reflect.DeepEqual(expected, actual) {
		t.Fatalf("\nwant:\n%s\nhave:\n%s", hex.Dump(expected), hex.Dump(actual))
	}
}

func TestSerializeInt(t *testing.T) {
	var expected = []byte{
		0x00, 0x00, 0x00, 0x02,
		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2a,
	}
	var buffer bytes.Buffer

	var err = serializeXmmsValue(XmmsInt(42), &buffer)
	if err != nil {
		t.Fatal(err)
	}

	checkBuffer(t, expected, buffer.Bytes())
}

func TestSerializeString(t *testing.T) {
	var expected = []byte{
		0x00, 0x00, 0x00, 0x03,
		0x00, 0x00, 0x00, 0x05,
		0x74, 0x65, 0x73, 0x74, 0x00,
	}
	var buffer bytes.Buffer

	var err = serializeXmmsValue(XmmsString("test"), &buffer)
	if err != nil {
		t.Fatal(err)
	}

	checkBuffer(t, expected, buffer.Bytes())
}

func TestSerializeList(t *testing.T) {
	var expected = []byte{
		0x00, 0x00, 0x00, 0x06,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x02,
		0x00, 0x00, 0x00, 0x02,
		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2a,
		0x00, 0x00, 0x00, 0x03,
		0x00, 0x00, 0x00, 0x05,
		0x74, 0x65, 0x73, 0x74, 0x00,
	}
	var buffer bytes.Buffer

	var err = serializeXmmsValue(XmmsList{XmmsInt(42), XmmsString("test")}, &buffer)
	if err != nil {
		t.Fatal(err)
	}

	checkBuffer(t, expected, buffer.Bytes())
}

func TestSerializeDict(t *testing.T) {
	var expected = []byte{
		0x00, 0x00, 0x00, 0x07,
		0x00, 0x00, 0x00, 0x01,
		0x00, 0x00, 0x00, 0x04,
		0x69, 0x6e, 0x74, 0x00,
		0x00, 0x00, 0x00, 0x02,
		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2a,
	}
	var buffer bytes.Buffer

	var err = serializeXmmsValue(XmmsDict{"int": XmmsInt(42)}, &buffer)
	if err != nil {
		t.Fatal(err)
	}

	checkBuffer(t, expected, buffer.Bytes())
}

func TestSerializeColl(t *testing.T) {
	expected := []byte{
		0x00, 0x00, 0x00, 0x04, /* XMMSV_TYPE_COLL */
		0x00, 0x00, 0x00, 0x06, /* XMMS_COLLECTION_TYPE_MATCH */
		0x00, 0x00, 0x00, 0x01, /* number of attributes*/

		0x00, 0x00, 0x00, 0x05, /* attr[0] key length */
		0x73, 0x65, 0x65, 0x64, /* attr[0] key "seed" */
		0x00, /*                               "\0"   */

		0x00, 0x00, 0x00, 0x02, /* attr[0] value type  */
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x7a, 0x69, /* attr[0] value 31337 */

		0x00, 0x00, 0x00, 0x02, /* idlist: restrict type XMMSV_TYPE_INT64 */
		0x00, 0x00, 0x00, 0x00, /* idlist: count */
		0x00, 0x00, 0x00, 0x04, /* operands: restrict type XMMSV_TYPE_COLL */
		0x00, 0x00, 0x00, 0x01, /* operands: number of operands */

		0x00, 0x00, 0x00, 0x01, /* operand[0]: coll type (_UNIVERSE) */

		0x00, 0x00, 0x00, 0x00, /* number of attributes*/
		0x00, 0x00, 0x00, 0x02, /* idlist: restrict type XMMSV_TYPE_INT64 */
		0x00, 0x00, 0x00, 0x00, /* idlist: count */
		0x00, 0x00, 0x00, 0x04, /* operands: restrict type XMMSV_TYPE_COLL */
		0x00, 0x00, 0x00, 0x00, /* operands: count */
	}
	var buffer bytes.Buffer

	match := XmmsColl{
		Type:       CollectionTypeMatch,
		Attributes: XmmsDict{"seed": XmmsInt(31337)},
		Operands: []XmmsColl{
			XmmsColl{Type: CollectionTypeUniverse},
		},
	}

	var err = serializeXmmsValue(match, &buffer)
	if err != nil {
		t.Fatal(err)
	}

	checkBuffer(t, expected, buffer.Bytes())
}
<commit_msg>Add test for serializing XmmsError.<commit_after>package xmmsclient

import (
	"bytes"
	"encoding/hex"
	"reflect"
	"testing"
)

func checkBuffer(t *testing.T, expected []byte, actual []byte) {
	if !reflect.DeepEqual(expected, actual) {
		t.Fatalf("\nwant:\n%s\nhave:\n%s", hex.Dump(expected), hex.Dump(actual))
	}
}

func TestSerializeInt(t *testing.T) {
	var expected = []byte{
		0x00, 0x00, 0x00, 0x02,
		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2a,
	}
	var buffer bytes.Buffer

	var err = serializeXmmsValue(XmmsInt(42), &buffer)
	if err != nil {
		t.Fatal(err)
	}

	checkBuffer(t, expected, buffer.Bytes())
}

func TestSerializeString(t *testing.T) {
	var expected = []byte{
		0x00, 0x00, 0x00, 0x03,
		0x00, 0x00, 0x00, 0x05,
		0x74, 0x65, 0x73, 0x74, 0x00,
	}
	var buffer bytes.Buffer

	var err = serializeXmmsValue(XmmsString("test"), &buffer)
	if err != nil {
		t.Fatal(err)
	}

	checkBuffer(t, expected, buffer.Bytes())
}

func TestSerializeError(t *testing.T) {
	var expected = []byte{
		0x00, 0x00, 0x00, 0x01,
		0x00, 0x00, 0x00, 0x05,
		0x74, 0x65, 0x73, 0x74, 0x00,
	}
	var buffer bytes.Buffer

	var err = serializeXmmsValue(XmmsError("test"), &buffer)
	if err != nil {
		t.Fatal(err)
	}

	checkBuffer(t, expected, buffer.Bytes())
}

func TestSerializeList(t *testing.T) {
	var expected = []byte{
		0x00, 0x00, 0x00, 0x06,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x02,
		0x00, 0x00, 0x00, 0x02,
		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2a,
		0x00, 0x00, 0x00, 0x03,
		0x00, 0x00, 0x00, 0x05,
		0x74, 0x65, 0x73, 0x74, 0x00,
	}
	var buffer bytes.Buffer

	var err = serializeXmmsValue(XmmsList{XmmsInt(42), XmmsString("test")}, &buffer)
	if err != nil {
		t.Fatal(err)
	}

	checkBuffer(t, expected, buffer.Bytes())
}

func TestSerializeDict(t *testing.T) {
	var expected = []byte{
		0x00, 0x00, 0x00, 0x07,
		0x00, 0x00, 0x00, 0x01,
		0x00, 0x00, 0x00, 0x04,
		0x69, 0x6e, 0x74, 0x00,
		0x00, 0x00, 0x00, 0x02,
		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2a,
	}
	var buffer bytes.Buffer

	var err = serializeXmmsValue(XmmsDict{"int": XmmsInt(42)}, &buffer)
	if err != nil {
		t.Fatal(err)
	}

	checkBuffer(t, expected, buffer.Bytes())
}

func TestSerializeColl(t *testing.T) {
	expected := []byte{
		0x00, 0x00, 0x00, 0x04, /* XMMSV_TYPE_COLL */
		0x00, 0x00, 0x00, 0x06, /* XMMS_COLLECTION_TYPE_MATCH */
		0x00, 0x00, 0x00, 0x01, /* number of attributes*/

		0x00, 0x00, 0x00, 0x05, /* attr[0] key length */
		0x73, 0x65, 0x65, 0x64, /* attr[0] key "seed" */
		0x00, /*                               "\0"   */

		0x00, 0x00, 0x00, 0x02, /* attr[0] value type  */
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x7a, 0x69, /* attr[0] value 31337 */

		0x00, 0x00, 0x00, 0x02, /* idlist: restrict type XMMSV_TYPE_INT64 */
		0x00, 0x00, 0x00, 0x00, /* idlist: count */
		0x00, 0x00, 0x00, 0x04, /* operands: restrict type XMMSV_TYPE_COLL */
		0x00, 0x00, 0x00, 0x01, /* operands: number of operands */

		0x00, 0x00, 0x00, 0x01, /* operand[0]: coll type (_UNIVERSE) */

		0x00, 0x00, 0x00, 0x00, /* number of attributes*/
		0x00, 0x00, 0x00, 0x02, /* idlist: restrict type XMMSV_TYPE_INT64 */
		0x00, 0x00, 0x00, 0x00, /* idlist: count */
		0x00, 0x00, 0x00, 0x04, /* operands: restrict type XMMSV_TYPE_COLL */
		0x00, 0x00, 0x00, 0x00, /* operands: count */
	}
	var buffer bytes.Buffer

	match := XmmsColl{
		Type:       CollectionTypeMatch,
		Attributes: XmmsDict{"seed": XmmsInt(31337)},
		Operands: []XmmsColl{
			XmmsColl{Type: CollectionTypeUniverse},
		},
	}

	var err = serializeXmmsValue(match, &buffer)
	if err != nil {
		t.Fatal(err)
	}

	checkBuffer(t, expected, buffer.Bytes())
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"runtime"
	"time"

	"github.com/jasonmoo/wc"
	"github.com/reducedb/bloom"
	"github.com/reducedb/bloom/scalable"
)

var (
	intersection = flag.Bool("i", false, "calculate the intersection")
	diff         = flag.Bool("d", false, "calculate the difference")
	union        = flag.Bool("u", false, "calculate the union")

	// bloom processing
	large           = flag.Bool("large", false, "use bloom filters for large data size (may be lossy)")
	estimated_lines = flag.Uint64("estimated_lines", 0, "estimate used to size bloom filters (set this to avoid prescan)")

	// options
	trim          = flag.Bool("trim", false, "trim each line")
	match_regex   = flag.String("match", "", "only process matching lines")
	capture_regex = flag.String("capture", "", "only process captured data")

	// for fs opts
	devnull     = flag.Bool("devnull", false, "do not output tokens, just counts")
	buffer_size = flag.Int("buffer_size", 1<<20, "buffered io chunk size")

	// totals
	total_tokens_emitted  uint64
	total_bytes_processed uint64
	total_lines_scanned   uint64
)

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
}

func main() {

	start := time.Now()

	flag.Parse()

	var stdout WriteFlusher

	if *devnull {
		stdout = new(DevNullWriter)
	} else {
		// buffered io
		stdout = bufio.NewWriterSize(os.Stdout, *buffer_size)
	}

	defer func() {
		stdout.Flush()
		fmt.Fprintln(os.Stderr, "** Token Report **")
		fmt.Fprintln(os.Stderr, "Lines scanned: ", total_lines_scanned)
		fmt.Fprintln(os.Stderr, "Tokens emitted: ", total_tokens_emitted)
		fmt.Fprintln(os.Stderr, "Time: ", time.Since(start))
	}()

	if !*intersection && !*diff && !*union {
		fmt.Println(`Usage: tt -[i,d,u] [-trim] [-match "regex"] [-capture "regex"] [-large [-estimated_lines N]] file1 file2[ file3..]`)
		flag.PrintDefaults()
		os.Exit(1)
	}

	file_paths := flag.Args()

	fmt.Fprintln(os.Stderr, "tt starting up")

	// if no estimate supplied, count lines
	if *large && *estimated_lines == 0 {

		var bytes_to_process uint64

		for _, file_path := range file_paths {

			file, err := os.Open(file_path)
			if err != nil {
				log.Fatal(err)
			}

			counter := wc.NewCounter(file)
			err = counter.Count(false, true, true, false)
			if err != nil {
				log.Fatal(err)
			}

			*estimated_lines += counter.Lines
			bytes_to_process += counter.Bytes

			file.Close()
		}

		fmt.Fprintln(os.Stderr, "Bytes to process: ", bytes_to_process)
		fmt.Fprintln(os.Stderr, "Lines to process: ", *estimated_lines)
	}

	if *large {

		if *union {

			unique_set := NewScalableBloom(*estimated_lines)

			for _, file_path := range file_paths {

				e, err := NewEmitter(file_path, *match_regex, *capture_regex, *buffer_size)
				if err != nil {
					log.Fatal(err)
				}

				for e.Scan() {
					token := e.Bytes()
					if !unique_set.Check(token) {
						total_tokens_emitted++
						stdout.Write(token)
						stdout.WriteByte('\n')
						unique_set.Add(token)
					}
				}

				e.Close()

				total_lines_scanned += e.LinesScanned

			}

			return
		}

		// multi file handling below
		sets := make([]bloom.Bloom, len(file_paths))

		// may require throttling due to disk thrashing
		// initial scan to fill the bloom filters
		for i, file_path := range file_paths {

			set := NewScalableBloom(*estimated_lines)

			e, err := NewEmitter(file_path, *match_regex, *capture_regex, *buffer_size)
			if err != nil {
				log.Fatal(err)
			}

			for e.Scan() {
				set.Add(e.Bytes())
			}

			e.Close()

			sets[i] = set

		}

		// do the work
		switch {

		// unique set of tokens that exist in all files
		case *intersection:

			echoed_set := NewScalableBloom(*estimated_lines)

			for _, file_path := range file_paths {

				e, err := NewEmitter(file_path, *match_regex, *capture_regex, *buffer_size)
				if err != nil {
					log.Fatal(err)
				}

			NEXT_TOKEN:
				for e.Scan() {

					token := e.Bytes()

					if echoed_set.Check(token) {
						goto NEXT_TOKEN
					}

					for _, set := range sets {
						if !set.Check(token) {
							goto NEXT_TOKEN
						}
					}

					total_tokens_emitted++
					stdout.Write(token)
					stdout.WriteByte('\n')
					echoed_set.Add(token)

				}

				total_lines_scanned += e.LinesScanned

				e.Close()

			}

		// unique set of tokens not in the intersection
		case *diff:

			echoed_set := NewScalableBloom(*estimated_lines)

			for _, file_path := range file_paths {

				e, err := NewEmitter(file_path, *match_regex, *capture_regex, *buffer_size)
				if err != nil {
					log.Fatal(err)
				}

				for e.Scan() {

					token := e.Bytes()

					if echoed_set.Check(token) {
						continue
					}

					for _, set := range sets {
						if !set.Check(token) {
							total_tokens_emitted++
							stdout.Write(token)
							stdout.WriteByte('\n')
							echoed_set.Add(token)
						}
					}

				}

				total_lines_scanned += e.LinesScanned

				e.Close()

			}
		}

		// defaults to map solution
	} else {

		if *union {

			unique_set := make(map[string]bool)

			for _, file_path := range file_paths {

				e, err := NewEmitter(file_path, *match_regex, *capture_regex, *buffer_size)
				if err != nil {
					log.Fatal(err)
				}

				for e.Scan() {
					token := e.Text()
					if _, exists := unique_set[token]; !exists {
						total_tokens_emitted++
						stdout.WriteString(token)
						stdout.WriteByte('\n')

						unique_set[token] = true
					}
				}

				total_lines_scanned += e.LinesScanned

				e.Close()

			}

			return
		}

		// multi file handling below
		sets := make([]map[string]bool, len(file_paths))

		// may require throttling due to disk thrashing
		// initial scan to fill the bloom filters
		for i, file_path := range file_paths {

			set := make(map[string]bool)

			e, err := NewEmitter(file_path, *match_regex, *capture_regex, *buffer_size)
			if err != nil {
				log.Fatal(err)
			}

			for e.Scan() {
				set[e.Text()] = true
			}

			e.Close()

			sets[i] = set

		}

		// do the work
		switch {

		// unique set of tokens that exist in all files
		case *intersection:

			echoed_set := make(map[string]bool)

			for _, file_path := range file_paths {

				e, err := NewEmitter(file_path, *match_regex, *capture_regex, *buffer_size)
				if err != nil {
					log.Fatal(err)
				}

			NEXT_TOKEN2:
				for e.Scan() {

					token := e.Text()

					if _, echoed := echoed_set[token]; echoed {
						goto NEXT_TOKEN2
					}

					for _, set := range sets {
						if _, in_this_set := set[token]; !in_this_set {
							goto NEXT_TOKEN2
						}
					}

					total_tokens_emitted++
					stdout.WriteString(token)
					stdout.WriteByte('\n')

					echoed_set[token] = true

				}

				total_lines_scanned += e.LinesScanned

				e.Close()

			}

		// unique set of tokens not in the intersection
		case *diff:

			echoed_set := make(map[string]bool)

			for _, file_path := range file_paths {

				e, err := NewEmitter(file_path, *match_regex, *capture_regex, *buffer_size)
				if err != nil {
					log.Fatal(err)
				}

				for e.Scan() {

					token := e.Text()

					if _, echoed := echoed_set[token]; echoed {
						continue
					}

					for _, set := range sets {
						if _, in_this_set := set[token]; !in_this_set {
							total_tokens_emitted++
							stdout.WriteString(token)
							stdout.WriteByte('\n')

							echoed_set[token] = true
							break
						}
					}

				}

				total_lines_scanned += e.LinesScanned

				e.Close()

			}
		}

	}

}

func NewScalableBloom(size uint64) bloom.Bloom {
	return scalable.New(uint(size))
}
<commit_msg>Adding counts for non-large unions<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"runtime"
	"time"

	"github.com/jasonmoo/wc"
	"github.com/reducedb/bloom"
	"github.com/reducedb/bloom/scalable"
)

var (
	intersection = flag.Bool("i", false, "calculate the intersection")
	diff         = flag.Bool("d", false, "calculate the difference")
	union        = flag.Bool("u", false, "calculate the union")

	count = flag.Bool("c", false, "output counts of each token on non-large unions")

	// bloom processing
	large           = flag.Bool("large", false, "use bloom filters for large data size (may be lossy)")
	estimated_lines = flag.Uint64("estimated_lines", 0, "estimate used to size bloom filters (set this to avoid prescan)")

	// options
	trim          = flag.Bool("trim", false, "trim each line")
	match_regex   = flag.String("match", "", "only process matching lines")
	capture_regex = flag.String("capture", "", "only process captured data")

	// for fs opts
	devnull     = flag.Bool("devnull", false, "do not output tokens, just counts")
	buffer_size = flag.Int("buffer_size", 1<<20, "buffered io chunk size")

	// totals
	total_tokens_emitted  uint64
	total_bytes_processed uint64
	total_lines_scanned   uint64
)

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
}

func main() {

	flag.Parse()

	if !*intersection && !*diff && !*union {
		fmt.Println(`Usage: tt -[i,d,u] [-c] [-trim] [-match "regex"] [-capture "regex"] [-large [-estimated_lines N]] file1 file2[ file3..]`)
		flag.PrintDefaults()
		os.Exit(1)
	}

	start := time.Now()

	var stdout WriteFlusher

	if *devnull {
		stdout = new(DevNullWriter)
	} else {
		// buffered io
		stdout = bufio.NewWriterSize(os.Stdout, *buffer_size)
	}

	defer func() {
		stdout.Flush()
		fmt.Fprintln(os.Stderr, "** Token Report **")
		fmt.Fprintln(os.Stderr, "Lines scanned: ", total_lines_scanned)
		fmt.Fprintln(os.Stderr, "Tokens emitted: ", total_tokens_emitted)
		fmt.Fprintln(os.Stderr, "Time: ", time.Since(start))
	}()

	file_paths := flag.Args()

	fmt.Fprintln(os.Stderr, "tt starting up")

	// if no estimate supplied, count lines
	if *large && *estimated_lines == 0 {

		var bytes_to_process uint64

		for _, file_path := range file_paths {

			file, err := os.Open(file_path)
			if err != nil {
				log.Fatal(err)
			}

			counter := wc.NewCounter(file)
			err = counter.Count(false, true, true, false)
			if err != nil {
				log.Fatal(err)
			}

			*estimated_lines += counter.Lines
			bytes_to_process += counter.Bytes

			file.Close()
		}

		fmt.Fprintln(os.Stderr, "Bytes to process: ", bytes_to_process)
		fmt.Fprintln(os.Stderr, "Lines to process: ", *estimated_lines)
	}

	if *large {

		if *union {

			unique_set := NewScalableBloom(*estimated_lines)

			for _, file_path := range file_paths {

				e, err := NewEmitter(file_path, *match_regex, *capture_regex, *buffer_size)
				if err != nil {
					log.Fatal(err)
				}

				for e.Scan() {
					token := e.Bytes()
					if !unique_set.Check(token) {
						total_tokens_emitted++
						stdout.Write(token)
						stdout.WriteByte('\n')
						unique_set.Add(token)
					}
				}

				e.Close()

				total_lines_scanned += e.LinesScanned

			}

			return
		}

		// multi file handling below
		sets := make([]bloom.Bloom, len(file_paths))

		// may require throttling due to disk thrashing
		// initial scan to fill the bloom filters
		for i, file_path := range file_paths {

			set := NewScalableBloom(*estimated_lines)

			e, err := NewEmitter(file_path, *match_regex, *capture_regex, *buffer_size)
			if err != nil {
				log.Fatal(err)
			}

			for e.Scan() {
				set.Add(e.Bytes())
			}

			e.Close()

			sets[i] = set

		}

		// do the work
		switch {

		// unique set of tokens that exist in all files
		case *intersection:

			echoed_set := NewScalableBloom(*estimated_lines)

			for _, file_path := range file_paths {

				e, err := NewEmitter(file_path, *match_regex, *capture_regex, *buffer_size)
				if err != nil {
					log.Fatal(err)
				}

			NEXT_TOKEN:
				for e.Scan() {

					token := e.Bytes()

					if echoed_set.Check(token) {
						goto NEXT_TOKEN
					}

					for _, set := range sets {
						if !set.Check(token) {
							goto NEXT_TOKEN
						}
					}

					total_tokens_emitted++
					stdout.Write(token)
					stdout.WriteByte('\n')
					echoed_set.Add(token)

				}

				total_lines_scanned += e.LinesScanned

				e.Close()

			}

		// unique set of tokens not in the intersection
		case *diff:

			echoed_set := NewScalableBloom(*estimated_lines)

			for _, file_path := range file_paths {

				e, err := NewEmitter(file_path, *match_regex, *capture_regex, *buffer_size)
				if err != nil {
					log.Fatal(err)
				}

				for e.Scan() {

					token := e.Bytes()

					if echoed_set.Check(token) {
						continue
					}

					for _, set := range sets {
						if !set.Check(token) {
							total_tokens_emitted++
							stdout.Write(token)
							stdout.WriteByte('\n')
							echoed_set.Add(token)
						}
					}

				}

				total_lines_scanned += e.LinesScanned

				e.Close()

			}
		}

		// defaults to map solution
	} else {

		if *union {

			unique_set := make(map[string]int)

			for _, file_path := range file_paths {

				e, err := NewEmitter(file_path, *match_regex, *capture_regex, *buffer_size)
				if err != nil {
					log.Fatal(err)
				}

				for e.Scan() {
					unique_set[e.Text()]++
				}

				total_lines_scanned += e.LinesScanned

				e.Close()

			}

			if *count {
				for token, ct := range unique_set {
					total_tokens_emitted++
					fmt.Fprintf(stdout, "%d: %s\n", ct, token)
				}
			} else {
				for token, _ := range unique_set {
					total_tokens_emitted++
					stdout.WriteString(token)
					stdout.WriteByte('\n')
				}
			}

			return
		}

		// multi file handling below
		sets := make([]map[string]bool, len(file_paths))

		// may require throttling due to disk thrashing
		// initial scan to fill the bloom filters
		for i, file_path := range file_paths {

			set := make(map[string]bool)

			e, err := NewEmitter(file_path, *match_regex, *capture_regex, *buffer_size)
			if err != nil {
				log.Fatal(err)
			}

			for e.Scan() {
				set[e.Text()] = true
			}

			e.Close()

			sets[i] = set

		}

		// do the work
		switch {

		// unique set of tokens that exist in all files
		case *intersection:

			echoed_set := make(map[string]bool)

			for _, file_path := range file_paths {

				e, err := NewEmitter(file_path, *match_regex, *capture_regex, *buffer_size)
				if err != nil {
					log.Fatal(err)
				}

			NEXT_TOKEN2:
				for e.Scan() {

					token := e.Text()

					if _, echoed := echoed_set[token]; echoed {
						goto NEXT_TOKEN2
					}

					for _, set := range sets {
						if _, in_this_set := set[token]; !in_this_set {
							goto NEXT_TOKEN2
						}
					}

					total_tokens_emitted++
					stdout.WriteString(token)
					stdout.WriteByte('\n')

					echoed_set[token] = true

				}

				total_lines_scanned += e.LinesScanned

				e.Close()

			}

		// unique set of tokens not in the intersection
		case *diff:

			echoed_set := make(map[string]bool)

			for _, file_path := range file_paths {

				e, err := NewEmitter(file_path, *match_regex, *capture_regex, *buffer_size)
				if err != nil {
					log.Fatal(err)
				}

				for e.Scan() {

					token := e.Text()

					if _, echoed := echoed_set[token]; echoed {
						continue
					}

					for _, set := range sets {
						if _, in_this_set := set[token]; !in_this_set {
							total_tokens_emitted++
							stdout.WriteString(token)
							stdout.WriteByte('\n')

							echoed_set[token] = true
							break
						}
					}

				}

				total_lines_scanned += e.LinesScanned

				e.Close()

			}
		}

	}

}

func NewScalableBloom(size uint64) bloom.Bloom {
	return scalable.New(uint(size))
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"github.com/reducedb/bloom"
	"github.com/reducedb/bloom/scalable"

	// "crypto/sha1"
	// "github.com/spaolacci/murmur3"

	"log"
	"os"
	"runtime"
	"time"
)

var (
	intersection = flag.Bool("i", false, "calculate the intersection")
	diff         = flag.Bool("d", false, "calculate the difference")
	union        = flag.Bool("u", false, "calculate the union")

	hint = flag.Uint("hint", 4096, "min number of tokens per file")

	// buffered io
	stdout = bufio.NewWriterSize(os.Stdout, 4096)

	// unique filter
	unique_set = NewScalableBloom(*hint)

	// total tokens in output
	total uint64
)

func main() {

	start := time.Now()

	defer func() {
		stdout.Flush()
		fmt.Fprintln(os.Stderr, "** Token Report **")
		fmt.Fprintln(os.Stderr, "Tokens output: ", total)
		fmt.Fprintln(os.Stderr, "Total time: ", time.Since(start))
	}()

	runtime.GOMAXPROCS(runtime.NumCPU())

	flag.Parse()

	file_paths := flag.Args()

	// may omit entries due to false positives
	// todo(jason): try crypto hash or use dual filters
	if *union {

		for _, file_path := range file_paths {

			file, err := os.Open(file_path)
			if err != nil {
				log.Fatal(err)
			}

			scanner := bufio.NewScanner(file)

			for scanner.Scan() {
				token := scanner.Bytes()
				if !unique_set.Check(token) {
					stdout.Write(token)
					stdout.WriteByte('\n')
					total++
					unique_set.Add(token)
				}
			}

			file.Close()

		}

		return
	}

	// multi file handling below
	sets := make([]bloom.Bloom, len(file_paths))

	// may require throttling due to disk thrashing
	// initial scan to fill the bloom filters
	for i, file_path := range file_paths {

		set := NewScalableBloom(*hint)

		file, err := os.Open(file_path)
		if err != nil {
			log.Fatal(err)
		}

		scanner := bufio.NewScanner(file)

		for scanner.Scan() {
			set.Add(scanner.Bytes())
		}

		file.Close()

		sets[i] = set

	}

	// do the work
	switch {

	// unique set of tokens that exist in all files
	case *intersection:
		for _, file_path := range file_paths {

			file, err := os.Open(file_path)
			if err != nil {
				log.Fatal(err)
			}

			scanner := bufio.NewScanner(file)

		NEXT_TOKEN:
			for scanner.Scan() {
				token := scanner.Bytes()
				for _, set := range sets {
					if !set.Check(token) || unique_set.Check(token) {
						goto NEXT_TOKEN
					}
				}
				stdout.Write(token)
				stdout.WriteByte('\n')
				total++
				unique_set.Add(token)
			}

			file.Close()

		}

	// unique set of tokens not in the intersection
	case *diff:
		for _, file_path := range file_paths {

			file, err := os.Open(file_path)
			if err != nil {
				log.Fatal(err)
			}

			scanner := bufio.NewScanner(file)

			for scanner.Scan() {
				token := scanner.Bytes()
				for _, set := range sets {
					if !set.Check(token) && !unique_set.Check(token) {
						stdout.Write(token)
						stdout.WriteByte('\n')
						total++
						unique_set.Add(token)
					}
				}
			}

			file.Close()

		}
	default:
		fmt.Println("Usage: tt -[i,d,u] file1 file2[ file3..]")
		flag.PrintDefaults()
		os.Exit(1)
	}

}

type (
	Bloomer struct {
		bloom.Bloom
		filters []bloom.Bloom
	}
)

func NewScalableBloom(size uint) bloom.Bloom {

	filters := make([]bloom.Bloom, 2)

	for i, _ := range filters {
		filter := scalable.New(size)
		filter.Reset()
		filters[i] = filter
	}

	return &Bloomer{
		filters: filters,
	}

}

func (b *Bloomer) Add(token []byte) bloom.Bloom {
	for _, filter := range b.filters {
		filter.Add(token)
		token = mash(token)
	}
	return b
}

func (b *Bloomer) Check(token []byte) bool {
	for _, filter := range b.filters {
		if !filter.Check(token) {
			return false
		}
		token = mash(token)
	}
	return true
}

// modifies the underlying structure
func mash(token []byte) []byte {
	for i, c := range token {
		c ^= 0xA * (c<<2 ^ c)
		token[i] = c
	}
	return token
}
<commit_msg>better mash function and fix for token side effects<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"github.com/reducedb/bloom"
	"github.com/reducedb/bloom/scalable"

	// "crypto/sha1"
	// "github.com/spaolacci/murmur3"
	// "github.com/zhenjl/cityhash"

	"log"
	"os"
	"runtime"
	"time"
)

var (
	intersection = flag.Bool("i", false, "calculate the intersection")
	diff         = flag.Bool("d", false, "calculate the difference")
	union        = flag.Bool("u", false, "calculate the union")

	hint = flag.Uint("hint", 4096, "min number of tokens per file")

	// buffered io
	stdout = bufio.NewWriterSize(os.Stdout, 4096)

	// unique filter
	unique_set = NewScalableBloom(*hint)

	// total tokens in output
	total uint64
)

func main() {

	start := time.Now()

	defer func() {
		stdout.Flush()
		fmt.Fprintln(os.Stderr, "** Token Report **")
		fmt.Fprintln(os.Stderr, "Tokens output: ", total)
		fmt.Fprintln(os.Stderr, "Total time: ", time.Since(start))
	}()

	runtime.GOMAXPROCS(runtime.NumCPU())

	flag.Parse()

	file_paths := flag.Args()

	// may omit entries due to false positives
	// todo(jason): try crypto hash or use dual filters
	if *union {

		for _, file_path := range file_paths {

			file, err := os.Open(file_path)
			if err != nil {
				log.Fatal(err)
			}

			scanner := bufio.NewScanner(file)

			for scanner.Scan() {
				token := scanner.Bytes()
				if !unique_set.Check(token) {
					stdout.Write(token)
					stdout.WriteByte('\n')
					total++
					unique_set.Add(token)
				}
			}

			file.Close()

		}

		return
	}

	// multi file handling below
	sets := make([]bloom.Bloom, len(file_paths))

	// may require throttling due to disk thrashing
	// initial scan to fill the bloom filters
	for i, file_path := range file_paths {

		set := NewScalableBloom(*hint)

		file, err := os.Open(file_path)
		if err != nil {
			log.Fatal(err)
		}

		scanner := bufio.NewScanner(file)

		for scanner.Scan() {
			set.Add(scanner.Bytes())
		}

		file.Close()

		sets[i] = set

	}

	// do the work
	switch {

	// unique set of tokens that exist in all files
	case *intersection:
		for _, file_path := range file_paths {

			file, err := os.Open(file_path)
			if err != nil {
				log.Fatal(err)
			}

			scanner := bufio.NewScanner(file)

		NEXT_TOKEN:
			for scanner.Scan() {
				token := scanner.Bytes()
				for _, set := range sets {
					if !set.Check(token) || unique_set.Check(token) {
						goto NEXT_TOKEN
					}
				}
				stdout.Write(token)
				stdout.WriteByte('\n')
				total++
				unique_set.Add(token)
			}

			file.Close()

		}

	// unique set of tokens not in the intersection
	case *diff:
		for _, file_path := range file_paths {

			file, err := os.Open(file_path)
			if err != nil {
				log.Fatal(err)
			}

			scanner := bufio.NewScanner(file)

			for scanner.Scan() {
				token := scanner.Bytes()
				for _, set := range sets {
					if !set.Check(token) && !unique_set.Check(token) {
						stdout.Write(token)
						stdout.WriteByte('\n')
						total++
						unique_set.Add(token)
					}
				}
			}

			file.Close()

		}
	default:
		fmt.Println("Usage: tt -[i,d,u] file1 file2[ file3..]")
		flag.PrintDefaults()
		os.Exit(1)
	}

}

type (
	Bloomer struct {
		bloom.Bloom
		filters []bloom.Bloom
	}
)

func NewScalableBloom(size uint) bloom.Bloom {

	filters := make([]bloom.Bloom, 2)

	for i, _ := range filters {
		filter := scalable.New(size)
		// filter.SetHasher(adler32.New())
		filter.Reset()
		filters[i] = filter
	}

	return &Bloomer{
		filters: filters,
	}

}

func (b *Bloomer) Add(token []byte) bloom.Bloom {
	token = append(make([]byte, len(token)), token...)
	for _, filter := range b.filters {
		filter.Add(token)
		token = mash(token)
	}
	return b
}

func (b *Bloomer) Check(token []byte) bool {
	token = append(make([]byte, len(token)), token...)
	for _, filter := range b.filters {
		if !filter.Check(token) {
			return false
		}
		token = mash(token)
	}
	return true
}

// modifies the underlying structure
func mash(token []byte) []byte {
	for i, c := range token {
		c ^= (0xA * c) << 1
		token[i] = c
	}
	return token
}
<|endoftext|>


<commit_before>package pg

// Not thread-safe.
type Tx struct {
	db  *DB
	_cn *conn

	err  error
	done bool
}

func (db *DB) Begin() (*Tx, error) {
	cn, err := db.conn()
	if err != nil {
		return nil, err
	}

	tx := &Tx{
		db:  db,
		_cn: cn,
	}
	if _, err := tx.Exec("BEGIN"); err != nil {
		tx.close()
		return nil, err
	}
	return tx, nil
}

// RunInTransaction runs a function in a transaction. If function
// returns an error transaction is rollbacked, otherwise transaction
// is committed.
func (db *DB) RunInTransaction(fn func(*Tx) error) error {
	tx, err := db.Begin()
	if err != nil {
		return err
	}

	if err := fn(tx); err != nil {
		tx.Rollback()
		return err
	}
	return tx.Commit()
}

func (tx *Tx) conn() *conn {
	tx._cn.SetReadTimeout(tx.db.opt.ReadTimeout)
	tx._cn.SetWriteTimeout(tx.db.opt.WriteTimeout)
	return tx._cn
}

// TODO(vmihailenco): track and close prepared statements
func (tx *Tx) Prepare(q string) (*Stmt, error) {
	if tx.done {
		return nil, errTxDone
	}

	cn := tx.conn()
	return prepare(tx.db, cn, q)
}

func (tx *Tx) Exec(q string, args ...interface{}) (*Result, error) {
	if tx.done {
		return nil, errTxDone
	}

	cn := tx.conn()

	res, err := simpleQuery(cn, q, args...)
	if err != nil {
		tx.setErr(err)
		return nil, err
	}
	return res, nil
}

func (tx *Tx) ExecOne(q string, args ...interface{}) (*Result, error) {
	res, err := tx.Exec(q, args...)
	if err != nil {
		return nil, err
	}
	return assertOneAffected(res, nil)
}

func (tx *Tx) Query(coll Collection, q string, args ...interface{}) (*Result, error) {
	if tx.done {
		return nil, errTxDone
	}

	cn := tx.conn()
	res, err := simpleQueryData(cn, coll, q, args...)
	if err != nil {
		tx.setErr(err)
		return nil, err
	}
	return res, nil
}

func (tx *Tx) QueryOne(record interface{}, q string, args ...interface{}) (*Result, error) {
	coll := &singleRecordCollection{record: record}
	res, err := tx.Query(coll, q, args...)
	if err != nil {
		return nil, err
	}
	return assertOneAffected(res, coll)
}

func (tx *Tx) Commit() (err error) {
	if tx.done {
		return errTxDone
	}

	_, err = tx.Exec("COMMIT")
	if err != nil {
		tx.setErr(err)
	}

	tx.close()
	return err
}

func (tx *Tx) Rollback() (err error) {
	if tx.done {
		return errTxDone
	}

	_, err = tx.Exec("ROLLBACK")
	if err != nil {
		tx.setErr(err)
	}

	tx.close()
	return err
}

func (tx *Tx) setErr(e error) {
	tx.err = e
}

func (tx *Tx) close() error {
	if tx.done {
		return nil
	}
	tx.done = true
	return tx.db.freeConn(tx._cn, tx.err)
}
<commit_msg>Add GO_PG_NO_TX environment variable.<commit_after>package pg

import "os"

// When true Tx does not issue BEGIN, COMMIT, and ROLLBACK.
// It is primarily useful for testing and can be enabled with
// GO_PG_NO_TX environment variable.
var noTx bool

func init() {
	noTx = os.Getenv("GO_PG_NO_TX") != ""
}

// Not thread-safe.
type Tx struct {
	db  *DB
	_cn *conn

	err  error
	done bool
}

func (db *DB) Begin() (*Tx, error) {
	cn, err := db.conn()
	if err != nil {
		return nil, err
	}

	tx := &Tx{
		db:  db,
		_cn: cn,
	}
	if !noTx {
		if _, err := tx.Exec("BEGIN"); err != nil {
			tx.close()
			return nil, err
		}
	}
	return tx, nil
}

// RunInTransaction runs a function in a transaction. If function
// returns an error transaction is rollbacked, otherwise transaction
// is committed.
func (db *DB) RunInTransaction(fn func(*Tx) error) error {
	tx, err := db.Begin()
	if err != nil {
		return err
	}

	if err := fn(tx); err != nil {
		tx.Rollback()
		return err
	}
	return tx.Commit()
}

func (tx *Tx) conn() *conn {
	tx._cn.SetReadTimeout(tx.db.opt.ReadTimeout)
	tx._cn.SetWriteTimeout(tx.db.opt.WriteTimeout)
	return tx._cn
}

// TODO(vmihailenco): track and close prepared statements
func (tx *Tx) Prepare(q string) (*Stmt, error) {
	if tx.done {
		return nil, errTxDone
	}

	cn := tx.conn()
	return prepare(tx.db, cn, q)
}

func (tx *Tx) Exec(q string, args ...interface{}) (*Result, error) {
	if tx.done {
		return nil, errTxDone
	}

	cn := tx.conn()

	res, err := simpleQuery(cn, q, args...)
	if err != nil {
		tx.setErr(err)
		return nil, err
	}
	return res, nil
}

func (tx *Tx) ExecOne(q string, args ...interface{}) (*Result, error) {
	res, err := tx.Exec(q, args...)
	if err != nil {
		return nil, err
	}
	return assertOneAffected(res, nil)
}

func (tx *Tx) Query(coll Collection, q string, args ...interface{}) (*Result, error) {
	if tx.done {
		return nil, errTxDone
	}

	cn := tx.conn()
	res, err := simpleQueryData(cn, coll, q, args...)
	if err != nil {
		tx.setErr(err)
		return nil, err
	}
	return res, nil
}

func (tx *Tx) QueryOne(record interface{}, q string, args ...interface{}) (*Result, error) {
	coll := &singleRecordCollection{record: record}
	res, err := tx.Query(coll, q, args...)
	if err != nil {
		return nil, err
	}
	return assertOneAffected(res, coll)
}

func (tx *Tx) Commit() (err error) {
	if tx.done {
		return errTxDone
	}

	if !noTx {
		_, err = tx.Exec("COMMIT")
		if err != nil {
			tx.setErr(err)
		}
	}

	tx.close()
	return err
}

func (tx *Tx) Rollback() (err error) {
	if tx.done {
		return errTxDone
	}

	if !noTx {
		_, err = tx.Exec("ROLLBACK")
		if err != nil {
			tx.setErr(err)
		}
	}

	tx.close()
	return err
}

func (tx *Tx) setErr(e error) {
	tx.err = e
}

func (tx *Tx) close() error {
	if tx.done {
		return nil
	}
	tx.done = true
	return tx.db.freeConn(tx._cn, tx.err)
}
<|endoftext|>


<commit_before>// Copyright 2014 Rana Ian. All rights reserved.
// Use of this source code is governed by The MIT License
// found in the accompanying LICENSE file.

package ora

/*
#include <oci.h>
#include <stdlib.h>
*/
import "C"
import (
	"container/list"
	"fmt"
)

// LogTxCfg represents Tx logging configuration values.
type LogTxCfg struct {
	// Commit determines whether the Tx.Commit method is logged.
	//
	// The default is true.
	Commit bool

	// Rollback determines whether the Tx.Rollback method is logged.
	//
	// The default is true.
	Rollback bool
}

// NewLogTxCfg creates a LogTxCfg with default values.
func NewLogTxCfg() LogTxCfg {
	c := LogTxCfg{}
	c.Commit = true
	c.Rollback = true
	return c
}

// Tx represents an Oracle transaction associated with a session.
//
// Implements the driver.Tx interface.
type Tx struct {
	id   uint64
	ses  *Ses
	elem *list.Element
}

// checkIsOpen validates that the session is open.
func (tx *Tx) checkIsOpen() error {
	if tx == nil || tx.ses == nil {
		return er("Tx is closed.")
	}
	return tx.ses.checkClosed()
}

func (tx *Tx) close() {
	if tx.ses != nil {
		tx.ses.openTxs.Remove(tx.elem)
		tx.ses = nil
		tx.elem = nil
		_drv.txPool.Put(tx)
	}
}

// Commit commits the transaction.
//
// Commit is a member of the driver.Tx interface.
func (tx *Tx) Commit() (err error) {
	if tx == nil {
		return nil
	}
	tx.log(_drv.cfg.Log.Tx.Commit)
	if err = tx.checkIsOpen(); err != nil {
		return err
	}
	defer tx.close()
	r := C.OCITransCommit(
		tx.ses.srv.ocisvcctx,  //OCISvcCtx    *svchp,
		tx.ses.srv.env.ocierr, //OCIError     *errhp,
		C.OCI_DEFAULT)         //ub4          flags );
	if r == C.OCI_ERROR {
		return tx.ses.srv.env.ociError()
	}
	return nil
}

// Rollback rolls back a transaction.
//
// Rollback is a member of the driver.Tx interface.
func (tx *Tx) Rollback() (err error) {
	if tx == nil {
		return nil
	}
	tx.log(_drv.cfg.Log.Tx.Rollback)
	if err = tx.checkIsOpen(); err != nil {
		return err
	}
	if tx.ses == nil || tx.ses.srv == nil {
		return nil
	}
	defer tx.close()
	r := C.OCITransRollback(
		tx.ses.srv.ocisvcctx,  //OCISvcCtx    *svchp,
		tx.ses.srv.env.ocierr, //OCIError     *errhp,
		C.OCI_DEFAULT)         //ub4          flags );
	if r == C.OCI_ERROR {
		return tx.ses.srv.env.ociError()
	}
	return nil
}

// sysName returns a string representing the Tx.
func (tx *Tx) sysName() string {
	if tx == nil {
		return "E_S_S_T_"
	}
	return tx.ses.sysName() + fmt.Sprintf("T%v", tx.id)
}

// log writes a message with an Tx system name and caller info.
func (tx *Tx) log(enabled bool, v ...interface{}) {
	if enabled {
		if len(v) == 0 {
			_drv.cfg.Log.Logger.Infof("%v %v", tx.sysName(), callInfo(1))
		} else {
			_drv.cfg.Log.Logger.Infof("%v %v %v", tx.sysName(), callInfo(1), fmt.Sprint(v...))
		}
	}
}

// log writes a formatted message with an Tx system name and caller info.
func (tx *Tx) logF(enabled bool, format string, v ...interface{}) {
	if enabled {
		if len(v) == 0 {
			_drv.cfg.Log.Logger.Infof("%v %v", tx.sysName(), callInfo(1))
		} else {
			_drv.cfg.Log.Logger.Infof("%v %v %v", tx.sysName(), callInfo(1), fmt.Sprintf(format, v...))
		}
	}
}
<commit_msg>revised openTxs, added mutext<commit_after>// Copyright 2014 Rana Ian. All rights reserved.
// Use of this source code is governed by The MIT License
// found in the accompanying LICENSE file.

package ora

/*
#include <oci.h>
#include <stdlib.h>
*/
import "C"
import (
	"fmt"
	"sync"
)

// LogTxCfg represents Tx logging configuration values.
type LogTxCfg struct {
	// Commit determines whether the Tx.Commit method is logged.
	//
	// The default is true.
	Commit bool

	// Rollback determines whether the Tx.Rollback method is logged.
	//
	// The default is true.
	Rollback bool
}

// NewLogTxCfg creates a LogTxCfg with default values.
func NewLogTxCfg() LogTxCfg {
	c := LogTxCfg{}
	c.Commit = true
	c.Rollback = true
	return c
}

// Tx represents an Oracle transaction associated with a session.
//
// Implements the driver.Tx interface.
type Tx struct {
	id  uint64
	ses *Ses
	mu  sync.Mutex
}

// checkIsOpen validates that the session is open.
func (tx *Tx) checkIsOpen() error {
	if tx == nil || tx.ses == nil {
		return er("Tx is closed.")
	}
	return tx.ses.checkClosed()
}

// closeWithRemove releases allocated resources and removes the Tx from the
// Ses.openTxss list.
func (tx *Tx) closeWithRemove() (err error) {
	tx.ses.openTxs.remove(tx)
	return tx.close()
}

// close releases allocated resources.
func (tx *Tx) close() (err error) {
	if tx.ses != nil {
		tx.ses = nil
		_drv.txPool.Put(tx)
	}
	return nil
}

// Commit commits the transaction.
//
// Commit is a member of the driver.Tx interface.
func (tx *Tx) Commit() (err error) {
	tx.mu.Lock()
	defer tx.mu.Unlock()
	if tx == nil {
		return nil
	}
	tx.log(_drv.cfg.Log.Tx.Commit)
	if err = tx.checkIsOpen(); err != nil {
		return err
	}
	defer tx.closeWithRemove()
	r := C.OCITransCommit(
		tx.ses.srv.ocisvcctx,  //OCISvcCtx    *svchp,
		tx.ses.srv.env.ocierr, //OCIError     *errhp,
		C.OCI_DEFAULT)         //ub4          flags );
	if r == C.OCI_ERROR {
		return tx.ses.srv.env.ociError()
	}
	return nil
}

// Rollback rolls back a transaction.
//
// Rollback is a member of the driver.Tx interface.
func (tx *Tx) Rollback() (err error) {
	tx.mu.Lock()
	defer tx.mu.Unlock()
	if tx == nil {
		return nil
	}
	tx.log(_drv.cfg.Log.Tx.Rollback)
	if err = tx.checkIsOpen(); err != nil {
		return err
	}
	if tx.ses == nil || tx.ses.srv == nil {
		return nil
	}
	defer tx.closeWithRemove()
	r := C.OCITransRollback(
		tx.ses.srv.ocisvcctx,  //OCISvcCtx    *svchp,
		tx.ses.srv.env.ocierr, //OCIError     *errhp,
		C.OCI_DEFAULT)         //ub4          flags );
	if r == C.OCI_ERROR {
		return tx.ses.srv.env.ociError()
	}
	return nil
}

// sysName returns a string representing the Tx.
func (tx *Tx) sysName() string {
	if tx == nil {
		return "E_S_S_T_"
	}
	return tx.ses.sysName() + fmt.Sprintf("T%v", tx.id)
}

// log writes a message with an Tx system name and caller info.
func (tx *Tx) log(enabled bool, v ...interface{}) {
	if enabled {
		if len(v) == 0 {
			_drv.cfg.Log.Logger.Infof("%v %v", tx.sysName(), callInfo(1))
		} else {
			_drv.cfg.Log.Logger.Infof("%v %v %v", tx.sysName(), callInfo(1), fmt.Sprint(v...))
		}
	}
}

// log writes a formatted message with an Tx system name and caller info.
func (tx *Tx) logF(enabled bool, format string, v ...interface{}) {
	if enabled {
		if len(v) == 0 {
			_drv.cfg.Log.Logger.Infof("%v %v", tx.sysName(), callInfo(1))
		} else {
			_drv.cfg.Log.Logger.Infof("%v %v %v", tx.sysName(), callInfo(1), fmt.Sprintf(format, v...))
		}
	}
}
<|endoftext|>


<commit_before>/**
 * ws.go - A light weight webserver for static content
 * development and prototyping route based web API.
 *
 * Supports both http and https protocols. Dynamic route
 * processing available via Otto JavaScript virtual machines.
 *
 * @author R. S. Doiel, <rsdoiel@yahoo.com>
 * copyright (c) 2014
 * All rights reserved.
 * @license BSD 2-Clause License
 */
package main

import (
    "./fsengine"
	"./ottoengine"
	"./wslog"
    "./app"
    "./keygen"
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"path/filepath"
)

var REVISION = "v0.0.0-alpha"

// command line parameters that override environment variables
var (
	cli_use_tls   *bool
	cli_docroot   *string
	cli_host      *string
	cli_port      *string
	cli_cert      *string
	cli_key       *string
	cli_otto      *bool
	cli_otto_path *string
	cli_version   *bool
	cli_keygen   = flag.Bool("keygen", false, "Interactive tool to generate TLS certificates and keys")
    cli_init     = flag.Bool("init", false, "Creates a basic project structure in the current working directory")
)

var Usage = func() {
	flag.PrintDefaults()
}

func request_log(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		wslog.LogRequest(r.Method, r.URL, r.RemoteAddr, r.Proto, r.Referer(), r.UserAgent())
		handler.ServeHTTP(w, r)
	})
}

func Webserver(profile *app.Profile) error {
	// If otto is enabled add routes and handle them.
	if profile.Otto == true {
		otto_path, err := filepath.Abs(profile.Otto_Path)
		if err != nil {
			log.Fatalf("Can't read %s: %s\n", profile.Otto_Path, err)
		}
		programs, err := ottoengine.Load(otto_path)
		if err != nil {
			log.Fatalf("Load error: %s\n", err)
		}
		ottoengine.AddRoutes(programs)
	}

	// Restricted FileService excluding dot files and directories
	http.HandleFunc("/", func (w http.ResponseWriter, r *http.Request) {
        // hande off this request/response pair to the fsengine
        fsengine.Engine(profile, w, r)
    })

	// Now start up the server and log transactions
	if profile.Use_TLS == true {
		if profile.Cert == "" || profile.Key == "" {
			log.Fatalf("TLS set true but missing key or certificate")
		}
		log.Println("Starting https://" + net.JoinHostPort(profile.Hostname, profile.Port))
		return http.ListenAndServeTLS(net.JoinHostPort(profile.Hostname, profile.Port), profile.Cert, profile.Key, request_log(http.DefaultServeMux))
	}
	log.Println("Starting http://" + net.JoinHostPort(profile.Hostname, profile.Port))
	// Now start up the server and log transactions
	return http.ListenAndServe(net.JoinHostPort(profile.Hostname, profile.Port), request_log(http.DefaultServeMux))
}

func init() {
	// command line parameters that override environment variables, many have short forms too.
	shortform := " (short form)"

	// No short form
	cli_use_tls = flag.Bool("tls", false, "When true this turns on TLS (https) support.")
	cli_key = flag.String("key", "", "path to your SSL key pem file.")
	cli_cert = flag.String("cert", "", "path to your SSL cert pem file.")

	// These have short forms too
	msg := "This is your document root for static files."
	cli_docroot = flag.String("docroot", "", msg)
	flag.StringVar(cli_docroot, "D", "", msg+shortform)

	msg = "Set this hostname for webserver."
	cli_host = flag.String("host", "", msg)
	flag.StringVar(cli_host, "H", "", msg+shortform)

	msg = "Set the port number to listen on."
	cli_port = flag.String("port", "", msg)
	flag.StringVar(cli_port, "P", "", msg+shortform)

	msg = "When true this option turns on ottoengine. Uses the path defined by WS_OTTO_PATH environment variable or one provided by -O option."
	cli_otto = flag.Bool("otto", false, msg)
	flag.BoolVar(cli_otto, "o", false, msg+shortform)

	msg = "Turns on otto engine using the path for route JavaScript route handlers"
	cli_otto_path = flag.String("otto-path", "", msg)
	flag.StringVar(cli_otto_path, "O", "", msg+shortform)

	msg = "Display the version number of ws command."
	cli_version = flag.Bool("version", false, msg)
	flag.BoolVar(cli_version, "v", false, msg+shortform)

}

func main() {
	flag.Parse()

	if *cli_version == true {
		fmt.Println(REVISION)
		os.Exit(0)
	}

	profile, _ := app.LoadProfile(*cli_docroot, *cli_host, *cli_port, *cli_use_tls, *cli_cert, *cli_key, *cli_otto, *cli_otto_path)
	if *cli_keygen == true {
		certFilename, keyFilename, err := keygen.Keygen("etc/ssl", "cert.pem", "key.pem")
		if err != nil {
			log.Fatalf("%s\n", err)
		}
        fmt.Printf("Wrote %s and %s\n", certFilename, keyFilename)
		os.Exit(0)
	}

    if *cli_init == true {
        err := app.Init()
        if err != nil {
            log.Fatalf("%s\n", err)
        }
        os.Exit(0)
    }

	fmt.Printf("\n\n"+
		"          TLS: %t\n"+
		"         Cert: %s\n"+
		"          Key: %s\n"+
		"      Docroot: %s\n"+
		"         Host: %s\n"+
		"         Port: %s\n"+
		"       Run as: %s\n\n"+
		" Otto enabled: %t\n"+
		"         Path: %s\n"+
		"\n\n",
		profile.Use_TLS,
		profile.Cert,
		profile.Key,
		profile.Docroot,
		profile.Hostname,
		profile.Port,
		profile.Username,
		profile.Otto,
		profile.Otto_Path)
	err := Webserver(profile)
	if err != nil {
		log.Fatal(err)
	}
	os.Exit(0)
}
<commit_msg>reformatted ws.go<commit_after>/**
 * ws.go - A light weight webserver for static content
 * development and prototyping route based web API.
 *
 * Supports both http and https protocols. Dynamic route
 * processing available via Otto JavaScript virtual machines.
 *
 * @author R. S. Doiel, <rsdoiel@yahoo.com>
 * copyright (c) 2014
 * All rights reserved.
 * @license BSD 2-Clause License
 */
package main

import (
	"./app"
	"./fsengine"
	"./keygen"
	"./ottoengine"
	"./wslog"
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"path/filepath"
)

var REVISION = "v0.0.0-alpha"

// command line parameters that override environment variables
var (
	cli_use_tls   *bool
	cli_docroot   *string
	cli_host      *string
	cli_port      *string
	cli_cert      *string
	cli_key       *string
	cli_otto      *bool
	cli_otto_path *string
	cli_version   *bool
	cli_keygen    = flag.Bool("keygen", false, "Interactive tool to generate TLS certificates and keys")
	cli_init      = flag.Bool("init", false, "Creates a basic project structure in the current working directory")
)

var Usage = func() {
	flag.PrintDefaults()
}

func request_log(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		wslog.LogRequest(r.Method, r.URL, r.RemoteAddr, r.Proto, r.Referer(), r.UserAgent())
		handler.ServeHTTP(w, r)
	})
}

func Webserver(profile *app.Profile) error {
	// If otto is enabled add routes and handle them.
	if profile.Otto == true {
		otto_path, err := filepath.Abs(profile.Otto_Path)
		if err != nil {
			log.Fatalf("Can't read %s: %s\n", profile.Otto_Path, err)
		}
		programs, err := ottoengine.Load(otto_path)
		if err != nil {
			log.Fatalf("Load error: %s\n", err)
		}
		ottoengine.AddRoutes(programs)
	}

	// Restricted FileService excluding dot files and directories
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		// hande off this request/response pair to the fsengine
		fsengine.Engine(profile, w, r)
	})

	// Now start up the server and log transactions
	if profile.Use_TLS == true {
		if profile.Cert == "" || profile.Key == "" {
			log.Fatalf("TLS set true but missing key or certificate")
		}
		log.Println("Starting https://" + net.JoinHostPort(profile.Hostname, profile.Port))
		return http.ListenAndServeTLS(net.JoinHostPort(profile.Hostname, profile.Port), profile.Cert, profile.Key, request_log(http.DefaultServeMux))
	}
	log.Println("Starting http://" + net.JoinHostPort(profile.Hostname, profile.Port))
	// Now start up the server and log transactions
	return http.ListenAndServe(net.JoinHostPort(profile.Hostname, profile.Port), request_log(http.DefaultServeMux))
}

func init() {
	// command line parameters that override environment variables, many have short forms too.
	shortform := " (short form)"

	// No short form
	cli_use_tls = flag.Bool("tls", false, "When true this turns on TLS (https) support.")
	cli_key = flag.String("key", "", "path to your SSL key pem file.")
	cli_cert = flag.String("cert", "", "path to your SSL cert pem file.")

	// These have short forms too
	msg := "This is your document root for static files."
	cli_docroot = flag.String("docroot", "", msg)
	flag.StringVar(cli_docroot, "D", "", msg+shortform)

	msg = "Set this hostname for webserver."
	cli_host = flag.String("host", "", msg)
	flag.StringVar(cli_host, "H", "", msg+shortform)

	msg = "Set the port number to listen on."
	cli_port = flag.String("port", "", msg)
	flag.StringVar(cli_port, "P", "", msg+shortform)

	msg = "When true this option turns on ottoengine. Uses the path defined by WS_OTTO_PATH environment variable or one provided by -O option."
	cli_otto = flag.Bool("otto", false, msg)
	flag.BoolVar(cli_otto, "o", false, msg+shortform)

	msg = "Turns on otto engine using the path for route JavaScript route handlers"
	cli_otto_path = flag.String("otto-path", "", msg)
	flag.StringVar(cli_otto_path, "O", "", msg+shortform)

	msg = "Display the version number of ws command."
	cli_version = flag.Bool("version", false, msg)
	flag.BoolVar(cli_version, "v", false, msg+shortform)

}

func main() {
	flag.Parse()

	if *cli_version == true {
		fmt.Println(REVISION)
		os.Exit(0)
	}

	profile, _ := app.LoadProfile(*cli_docroot, *cli_host, *cli_port, *cli_use_tls, *cli_cert, *cli_key, *cli_otto, *cli_otto_path)
	if *cli_keygen == true {
		certFilename, keyFilename, err := keygen.Keygen("etc/ssl", "cert.pem", "key.pem")
		if err != nil {
			log.Fatalf("%s\n", err)
		}
		fmt.Printf("Wrote %s and %s\n", certFilename, keyFilename)
		os.Exit(0)
	}

	if *cli_init == true {
		err := app.Init()
		if err != nil {
			log.Fatalf("%s\n", err)
		}
		os.Exit(0)
	}

	fmt.Printf("\n\n"+
		"          TLS: %t\n"+
		"         Cert: %s\n"+
		"          Key: %s\n"+
		"      Docroot: %s\n"+
		"         Host: %s\n"+
		"         Port: %s\n"+
		"       Run as: %s\n\n"+
		" Otto enabled: %t\n"+
		"         Path: %s\n"+
		"\n\n",
		profile.Use_TLS,
		profile.Cert,
		profile.Key,
		profile.Docroot,
		profile.Hostname,
		profile.Port,
		profile.Username,
		profile.Otto,
		profile.Otto_Path)
	err := Webserver(profile)
	if err != nil {
		log.Fatal(err)
	}
	os.Exit(0)
}
<|endoftext|>


<commit_before>package http

import (
	"fmt"
	"net"
	"net/http"

	"github.com/SparrowDb/sparrowdb/db"
	"github.com/SparrowDb/sparrowdb/slog"
	"github.com/SparrowDb/sparrowdb/spql"
	"github.com/gin-gonic/gin"
)

// HTTPServer holds HTTP server configuration and routes
type HTTPServer struct {
	Config        *db.SparrowConfig
	router        *gin.Engine
	dbManager     *db.DBManager
	queryExecutor *spql.QueryExecutor
	listener      net.Listener
}

func (httpServer *HTTPServer) basicMiddleware() gin.HandlerFunc {
	return func(c *gin.Context) {
		c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
		c.Writer.Header().Set("Server", "SparrowDb")
		c.Next()
	}
}

// Start starts HTTP server listener
func (httpServer *HTTPServer) Start() {
	var err error
	httpServer.listener, err = net.Listen("tcp", fmt.Sprintf("%s:%s", httpServer.Config.HTTPHost, httpServer.Config.HTTPPort))
	if err != nil {
		slog.Fatalf(err.Error())
	}

	handler := NewServeHandler(httpServer.dbManager, httpServer.queryExecutor)

	httpServer.router.Use(httpServer.basicMiddleware())

	httpServer.router.GET("/ping", handler.ping)
	httpServer.router.POST("/query", handler.serveQuery)
	httpServer.router.POST("/upload", handler.upload)
	httpServer.router.GET("/g/:dbname/:key", handler.get)

	http.Serve(httpServer.listener, httpServer.router)
}

// Stop stops HTTP server listener
func (httpServer *HTTPServer) Stop() {
	slog.Infof("Stopping HTTP Server")
	httpServer.listener.Close()
}

// NewHTTPServer returns new HTTPServer
func NewHTTPServer(config *db.SparrowConfig, dbm *db.DBManager) HTTPServer {
	gin.SetMode(gin.ReleaseMode)

	return HTTPServer{
		Config:        config,
		dbManager:     dbm,
		queryExecutor: spql.NewQueryExecutor(dbm),
		router:        gin.New(),
	}
}
<commit_msg>access to url<commit_after>package http

import (
	"fmt"
	"net"
	"net/http"

	"github.com/SparrowDb/sparrowdb/db"
	"github.com/SparrowDb/sparrowdb/slog"
	"github.com/SparrowDb/sparrowdb/spql"
	"github.com/gin-gonic/gin"
)

// HTTPServer holds HTTP server configuration and routes
type HTTPServer struct {
	Config        *db.SparrowConfig
	router        *gin.Engine
	dbManager     *db.DBManager
	queryExecutor *spql.QueryExecutor
	listener      net.Listener
}

func (httpServer *HTTPServer) basicMiddleware() gin.HandlerFunc {
	return func(c *gin.Context) {
		c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
		c.Writer.Header().Set("Server", "SparrowDb")
		c.Next()
	}
}

// Start starts HTTP server listener
func (httpServer *HTTPServer) Start() {
	var err error
	httpServer.listener, err = net.Listen("tcp", fmt.Sprintf("%s:%s", httpServer.Config.HTTPHost, httpServer.Config.HTTPPort))
	if err != nil {
		slog.Fatalf(err.Error())
	}

	handler := NewServeHandler(httpServer.dbManager, httpServer.queryExecutor)

	httpServer.router.Use(httpServer.basicMiddleware())

	r, w, q := httpServer.Config.GetMode()

	httpServer.router.GET("/ping", handler.ping)

	if r == true {
		httpServer.router.GET("/g/:dbname/:key", handler.get)
	}
	if w == true {
		httpServer.router.POST("/upload", handler.upload)
	}
	if q == true {
		httpServer.router.POST("/query", handler.serveQuery)
	}

	http.Serve(httpServer.listener, httpServer.router)
}

// Stop stops HTTP server listener
func (httpServer *HTTPServer) Stop() {
	slog.Infof("Stopping HTTP Server")
	httpServer.listener.Close()
}

// NewHTTPServer returns new HTTPServer
func NewHTTPServer(config *db.SparrowConfig, dbm *db.DBManager) HTTPServer {
	gin.SetMode(gin.ReleaseMode)

	return HTTPServer{
		Config:        config,
		dbManager:     dbm,
		queryExecutor: spql.NewQueryExecutor(dbm),
		router:        gin.New(),
	}
}
<|endoftext|>


<commit_before>package database

import (
	"encoding/json"
	"reflect"
	"strings"

	"github.com/pkg/errors"
)

const (
	CustomJSONOperationIDFollow = "follow"
)

var customJSONOpBodyObjects = map[string]interface{}{
	CustomJSONOperationIDFollow: &FollowOperation{},
}

// FC_REFLECT( steemit::chain::custom_json_operation,
//             (required_auths)
//             (required_posting_auths)
//             (id)
//             (json) )

type CustomJSONOperation struct {
	RequiredAuths        []string `json:"required_auths"`
	RequiredPostingAuths []string `json:"required_posting_auths"`
	ID                   string   `json:"id"`
	JSON                 string   `json:"json"`
}

func (op *CustomJSONOperation) UnmarshalBody() (interface{}, error) {
	// Get the corresponding operation object template.
	bodyTemplate, ok := customJSONOpBodyObjects[op.ID]
	if !ok {
		// In case there is no corresponding template, return unquoted data.
		return op.JSON, nil
	}

	// Clone the template.
	body := reflect.New(reflect.Indirect(reflect.ValueOf(bodyTemplate)).Type()).Interface()

	// Unmarshal into the new object instance.
	if err := json.NewDecoder(strings.NewReader(op.JSON)).Decode(body); err != nil {
		return nil, errors.Wrap(err, "failed to unmarshal CustomJSONOperation.JSON")
	}

	return body, nil
}

type FollowOperation struct {
	Follower  string   `json:"follower"`
	Following string   `json:"following"`
	What      []string `json:"what"`
}
<commit_msg>Fix custom_json operation unmarshalling<commit_after>package database

import (
	"encoding/json"
	"reflect"
	"strings"

	"github.com/pkg/errors"
)

const (
	CustomJSONOperationIDFollow = "follow"
)

var customJSONOpBodyObjects = map[string]interface{}{
	CustomJSONOperationIDFollow: &FollowOperation{},
}

// FC_REFLECT( steemit::chain::custom_json_operation,
//             (required_auths)
//             (required_posting_auths)
//             (id)
//             (json) )

type CustomJSONOperation struct {
	RequiredAuths        []string `json:"required_auths"`
	RequiredPostingAuths []string `json:"required_posting_auths"`
	ID                   string   `json:"id"`
	JSON                 string   `json:"json"`
}

func (op *CustomJSONOperation) UnmarshalBody() (interface{}, error) {
	// Get the corresponding operation object template.
	bodyTemplate, ok := customJSONOpBodyObjects[op.ID]
	if !ok {
		// In case there is no corresponding template, return unquoted data.
		return op.JSON, nil
	}

	// Clone the template.
	body := reflect.New(reflect.Indirect(reflect.ValueOf(bodyTemplate)).Type()).Interface()

	// Prepare the whole operation tuple.
	rawTuple := make([]json.RawMessage, 2)
	if err := json.NewDecoder(strings.NewReader(op.JSON)).Decode(&rawTuple); err != nil {
		return nil, errors.Wrapf(err, "failed to unmarshal CustomJSONOperation.JSON: \n%v", op.JSON)
	}
	if rawTuple[1] == nil {
		return nil, errors.Errorf("invalid CustomJSONOperation.JSON: \n%v", op.JSON)
	}
	data := []byte(rawTuple[1])

	// Unmarshal into the new object instance.
	if err := json.Unmarshal(data, body); err != nil {
		return nil, errors.Wrapf(err, "failed to unmarshal CustomJSONOperation.JSON: \n%v", op.JSON)
	}

	return body, nil
}

type FollowOperation struct {
	Follower  string   `json:"follower"`
	Following string   `json:"following"`
	What      []string `json:"what"`
}
<|endoftext|>


<commit_before>package config

import "flag"

/*
	Gets config from command line arguments
*/
func FromFlags() *Config {
	var expConf ExportConfig

	flag.StringVar(&expConf.URL, "url", "", "Export target")
	flag.StringVar(&expConf.Token, "token", "", "Authentication token")
	flag.Parse()

	return &Config{
		Export:      expConf,
		Restaurants: staticRestaurantConfigs(),
	}
}
<commit_msg>Update command line argument help text<commit_after>package config

import "flag"

/*
	Gets config from command line arguments
*/
func FromFlags() *Config {
	var expConf ExportConfig

	flag.StringVar(&expConf.URL, "url", "", "Export target")
	flag.StringVar(&expConf.Token, "token", "", "Export authentication token")
	flag.Parse()

	return &Config{
		Export:      expConf,
		Restaurants: staticRestaurantConfigs(),
	}
}
<|endoftext|>


<commit_before>package main

import (
	"net/http/httptest"
	"strings"
	"testing"
	"time"

	ws "github.com/gorilla/websocket"
)

var dialer = ws.Dialer{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
}

type EmptyBroadcaster struct {}
func (*EmptyBroadcaster) AddConn(c *ws.Conn){}
func (*EmptyBroadcaster) ManageUsers(){}
func (*EmptyBroadcaster) DisconnectConn(id string){}
func (*EmptyBroadcaster) MsgFromConn(msg []byte){}

//Construct the /ws WebSocket URL that connects to the WebSocket server.
func getWebSocketURL(svr *httptest.Server) string {
	urlNoProtocol := strings.TrimPrefix(svr.URL, "http")
	return "ws" + urlNoProtocol + "/ws"
}

func initServerWithEmptyBroadcaster() *httptest.Server{
	return httptest.NewServer(initMux(&EmptyBroadcaster{}))
}

func initTestServer() (*httptest.Server, *MapBroadcaster) {
	m := initMapBroadcaster()
	svr := httptest.NewServer(initMux(m))
	return svr, m
}

func makeWebSocketConn(t *testing.T, svr *httptest.Server) *ws.Conn {
	wsURL := getWebSocketURL(svr)
	conn, res, err := dialer.Dial(wsURL, nil)
	if err != nil {
		t.Errorf("%v", res.Status)
		t.Fatalf("Dial failed, %v", err)
	}

	return conn
}

//Test that a conection is started when connecting to ws
func TestConn(t *testing.T) {
	svr := initServerWithEmptyBroadcaster()
	defer svr.Close()

	conn := makeWebSocketConn(t, svr)
	conn.Close()
}

//Verify that two connections can be added to the broadcaster
func TestTwoConns(t *testing.T) {
	svr, broadcaster := initTestServer()
	defer svr.Close()

	//Make the first connection
	conn1 := makeWebSocketConn(t, svr)
	defer conn1.Close()

	//Make sure first connection was added to broadcaster
	if broadcaster.highestId != 1 {
		t.Fatalf("broadcaster.highestId: Expected %d, got %d",
			1, broadcaster.highestId)
	}

	//Make the second connection
	conn2 := makeWebSocketConn(t, svr)
	defer conn2.Close()

	//Make sure second connection was added to broadcaster
	if broadcaster.highestId != 2 {
		t.Fatalf("broadcaster.highestId: Expected %d, got %d",
			2, broadcaster.highestId)
	}
}

//Makes sure when a user joins the server they get a message with their ID
//number
func TestGetIdMessage(t *testing.T) {
	svr, _ := initTestServer()
	defer svr.Close()

	yourIdJSON := `{"msgType":"Your ID","data":"1"}`

	//Make a connection
	conn := makeWebSocketConn(t, svr)
	defer conn.Close()

	_, msg, err := conn.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}
	if string(msg) != yourIdJSON {
		t.Fatalf("msg: Expected %s, got %s", yourIdJSON, msg)
	}
}

//Makes sure when a user joins they get a message with the ID numbers and
//coordinates of all users
func TestGetEveryoneMessage(t *testing.T) {
	twoUsersJSON := `{"msgType":"Everyone","data":{"users":{` +
		`"1":{"id":"1","lat":0,"lng":0},` +
		`"2":{"id":"2","lat":0,"lng":0}}}}`

	svr, _ := initTestServer()
	defer svr.Close()

	//Make two connections
	conn1 := makeWebSocketConn(t, svr)
	defer conn1.Close()
	conn2 := makeWebSocketConn(t, svr)
	defer conn2.Close()

	_, _, err := conn2.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}
	_, everyone, err := conn2.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}
	if string(everyone) != twoUsersJSON {
		t.Fatalf("everyone: Expected %s, got %s", twoUsersJSON,  everyone)
	}
}

//Makes sure when a user disconnects they are removed from the broadcaster's
//Users map.
func TestDisconnect(t *testing.T) {
	svr, broadcaster := initTestServer()
	defer svr.Close()

	conn1 := makeWebSocketConn(t, svr)
	defer conn1.Close()
	conn2 := makeWebSocketConn(t, svr)
	conn2.Close()

	//Make sure second connection was removed from broadcaster
	time.Sleep(100)
	if len(broadcaster.Users) != 1 {
		t.Errorf("len(broadcaster.Users): Expected %d, got %d",
			1, len(broadcaster.Users))
	}
}

//Makes sure when a user joins the server all users except the user that joined
//get a "user joined" message
func TestUserJoinedMessage(t *testing.T) {
	svr, _ := initTestServer()
	defer svr.Close()

	//Make the first connection
	conn1 := makeWebSocketConn(t, svr)
	defer conn1.Close()

	//Have the first connection read its ID and everyone messages
	_, msg, err := conn1.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}
	_, msg, err = conn1.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}

	//Make the second connection
	conn2 := makeWebSocketConn(t, svr)
	defer conn2.Close()

	//Have the second connection read its ID and everyone messages
	_, msg, err = conn2.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}
	_, msg, err = conn2.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}

	//Make sure the first user gets the message that the second user joined
	secondUserJoinedJSON := `{"msgType":"User joined","data":"2"}`
	_, msg, err = conn1.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}
	if string(msg) != secondUserJoinedJSON {
		t.Fatalf("msg: Expected %s, got %s", secondUserJoinedJSON, msg)
	}

	//Make sure the message that the second user joined isn't broadcasted to the
	//first user.
	conn2.SetReadDeadline(time.Now().Add(500*time.Millisecond))
	_, msg, err = conn2.ReadMessage()
	if err == nil {
		t.Fatalf("Expected error reading message, msg = %s", msg)
	}
}

//Makes sure when a user disconnects they are removed from the broadcaster's
//Users map.
func TestDisconnectBroadcast(t *testing.T) {
	svr, _ := initTestServer()
	defer svr.Close()

	//Make two connections
	conn1 := makeWebSocketConn(t, svr)
	defer conn1.Close()

	//Have the first connection read its ID and everyone messages
	_, msg, err := conn1.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}

	_, msg, err = conn1.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}

	conn2 := makeWebSocketConn(t, svr)

	//Have the second connection read its ID and everyone messages
	_, msg, err = conn2.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}
	_, msg, err = conn2.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}

	//Read in the message on the first connection that the second user connected
	_, msg, err = conn1.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}

	conn2.Close()

	//Read in the message on the first connection that the second user
	//disconnected
	secondUserDisconnectedJSON := `{"msgType":"User disconnected","data":"2"}`
	_, msg, err = conn1.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}
	if string(msg) != secondUserDisconnectedJSON {
		t.Fatalf("msg: Expected %s, got %s", secondUserDisconnectedJSON, msg)
	}
}

func TestBroadcastCoords(t *testing.T){
	svr, _ := initTestServer()
	defer svr.Close()

	//Make two connections
	conn1 := makeWebSocketConn(t, svr)
	defer conn1.Close()

	//Have the first connection read its ID and everyone messages
	_, msg, err := conn1.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}

	_, msg, err = conn1.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}

	conn2 := makeWebSocketConn(t, svr)
	defer conn2.Close()

	//Have the second connection read its ID and everyone messages
	_, msg, err = conn2.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}
	_, msg, err = conn2.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}
	_, msg, err = conn1.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}

	firstUserAtFreshPondJSON :=
		`{"msgType":"Update coordinates","data":`+
			`{"id":"1","lat":42.388282,"lng":-71.153968}}`

	conn1.WriteMessage(ws.TextMessage,
		[]byte(`{"id":"1","lat":42.388282,"lng":-71.153968}`))

	//Make sure the first user's location is sent to the second user and not the
	//first user.
	_, msg, err = conn2.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}
	if string(msg) != string(firstUserAtFreshPondJSON) {
		t.Fatalf("msg: Expected %s, got %s", firstUserAtFreshPondJSON, msg)
	}

	//Make sure the first user's location wasn't broadcasted to the first user
	conn1.SetReadDeadline(time.Now().Add(500*time.Millisecond))
	_, msg, err = conn1.ReadMessage()
	if err == nil {
		t.Fatalf("Expected error reading message, msg = %s", msg)
	}
}
<commit_msg>Added more documentation to connect_test.go<commit_after>package main

import (
	"net/http/httptest"
	"strings"
	"testing"
	"time"

	ws "github.com/gorilla/websocket"
)

var dialer = ws.Dialer{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
}

type EmptyBroadcaster struct {}
func (*EmptyBroadcaster) AddConn(c *ws.Conn){}
func (*EmptyBroadcaster) ManageUsers(){}
func (*EmptyBroadcaster) DisconnectConn(id string){}
func (*EmptyBroadcaster) MsgFromConn(msg []byte){}

//Construct the /ws WebSocket URL that connects to the WebSocket server.
func getWebSocketURL(svr *httptest.Server) string {
	urlNoProtocol := strings.TrimPrefix(svr.URL, "http")
	return "ws" + urlNoProtocol + "/ws"
}

//initServerWithEmptyBroadcaster initializes an httptest server with the routes
//defined in initMux and an EmptyBroadcaster as the server's Broadcaster and
//returns the server.
func initServerWithEmptyBroadcaster() *httptest.Server{
	return httptest.NewServer(initMux(&EmptyBroadcaster{}))
}

//initTestServer initializes a MapBroadcaster and an httptest server that uses
//that MapBroadcaster and the routes defined in initMux and returns the server
//and the MapBroadcaster.
func initTestServer() (*httptest.Server, *MapBroadcaster) {
	m := initMapBroadcaster()
	svr := httptest.NewServer(initMux(m))
	return svr, m
}

//makeWebSocketConn takes in a testing.T and an httptest Server and returns a
//WebSocket connection to the server. If the connection fails, the test stops,
//sending an error for the response's HTTP status (supposed to be 101) and
//the text of the connection error.
func makeWebSocketConn(t *testing.T, svr *httptest.Server) *ws.Conn {
	wsURL := getWebSocketURL(svr)
	conn, res, err := dialer.Dial(wsURL, nil)
	if err != nil {
		t.Errorf("%v", res.Status)
		t.Fatalf("Dial failed, %v", err)
	}

	return conn
}

//Test that a conection is started when connecting to ws
func TestConn(t *testing.T) {
	svr := initServerWithEmptyBroadcaster()
	defer svr.Close()

	conn := makeWebSocketConn(t, svr)
	conn.Close()
}

//Verify that two connections can be added to the broadcaster
func TestTwoConns(t *testing.T) {
	svr, broadcaster := initTestServer()
	defer svr.Close()

	//Make the first connection
	conn1 := makeWebSocketConn(t, svr)
	defer conn1.Close()

	//Make sure first connection was added to broadcaster
	if broadcaster.highestId != 1 {
		t.Fatalf("broadcaster.highestId: Expected %d, got %d",
			1, broadcaster.highestId)
	}

	//Make the second connection
	conn2 := makeWebSocketConn(t, svr)
	defer conn2.Close()

	//Make sure second connection was added to broadcaster
	if broadcaster.highestId != 2 {
		t.Fatalf("broadcaster.highestId: Expected %d, got %d",
			2, broadcaster.highestId)
	}
}

//Makes sure when a user joins the server they get a message with their ID
//number
func TestGetIdMessage(t *testing.T) {
	svr, _ := initTestServer()
	defer svr.Close()

	yourIdJSON := `{"msgType":"Your ID","data":"1"}`

	//Make a connection
	conn := makeWebSocketConn(t, svr)
	defer conn.Close()

	_, msg, err := conn.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}
	if string(msg) != yourIdJSON {
		t.Fatalf("msg: Expected %s, got %s", yourIdJSON, msg)
	}
}

//Makes sure when a user joins they get a message with the ID numbers and
//coordinates of all users
func TestGetEveryoneMessage(t *testing.T) {
	twoUsersJSON := `{"msgType":"Everyone","data":{"users":{` +
		`"1":{"id":"1","lat":0,"lng":0},` +
		`"2":{"id":"2","lat":0,"lng":0}}}}`

	svr, _ := initTestServer()
	defer svr.Close()

	//Make two connections
	conn1 := makeWebSocketConn(t, svr)
	defer conn1.Close()
	conn2 := makeWebSocketConn(t, svr)
	defer conn2.Close()

	_, _, err := conn2.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}
	_, everyone, err := conn2.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}
	if string(everyone) != twoUsersJSON {
		t.Fatalf("everyone: Expected %s, got %s", twoUsersJSON,  everyone)
	}
}

//Makes sure when a user disconnects they are removed from the broadcaster's
//Users map.
func TestDisconnect(t *testing.T) {
	svr, broadcaster := initTestServer()
	defer svr.Close()

	conn1 := makeWebSocketConn(t, svr)
	defer conn1.Close()
	conn2 := makeWebSocketConn(t, svr)
	conn2.Close()

	//Make sure second connection was removed from broadcaster
	time.Sleep(100)
	if len(broadcaster.Users) != 1 {
		t.Errorf("len(broadcaster.Users): Expected %d, got %d",
			1, len(broadcaster.Users))
	}
}

//Makes sure when a user joins the server all users except the user that joined
//get a "user joined" message
func TestUserJoinedMessage(t *testing.T) {
	svr, _ := initTestServer()
	defer svr.Close()

	//Make the first connection
	conn1 := makeWebSocketConn(t, svr)
	defer conn1.Close()

	//Have the first connection read its ID and everyone messages
	_, msg, err := conn1.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}
	_, msg, err = conn1.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}

	//Make the second connection
	conn2 := makeWebSocketConn(t, svr)
	defer conn2.Close()

	//Have the second connection read its ID and everyone messages
	_, msg, err = conn2.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}
	_, msg, err = conn2.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}

	//Make sure the first user gets the message that the second user joined
	secondUserJoinedJSON := `{"msgType":"User joined","data":"2"}`
	_, msg, err = conn1.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}
	if string(msg) != secondUserJoinedJSON {
		t.Fatalf("msg: Expected %s, got %s", secondUserJoinedJSON, msg)
	}

	//Make sure the message that the second user joined isn't broadcasted to the
	//first user.
	conn2.SetReadDeadline(time.Now().Add(500*time.Millisecond))
	_, msg, err = conn2.ReadMessage()
	if err == nil {
		t.Fatalf("Expected error reading message, msg = %s", msg)
	}
}

//Makes sure when a user disconnects they are removed from the broadcaster's
//Users map.
func TestDisconnectBroadcast(t *testing.T) {
	svr, _ := initTestServer()
	defer svr.Close()

	//Make two connections
	conn1 := makeWebSocketConn(t, svr)
	defer conn1.Close()

	//Have the first connection read its ID and everyone messages
	_, msg, err := conn1.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}

	_, msg, err = conn1.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}

	conn2 := makeWebSocketConn(t, svr)

	//Have the second connection read its ID and everyone messages
	_, msg, err = conn2.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}
	_, msg, err = conn2.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}

	//Read in the message on the first connection that the second user connected
	_, msg, err = conn1.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}

	conn2.Close()

	//Read in the message on the first connection that the second user
	//disconnected
	secondUserDisconnectedJSON := `{"msgType":"User disconnected","data":"2"}`
	_, msg, err = conn1.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}
	if string(msg) != secondUserDisconnectedJSON {
		t.Fatalf("msg: Expected %s, got %s", secondUserDisconnectedJSON, msg)
	}
}

//Makes sure when a user sends a JSON message containing their ID number and
//coordinates that it is broadcasted to every user connected to the broadcaster
//except the user that sent the message.
func TestBroadcastCoords(t *testing.T){
	svr, _ := initTestServer()
	defer svr.Close()

	//Make two connections
	conn1 := makeWebSocketConn(t, svr)
	defer conn1.Close()

	//Have the first connection read its ID and everyone messages
	_, msg, err := conn1.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}

	_, msg, err = conn1.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}

	conn2 := makeWebSocketConn(t, svr)
	defer conn2.Close()

	//Have the second connection read its ID and everyone messages
	_, msg, err = conn2.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}
	_, msg, err = conn2.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}
	_, msg, err = conn1.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}

	firstUserAtFreshPondJSON :=
		`{"msgType":"Update coordinates","data":`+
			`{"id":"1","lat":42.388282,"lng":-71.153968}}`

	conn1.WriteMessage(ws.TextMessage,
		[]byte(`{"id":"1","lat":42.388282,"lng":-71.153968}`))

	//Make sure the first user's location is sent to the second user and not the
	//first user.
	_, msg, err = conn2.ReadMessage()
	if err != nil {
		t.Fatalf("Read message failed, %v", err)
	}
	if string(msg) != string(firstUserAtFreshPondJSON) {
		t.Fatalf("msg: Expected %s, got %s", firstUserAtFreshPondJSON, msg)
	}

	//Make sure the first user's location wasn't broadcasted to the first user
	conn1.SetReadDeadline(time.Now().Add(500*time.Millisecond))
	_, msg, err = conn1.ReadMessage()
	if err == nil {
		t.Fatalf("Expected error reading message, msg = %s", msg)
	}
}
<|endoftext|>


<commit_before>package sql

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"

	"gnd.la/orm/driver"
	"gnd.la/util/generic"
)

type ConstraintType int

const (
	ConstraintNotNull = 1 + iota
	ConstraintUnique
	ConstraintPrimaryKey
	ConstraintForeignKey
)

type FieldOption int

const (
	OptionAutoIncrement = 1 + iota
)

type Reference string

const refSep = "|"

func (r Reference) Table() string {
	s := string(r)
	pos := strings.Index(s, refSep)
	return s[:pos]
}

func (r Reference) Field() string {
	s := string(r)
	pos := strings.Index(s, refSep)
	return s[pos+1:]
}

func MakeReference(table string, field string) Reference {
	return Reference(table + refSep + field)
}

type Field struct {
	Name        string
	Type        string
	Default     string
	Options     []FieldOption
	Constraints []*Constraint
}

func (f *Field) AddOption(opt FieldOption) {
	f.Options = append(f.Options, opt)
}

func (f *Field) HasOption(opt FieldOption) bool {
	for _, v := range f.Options {
		if v == opt {
			return true
		}
	}
	return false
}

func (f *Field) AddConstraint(ct ConstraintType) {
	f.Constraints = append(f.Constraints, &Constraint{Type: ct})
}

func (f *Field) Constraint(ct ConstraintType) *Constraint {
	for _, v := range f.Constraints {
		if v.Type == ct {
			return v
		}
	}
	return nil
}

func (f *Field) SQL(db *DB, m driver.Model, table *Table) (string, []string, error) {
	return db.Backend().DefineField(db, m, table, f)
}

func (f *Field) Copy() *Field {
	fc := *f
	fc.Options = make([]FieldOption, len(f.Options))
	copy(fc.Options, f.Options)
	fc.Constraints = make([]*Constraint, len(f.Constraints))
	copy(fc.Constraints, f.Constraints)
	return &fc
}

type Constraint struct {
	Type       ConstraintType
	References Reference
}

func (c *Constraint) String() string {
	switch c.Type {
	case ConstraintNotNull:
		return "NOT_NULL"
	case ConstraintUnique:
		return "UNIQUE"
	case ConstraintPrimaryKey:
		return "PRIMARY_KEY"
	case ConstraintForeignKey:
		return fmt.Sprintf("FOREIGN_KEY %s", string(c.References))
	}
	return fmt.Sprintf("unknown constraint type %d", int(c.Type))
}

type Table struct {
	Fields      []*Field
	Constraints []*Constraint
}

func (t *Table) PrimaryKeys() []string {
	var keys []string
	for _, v := range t.Fields {
		if v.Constraint(ConstraintPrimaryKey) != nil {
			keys = append(keys, v.Name)
		}
	}
	return keys
}

func (t *Table) definePks(db *DB, m driver.Model) (string, error) {
	pks := t.PrimaryKeys()
	if len(pks) < 2 {
		return "", nil
	}
	pkFields := generic.Map(pks, db.QuoteIdentifier).([]string)
	return fmt.Sprintf("PRIMARY KEY(%s)", strings.Join(pkFields, ", ")), nil
}

func (t *Table) SQL(db *DB, b Backend, m driver.Model, name string) (string, error) {
	var lines []string
	var constraints []string
	for _, v := range t.Fields {
		def, cons, err := v.SQL(db, m, t)
		if err != nil {
			return "", err
		}
		lines = append(lines, def)
		constraints = append(constraints, cons...)
	}
	pk, err := t.definePks(db, m)
	if err != nil {
		return "", err
	}
	if pk != "" {
		lines = append(lines, pk)
	}
	lines = append(lines, constraints...)
	if name == "" {
		name = m.Table()
	}
	sql := fmt.Sprintf("\nCREATE TABLE %s (\n\t%s\n)", db.QuoteIdentifier(name), strings.Join(lines, ",\n\t"))
	return sql, nil
}

type Kind int

const (
	KindInvalid = iota
	KindInteger
	KindFloat
	KindDecimal
	KindChar
	KindVarchar
	KindText
	KindBlob
	KindTime
)

var (
	lengthRe = regexp.MustCompile(`\((\d+)\)`)
)

func fieldLength(typ string) int {
	m := lengthRe.FindStringSubmatch(typ)
	if len(m) > 1 {
		val, _ := strconv.Atoi(m[1])
		return val
	}
	return 0
}

func TypeKind(typ string) (Kind, int) {
	t := strings.ToUpper(typ)
	switch {
	case strings.Contains(t, "INT") || strings.Contains(t, "SERIAL"):
		return KindInteger, 0
	case strings.HasPrefix(t, "VARCHAR") || strings.HasPrefix(t, "CHARACTER VARYING"):
		return KindVarchar, fieldLength(t)
	case strings.HasPrefix(t, "CHAR"):
		return KindChar, fieldLength(t)
	case strings.HasPrefix(t, "BLOB"):
		return KindBlob, 0
	case strings.HasPrefix(t, "TEXT"):
		return KindText, 0
	case t == "DATETIME" || strings.Contains(t, "TIMESTAMP"):
		return KindTime, 0
	}
	return KindInvalid, 0
}
<commit_msg>Use IF NOT EXISTS to create tables<commit_after>package sql

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"

	"gnd.la/orm/driver"
	"gnd.la/util/generic"
)

type ConstraintType int

const (
	ConstraintNotNull = 1 + iota
	ConstraintUnique
	ConstraintPrimaryKey
	ConstraintForeignKey
)

type FieldOption int

const (
	OptionAutoIncrement = 1 + iota
)

type Reference string

const refSep = "|"

func (r Reference) Table() string {
	s := string(r)
	pos := strings.Index(s, refSep)
	return s[:pos]
}

func (r Reference) Field() string {
	s := string(r)
	pos := strings.Index(s, refSep)
	return s[pos+1:]
}

func MakeReference(table string, field string) Reference {
	return Reference(table + refSep + field)
}

type Field struct {
	Name        string
	Type        string
	Default     string
	Options     []FieldOption
	Constraints []*Constraint
}

func (f *Field) AddOption(opt FieldOption) {
	f.Options = append(f.Options, opt)
}

func (f *Field) HasOption(opt FieldOption) bool {
	for _, v := range f.Options {
		if v == opt {
			return true
		}
	}
	return false
}

func (f *Field) AddConstraint(ct ConstraintType) {
	f.Constraints = append(f.Constraints, &Constraint{Type: ct})
}

func (f *Field) Constraint(ct ConstraintType) *Constraint {
	for _, v := range f.Constraints {
		if v.Type == ct {
			return v
		}
	}
	return nil
}

func (f *Field) SQL(db *DB, m driver.Model, table *Table) (string, []string, error) {
	return db.Backend().DefineField(db, m, table, f)
}

func (f *Field) Copy() *Field {
	fc := *f
	fc.Options = make([]FieldOption, len(f.Options))
	copy(fc.Options, f.Options)
	fc.Constraints = make([]*Constraint, len(f.Constraints))
	copy(fc.Constraints, f.Constraints)
	return &fc
}

type Constraint struct {
	Type       ConstraintType
	References Reference
}

func (c *Constraint) String() string {
	switch c.Type {
	case ConstraintNotNull:
		return "NOT_NULL"
	case ConstraintUnique:
		return "UNIQUE"
	case ConstraintPrimaryKey:
		return "PRIMARY_KEY"
	case ConstraintForeignKey:
		return fmt.Sprintf("FOREIGN_KEY %s", string(c.References))
	}
	return fmt.Sprintf("unknown constraint type %d", int(c.Type))
}

type Table struct {
	Fields      []*Field
	Constraints []*Constraint
}

func (t *Table) PrimaryKeys() []string {
	var keys []string
	for _, v := range t.Fields {
		if v.Constraint(ConstraintPrimaryKey) != nil {
			keys = append(keys, v.Name)
		}
	}
	return keys
}

func (t *Table) definePks(db *DB, m driver.Model) (string, error) {
	pks := t.PrimaryKeys()
	if len(pks) < 2 {
		return "", nil
	}
	pkFields := generic.Map(pks, db.QuoteIdentifier).([]string)
	return fmt.Sprintf("PRIMARY KEY(%s)", strings.Join(pkFields, ", ")), nil
}

func (t *Table) SQL(db *DB, b Backend, m driver.Model, name string) (string, error) {
	var lines []string
	var constraints []string
	for _, v := range t.Fields {
		def, cons, err := v.SQL(db, m, t)
		if err != nil {
			return "", err
		}
		lines = append(lines, def)
		constraints = append(constraints, cons...)
	}
	pk, err := t.definePks(db, m)
	if err != nil {
		return "", err
	}
	if pk != "" {
		lines = append(lines, pk)
	}
	lines = append(lines, constraints...)
	if name == "" {
		name = m.Table()
	}
	// Use IF NOT EXISTS, since the DB user might not have
	// the privileges to inspect the database but still
	// be allowed to read and write from the tables (e.g.
	// Postgres only allows superusers and the owner to
	// inspect the database).
	sql := fmt.Sprintf("\nCREATE TABLE IF NOT EXISTS %s (\n\t%s\n)", db.QuoteIdentifier(name), strings.Join(lines, ",\n\t"))
	return sql, nil
}

type Kind int

const (
	KindInvalid = iota
	KindInteger
	KindFloat
	KindDecimal
	KindChar
	KindVarchar
	KindText
	KindBlob
	KindTime
)

var (
	lengthRe = regexp.MustCompile(`\((\d+)\)`)
)

func fieldLength(typ string) int {
	m := lengthRe.FindStringSubmatch(typ)
	if len(m) > 1 {
		val, _ := strconv.Atoi(m[1])
		return val
	}
	return 0
}

func TypeKind(typ string) (Kind, int) {
	t := strings.ToUpper(typ)
	switch {
	case strings.Contains(t, "INT") || strings.Contains(t, "SERIAL"):
		return KindInteger, 0
	case strings.HasPrefix(t, "VARCHAR") || strings.HasPrefix(t, "CHARACTER VARYING"):
		return KindVarchar, fieldLength(t)
	case strings.HasPrefix(t, "CHAR"):
		return KindChar, fieldLength(t)
	case strings.HasPrefix(t, "BLOB"):
		return KindBlob, 0
	case strings.HasPrefix(t, "TEXT"):
		return KindText, 0
	case t == "DATETIME" || strings.Contains(t, "TIMESTAMP"):
		return KindTime, 0
	}
	return KindInvalid, 0
}
<|endoftext|>


<commit_before>package main
import (
    "fmt"
    "strings"
)
// Determin the error type
// Print correct response

func checkError(err error) {
  // IF a port scan
  if strings.Contains(err.Error(), "connection reset by peer") {
    fmt.Println("Error: " + err.Error())
    fmt.Println("* * * * Most likely a port scan! * * * *")

  // Print any other error
  } else if err != nil {
    fmt.Printf("ERROR: " + err.Error() + "\n\n")
  }
}
<commit_msg>re-arrange comment<commit_after>package main
import (
    "fmt"
    "strings"
)
// Determin the error type
// Print correct response

func checkError(err error) {
  // IF a port scan
  if strings.Contains(err.Error(), "connection reset by peer") {
    fmt.Println("Error: " + err.Error())
    fmt.Println("* * * * Most likely a port scan! * * * *")

  // Print any other error
  } else if err != nil {
    fmt.Printf("ERROR: " + err.Error() + "\n\n")
  }
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

import (
	"flag"
	"fmt"
	"io"
	"strings"
	"syscall"
	"testing"
)

// If an IPv6 tunnel is running, we can try dialing a real IPv6 address.
var ipv6 = flag.Bool("ipv6", false, "assume ipv6 tunnel is present")

// fd is already connected to the destination, port 80.
// Run an HTTP request to fetch the appropriate page.
func fetchGoogle(t *testing.T, fd Conn, network, addr string) {
	req := []byte("GET /intl/en/privacy/ HTTP/1.0\r\nHost: www.google.com\r\n\r\n")
	n, err := fd.Write(req)

	buf := make([]byte, 1000)
	n, err = io.ReadFull(fd, buf)

	if n < 1000 {
		t.Errorf("fetchGoogle: short HTTP read from %s %s - %v", network, addr, err)
		return
	}
}

func doDial(t *testing.T, network, addr string) {
	fd, err := Dial(network, addr)
	if err != nil {
		t.Errorf("Dial(%q, %q, %q) = _, %v", network, "", addr, err)
		return
	}
	fetchGoogle(t, fd, network, addr)
	fd.Close()
}

var googleaddrs = []string{
	"%d.%d.%d.%d:80",
	"www.google.com:80",
	"%d.%d.%d.%d:http",
	"www.google.com:http",
	"%03d.%03d.%03d.%03d:0080",
	"[::ffff:%d.%d.%d.%d]:80",
	"[::ffff:%02x%02x:%02x%02x]:80",
	"[0:0:0:0:0000:ffff:%d.%d.%d.%d]:80",
	"[0:0:0:0:000000:ffff:%d.%d.%d.%d]:80",
	"[0:0:0:0:0:ffff::%d.%d.%d.%d]:80",
	"[2001:4860:0:2001::68]:80", // ipv6.google.com; removed if ipv6 flag not set
}

func TestLookupCNAME(t *testing.T) {
	if testing.Short() {
		// Don't use external network.
		t.Logf("skipping external network test during -short")
		return
	}
	cname, err := LookupCNAME("www.google.com")
	if !strings.HasSuffix(cname, ".l.google.com.") || err != nil {
		t.Errorf(`LookupCNAME("www.google.com.") = %q, %v, want "*.l.google.com.", nil`, cname, err)
	}
}

func TestDialGoogle(t *testing.T) {
	if testing.Short() {
		// Don't use external network.
		t.Logf("skipping external network test during -short")
		return
	}
	// If no ipv6 tunnel, don't try the last address.
	if !*ipv6 {
		googleaddrs[len(googleaddrs)-1] = ""
	}

	// Insert an actual IPv4 address for google.com
	// into the table.
	addrs, err := LookupIP("www.google.com")
	if err != nil {
		t.Fatalf("lookup www.google.com: %v", err)
	}
	var ip IP
	for _, addr := range addrs {
		if x := addr.To4(); x != nil {
			ip = x
			break
		}
	}
	if ip == nil {
		t.Fatalf("no IPv4 addresses for www.google.com")
	}

	for i, s := range googleaddrs {
		if strings.Contains(s, "%") {
			googleaddrs[i] = fmt.Sprintf(s, ip[0], ip[1], ip[2], ip[3])
		}
	}

	for i := 0; i < len(googleaddrs); i++ {
		addr := googleaddrs[i]
		if addr == "" {
			continue
		}
		t.Logf("-- %s --", addr)
		doDial(t, "tcp", addr)
		if addr[0] != '[' {
			doDial(t, "tcp4", addr)

			if !preferIPv4 {
				// make sure preferIPv4 flag works.
				preferIPv4 = true
				syscall.SocketDisableIPv6 = true
				doDial(t, "tcp4", addr)
				syscall.SocketDisableIPv6 = false
				preferIPv4 = false
			}
		}

		// Only run tcp6 if the kernel will take it.
		if kernelSupportsIPv6() {
			doDial(t, "tcp6", addr)
		}
	}
}
<commit_msg>net: fix ipv6 test<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

import (
	"flag"
	"fmt"
	"io"
	"strings"
	"syscall"
	"testing"
)

// If an IPv6 tunnel is running, we can try dialing a real IPv6 address.
var ipv6 = flag.Bool("ipv6", false, "assume ipv6 tunnel is present")

// fd is already connected to the destination, port 80.
// Run an HTTP request to fetch the appropriate page.
func fetchGoogle(t *testing.T, fd Conn, network, addr string) {
	req := []byte("GET /intl/en/privacy/ HTTP/1.0\r\nHost: www.google.com\r\n\r\n")
	n, err := fd.Write(req)

	buf := make([]byte, 1000)
	n, err = io.ReadFull(fd, buf)

	if n < 1000 {
		t.Errorf("fetchGoogle: short HTTP read from %s %s - %v", network, addr, err)
		return
	}
}

func doDial(t *testing.T, network, addr string) {
	fd, err := Dial(network, addr)
	if err != nil {
		t.Errorf("Dial(%q, %q, %q) = _, %v", network, "", addr, err)
		return
	}
	fetchGoogle(t, fd, network, addr)
	fd.Close()
}

func TestLookupCNAME(t *testing.T) {
	if testing.Short() {
		// Don't use external network.
		t.Logf("skipping external network test during -short")
		return
	}
	cname, err := LookupCNAME("www.google.com")
	if !strings.HasSuffix(cname, ".l.google.com.") || err != nil {
		t.Errorf(`LookupCNAME("www.google.com.") = %q, %v, want "*.l.google.com.", nil`, cname, err)
	}
}

var googleaddrsipv4 = []string{
	"%d.%d.%d.%d:80",
	"www.google.com:80",
	"%d.%d.%d.%d:http",
	"www.google.com:http",
	"%03d.%03d.%03d.%03d:0080",
	"[::ffff:%d.%d.%d.%d]:80",
	"[::ffff:%02x%02x:%02x%02x]:80",
	"[0:0:0:0:0000:ffff:%d.%d.%d.%d]:80",
	"[0:0:0:0:000000:ffff:%d.%d.%d.%d]:80",
	"[0:0:0:0:0:ffff::%d.%d.%d.%d]:80",
}

func TestDialGoogleIPv4(t *testing.T) {
	if testing.Short() {
		// Don't use external network.
		t.Logf("skipping external network test during -short")
		return
	}

	// Insert an actual IPv4 address for google.com
	// into the table.
	addrs, err := LookupIP("www.google.com")
	if err != nil {
		t.Fatalf("lookup www.google.com: %v", err)
	}
	var ip IP
	for _, addr := range addrs {
		if x := addr.To4(); x != nil {
			ip = x
			break
		}
	}
	if ip == nil {
		t.Fatalf("no IPv4 addresses for www.google.com")
	}

	for i, s := range googleaddrsipv4 {
		if strings.Contains(s, "%") {
			googleaddrsipv4[i] = fmt.Sprintf(s, ip[0], ip[1], ip[2], ip[3])
		}
	}

	for i := 0; i < len(googleaddrsipv4); i++ {
		addr := googleaddrsipv4[i]
		if addr == "" {
			continue
		}
		t.Logf("-- %s --", addr)
		doDial(t, "tcp", addr)
		if addr[0] != '[' {
			doDial(t, "tcp4", addr)
			if !preferIPv4 {
				// make sure preferIPv4 flag works.
				preferIPv4 = true
				syscall.SocketDisableIPv6 = true
				doDial(t, "tcp", addr)
				doDial(t, "tcp4", addr)
				syscall.SocketDisableIPv6 = false
				preferIPv4 = false
			}
		}
	}
}

var googleaddrsipv6 = []string{
	"[%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x]:80",
	"ipv6.google.com:80",
	"[%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x]:http",
	"ipv6.google.com:http",
}

func TestDialGoogleIPv6(t *testing.T) {
	if testing.Short() {
		// Don't use external network.
		t.Logf("skipping external network test during -short")
		return
	}
	// Only run tcp6 if the kernel will take it.
	if !*ipv6 || !kernelSupportsIPv6() {
		return
	}

	// Insert an actual IPv6 address for ipv6.google.com
	// into the table.
	addrs, err := LookupIP("ipv6.google.com")
	if err != nil {
		t.Fatalf("lookup ipv6.google.com: %v", err)
	}
	var ip IP
	for _, addr := range addrs {
		if x := addr.To16(); x != nil {
			ip = x
			break
		}
	}
	if ip == nil {
		t.Fatalf("no IPv6 addresses for ipv6.google.com")
	}

	for i, s := range googleaddrsipv6 {
		if strings.Contains(s, "%") {
			googleaddrsipv6[i] = fmt.Sprintf(s, ip[0], ip[1], ip[2], ip[3], ip[4], ip[5], ip[6], ip[7], ip[8], ip[9], ip[10], ip[11], ip[12], ip[13], ip[14], ip[15])
		}
	}

	for i := 0; i < len(googleaddrsipv6); i++ {
		addr := googleaddrsipv6[i]
		if addr == "" {
			continue
		}
		t.Logf("-- %s --", addr)
		doDial(t, "tcp", addr)
		doDial(t, "tcp6", addr)
	}
}
<|endoftext|>