<commit_before>package dbf

import (
	"fmt"
	"strings"
	"testing"
)

const (
	TEST_DBF_PATH  = "./testdbf/TEST.DBF"
	BENCH_DBF_PATH = "C:/DATA/KENTEKEN.DBF" //"./testdbf/TEST.DBF" //For real benchmarks replace this with the path to a large DBF/FPT combo
)

var test_dbf *DBF

func TestOpenFile(t *testing.T) {
	var err error
	test_dbf, err = OpenFile(TEST_DBF_PATH)
	if err != nil {
		t.Fatal(err)
	}
}

//Quick check if the first field matches
func TestFieldHeader(t *testing.T) {
	want := "{Name:[73 68 0 0 0 0 0 0 0 0 0] Type:73 Pos:1 Len:4 Decimals:0 Flags:0 Next:5 Step:1 Reserved:[0 0 0 0 0 0 0 78]}"
	have := fmt.Sprintf("%+v", test_dbf.fields[0])
	if have != want {
		t.Errorf("First field from header does not match signature: Want %s, have %s", want, have)
	}
}

//Test if the modified date of Stat() matches the header
//This is therefore also a header test, these dates should be equal, but not sure if this is always true on every OS
//Update: Disable for now, fails on other timezones
/*
func TestStat(t *testing.T) {
	stat, err := test_dbf.Stat()
	if err != nil {
		t.Fatal(err)
	}
	stat_mod := stat.ModTime()
	hdr_mod := test_dbf.header.Modified()
	format := "20060102"
	if stat_mod.Format(format) != hdr_mod.Format(format) {
		t.Errorf("Modified date in header (%s) not equal to modified date in OS (%s)", hdr_mod.Format(format), stat_mod.Format(format))
	}
}*/
//test with size instead
func TestStatAndFileSize(t *testing.T) {
	stat, err := test_dbf.Stat()
	if err != nil {
		t.Fatal(err)
	}
	stat_size := stat.Size()
	hdr_size := test_dbf.header.FileSize()
	if stat_size != hdr_size {
		t.Errorf("Calculated header size: %d, stat size: %d", hdr_size, stat_size)
	}
}

//Tests if field headers have been parsed, fails if there are no fields
func TestFieldNames(t *testing.T) {
	fieldnames := test_dbf.FieldNames()
	want := 13
	if len(fieldnames) != want {
		t.Errorf("Expected %d fields, have %d", want, len(fieldnames))
	}
	//t.Log(fieldnames)
}

func TestNumFields(t *testing.T) {
	header := test_dbf.NumFields()
	header_calc := test_dbf.Header().NumFields()
	if header != header_calc {
		t.Errorf("NumFields not equal. DBF NumFields: %d, DBF Header NumField: %d", header, header_calc)
	}
}

func TestGoTo(t *testing.T) {
	err := test_dbf.GoTo(0)
	if err != nil {
		t.Error(err)
	}
	if !test_dbf.BOF() {
		t.Error("Expected to be at BOF")
	}
	err = test_dbf.GoTo(1)
	if err != nil {
		t.Error(err)
	}
	if test_dbf.EOF() {
		t.Error("Did not expect to be at EOF")
	}
	err = test_dbf.GoTo(4)
	if err != nil {
		if err != ErrEOF {
			t.Error(err)
		}
	}
	if !test_dbf.EOF() {
		t.Error("Expected to be at EOF")
	}
}

func TestSkip(t *testing.T) {
	test_dbf.GoTo(0)

	err := test_dbf.Skip(1)
	if err != nil {
		t.Error(err)
	}
	if test_dbf.EOF() {
		t.Error("Did not expect to be at EOF")
	}
	err = test_dbf.Skip(3)
	if err != nil {
		if err != ErrEOF {
			t.Error(err)
		}
	}
	if !test_dbf.EOF() {
		t.Error("Expected to be at EOF")
	}
	err = test_dbf.Skip(-20)
	if err != nil {
		if err != ErrBOF {
			t.Error(err)
		}
	}
	if !test_dbf.BOF() {
		t.Error("Expected to be at BOF")
	}
}

var want_values = []struct {
	pos                   int
	name, strval, strtype string
}{
	{0, "ID", "2", "int32"},
	{10, "NUMBER", "1.2345678999e+08", "float64"},
	{12, "BOOL", "true", "bool"},
	{2, "DATUM", "2015-02-03 00:00:00 +0000 UTC", "time.Time"},
	{7, "COMP_NAME", "TEST2", "string"},
}

func TestFieldPos(t *testing.T) {

	for _, want := range want_values {
		pos := test_dbf.FieldPos(want.name)
		if pos != want.pos {
			t.Errorf("Wanted fieldpos %d for field %s, have pos %d", want.pos, want.name, pos)
		}
	}
}

//Tests a complete record read, reads the second record which is also deleted,
//also tests getting field values from record object
func TestRecord(t *testing.T) {
	err := test_dbf.GoTo(1)
	if err != nil {
		t.Fatal(err)
	}
	rec, err := test_dbf.Record()
	if err != nil {
		t.Fatal(err)
	}
	t.Log(rec.data)

	//Get fields by pos
	for _, want := range want_values {
		val, err := rec.Field(want.pos)
		if err != nil {
			t.Error(err)
		}
		strval := strings.TrimSpace(fmt.Sprintf("%v", val))
		strtype := fmt.Sprintf("%T", val)

		if want.strval != strval || want.strtype != strtype {
			t.Errorf("Wanted value %s with type %s, have value %s with type %s", want.strval, want.strtype, strval, strtype)
		}
	}
}

//Test reading fields field by field
func TestField(t *testing.T) {
	for _, want := range want_values {
		val, err := test_dbf.Field(want.pos)
		if err != nil {
			t.Error(err)
		}
		strval := strings.TrimSpace(fmt.Sprintf("%v", val))
		strtype := fmt.Sprintf("%T", val)

		if want.strval != strval || want.strtype != strtype {
			t.Errorf("Wanted value %s with type %s, have value %s with type %s", want.strval, want.strtype, strval, strtype)
		}
	}
}

//Close file handles
func TestClose(t *testing.T) {
	err := test_dbf.Close()
	if err != nil {
		t.Fatal(err)
	}
}

//Benchmark for reading all records sequentially
//Use a large DBF/FPT combo for more realistic results
func BenchmarkReadRecords(b *testing.B) {
	for n := 0; n < b.N; n++ {
		err := func() error {
			dbf, err := OpenFile(BENCH_DBF_PATH)
			if err != nil {
				return err
			}
			defer dbf.Close()
			for i := uint32(0); i < dbf.NumRecords(); i++ {
				_, err := dbf.Record()
				if err != nil {
					return err
				}
				dbf.Skip(1)
			}
			return nil
		}()
		if err != nil {
			b.Fatal(err)
		}
	}
}
<commit_msg>Fix tests<commit_after>package dbf

import (
	"fmt"
	"strings"
	"testing"
)

const (
	TEST_DBF_PATH  = "./testdbf/TEST.DBF"
	BENCH_DBF_PATH = "./testdbf/TEST.DBF" //For real benchmarks replace this with the path to a large DBF/FPT combo
)

var test_dbf *DBF

func TestOpenFile(t *testing.T) {
	var err error
	test_dbf, err = OpenFile(TEST_DBF_PATH)
	if err != nil {
		t.Fatal(err)
	}
}

//Quick check if the first field matches
func TestFieldHeader(t *testing.T) {
	want := "{Name:[73 68 0 0 0 0 0 0 0 0 0] Type:73 Pos:1 Len:4 Decimals:0 Flags:0 Next:5 Step:1 Reserved:[0 0 0 0 0 0 0 78]}"
	have := fmt.Sprintf("%+v", test_dbf.fields[0])
	if have != want {
		t.Errorf("First field from header does not match signature: Want %s, have %s", want, have)
	}
}

//Test if the modified date of Stat() matches the header
//This is therefore also a header test, these dates should be equal, but not sure if this is always true on every OS
//Update: Disable for now, fails on other timezones
/*
func TestStat(t *testing.T) {
	stat, err := test_dbf.Stat()
	if err != nil {
		t.Fatal(err)
	}
	stat_mod := stat.ModTime()
	hdr_mod := test_dbf.header.Modified()
	format := "20060102"
	if stat_mod.Format(format) != hdr_mod.Format(format) {
		t.Errorf("Modified date in header (%s) not equal to modified date in OS (%s)", hdr_mod.Format(format), stat_mod.Format(format))
	}
}*/
//test with size instead
func TestStatAndFileSize(t *testing.T) {
	stat, err := test_dbf.Stat()
	if err != nil {
		t.Fatal(err)
	}
	stat_size := stat.Size()
	hdr_size := test_dbf.header.FileSize()
	if stat_size != hdr_size {
		t.Errorf("Calculated header size: %d, stat size: %d", hdr_size, stat_size)
	}
}

//Tests if field headers have been parsed, fails if there are no fields
func TestFieldNames(t *testing.T) {
	fieldnames := test_dbf.FieldNames()
	want := 13
	if len(fieldnames) != want {
		t.Errorf("Expected %d fields, have %d", want, len(fieldnames))
	}
	//t.Log(fieldnames)
}

func TestNumFields(t *testing.T) {
	header := test_dbf.NumFields()
	header_calc := test_dbf.Header().NumFields()
	if header != header_calc {
		t.Errorf("NumFields not equal. DBF NumFields: %d, DBF Header NumField: %d", header, header_calc)
	}
}

func TestGoTo(t *testing.T) {
	err := test_dbf.GoTo(0)
	if err != nil {
		t.Error(err)
	}
	if !test_dbf.BOF() {
		t.Error("Expected to be at BOF")
	}
	err = test_dbf.GoTo(1)
	if err != nil {
		t.Error(err)
	}
	if test_dbf.EOF() {
		t.Error("Did not expect to be at EOF")
	}
	err = test_dbf.GoTo(4)
	if err != nil {
		if err != ErrEOF {
			t.Error(err)
		}
	}
	if !test_dbf.EOF() {
		t.Error("Expected to be at EOF")
	}
}

func TestSkip(t *testing.T) {
	test_dbf.GoTo(0)

	err := test_dbf.Skip(1)
	if err != nil {
		t.Error(err)
	}
	if test_dbf.EOF() {
		t.Error("Did not expect to be at EOF")
	}
	err = test_dbf.Skip(3)
	if err != nil {
		if err != ErrEOF {
			t.Error(err)
		}
	}
	if !test_dbf.EOF() {
		t.Error("Expected to be at EOF")
	}
	err = test_dbf.Skip(-20)
	if err != nil {
		if err != ErrBOF {
			t.Error(err)
		}
	}
	if !test_dbf.BOF() {
		t.Error("Expected to be at BOF")
	}
}

var want_values = []struct {
	pos                   int
	name, strval, strtype string
}{
	{0, "ID", "2", "int32"},
	{10, "NUMBER", "1.2345678999e+08", "float64"},
	{12, "BOOL", "true", "bool"},
	{2, "DATUM", "2015-02-03 00:00:00 +0000 UTC", "time.Time"},
	{7, "COMP_NAME", "TEST2", "string"},
}

func TestFieldPos(t *testing.T) {

	for _, want := range want_values {
		pos := test_dbf.FieldPos(want.name)
		if pos != want.pos {
			t.Errorf("Wanted fieldpos %d for field %s, have pos %d", want.pos, want.name, pos)
		}
	}
}

//Tests a complete record read, reads the second record which is also deleted,
//also tests getting field values from record object
func TestRecord(t *testing.T) {
	err := test_dbf.GoTo(1)
	if err != nil {
		t.Fatal(err)
	}
	rec, err := test_dbf.Record()
	if err != nil {
		t.Fatal(err)
	}
	//t.Log(rec.data)

	//Get fields by pos
	for _, want := range want_values {
		val, err := rec.Field(want.pos)
		if err != nil {
			t.Error(err)
		}
		strval := strings.TrimSpace(fmt.Sprintf("%v", val))
		strtype := fmt.Sprintf("%T", val)

		if want.strval != strval || want.strtype != strtype {
			t.Errorf("Wanted value %s with type %s, have value %s with type %s", want.strval, want.strtype, strval, strtype)
		}
	}
}

//Test reading fields field by field
func TestField(t *testing.T) {
	for _, want := range want_values {
		val, err := test_dbf.Field(want.pos)
		if err != nil {
			t.Error(err)
		}
		strval := strings.TrimSpace(fmt.Sprintf("%v", val))
		strtype := fmt.Sprintf("%T", val)

		if want.strval != strval || want.strtype != strtype {
			t.Errorf("Wanted value %s with type %s, have value %s with type %s", want.strval, want.strtype, strval, strtype)
		}
	}
}

//Close file handles
func TestClose(t *testing.T) {
	err := test_dbf.Close()
	if err != nil {
		t.Fatal(err)
	}
}

//Benchmark for reading all records sequentially
//Use a large DBF/FPT combo for more realistic results
func BenchmarkReadRecords(b *testing.B) {
	for n := 0; n < b.N; n++ {
		err := func() error {
			dbf, err := OpenFile(BENCH_DBF_PATH)
			if err != nil {
				return err
			}
			defer dbf.Close()
			for i := uint32(0); i < dbf.NumRecords(); i++ {
				_, err := dbf.Record()
				if err != nil {
					return err
				}
				dbf.Skip(1)
			}
			return nil
		}()
		if err != nil {
			b.Fatal(err)
		}
	}
}
<|endoftext|>


<commit_before>package record

import (
	"encoding/json"
	"errors"
	"fmt"
	"strings"

	ch "github.com/BatchLabs/charlatan"
)

// JSONRecord is a record for JSON objects.
//
// It supports the special field "*", as in "SELECT * FROM x WHERE y", which
// returns the JSON as-is, except that the keys order is not garanteed.
//
// If the SoftMatching attribute is set to true, non-existing fields are
// returned as null contants instead of failing with an error.
type JSONRecord struct {
	attrs        map[string]*json.RawMessage
	SoftMatching bool
}

var _ ch.Record = &JSONRecord{}

var errEmptyField = errors.New("Empty field name")

// NewJSONRecordFromDecoder creates a new JSONRecord from a JSON decoder
func NewJSONRecordFromDecoder(dec *json.Decoder) (*JSONRecord, error) {
	attrs := make(map[string]*json.RawMessage)

	if err := dec.Decode(&attrs); err != nil {
		return nil, err
	}

	return &JSONRecord{attrs: attrs}, nil
}

// Find implements the charlatan.Record interface
func (r *JSONRecord) Find(field *ch.Field) (*ch.Const, error) {
	var ok bool
	var partial *json.RawMessage
	var name string

	if name = field.Name(); len(name) == 0 {
		return nil, errEmptyField
	}

	// support for "SELECT *"
	if name == "*" {
		b, err := json.Marshal(r.attrs)
		if err != nil {
			return nil, err
		}
		return ch.StringConst(string(b)), nil
	}

	attrs := r.attrs
	parts := strings.Split(name, ".")

	for i, k := range parts {
		partial, ok = attrs[k]

		if !ok {
			if r.SoftMatching {
				return ch.NullConst(), nil
			}

			return nil, fmt.Errorf("Unknown '%s' field (in '%s')", k, field.Name())
		}

		// update the attrs if we need to go deeper
		if i < len(parts)-1 {
			attrs = make(map[string]*json.RawMessage)
			if err := json.Unmarshal(*partial, &attrs); err != nil {
				if r.SoftMatching {
					return ch.NullConst(), nil
				}
				return nil, err
			}
		}
	}

	return jsonToConst(partial)
}

func jsonToConst(partial *json.RawMessage) (*ch.Const, error) {
	var value string

	if partial == nil {
		return ch.NullConst(), nil
	}

	asString := string(*partial)

	if asString == "null" {
		return ch.NullConst(), nil
	}

	if err := json.Unmarshal(*partial, &value); err != nil {
		if err, ok := err.(*json.UnmarshalTypeError); ok {
			// we failed to unmarshal into a string, let's try the other types
			switch err.Value {
			case "number":
				var n json.Number
				if err := json.Unmarshal(*partial, &n); err != nil {
					return nil, err
				}

				value = n.String()

			case "bool":
				value = asString

			default:
				return nil, err
			}
		}
	}

	return ch.ConstFromString(value), nil
}
<commit_msg>record: treat objects and arrays as a string<commit_after>package record

import (
	"encoding/json"
	"errors"
	"fmt"
	"strings"

	ch "github.com/BatchLabs/charlatan"
)

// JSONRecord is a record for JSON objects.
//
// It supports the special field "*", as in "SELECT * FROM x WHERE y", which
// returns the JSON as-is, except that the keys order is not garanteed.
//
// If the SoftMatching attribute is set to true, non-existing fields are
// returned as null contants instead of failing with an error.
type JSONRecord struct {
	attrs        map[string]*json.RawMessage
	SoftMatching bool
}

var _ ch.Record = &JSONRecord{}

var errEmptyField = errors.New("Empty field name")

// NewJSONRecordFromDecoder creates a new JSONRecord from a JSON decoder
func NewJSONRecordFromDecoder(dec *json.Decoder) (*JSONRecord, error) {
	attrs := make(map[string]*json.RawMessage)

	if err := dec.Decode(&attrs); err != nil {
		return nil, err
	}

	return &JSONRecord{attrs: attrs}, nil
}

// Find implements the charlatan.Record interface
func (r *JSONRecord) Find(field *ch.Field) (*ch.Const, error) {
	var ok bool
	var partial *json.RawMessage
	var name string

	if name = field.Name(); len(name) == 0 {
		return nil, errEmptyField
	}

	// support for "SELECT *"
	if name == "*" {
		b, err := json.Marshal(r.attrs)
		if err != nil {
			return nil, err
		}
		return ch.StringConst(string(b)), nil
	}

	attrs := r.attrs
	parts := strings.Split(name, ".")

	for i, k := range parts {
		partial, ok = attrs[k]

		if !ok {
			if r.SoftMatching {
				return ch.NullConst(), nil
			}

			return nil, fmt.Errorf("Unknown '%s' field (in '%s')", k, field.Name())
		}

		// update the attrs if we need to go deeper
		if i < len(parts)-1 {
			attrs = make(map[string]*json.RawMessage)
			if err := json.Unmarshal(*partial, &attrs); err != nil {
				if r.SoftMatching {
					return ch.NullConst(), nil
				}
				return nil, err
			}
		}
	}

	return jsonToConst(partial)
}

func jsonToConst(partial *json.RawMessage) (*ch.Const, error) {
	var value string

	if partial == nil {
		return ch.NullConst(), nil
	}

	asString := string(*partial)

	if asString == "null" {
		return ch.NullConst(), nil
	}

	if err := json.Unmarshal(*partial, &value); err != nil {
		if err, ok := err.(*json.UnmarshalTypeError); ok {
			// we failed to unmarshal into a string, let's try the other types
			switch err.Value {
			case "number":
				var n json.Number
				if err := json.Unmarshal(*partial, &n); err != nil {
					return nil, err
				}

				value = n.String()

			case "bool", "object", "array":
				value = asString

			default:
				return nil, err
			}
		}
	}

	return ch.ConstFromString(value), nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Stuart Glenn, OMRF. All rights reserved.
// Use of this code is governed by a 3 clause BSD style license
// Full license details in LICENSE file distributed with this software

package mmatcher

import (
	"strings"
	"testing"
)

func TestMatch(t *testing.T) {
	a := &Record{ID: "a", Atts: []Atter{TextAtt{"text"}, NumericAtt{1}, TextAtt{"green"}}}
	b := &Record{ID: "b", Atts: []Atter{TextAtt{"text"}, NumericAtt{1}, TextAtt{"red"}}}
	if a.IsMatch(b) {
		t.Error("A should not match b on all attributes")
	}
	if !a.IsMatch(a) {
		t.Error("A should match itself on all attributes")
	}
	if !a.IsMatch(b, 0) {
		t.Error("A should match b on just first attribute")
	}
	if !b.IsMatch(a, []int{0, 1}...) {
		t.Error("B should match a on first two attributes")
	}
	if a.IsMatch(b, 2) {
		t.Error("A should not match B on third attribute")
	}
	if a.IsMatch(a, 100) {
		t.Error("A should not match itself with attribute out of bounds")
	}
	if !a.IsMatch(a, 0, 1, 2) {
		t.Error("A should match itself with all attributes specified")
	}
}

func TestMatchAtt(t *testing.T) {
	a := &Record{ID: "a", Atts: []Atter{TextAtt{"first"}, NumericAtt{1}, NumericAtt{8}, TextAtt{"yes"}}}
	b := &Record{ID: "b", Atts: []Atter{TextAtt{"second"}, NumericAtt{4.2}, NumericAtt{8}, TextAtt{"yes"}}}
	if !a.isMatchAt(b, NumericAtt{4}, 1) {
		t.Error("A numeric att should match b with an epsilon")
	}
	if !a.isMatchAt(b, NumericAtt{3.2}, 1) {
		t.Error("A numeric att should match b with an epsilon")
	}
	if a.isMatchAt(b, NumericAtt{2}, 1) {
		t.Error("A numeric att should not match b with a small epsilon")
	}
	e := make([]Atter, len(a.Atts))
	e[1] = NumericAtt{2}
	if a.IsMatchWithRanges(b, e, 1) {
		t.Error("A should not match b with small range on one index")
	}
	if a.IsMatchWithRanges(b, e, 1, 2, 3) {
		t.Error("A should not match on multi indices b with small range on one index")
	}
	e[1] = NumericAtt{3.2}
	tests := map[int]bool{
		0: false,
		1: true,
		2: true,
		3: true,
	}
	for index, result := range tests {
		if result != a.isMatchAt(b, e[index], index) {
			t.Errorf("A:%v att:%v match? b:%v with a %v", a, index, b, e[index])
		}
	}
}

func TestMatchWrongSizes(t *testing.T) {
	a := &Record{ID: "a", Atts: []Atter{TextAtt{"first"}, NumericAtt{1}}}
	b := &Record{ID: "b", Atts: []Atter{TextAtt{"second"}, NumericAtt{4.2}, NumericAtt{8}, TextAtt{"yes"}}}
	if a.IsMatchWithRanges(b, make([]Atter, len(a.Atts))) {
		t.Error("A longer record should not match a shorter record")
	}
	if a.IsMatchWithRanges(a, make([]Atter, 0)) {
		t.Error("A record cannot match itself is the []range is too short")
	}
	if !a.IsMatchWithRanges(a, make([]Atter, len(a.Atts))) {
		t.Error("A record should match itself completely")
	}
	if a.isMatchAt(a, TextAtt{}, len(a.Atts)+1) {
		t.Error("A record cannot equal even itself at a position past the record")
	}
	if a.isMatchAt(a, TextAtt{}, -1) {
		t.Error("A record cannot equal even itself at a position before the record")
	}
}

func TestMatchRange(t *testing.T) {
	a := &Record{ID: "a", Atts: []Atter{TextAtt{"first"}, NumericAtt{1}, NumericAtt{8}, TextAtt{"yes"}}}
	b := &Record{ID: "b", Atts: []Atter{TextAtt{"second"}, NumericAtt{4.2}, NumericAtt{8}, TextAtt{"yes"}}}
	if a.IsMatch(b) {
		t.Error("A and b should not match")
	}
	e := make([]Atter, len(a.Atts))
	e[1] = NumericAtt{2}
	if a.IsMatchWithRanges(b, e, 1) {
		t.Error("A should not match b with small range on one index")
	}
	if a.IsMatchWithRanges(b, e, 1, 2, 3) {
		t.Error("A should not match on multi indices b with small range on one index")
	}
	e[1] = NumericAtt{3.2}
	i := []int{1}
	if !a.IsMatchWithRanges(b, e, i...) {
		t.Errorf("A:%v should match b:%v with correct range:%v on %v", a, b, e, i)
	}
	i = []int{1, 2, 3}
	if !a.IsMatchWithRanges(b, e, i...) {
		t.Errorf("A:%v should match b:%v with correct range:%v on %v", a, b, e, i)
	}
	e[1] = NumericAtt{3.0}
	if a.IsMatchWithRanges(b, e, i...) {
		t.Errorf("A:%v should not match b:%v with small range:%v on %v", a, b, e, i)
	}
	i = []int{2, 3}
	if !a.IsMatchWithRanges(b, e, i...) {
		t.Errorf("A:%v should match b:%v with small range:%v on %v", a, b, e, i)
	}
}

func TestMatchesAll(t *testing.T) {
	a := Records{
		Record{ID: "a1", Atts: []Atter{TextAtt{"red"}, NumericAtt{1}}},
		Record{ID: "a2", Atts: []Atter{TextAtt{"red"}, NumericAtt{20}}},
		Record{ID: "a3", Atts: []Atter{TextAtt{"green"}, NumericAtt{30}}},
	}
	b := Records{
		Record{ID: "b1", Atts: []Atter{TextAtt{"green"}, NumericAtt{5}}},
		Record{ID: "b2", Atts: []Atter{TextAtt{"red"}, NumericAtt{25}}},
		Record{ID: "b3", Atts: []Atter{TextAtt{"red"}, NumericAtt{35}}},
		Record{ID: "b4", Atts: []Atter{TextAtt{"green"}, NumericAtt{30}}},
		Record{ID: "b5", Atts: []Atter{TextAtt{"green"}, NumericAtt{31}}},
		Record{ID: "b6", Atts: []Atter{TextAtt{"red"}, NumericAtt{20}}},
		Record{ID: "b7", Atts: []Atter{TextAtt{"red"}, NumericAtt{20}}},
	}

	if m := a[0].MatchesAll(b); len(m) != 0 {
		t.Errorf("%v should not have found any matches in %v", a[0], b)
	}
	m := a[2].MatchesAll(b)
	if len(m) != 1 {
		t.Errorf("%v should have found one match in %v, but found %v", a[2], b, m)
	}
	if 3 != m[0] {
		t.Errorf("%v should have found one at 1, but found it at %v", a[2], m[0])
	}

	m = a[1].MatchesAll(b)
	if len(m) != 2 {
		t.Errorf("%v should have found two matches in %v, but found %v", a[1], b, m)
	}
	if 5 != m[0] && 6 != m[1] {
		t.Errorf("%v should have found one at 5 & 6, but found it at %v", a[1], m)
	}

	e := []Atter{TextAtt{}, NumericAtt{5}}
	m = a[0].MatchesAll(b, e...)
	if 0 != len(m) {
		t.Errorf("%v should not have found any matches in %v using %v", a[0], b, e)
	}
	m = a[1].MatchesAll(b, e...)
	if 3 != len(m) {
		t.Errorf("%v should have found 3 in %v using %v, but found %v", a[0], b, e, m)
	}
	m = a[2].MatchesAll(b, e...)
	if 2 != len(m) {
		t.Errorf("%v should have found 2 in %v using %v, but found %v", a[0], b, e, m)
	}
}

func TestMatchesColumns(t *testing.T) {
	a := Records{
		Record{ID: "a0", Atts: []Atter{TextAtt{"red"}, NumericAtt{1}}},
		Record{ID: "a1", Atts: []Atter{TextAtt{"red"}, NumericAtt{20}}},
		Record{ID: "a2", Atts: []Atter{TextAtt{"green"}, NumericAtt{30}}},
	}
	b := Records{
		Record{ID: "b0", Atts: []Atter{TextAtt{"green"}, NumericAtt{5}}},
		Record{ID: "b1", Atts: []Atter{TextAtt{"red"}, NumericAtt{25}}},
		Record{ID: "b2", Atts: []Atter{TextAtt{"red"}, NumericAtt{35}}},
		Record{ID: "b3", Atts: []Atter{TextAtt{"green"}, NumericAtt{30}}},
		Record{ID: "b4", Atts: []Atter{TextAtt{"green"}, NumericAtt{31}}},
		Record{ID: "b5", Atts: []Atter{TextAtt{"red"}, NumericAtt{20}}},
		Record{ID: "b6", Atts: []Atter{TextAtt{"red"}, NumericAtt{20}}},
	}

	c := []int{10}
	if m := a[0].Matches(b, c); len(m) != 0 {
		t.Errorf("%v should have found 0 matches using column %v in %v, instead found %v", a[0], c, b, m)
	}

	c = []int{0}
	if m := a[0].Matches(b, c); len(m) != 4 {
		t.Errorf("%v should have found 4 matches in %v, instead found %v", a[0], b, m)
	}
	m := a[2].Matches(b, c)
	if len(m) != 3 {
		t.Errorf("%v should have found 3 matches in %v, but found %v", a[2], b, m)
	}
	if 0 != m[0] {
		t.Errorf("%v should have found one at 1, but found it at %v", a[2], m[0])
	}
	if 3 != m[1] {
		t.Errorf("%v should have found one at 1, but found it at %v", a[2], m[1])
	}
	if 4 != m[2] {
		t.Errorf("%v should have found one at 1, but found it at %v", a[2], m[2])
	}
	c = []int{1}
	m = a[2].Matches(b, c)
	if len(m) != 1 {
		t.Errorf("%v should have found 1 matches in %v, but found %v", a[2], b, m)
	}
	if 3 != m[0] {
		t.Errorf("%v should have found one at 1, but found it at %v", a[2], m[0])
	}

	m = a[1].Matches(b, c)
	if len(m) != 2 {
		t.Errorf("%v should have found two matches in %v, but found %v", a[1], b, m)
	}
	if 5 != m[0] && 6 != m[1] {
		t.Errorf("%v should have found one at 5 & 6, but found it at %v", a[1], m)
	}

	e := []Atter{TextAtt{}, NumericAtt{5}}
	m = a[0].Matches(b, c, e...)
	if 1 != len(m) {
		t.Errorf("%v should have found 1 matches in %v using %v, but found %v", a[0], b, e)
	}
	m = a[1].Matches(b, c, e...)
	if 3 != len(m) {
		t.Errorf("%v should have found 3 in %v using %v, but found %v", a[0], b, e, m)
	}
	m = a[2].Matches(b, c, e...)
	if 4 != len(m) {
		t.Errorf("%v should have found 4 in %v using %v, but found %v", a[0], b, e, m)
	}
}

func TestCSVParsing(t *testing.T) {
	csv := `item,type,color,count
a1,m,red,25`
	r, err := NewRecordsFromCSV(strings.NewReader(csv))
	if err != nil {
		t.Error("Expected no error parsing, but got ", err)
	}
	if 1 != len(r) {
		t.Error("Expected 1 record from", r)
	}
	if 3 != len(r[0].Atts) {
		t.Error("Expedted 3 attributes from", r[0].Atts)
	}

	csv = `item,type,color,count
a1,f,red,15
a2,m,red,25`
	r, err = NewRecordsFromCSV(strings.NewReader(csv), 2)
	if err != nil {
		t.Error("Expected no error parsing, but got ", err)
	}
	if 2 != len(r) {
		t.Error("Expected 1 record from", r)
	}
	if !(NumericAtt{15}).Equal(r[0].Atts[2], NumericAtt{}) {
		t.Error("Expected last attribute to be numeric equal to 15, but was not in", r[0].Atts[2])
	}

	csv = "item,type,color,count"
	r, err = NewRecordsFromCSV(strings.NewReader(csv))
	if err != nil {
		t.Error("Expected no error parsing, but got ", err)
	}
	if 0 != len(r) {
		t.Error("Expected 0 record from", r)
	}
}
<commit_msg>Add test for invalid numeric conversion<commit_after>// Copyright 2015 Stuart Glenn, OMRF. All rights reserved.
// Use of this code is governed by a 3 clause BSD style license
// Full license details in LICENSE file distributed with this software

package mmatcher

import (
	"strings"
	"testing"
)

func TestMatch(t *testing.T) {
	a := &Record{ID: "a", Atts: []Atter{TextAtt{"text"}, NumericAtt{1}, TextAtt{"green"}}}
	b := &Record{ID: "b", Atts: []Atter{TextAtt{"text"}, NumericAtt{1}, TextAtt{"red"}}}
	if a.IsMatch(b) {
		t.Error("A should not match b on all attributes")
	}
	if !a.IsMatch(a) {
		t.Error("A should match itself on all attributes")
	}
	if !a.IsMatch(b, 0) {
		t.Error("A should match b on just first attribute")
	}
	if !b.IsMatch(a, []int{0, 1}...) {
		t.Error("B should match a on first two attributes")
	}
	if a.IsMatch(b, 2) {
		t.Error("A should not match B on third attribute")
	}
	if a.IsMatch(a, 100) {
		t.Error("A should not match itself with attribute out of bounds")
	}
	if !a.IsMatch(a, 0, 1, 2) {
		t.Error("A should match itself with all attributes specified")
	}
}

func TestMatchAtt(t *testing.T) {
	a := &Record{ID: "a", Atts: []Atter{TextAtt{"first"}, NumericAtt{1}, NumericAtt{8}, TextAtt{"yes"}}}
	b := &Record{ID: "b", Atts: []Atter{TextAtt{"second"}, NumericAtt{4.2}, NumericAtt{8}, TextAtt{"yes"}}}
	if !a.isMatchAt(b, NumericAtt{4}, 1) {
		t.Error("A numeric att should match b with an epsilon")
	}
	if !a.isMatchAt(b, NumericAtt{3.2}, 1) {
		t.Error("A numeric att should match b with an epsilon")
	}
	if a.isMatchAt(b, NumericAtt{2}, 1) {
		t.Error("A numeric att should not match b with a small epsilon")
	}
	e := make([]Atter, len(a.Atts))
	e[1] = NumericAtt{2}
	if a.IsMatchWithRanges(b, e, 1) {
		t.Error("A should not match b with small range on one index")
	}
	if a.IsMatchWithRanges(b, e, 1, 2, 3) {
		t.Error("A should not match on multi indices b with small range on one index")
	}
	e[1] = NumericAtt{3.2}
	tests := map[int]bool{
		0: false,
		1: true,
		2: true,
		3: true,
	}
	for index, result := range tests {
		if result != a.isMatchAt(b, e[index], index) {
			t.Errorf("A:%v att:%v match? b:%v with a %v", a, index, b, e[index])
		}
	}
}

func TestMatchWrongSizes(t *testing.T) {
	a := &Record{ID: "a", Atts: []Atter{TextAtt{"first"}, NumericAtt{1}}}
	b := &Record{ID: "b", Atts: []Atter{TextAtt{"second"}, NumericAtt{4.2}, NumericAtt{8}, TextAtt{"yes"}}}
	if a.IsMatchWithRanges(b, make([]Atter, len(a.Atts))) {
		t.Error("A longer record should not match a shorter record")
	}
	if a.IsMatchWithRanges(a, make([]Atter, 0)) {
		t.Error("A record cannot match itself is the []range is too short")
	}
	if !a.IsMatchWithRanges(a, make([]Atter, len(a.Atts))) {
		t.Error("A record should match itself completely")
	}
	if a.isMatchAt(a, TextAtt{}, len(a.Atts)+1) {
		t.Error("A record cannot equal even itself at a position past the record")
	}
	if a.isMatchAt(a, TextAtt{}, -1) {
		t.Error("A record cannot equal even itself at a position before the record")
	}
}

func TestMatchRange(t *testing.T) {
	a := &Record{ID: "a", Atts: []Atter{TextAtt{"first"}, NumericAtt{1}, NumericAtt{8}, TextAtt{"yes"}}}
	b := &Record{ID: "b", Atts: []Atter{TextAtt{"second"}, NumericAtt{4.2}, NumericAtt{8}, TextAtt{"yes"}}}
	if a.IsMatch(b) {
		t.Error("A and b should not match")
	}
	e := make([]Atter, len(a.Atts))
	e[1] = NumericAtt{2}
	if a.IsMatchWithRanges(b, e, 1) {
		t.Error("A should not match b with small range on one index")
	}
	if a.IsMatchWithRanges(b, e, 1, 2, 3) {
		t.Error("A should not match on multi indices b with small range on one index")
	}
	e[1] = NumericAtt{3.2}
	i := []int{1}
	if !a.IsMatchWithRanges(b, e, i...) {
		t.Errorf("A:%v should match b:%v with correct range:%v on %v", a, b, e, i)
	}
	i = []int{1, 2, 3}
	if !a.IsMatchWithRanges(b, e, i...) {
		t.Errorf("A:%v should match b:%v with correct range:%v on %v", a, b, e, i)
	}
	e[1] = NumericAtt{3.0}
	if a.IsMatchWithRanges(b, e, i...) {
		t.Errorf("A:%v should not match b:%v with small range:%v on %v", a, b, e, i)
	}
	i = []int{2, 3}
	if !a.IsMatchWithRanges(b, e, i...) {
		t.Errorf("A:%v should match b:%v with small range:%v on %v", a, b, e, i)
	}
}

func TestMatchesAll(t *testing.T) {
	a := Records{
		Record{ID: "a1", Atts: []Atter{TextAtt{"red"}, NumericAtt{1}}},
		Record{ID: "a2", Atts: []Atter{TextAtt{"red"}, NumericAtt{20}}},
		Record{ID: "a3", Atts: []Atter{TextAtt{"green"}, NumericAtt{30}}},
	}
	b := Records{
		Record{ID: "b1", Atts: []Atter{TextAtt{"green"}, NumericAtt{5}}},
		Record{ID: "b2", Atts: []Atter{TextAtt{"red"}, NumericAtt{25}}},
		Record{ID: "b3", Atts: []Atter{TextAtt{"red"}, NumericAtt{35}}},
		Record{ID: "b4", Atts: []Atter{TextAtt{"green"}, NumericAtt{30}}},
		Record{ID: "b5", Atts: []Atter{TextAtt{"green"}, NumericAtt{31}}},
		Record{ID: "b6", Atts: []Atter{TextAtt{"red"}, NumericAtt{20}}},
		Record{ID: "b7", Atts: []Atter{TextAtt{"red"}, NumericAtt{20}}},
	}

	if m := a[0].MatchesAll(b); len(m) != 0 {
		t.Errorf("%v should not have found any matches in %v", a[0], b)
	}
	m := a[2].MatchesAll(b)
	if len(m) != 1 {
		t.Errorf("%v should have found one match in %v, but found %v", a[2], b, m)
	}
	if 3 != m[0] {
		t.Errorf("%v should have found one at 1, but found it at %v", a[2], m[0])
	}

	m = a[1].MatchesAll(b)
	if len(m) != 2 {
		t.Errorf("%v should have found two matches in %v, but found %v", a[1], b, m)
	}
	if 5 != m[0] && 6 != m[1] {
		t.Errorf("%v should have found one at 5 & 6, but found it at %v", a[1], m)
	}

	e := []Atter{TextAtt{}, NumericAtt{5}}
	m = a[0].MatchesAll(b, e...)
	if 0 != len(m) {
		t.Errorf("%v should not have found any matches in %v using %v", a[0], b, e)
	}
	m = a[1].MatchesAll(b, e...)
	if 3 != len(m) {
		t.Errorf("%v should have found 3 in %v using %v, but found %v", a[0], b, e, m)
	}
	m = a[2].MatchesAll(b, e...)
	if 2 != len(m) {
		t.Errorf("%v should have found 2 in %v using %v, but found %v", a[0], b, e, m)
	}
}

func TestMatchesColumns(t *testing.T) {
	a := Records{
		Record{ID: "a0", Atts: []Atter{TextAtt{"red"}, NumericAtt{1}}},
		Record{ID: "a1", Atts: []Atter{TextAtt{"red"}, NumericAtt{20}}},
		Record{ID: "a2", Atts: []Atter{TextAtt{"green"}, NumericAtt{30}}},
	}
	b := Records{
		Record{ID: "b0", Atts: []Atter{TextAtt{"green"}, NumericAtt{5}}},
		Record{ID: "b1", Atts: []Atter{TextAtt{"red"}, NumericAtt{25}}},
		Record{ID: "b2", Atts: []Atter{TextAtt{"red"}, NumericAtt{35}}},
		Record{ID: "b3", Atts: []Atter{TextAtt{"green"}, NumericAtt{30}}},
		Record{ID: "b4", Atts: []Atter{TextAtt{"green"}, NumericAtt{31}}},
		Record{ID: "b5", Atts: []Atter{TextAtt{"red"}, NumericAtt{20}}},
		Record{ID: "b6", Atts: []Atter{TextAtt{"red"}, NumericAtt{20}}},
	}

	c := []int{10}
	if m := a[0].Matches(b, c); len(m) != 0 {
		t.Errorf("%v should have found 0 matches using column %v in %v, instead found %v", a[0], c, b, m)
	}

	c = []int{0}
	if m := a[0].Matches(b, c); len(m) != 4 {
		t.Errorf("%v should have found 4 matches in %v, instead found %v", a[0], b, m)
	}
	m := a[2].Matches(b, c)
	if len(m) != 3 {
		t.Errorf("%v should have found 3 matches in %v, but found %v", a[2], b, m)
	}
	if 0 != m[0] {
		t.Errorf("%v should have found one at 1, but found it at %v", a[2], m[0])
	}
	if 3 != m[1] {
		t.Errorf("%v should have found one at 1, but found it at %v", a[2], m[1])
	}
	if 4 != m[2] {
		t.Errorf("%v should have found one at 1, but found it at %v", a[2], m[2])
	}
	c = []int{1}
	m = a[2].Matches(b, c)
	if len(m) != 1 {
		t.Errorf("%v should have found 1 matches in %v, but found %v", a[2], b, m)
	}
	if 3 != m[0] {
		t.Errorf("%v should have found one at 1, but found it at %v", a[2], m[0])
	}

	m = a[1].Matches(b, c)
	if len(m) != 2 {
		t.Errorf("%v should have found two matches in %v, but found %v", a[1], b, m)
	}
	if 5 != m[0] && 6 != m[1] {
		t.Errorf("%v should have found one at 5 & 6, but found it at %v", a[1], m)
	}

	e := []Atter{TextAtt{}, NumericAtt{5}}
	m = a[0].Matches(b, c, e...)
	if 1 != len(m) {
		t.Errorf("%v should have found 1 matches in %v using %v, but found %v", a[0], b, e)
	}
	m = a[1].Matches(b, c, e...)
	if 3 != len(m) {
		t.Errorf("%v should have found 3 in %v using %v, but found %v", a[0], b, e, m)
	}
	m = a[2].Matches(b, c, e...)
	if 4 != len(m) {
		t.Errorf("%v should have found 4 in %v using %v, but found %v", a[0], b, e, m)
	}
}

func TestCSVParsing(t *testing.T) {
	csv := `item,type,color,count
a1,m,red,25`
	r, err := NewRecordsFromCSV(strings.NewReader(csv))
	if err != nil {
		t.Error("Expected no error parsing, but got ", err)
	}
	if 1 != len(r) {
		t.Error("Expected 1 record from", r)
	}
	if 3 != len(r[0].Atts) {
		t.Error("Expedted 3 attributes from", r[0].Atts)
	}

	csv = `item,type,color,count
a1,f,red,15
a2,m,red,25`
	r, err = NewRecordsFromCSV(strings.NewReader(csv), 2)
	if err != nil {
		t.Error("Expected no error parsing, but got ", err)
	}
	if 2 != len(r) {
		t.Error("Expected 1 record from", r)
	}
	if !(NumericAtt{15}).Equal(r[0].Atts[2], NumericAtt{}) {
		t.Error("Expected last attribute to be numeric equal to 15, but was not in", r[0].Atts[2])
	}

	r, err = NewRecordsFromCSV(strings.NewReader(csv), 0)
	if err == nil {
		t.Error("Expect error parsing, but got ", err)
	}
	if 0 != len(r) {
		t.Error("Expected 0 record from", r)
	}

	csv = "item,type,color,count"
	r, err = NewRecordsFromCSV(strings.NewReader(csv))
	if err != nil {
		t.Error("Expected no error parsing, but got ", err)
	}
	if 0 != len(r) {
		t.Error("Expected 0 record from", r)
	}
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"
	"time"

	"github.com/hashicorp/aws-sdk-go/aws"
	"github.com/hashicorp/aws-sdk-go/gen/ec2"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsVpc() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsVpcCreate,
		Read:   resourceAwsVpcRead,
		Update: resourceAwsVpcUpdate,
		Delete: resourceAwsVpcDelete,

		Schema: map[string]*schema.Schema{
			"cidr_block": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"instance_tenancy": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},

			"enable_dns_hostnames": &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				Computed: true,
			},

			"enable_dns_support": &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				Computed: true,
			},

			"main_route_table_id": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"default_network_acl_id": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"default_security_group_id": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			//			"tags": tagsSchema(),
		},
	}
}

func resourceAwsVpcCreate(d *schema.ResourceData, meta interface{}) error {
	ec2conn := meta.(*AWSClient).awsEc2conn
	instance_tenancy := "default"
	if v, ok := d.GetOk("instance_tenancy"); ok {
		instance_tenancy = v.(string)
	}
	// Create the VPC
	createOpts := &ec2.CreateVPCRequest{
		CIDRBlock:       aws.String(d.Get("cidr_block").(string)),
		InstanceTenancy: &instance_tenancy,
	}
	log.Printf("[DEBUG] VPC create config: %#v", *createOpts)
	vpcResp, err := ec2conn.CreateVPC(createOpts)
	if err != nil {
		return fmt.Errorf("Error creating VPC: %s : %s", err)
	}

	// Get the ID and store it
	vpc := vpcResp.VPC
	d.SetId(*vpc.VPCID)
	log.Printf("[INFO] VPC ID: %s", d.Id())

	// Set partial mode and say that we setup the cidr block
	d.Partial(true)
	d.SetPartial("cidr_block")

	// Wait for the VPC to become available
	log.Printf(
		"[DEBUG] Waiting for VPC (%s) to become available",
		d.Id())
	stateConf := &resource.StateChangeConf{
		Pending: []string{"pending"},
		Target:  "available",
		Refresh: VPCStateRefreshFunc(ec2conn, d.Id()),
		Timeout: 10 * time.Minute,
	}
	if _, err := stateConf.WaitForState(); err != nil {
		return fmt.Errorf(
			"Error waiting for VPC (%s) to become available: %s",
			d.Id(), err)
	}

	// Update our attributes and return
	return resourceAwsVpcUpdate(d, meta)
}

func resourceAwsVpcRead(d *schema.ResourceData, meta interface{}) error {
	ec2conn := meta.(*AWSClient).awsEc2conn

	// Refresh the VPC state
	vpcRaw, _, err := VPCStateRefreshFunc(ec2conn, d.Id())()
	if err != nil {
		return err
	}
	if vpcRaw == nil {
		d.SetId("")
		return nil
	}

	// VPC stuff
	vpc := vpcRaw.(*ec2.VPC)
	vpcid := d.Id()
	d.Set("cidr_block", vpc.CIDRBlock)

	// Tags - TBD rmenn
	//d.Set("tags", tagsToMap(vpc.Tags))

	// Attributes
	attribute := "enableDnsSupport"
	DescribeAttrOpts := &ec2.DescribeVPCAttributeRequest{
		Attribute: &attribute,
		VPCID:     &vpcid,
	}
	resp, err := ec2conn.DescribeVPCAttribute(DescribeAttrOpts)
	if err != nil {
		return err
	}
	d.Set("enable_dns_support", *resp.EnableDNSSupport)
	attribute = "enableDnsHostnames"
	DescribeAttrOpts = &ec2.DescribeVPCAttributeRequest{
		Attribute: &attribute,
		VPCID:     &vpcid,
	}
	resp, err = ec2conn.DescribeVPCAttribute(DescribeAttrOpts)
	if err != nil {
		return err
	}
	d.Set("enable_dns_hostnames", *resp.EnableDNSHostnames)

	// Get the main routing table for this VPC
	// Really Ugly need to make this better - rmenn
	filter1 := &ec2.Filter{
		Name:   aws.String("association.main"),
		Values: []string{("true")},
	}
	filter2 := &ec2.Filter{
		Name:   aws.String("vpc-id"),
		Values: []string{(d.Id())},
	}
	DescribeRouteOpts := &ec2.DescribeRouteTablesRequest{
		Filters: []ec2.Filter{*filter1, *filter2},
	}
	routeResp, err := ec2conn.DescribeRouteTables(DescribeRouteOpts)
	if err != nil {
		return err
	}
	if v := routeResp.RouteTables; len(v) > 0 {
		d.Set("main_route_table_id", *v[0].RouteTableID)
	}

	resourceAwsVpcSetDefaultNetworkAcl(ec2conn, d)
	resourceAwsVpcSetDefaultSecurityGroup(ec2conn, d)

	return nil
}

func resourceAwsVpcUpdate(d *schema.ResourceData, meta interface{}) error {
	ec2conn := meta.(*AWSClient).awsEc2conn

	// Turn on partial mode
	d.Partial(true)
	vpcid := d.Id()
	modifyOpts := &ec2.ModifyVPCAttributeRequest{
		VPCID: &vpcid,
	}
	if d.HasChange("enable_dns_hostnames") {
		val := d.Get("enable_dns_hostnames").(bool)
		modifyOpts.EnableDNSHostnames = &ec2.AttributeBooleanValue{
			Value: &val,
		}

		log.Printf(
			"[INFO] Modifying enable_dns_hostnames vpc attribute for %s: %#v",
			d.Id(), modifyOpts)
		if err := ec2conn.ModifyVPCAttribute(modifyOpts); err != nil {
			return err
		}

		d.SetPartial("enable_dns_hostnames")
	}

	if d.HasChange("enable_dns_support") {
		val := d.Get("enable_dns_hostnames").(bool)
		modifyOpts.EnableDNSSupport = &ec2.AttributeBooleanValue{
			Value: &val,
		}

		log.Printf(
			"[INFO] Modifying enable_dns_support vpc attribute for %s: %#v",
			d.Id(), modifyOpts)
		if err := ec2conn.ModifyVPCAttribute(modifyOpts); err != nil {
			return err
		}

		d.SetPartial("enable_dns_support")
	}
	//Tagging Support need to be worked on - rmenn
	//	if err := setTags(ec2conn, d); err != nil {
	//		return err
	//	} else {
	//		d.SetPartial("tags")
	//	}

	d.Partial(false)
	return resourceAwsVpcRead(d, meta)
}

func resourceAwsVpcDelete(d *schema.ResourceData, meta interface{}) error {
	ec2conn := meta.(*AWSClient).awsEc2conn
	vpcID := d.Id()
	DeleteVpcOpts := &ec2.DeleteVPCRequest{
		VPCID: &vpcID,
	}
	log.Printf("[INFO] Deleting VPC: %s", d.Id())
	if err := ec2conn.DeleteVPC(DeleteVpcOpts); err != nil {
		ec2err, ok := err.(*aws.APIError)
		if ok && ec2err.Code == "InvalidVpcID.NotFound" {
			return nil
		}

		return fmt.Errorf("Error deleting VPC: %s", err)
	}

	return nil
}

// VPCStateRefreshFunc returns a resource.StateRefreshFunc that is used to watch
// a VPC.
func VPCStateRefreshFunc(conn *ec2.EC2, id string) resource.StateRefreshFunc {
	return func() (interface{}, string, error) {
		DescribeVpcOpts := &ec2.DescribeVPCsRequest{
			VPCIDs: []string{id},
		}
		resp, err := conn.DescribeVPCs(DescribeVpcOpts)
		if err != nil {
			if ec2err, ok := err.(*aws.APIError); ok && ec2err.Code == "InvalidVpcID.NotFound" {
				resp = nil
			} else {
				log.Printf("Error on VPCStateRefresh: %s", err)
				return nil, "", err
			}
		}

		if resp == nil {
			// Sometimes AWS just has consistency issues and doesn't see
			// our instance yet. Return an empty state.
			return nil, "", nil
		}

		vpc := &resp.VPCs[0]
		return vpc, *vpc.State, nil
	}
}

func resourceAwsVpcSetDefaultNetworkAcl(conn *ec2.EC2, d *schema.ResourceData) error {
	filter1 := &ec2.Filter{
		Name:   aws.String("default"),
		Values: []string{("true")},
	}
	filter2 := &ec2.Filter{
		Name:   aws.String("vpc-id"),
		Values: []string{(d.Id())},
	}
	DescribeNetworkACLOpts := &ec2.DescribeNetworkACLsRequest{
		Filters: []ec2.Filter{*filter1, *filter2},
	}
	networkAclResp, err := conn.DescribeNetworkACLs(DescribeNetworkACLOpts)

	if err != nil {
		return err
	}
	if v := networkAclResp.NetworkACLs; len(v) > 0 {
		d.Set("default_network_acl_id", v[0].NetworkACLID)
	}

	return nil
}

func resourceAwsVpcSetDefaultSecurityGroup(conn *ec2.EC2, d *schema.ResourceData) error {
	filter1 := &ec2.Filter{
		Name:   aws.String("group-name"),
		Values: []string{("default")},
	}
	filter2 := &ec2.Filter{
		Name:   aws.String("vpc-id"),
		Values: []string{(d.Id())},
	}
	DescribeSgOpts := &ec2.DescribeSecurityGroupsRequest{
		Filters: []ec2.Filter{*filter1, *filter2},
	}
	securityGroupResp, err := conn.DescribeSecurityGroups(DescribeSgOpts)

	if err != nil {
		return err
	}
	if v := securityGroupResp.SecurityGroups; len(v) > 0 {
		d.Set("default_security_group_id", v[0].GroupID)
	}

	return nil
}
<commit_msg>Removed additional variable for print, added for debugging<commit_after>package aws

import (
	"fmt"
	"log"
	"time"

	"github.com/hashicorp/aws-sdk-go/aws"
	"github.com/hashicorp/aws-sdk-go/gen/ec2"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsVpc() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsVpcCreate,
		Read:   resourceAwsVpcRead,
		Update: resourceAwsVpcUpdate,
		Delete: resourceAwsVpcDelete,

		Schema: map[string]*schema.Schema{
			"cidr_block": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"instance_tenancy": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},

			"enable_dns_hostnames": &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				Computed: true,
			},

			"enable_dns_support": &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				Computed: true,
			},

			"main_route_table_id": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"default_network_acl_id": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"default_security_group_id": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			//			"tags": tagsSchema(),
		},
	}
}

func resourceAwsVpcCreate(d *schema.ResourceData, meta interface{}) error {
	ec2conn := meta.(*AWSClient).awsEc2conn
	instance_tenancy := "default"
	if v, ok := d.GetOk("instance_tenancy"); ok {
		instance_tenancy = v.(string)
	}
	// Create the VPC
	createOpts := &ec2.CreateVPCRequest{
		CIDRBlock:       aws.String(d.Get("cidr_block").(string)),
		InstanceTenancy: &instance_tenancy,
	}
	log.Printf("[DEBUG] VPC create config: %#v", *createOpts)
	vpcResp, err := ec2conn.CreateVPC(createOpts)
	if err != nil {
		return fmt.Errorf("Error creating VPC: %s", err)
	}

	// Get the ID and store it
	vpc := vpcResp.VPC
	d.SetId(*vpc.VPCID)
	log.Printf("[INFO] VPC ID: %s", d.Id())

	// Set partial mode and say that we setup the cidr block
	d.Partial(true)
	d.SetPartial("cidr_block")

	// Wait for the VPC to become available
	log.Printf(
		"[DEBUG] Waiting for VPC (%s) to become available",
		d.Id())
	stateConf := &resource.StateChangeConf{
		Pending: []string{"pending"},
		Target:  "available",
		Refresh: VPCStateRefreshFunc(ec2conn, d.Id()),
		Timeout: 10 * time.Minute,
	}
	if _, err := stateConf.WaitForState(); err != nil {
		return fmt.Errorf(
			"Error waiting for VPC (%s) to become available: %s",
			d.Id(), err)
	}

	// Update our attributes and return
	return resourceAwsVpcUpdate(d, meta)
}

func resourceAwsVpcRead(d *schema.ResourceData, meta interface{}) error {
	ec2conn := meta.(*AWSClient).awsEc2conn

	// Refresh the VPC state
	vpcRaw, _, err := VPCStateRefreshFunc(ec2conn, d.Id())()
	if err != nil {
		return err
	}
	if vpcRaw == nil {
		d.SetId("")
		return nil
	}

	// VPC stuff
	vpc := vpcRaw.(*ec2.VPC)
	vpcid := d.Id()
	d.Set("cidr_block", vpc.CIDRBlock)

	// Tags - TBD rmenn
	//d.Set("tags", tagsToMap(vpc.Tags))

	// Attributes
	attribute := "enableDnsSupport"
	DescribeAttrOpts := &ec2.DescribeVPCAttributeRequest{
		Attribute: &attribute,
		VPCID:     &vpcid,
	}
	resp, err := ec2conn.DescribeVPCAttribute(DescribeAttrOpts)
	if err != nil {
		return err
	}
	d.Set("enable_dns_support", *resp.EnableDNSSupport)
	attribute = "enableDnsHostnames"
	DescribeAttrOpts = &ec2.DescribeVPCAttributeRequest{
		Attribute: &attribute,
		VPCID:     &vpcid,
	}
	resp, err = ec2conn.DescribeVPCAttribute(DescribeAttrOpts)
	if err != nil {
		return err
	}
	d.Set("enable_dns_hostnames", *resp.EnableDNSHostnames)

	// Get the main routing table for this VPC
	// Really Ugly need to make this better - rmenn
	filter1 := &ec2.Filter{
		Name:   aws.String("association.main"),
		Values: []string{("true")},
	}
	filter2 := &ec2.Filter{
		Name:   aws.String("vpc-id"),
		Values: []string{(d.Id())},
	}
	DescribeRouteOpts := &ec2.DescribeRouteTablesRequest{
		Filters: []ec2.Filter{*filter1, *filter2},
	}
	routeResp, err := ec2conn.DescribeRouteTables(DescribeRouteOpts)
	if err != nil {
		return err
	}
	if v := routeResp.RouteTables; len(v) > 0 {
		d.Set("main_route_table_id", *v[0].RouteTableID)
	}

	resourceAwsVpcSetDefaultNetworkAcl(ec2conn, d)
	resourceAwsVpcSetDefaultSecurityGroup(ec2conn, d)

	return nil
}

func resourceAwsVpcUpdate(d *schema.ResourceData, meta interface{}) error {
	ec2conn := meta.(*AWSClient).awsEc2conn

	// Turn on partial mode
	d.Partial(true)
	vpcid := d.Id()
	modifyOpts := &ec2.ModifyVPCAttributeRequest{
		VPCID: &vpcid,
	}
	if d.HasChange("enable_dns_hostnames") {
		val := d.Get("enable_dns_hostnames").(bool)
		modifyOpts.EnableDNSHostnames = &ec2.AttributeBooleanValue{
			Value: &val,
		}

		log.Printf(
			"[INFO] Modifying enable_dns_hostnames vpc attribute for %s: %#v",
			d.Id(), modifyOpts)
		if err := ec2conn.ModifyVPCAttribute(modifyOpts); err != nil {
			return err
		}

		d.SetPartial("enable_dns_hostnames")
	}

	if d.HasChange("enable_dns_support") {
		val := d.Get("enable_dns_hostnames").(bool)
		modifyOpts.EnableDNSSupport = &ec2.AttributeBooleanValue{
			Value: &val,
		}

		log.Printf(
			"[INFO] Modifying enable_dns_support vpc attribute for %s: %#v",
			d.Id(), modifyOpts)
		if err := ec2conn.ModifyVPCAttribute(modifyOpts); err != nil {
			return err
		}

		d.SetPartial("enable_dns_support")
	}
	//Tagging Support need to be worked on - rmenn
	//	if err := setTags(ec2conn, d); err != nil {
	//		return err
	//	} else {
	//		d.SetPartial("tags")
	//	}

	d.Partial(false)
	return resourceAwsVpcRead(d, meta)
}

func resourceAwsVpcDelete(d *schema.ResourceData, meta interface{}) error {
	ec2conn := meta.(*AWSClient).awsEc2conn
	vpcID := d.Id()
	DeleteVpcOpts := &ec2.DeleteVPCRequest{
		VPCID: &vpcID,
	}
	log.Printf("[INFO] Deleting VPC: %s", d.Id())
	if err := ec2conn.DeleteVPC(DeleteVpcOpts); err != nil {
		ec2err, ok := err.(*aws.APIError)
		if ok && ec2err.Code == "InvalidVpcID.NotFound" {
			return nil
		}

		return fmt.Errorf("Error deleting VPC: %s", err)
	}

	return nil
}

// VPCStateRefreshFunc returns a resource.StateRefreshFunc that is used to watch
// a VPC.
func VPCStateRefreshFunc(conn *ec2.EC2, id string) resource.StateRefreshFunc {
	return func() (interface{}, string, error) {
		DescribeVpcOpts := &ec2.DescribeVPCsRequest{
			VPCIDs: []string{id},
		}
		resp, err := conn.DescribeVPCs(DescribeVpcOpts)
		if err != nil {
			if ec2err, ok := err.(*aws.APIError); ok && ec2err.Code == "InvalidVpcID.NotFound" {
				resp = nil
			} else {
				log.Printf("Error on VPCStateRefresh: %s", err)
				return nil, "", err
			}
		}

		if resp == nil {
			// Sometimes AWS just has consistency issues and doesn't see
			// our instance yet. Return an empty state.
			return nil, "", nil
		}

		vpc := &resp.VPCs[0]
		return vpc, *vpc.State, nil
	}
}

func resourceAwsVpcSetDefaultNetworkAcl(conn *ec2.EC2, d *schema.ResourceData) error {
	filter1 := &ec2.Filter{
		Name:   aws.String("default"),
		Values: []string{("true")},
	}
	filter2 := &ec2.Filter{
		Name:   aws.String("vpc-id"),
		Values: []string{(d.Id())},
	}
	DescribeNetworkACLOpts := &ec2.DescribeNetworkACLsRequest{
		Filters: []ec2.Filter{*filter1, *filter2},
	}
	networkAclResp, err := conn.DescribeNetworkACLs(DescribeNetworkACLOpts)

	if err != nil {
		return err
	}
	if v := networkAclResp.NetworkACLs; len(v) > 0 {
		d.Set("default_network_acl_id", v[0].NetworkACLID)
	}

	return nil
}

func resourceAwsVpcSetDefaultSecurityGroup(conn *ec2.EC2, d *schema.ResourceData) error {
	filter1 := &ec2.Filter{
		Name:   aws.String("group-name"),
		Values: []string{("default")},
	}
	filter2 := &ec2.Filter{
		Name:   aws.String("vpc-id"),
		Values: []string{(d.Id())},
	}
	DescribeSgOpts := &ec2.DescribeSecurityGroupsRequest{
		Filters: []ec2.Filter{*filter1, *filter2},
	}
	securityGroupResp, err := conn.DescribeSecurityGroups(DescribeSgOpts)

	if err != nil {
		return err
	}
	if v := securityGroupResp.SecurityGroups; len(v) > 0 {
		d.Set("default_security_group_id", v[0].GroupID)
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"compress/gzip"
	"crypto/md5"
	"encoding/base64"
	"fmt"
	"io"
	"math/big"
	"mime"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"code.google.com/p/go.net/html"

	"log"

	"github.com/crowdmob/goamz/s3"
	"github.com/wsxiaoys/terminal/color"
)

const (
	SCRIPT = iota
	STYLE
)

const UPLOAD_WORKERS = 20

func hashFile(path string) []byte {
	hash := md5.New()
	io.WriteString(hash, path)
	io.WriteString(hash, "\n")

	// TODO: Encode type?

	ref := must(os.Open(path)).(*os.File)
	defer ref.Close()

	must(io.Copy(hash, ref))

	return hash.Sum(nil)
}

func hashBytes(data []byte) []byte {
	hash := md5.New()
	must(io.Copy(hash, bytes.NewReader(data)))
	return hash.Sum(nil)
}

func hashFiles(files []string) string {
	hash := new(big.Int)
	for _, file := range files {
		val := new(big.Int)
		val.SetBytes(hashFile(file))

		hash = hash.Xor(hash, val)
	}

	return fmt.Sprintf("%x", hash)
}

func getRef() string {
	gitPath := mustString(exec.LookPath("git"))

	cmd := exec.Command(gitPath, "rev-parse", "--verify", "HEAD")

	out := bytes.Buffer{}
	cmd.Stdout = &out
	panicIf(cmd.Run())

	return string(out.Bytes())
}

func guessContentType(file string) string {
	return mime.TypeByExtension(filepath.Ext(file))
}

func uploadFile(bucket *s3.Bucket, reader io.Reader, dest string, includeHash bool, caching int) string {
	buffer := bytes.NewBuffer([]byte{})
	writer := gzip.NewWriter(buffer)
	must(io.Copy(writer, reader))
	writer.Close()

	data := buffer.Bytes()

	hash := hashBytes(data)
	hashPrefix := fmt.Sprintf("%x", hash)[:12]
	s3Opts := s3.Options{
		ContentMD5:      base64.StdEncoding.EncodeToString(hash),
		ContentEncoding: "gzip",
		CacheControl:    fmt.Sprintf("public, max-age=%d", caching),
	}

	if includeHash {
		dest = filepath.Join(hashPrefix, dest)
	}

	log.Printf("Uploading to %s in %s (%s) [%d]\n", dest, bucket.Name, hashPrefix, caching)
	err := bucket.PutReader(dest, buffer, int64(len(data)), guessContentType(dest), s3.PublicRead, s3Opts)
	panicIf(err)

	return dest
}

type FileRef struct {
	LocalPath    string
	RemotePath   string
	UploadedPath string
}

type FileInst struct {
	File     *FileRef
	InstPath string
}

func writeFiles(options Options, includeHash bool, files chan *FileRef) {
	bucket := s3Session.Bucket(options.Bucket)

	for file := range files {
		handle := must(os.Open(file.LocalPath)).(*os.File)
		defer handle.Close()

		var ttl int
		ttl = FOREVER
		if !includeHash {
			ttl = LIMITED
		}

		(*file).UploadedPath = uploadFile(bucket, handle, file.RemotePath, includeHash, ttl)
	}
}

func deployFiles(options Options, includeHash bool, files []*FileRef) {
	ch := make(chan *FileRef)

	wg := new(sync.WaitGroup)
	for i := 0; i < UPLOAD_WORKERS; i++ {
		wg.Add(1)
		go func() {
			writeFiles(options, includeHash, ch)
			wg.Done()
		}()
	}

	for _, file := range files {
		if !includeHash && strings.HasSuffix(file.RemotePath, ".html") {
			panic(fmt.Sprintf("Cowardly refusing to deploy an html file (%s) without versioning.  Add the file to the --html list to deploy with versioning.", file.RemotePath))
		}

		ch <- file
	}

	close(ch)

	wg.Wait()
}

func addFiles(form uint8, parent *html.Node, files []string) {
	for _, file := range files {
		node := html.Node{
			Type: html.ElementNode,
		}
		switch form {
		case SCRIPT:
			node.Data = "script"
			node.Attr = []html.Attribute{
				html.Attribute{
					Key: "src",
					Val: file,
				},
			}

		case STYLE:
			node.Data = "link"
			node.Attr = []html.Attribute{
				html.Attribute{
					Key: "rel",
					Val: "stylesheet",
				},
				html.Attribute{
					Key: "href",
					Val: file,
				},
			}
		default:
			panic("Type not understood")
		}

		parent.AppendChild(&node)
	}
}

func isLocal(href string) bool {
	parsed := must(url.Parse(href)).(*url.URL)
	return parsed.Host == ""
}

func formatHref(path string) string {
	if !strings.HasPrefix(path, "/") {
		path = "/" + path
	}
	return path
}

func renderHTML(options Options, file HTMLFile) string {
	handle := must(os.Open(file.File.LocalPath)).(*os.File)
	defer handle.Close()

	doc := must(html.Parse(handle)).(*html.Node)

	var f func(*html.Node)
	f = func(n *html.Node) {
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			f(c)
		}

		if n.Type == html.ElementNode {
			switch n.Data {
			case "script":
				for i, a := range n.Attr {
					if a.Key == "src" {
						for _, dep := range file.Deps {
							if dep.InstPath == a.Val {
								n.Attr[i].Val = formatHref(dep.File.UploadedPath)
								break
							}
						}
					}
				}
			case "link":
				stylesheet := false
				for _, a := range n.Attr {
					if a.Key == "rel" {
						stylesheet = a.Val == "stylesheet"
						break
					}
				}
				if !stylesheet {
					return
				}

				for i, a := range n.Attr {
					if a.Key == "href" {
						for _, dep := range file.Deps {
							if dep.InstPath == a.Val {
								n.Attr[i].Val = formatHref(dep.File.UploadedPath)
								break
							}
						}
					}
				}
			}
		}
	}
	f(doc)

	buf := bytes.NewBuffer([]byte{})
	panicIf(html.Render(buf, doc))

	return buf.String()
}

func parseHTML(options Options, path string) (files []string, base string) {
	files = make([]string, 0)

	handle := must(os.Open(path)).(*os.File)
	defer handle.Close()

	doc := must(html.Parse(handle)).(*html.Node)

	var f func(*html.Node)
	f = func(n *html.Node) {
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			f(c)
		}

		if n.Type == html.ElementNode {
			switch n.Data {
			case "base":
				for _, a := range n.Attr {
					if a.Key == "href" {
						base = a.Val
					}
				}
			case "script":
				for _, a := range n.Attr {
					if a.Key == "src" {
						if isLocal(a.Val) {
							files = append(files, a.Val)
						}
					}
				}
			case "link":
				local := false
				stylesheet := false
				href := ""
				for _, a := range n.Attr {
					switch a.Key {
					case "href":
						local = isLocal(a.Val)
						href = a.Val
					case "rel":
						stylesheet = a.Val == "stylesheet"
					}
				}
				if local && stylesheet {
					files = append(files, href)
				}
			}
		}
	}
	f(doc)

	return
}

func deployHTML(options Options, id string, file HTMLFile) {
	data := renderHTML(options, file)

	internalPath, err := filepath.Rel(options.Root, file.File.LocalPath)
	if err != nil {
		panic(err)
	}

	permPath := filepath.Join(options.Dest, id, internalPath)
	curPath := filepath.Join(options.Dest, internalPath)

	bucket := s3Session.Bucket(options.Bucket)
	uploadFile(bucket, strings.NewReader(data), permPath, false, FOREVER)

	log.Println("Copying", permPath, "to", curPath)
	copyFile(bucket, permPath, curPath, "text/html", LIMITED)
}

func expandFiles(root string, glob string) []string {
	out := make([]string, 0)
	cases := strings.Split(glob, ",")

	for _, pattern := range cases {
		list := must(filepath.Glob(filepath.Join(root, pattern))).([]string)

		for _, file := range list {
			info := must(os.Stat(file)).(os.FileInfo)

			if info.IsDir() {
				filepath.Walk(file, func(path string, info os.FileInfo, err error) error {
					panicIf(err)

					if !info.IsDir() {
						out = append(out, path)
					}

					return nil
				})
			} else {
				out = append(out, file)
			}
		}
	}
	return out
}

func listFiles(options Options) []*FileRef {
	filePaths := expandFiles(options.Root, options.Files)

	files := make([]*FileRef, len(filePaths))
	for i, path := range filePaths {
		remotePath := filepath.Join(options.Dest, mustString(filepath.Rel(options.Root, path)))

		files[i] = &FileRef{
			LocalPath:  path,
			RemotePath: remotePath,
		}
	}

	return files
}

func ignoreFiles(full []*FileRef, rem []*FileRef) []*FileRef {
	out := make([]*FileRef, 0, len(full))

	for _, file := range full {
		ignore := false
		path := filepath.Clean(file.LocalPath)

		for _, remFile := range rem {
			if filepath.Clean(remFile.LocalPath) == path {
				ignore = true
				break
			}
		}

		if !ignore {
			out = append(out, file)
		}
	}

	return out
}

func extractFileList(options Options, pattern string) (files []string) {
	files = make([]string, 0)

	parts := strings.Split(pattern, ",")

	for _, part := range parts {
		matches, err := filepath.Glob(filepath.Join(options.Root, part))
		if err != nil {
			panic(err)
		}
		if matches == nil {
			panic(fmt.Sprintf("Pattern %s did not match any files", part))
		}

		files = append(files, matches...)
	}

	return files
}

func filesWithExtension(files []*FileRef, ext string) (outFiles []*FileRef) {
	outFiles = make([]*FileRef, 0)
	for _, file := range files {
		if filepath.Ext(file.LocalPath) == ext {
			outFiles = append(outFiles, file)
		}
	}

	return
}

type HTMLFile struct {
	File FileRef
	Deps []FileInst
	Base string
}

func (f HTMLFile) GetLocalPath() string {
	return f.File.LocalPath
}

func Deploy(options Options) {
	if s3Session == nil {
		s3Session = openS3(options.AWSKey, options.AWSSecret)
	}

	files := listFiles(options)

	htmlFileRefs := filesWithExtension(files, ".html")

	inclFiles := make(map[string]*FileRef)
	htmlFiles := make([]HTMLFile, len(htmlFileRefs))
	for i, file := range htmlFileRefs {
		dir := filepath.Dir(file.LocalPath)

		rel, err := filepath.Rel(options.Root, dir)
		if err != nil {
			panic(err)
		}

		paths, base := parseHTML(options, file.LocalPath)

		if strings.HasPrefix(strings.ToLower(base), "http") || strings.HasPrefix(base, "//") {
			panic("Absolute base tags are not supported")
		}

		htmlFiles[i] = HTMLFile{
			File: *file,
			Deps: make([]FileInst, len(paths)),
			Base: base,
		}

		for j, path := range paths {
			local := filepath.Join(options.Root, rel, base, path)
			remote := filepath.Join(options.Dest, rel, base, path)

			ref, ok := inclFiles[local]
			if !ok {
				ref = &FileRef{
					LocalPath:  local,
					RemotePath: remote,

					// Filled in after the deploy:
					UploadedPath: "",
				}

				inclFiles[local] = ref
			}

			use := FileInst{
				File:     ref,
				InstPath: path,
			}

			htmlFiles[i].Deps[j] = use
		}
	}

	inclFileList := make([]*FileRef, len(inclFiles))
	i := 0
	for _, ref := range inclFiles {
		inclFileList[i] = ref
		i++
	}

	hashPaths := make([]string, 0)
	for _, item := range inclFileList {
		hashPaths = append(hashPaths, item.LocalPath)
	}
	for _, item := range htmlFiles {
		hashPaths = append(hashPaths, item.File.LocalPath)
	}

	hash := hashFiles(hashPaths)
	id := hash[:12]

	deployFiles(options, true, inclFileList)
	deployFiles(options, false, ignoreFiles(files, htmlFileRefs))

	// Ensure that the new files exist in s3
	// Time based on "Eventual Consistency: How soon is eventual?"
	time.Sleep(1500 * time.Millisecond)

	wg := sync.WaitGroup{}
	for _, file := range htmlFiles {
		wg.Add(1)

		go func(file HTMLFile) {
			defer wg.Done()
			deployHTML(options, id, file)
		}(file)
	}

	wg.Wait()

	color.Printf(`
+------------------------------------+
|         @{g}Deploy Successful!@{|}         |
|                                    |
|       Deploy ID: @{?}%s@{|}      |
+------------------------------------+
`, id)

}

func deployCmd() {
	options, _ := parseOptions()
	loadConfigFile(&options)

	if options.Bucket == "" {
		panic("You must specify a bucket")
	}

	if options.AWSKey == "" || options.AWSSecret == "" {
		panic("You must specify your AWS credentials")
	}

	Deploy(options)
}
<commit_msg>Remove no longer correct error message<commit_after>package main

import (
	"bytes"
	"compress/gzip"
	"crypto/md5"
	"encoding/base64"
	"fmt"
	"io"
	"math/big"
	"mime"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"code.google.com/p/go.net/html"

	"log"

	"github.com/crowdmob/goamz/s3"
	"github.com/wsxiaoys/terminal/color"
)

const (
	SCRIPT = iota
	STYLE
)

const UPLOAD_WORKERS = 20

func hashFile(path string) []byte {
	hash := md5.New()
	io.WriteString(hash, path)
	io.WriteString(hash, "\n")

	// TODO: Encode type?

	ref := must(os.Open(path)).(*os.File)
	defer ref.Close()

	must(io.Copy(hash, ref))

	return hash.Sum(nil)
}

func hashBytes(data []byte) []byte {
	hash := md5.New()
	must(io.Copy(hash, bytes.NewReader(data)))
	return hash.Sum(nil)
}

func hashFiles(files []string) string {
	hash := new(big.Int)
	for _, file := range files {
		val := new(big.Int)
		val.SetBytes(hashFile(file))

		hash = hash.Xor(hash, val)
	}

	return fmt.Sprintf("%x", hash)
}

func getRef() string {
	gitPath := mustString(exec.LookPath("git"))

	cmd := exec.Command(gitPath, "rev-parse", "--verify", "HEAD")

	out := bytes.Buffer{}
	cmd.Stdout = &out
	panicIf(cmd.Run())

	return string(out.Bytes())
}

func guessContentType(file string) string {
	return mime.TypeByExtension(filepath.Ext(file))
}

func uploadFile(bucket *s3.Bucket, reader io.Reader, dest string, includeHash bool, caching int) string {
	buffer := bytes.NewBuffer([]byte{})
	writer := gzip.NewWriter(buffer)
	must(io.Copy(writer, reader))
	writer.Close()

	data := buffer.Bytes()

	hash := hashBytes(data)
	hashPrefix := fmt.Sprintf("%x", hash)[:12]
	s3Opts := s3.Options{
		ContentMD5:      base64.StdEncoding.EncodeToString(hash),
		ContentEncoding: "gzip",
		CacheControl:    fmt.Sprintf("public, max-age=%d", caching),
	}

	if includeHash {
		dest = filepath.Join(hashPrefix, dest)
	}

	log.Printf("Uploading to %s in %s (%s) [%d]\n", dest, bucket.Name, hashPrefix, caching)
	err := bucket.PutReader(dest, buffer, int64(len(data)), guessContentType(dest), s3.PublicRead, s3Opts)
	panicIf(err)

	return dest
}

type FileRef struct {
	LocalPath    string
	RemotePath   string
	UploadedPath string
}

type FileInst struct {
	File     *FileRef
	InstPath string
}

func writeFiles(options Options, includeHash bool, files chan *FileRef) {
	bucket := s3Session.Bucket(options.Bucket)

	for file := range files {
		handle := must(os.Open(file.LocalPath)).(*os.File)
		defer handle.Close()

		var ttl int
		ttl = FOREVER
		if !includeHash {
			ttl = LIMITED
		}

		(*file).UploadedPath = uploadFile(bucket, handle, file.RemotePath, includeHash, ttl)
	}
}

func deployFiles(options Options, includeHash bool, files []*FileRef) {
	ch := make(chan *FileRef)

	wg := new(sync.WaitGroup)
	for i := 0; i < UPLOAD_WORKERS; i++ {
		wg.Add(1)
		go func() {
			writeFiles(options, includeHash, ch)
			wg.Done()
		}()
	}

	for _, file := range files {
		if !includeHash && strings.HasSuffix(file.RemotePath, ".html") {
			panic(fmt.Sprintf("Cowardly refusing to deploy an html file (%s) without versioning.", file.RemotePath))
		}

		ch <- file
	}

	close(ch)

	wg.Wait()
}

func addFiles(form uint8, parent *html.Node, files []string) {
	for _, file := range files {
		node := html.Node{
			Type: html.ElementNode,
		}
		switch form {
		case SCRIPT:
			node.Data = "script"
			node.Attr = []html.Attribute{
				html.Attribute{
					Key: "src",
					Val: file,
				},
			}

		case STYLE:
			node.Data = "link"
			node.Attr = []html.Attribute{
				html.Attribute{
					Key: "rel",
					Val: "stylesheet",
				},
				html.Attribute{
					Key: "href",
					Val: file,
				},
			}
		default:
			panic("Type not understood")
		}

		parent.AppendChild(&node)
	}
}

func isLocal(href string) bool {
	parsed := must(url.Parse(href)).(*url.URL)
	return parsed.Host == ""
}

func formatHref(path string) string {
	if !strings.HasPrefix(path, "/") {
		path = "/" + path
	}
	return path
}

func renderHTML(options Options, file HTMLFile) string {
	handle := must(os.Open(file.File.LocalPath)).(*os.File)
	defer handle.Close()

	doc := must(html.Parse(handle)).(*html.Node)

	var f func(*html.Node)
	f = func(n *html.Node) {
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			f(c)
		}

		if n.Type == html.ElementNode {
			switch n.Data {
			case "script":
				for i, a := range n.Attr {
					if a.Key == "src" {
						for _, dep := range file.Deps {
							if dep.InstPath == a.Val {
								n.Attr[i].Val = formatHref(dep.File.UploadedPath)
								break
							}
						}
					}
				}
			case "link":
				stylesheet := false
				for _, a := range n.Attr {
					if a.Key == "rel" {
						stylesheet = a.Val == "stylesheet"
						break
					}
				}
				if !stylesheet {
					return
				}

				for i, a := range n.Attr {
					if a.Key == "href" {
						for _, dep := range file.Deps {
							if dep.InstPath == a.Val {
								n.Attr[i].Val = formatHref(dep.File.UploadedPath)
								break
							}
						}
					}
				}
			}
		}
	}
	f(doc)

	buf := bytes.NewBuffer([]byte{})
	panicIf(html.Render(buf, doc))

	return buf.String()
}

func parseHTML(options Options, path string) (files []string, base string) {
	files = make([]string, 0)

	handle := must(os.Open(path)).(*os.File)
	defer handle.Close()

	doc := must(html.Parse(handle)).(*html.Node)

	var f func(*html.Node)
	f = func(n *html.Node) {
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			f(c)
		}

		if n.Type == html.ElementNode {
			switch n.Data {
			case "base":
				for _, a := range n.Attr {
					if a.Key == "href" {
						base = a.Val
					}
				}
			case "script":
				for _, a := range n.Attr {
					if a.Key == "src" {
						if isLocal(a.Val) {
							files = append(files, a.Val)
						}
					}
				}
			case "link":
				local := false
				stylesheet := false
				href := ""
				for _, a := range n.Attr {
					switch a.Key {
					case "href":
						local = isLocal(a.Val)
						href = a.Val
					case "rel":
						stylesheet = a.Val == "stylesheet"
					}
				}
				if local && stylesheet {
					files = append(files, href)
				}
			}
		}
	}
	f(doc)

	return
}

func deployHTML(options Options, id string, file HTMLFile) {
	data := renderHTML(options, file)

	internalPath, err := filepath.Rel(options.Root, file.File.LocalPath)
	if err != nil {
		panic(err)
	}

	permPath := filepath.Join(options.Dest, id, internalPath)
	curPath := filepath.Join(options.Dest, internalPath)

	bucket := s3Session.Bucket(options.Bucket)
	uploadFile(bucket, strings.NewReader(data), permPath, false, FOREVER)

	log.Println("Copying", permPath, "to", curPath)
	copyFile(bucket, permPath, curPath, "text/html", LIMITED)
}

func expandFiles(root string, glob string) []string {
	out := make([]string, 0)
	cases := strings.Split(glob, ",")

	for _, pattern := range cases {
		list := must(filepath.Glob(filepath.Join(root, pattern))).([]string)

		for _, file := range list {
			info := must(os.Stat(file)).(os.FileInfo)

			if info.IsDir() {
				filepath.Walk(file, func(path string, info os.FileInfo, err error) error {
					panicIf(err)

					if !info.IsDir() {
						out = append(out, path)
					}

					return nil
				})
			} else {
				out = append(out, file)
			}
		}
	}
	return out
}

func listFiles(options Options) []*FileRef {
	filePaths := expandFiles(options.Root, options.Files)

	files := make([]*FileRef, len(filePaths))
	for i, path := range filePaths {
		remotePath := filepath.Join(options.Dest, mustString(filepath.Rel(options.Root, path)))

		files[i] = &FileRef{
			LocalPath:  path,
			RemotePath: remotePath,
		}
	}

	return files
}

func ignoreFiles(full []*FileRef, rem []*FileRef) []*FileRef {
	out := make([]*FileRef, 0, len(full))

	for _, file := range full {
		ignore := false
		path := filepath.Clean(file.LocalPath)

		for _, remFile := range rem {
			if filepath.Clean(remFile.LocalPath) == path {
				ignore = true
				break
			}
		}

		if !ignore {
			out = append(out, file)
		}
	}

	return out
}

func extractFileList(options Options, pattern string) (files []string) {
	files = make([]string, 0)

	parts := strings.Split(pattern, ",")

	for _, part := range parts {
		matches, err := filepath.Glob(filepath.Join(options.Root, part))
		if err != nil {
			panic(err)
		}
		if matches == nil {
			panic(fmt.Sprintf("Pattern %s did not match any files", part))
		}

		files = append(files, matches...)
	}

	return files
}

func filesWithExtension(files []*FileRef, ext string) (outFiles []*FileRef) {
	outFiles = make([]*FileRef, 0)
	for _, file := range files {
		if filepath.Ext(file.LocalPath) == ext {
			outFiles = append(outFiles, file)
		}
	}

	return
}

type HTMLFile struct {
	File FileRef
	Deps []FileInst
	Base string
}

func (f HTMLFile) GetLocalPath() string {
	return f.File.LocalPath
}

func Deploy(options Options) {
	if s3Session == nil {
		s3Session = openS3(options.AWSKey, options.AWSSecret)
	}

	files := listFiles(options)

	htmlFileRefs := filesWithExtension(files, ".html")

	inclFiles := make(map[string]*FileRef)
	htmlFiles := make([]HTMLFile, len(htmlFileRefs))
	for i, file := range htmlFileRefs {
		dir := filepath.Dir(file.LocalPath)

		rel, err := filepath.Rel(options.Root, dir)
		if err != nil {
			panic(err)
		}

		paths, base := parseHTML(options, file.LocalPath)

		if strings.HasPrefix(strings.ToLower(base), "http") || strings.HasPrefix(base, "//") {
			panic("Absolute base tags are not supported")
		}

		htmlFiles[i] = HTMLFile{
			File: *file,
			Deps: make([]FileInst, len(paths)),
			Base: base,
		}

		for j, path := range paths {
			local := filepath.Join(options.Root, rel, base, path)
			remote := filepath.Join(options.Dest, rel, base, path)

			ref, ok := inclFiles[local]
			if !ok {
				ref = &FileRef{
					LocalPath:  local,
					RemotePath: remote,

					// Filled in after the deploy:
					UploadedPath: "",
				}

				inclFiles[local] = ref
			}

			use := FileInst{
				File:     ref,
				InstPath: path,
			}

			htmlFiles[i].Deps[j] = use
		}
	}

	inclFileList := make([]*FileRef, len(inclFiles))
	i := 0
	for _, ref := range inclFiles {
		inclFileList[i] = ref
		i++
	}

	hashPaths := make([]string, 0)
	for _, item := range inclFileList {
		hashPaths = append(hashPaths, item.LocalPath)
	}
	for _, item := range htmlFiles {
		hashPaths = append(hashPaths, item.File.LocalPath)
	}

	hash := hashFiles(hashPaths)
	id := hash[:12]

	deployFiles(options, true, inclFileList)
	deployFiles(options, false, ignoreFiles(files, htmlFileRefs))

	// Ensure that the new files exist in s3
	// Time based on "Eventual Consistency: How soon is eventual?"
	time.Sleep(1500 * time.Millisecond)

	wg := sync.WaitGroup{}
	for _, file := range htmlFiles {
		wg.Add(1)

		go func(file HTMLFile) {
			defer wg.Done()
			deployHTML(options, id, file)
		}(file)
	}

	wg.Wait()

	color.Printf(`
+------------------------------------+
|         @{g}Deploy Successful!@{|}         |
|                                    |
|       Deploy ID: @{?}%s@{|}      |
+------------------------------------+
`, id)

}

func deployCmd() {
	options, _ := parseOptions()
	loadConfigFile(&options)

	if options.Bucket == "" {
		panic("You must specify a bucket")
	}

	if options.AWSKey == "" || options.AWSSecret == "" {
		panic("You must specify your AWS credentials")
	}

	Deploy(options)
}
<|endoftext|>


<commit_before>package pkcs11

// A test of using several pkcs11 sessions in parallel for signing across
// multiple goroutines. Access to the PKCS11 module is thread-safe because of
// the C.CKF_OS_LOCKING_OK param and nil mutex functions that the pkcs11
// package passes to C.Initialize, which indicate that the module should use OS
// locking primitives on its own.
//
// Note that while access to the module is thread-safe, sessions are not thread
// safe, and each session must be protected from simultaneous use by some
// synchronization mechanism. In this case we use a cache of sessions (as
// embodied by the `signer` struct), protected by a condition variable. So long
// as there is an available signer in the cache, it is popped off and used. If
// there are no signers available, the caller blocks until there is one
// available.
//
// Please set the appropiate env variables. See the init function.
import (
	"fmt"
	"log"
	"os"
	"sync"
	"testing"
)

var (
	module          = "/usr/lib/softhsm/libsofthsm.so"
	tokenLabel      = "softhsm token"
	privateKeyLabel = "my key"
	pin             = "1234"
)

func init() {
	if x := os.Getenv("SOFTHSM_LIB"); x != "" {
		module = x
	}
	if x := os.Getenv("SOFTHSM_TOKENLABEL"); x != "" {
		tokenLabel = x
	}
	if x := os.Getenv("SOFTHSM_PRIVKEYLABEL"); x != "" {
		privateKeyLabel = x
	}
	if x := os.Getenv("SOFTHSM_PIN"); x != "" {
		pin = x
	}
	wd, _ := os.Getwd()
	os.Setenv("SOFTHSM_CONF", wd+"/softhsm.conf")
}

func initPKCS11Context(modulePath string) (*Ctx, error) {
	context := New(modulePath)

	if context == nil {
		return nil, fmt.Errorf("unable to load PKCS#11 module")
	}

	err := context.Initialize()
	return context, err
}

func getSlot(p *Ctx, label string) (uint, error) {
	slots, err := p.GetSlotList(true)
	if err != nil {
		return 0, err
	}
	for _, slot := range slots {
		_, err := p.GetSlotInfo(slot)
		if err != nil {
			return 0, err
		}
		tokenInfo, err := p.GetTokenInfo(slot)
		if err != nil {
			return 0, err
		}
		if tokenInfo.Label == label {
			return slot, nil
		}
	}
	return 0, fmt.Errorf("Slot not found: %s", label)
}

func getPrivateKey(context *Ctx, session SessionHandle, label string) (ObjectHandle, error) {
	var noKey ObjectHandle
	template := []*Attribute{
		NewAttribute(CKA_CLASS, CKO_PRIVATE_KEY),
		NewAttribute(CKA_LABEL, label),
	}
	if err := context.FindObjectsInit(session, template); err != nil {
		return noKey, err
	}
	objs, _, err := context.FindObjects(session, 2)
	if err != nil {
		return noKey, err
	}
	if err = context.FindObjectsFinal(session); err != nil {
		return noKey, err
	}

	if len(objs) == 0 {
		err = fmt.Errorf("private key not found")
		return noKey, err
	}
	return objs[0], nil
}

type signer struct {
	context    *Ctx
	session    SessionHandle
	privateKey ObjectHandle
}

func makeSigner(context *Ctx) (*signer, error) {
	slot, err := getSlot(context, tokenLabel)
	if err != nil {
		return nil, err
	}
	session, err := context.OpenSession(slot, CKF_SERIAL_SESSION)
	if err != nil {
		return nil, err
	}

	if err = context.Login(session, CKU_USER, pin); err != nil {
		context.CloseSession(session)
		return nil, err
	}

	privateKey, err := getPrivateKey(context, session, privateKeyLabel)
	if err != nil {
		context.CloseSession(session)
		return nil, err
	}
	return &signer{context, session, privateKey}, nil
}

func (s *signer) sign(input []byte) ([]byte, error) {
	mechanism := []*Mechanism{NewMechanism(CKM_RSA_PKCS, nil)}
	if err := s.context.SignInit(s.session, mechanism, s.privateKey); err != nil {
		log.Fatalf("SignInit: %s", err)
	}

	signed, err := s.context.Sign(s.session, input)
	if err != nil {
		log.Fatalf("Sign: %s", err)
	}
	return signed, nil
}

type cache struct {
	signers []*signer
	// this variable signals the condition that there are signers available to be
	// used.
	cond *sync.Cond
}

func newCache(signers []*signer) cache {
	var mutex sync.Mutex
	return cache{
		signers: signers,
		cond:    sync.NewCond(&mutex),
	}
}

func (c *cache) get() *signer {
	c.cond.L.Lock()
	for len(c.signers) == 0 {
		c.cond.Wait()
	}

	instance := c.signers[len(c.signers)-1]
	c.signers = c.signers[:len(c.signers)-1]
	c.cond.L.Unlock()
	return instance
}

func (c *cache) put(instance *signer) {
	c.cond.L.Lock()
	c.signers = append(c.signers, instance)
	c.cond.Signal()
	c.cond.L.Unlock()
}

func (c *cache) sign(input []byte) ([]byte, error) {
	instance := c.get()
	defer c.put(instance)
	return instance.sign(input)
}

// TODO(miek): disabled for now. Fill out the correct values in hsm.db so we can use it.
func testParallel(t *testing.T) {
	if module == "" || tokenLabel == "" || pin == "" || privateKeyLabel == "" {
		t.Fatal("Must pass all flags: module, tokenLabel, pin, and privateKeyLabel")
		return
	}

	context, err := initPKCS11Context(module)
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		context.Finalize()
		context.Destroy()
	}()

	const nSigners = 100
	const nSignatures = 1000
	signers := make([]*signer, nSigners)
	for i := 0; i < nSigners; i++ {
		signers[i], err = makeSigner(context)
		if err != nil {
			t.Fatalf("Problem making signer: %s", err)
		}
	}
	pool := newCache(signers)

	output := make(chan []byte, nSignatures)
	for i := 0; i < nSignatures; i++ {
		go func() {
			result, err := pool.sign([]byte("hi"))
			if err != nil {
				t.Fatal(err)
			}
			output <- result
		}()
	}

	for i := 0; i < nSignatures; i++ {
		// Consume the output of the signers, but do nothing with it.
		<-output
	}

	for i := 0; i < nSigners; i++ {
		// Note: It is not necessary to call context.Logout. Closing the last
		// session will automatically log out, per PKCS#11 API.
		context.CloseSession(signers[i].session)
	}
}
<commit_msg>correct spelling mistake (#52)<commit_after>package pkcs11

// A test of using several pkcs11 sessions in parallel for signing across
// multiple goroutines. Access to the PKCS11 module is thread-safe because of
// the C.CKF_OS_LOCKING_OK param and nil mutex functions that the pkcs11
// package passes to C.Initialize, which indicate that the module should use OS
// locking primitives on its own.
//
// Note that while access to the module is thread-safe, sessions are not thread
// safe, and each session must be protected from simultaneous use by some
// synchronization mechanism. In this case we use a cache of sessions (as
// embodied by the `signer` struct), protected by a condition variable. So long
// as there is an available signer in the cache, it is popped off and used. If
// there are no signers available, the caller blocks until there is one
// available.
//
// Please set the appropriate env variables. See the init function.
import (
	"fmt"
	"log"
	"os"
	"sync"
	"testing"
)

var (
	module          = "/usr/lib/softhsm/libsofthsm.so"
	tokenLabel      = "softhsm token"
	privateKeyLabel = "my key"
	pin             = "1234"
)

func init() {
	if x := os.Getenv("SOFTHSM_LIB"); x != "" {
		module = x
	}
	if x := os.Getenv("SOFTHSM_TOKENLABEL"); x != "" {
		tokenLabel = x
	}
	if x := os.Getenv("SOFTHSM_PRIVKEYLABEL"); x != "" {
		privateKeyLabel = x
	}
	if x := os.Getenv("SOFTHSM_PIN"); x != "" {
		pin = x
	}
	wd, _ := os.Getwd()
	os.Setenv("SOFTHSM_CONF", wd+"/softhsm.conf")
}

func initPKCS11Context(modulePath string) (*Ctx, error) {
	context := New(modulePath)

	if context == nil {
		return nil, fmt.Errorf("unable to load PKCS#11 module")
	}

	err := context.Initialize()
	return context, err
}

func getSlot(p *Ctx, label string) (uint, error) {
	slots, err := p.GetSlotList(true)
	if err != nil {
		return 0, err
	}
	for _, slot := range slots {
		_, err := p.GetSlotInfo(slot)
		if err != nil {
			return 0, err
		}
		tokenInfo, err := p.GetTokenInfo(slot)
		if err != nil {
			return 0, err
		}
		if tokenInfo.Label == label {
			return slot, nil
		}
	}
	return 0, fmt.Errorf("Slot not found: %s", label)
}

func getPrivateKey(context *Ctx, session SessionHandle, label string) (ObjectHandle, error) {
	var noKey ObjectHandle
	template := []*Attribute{
		NewAttribute(CKA_CLASS, CKO_PRIVATE_KEY),
		NewAttribute(CKA_LABEL, label),
	}
	if err := context.FindObjectsInit(session, template); err != nil {
		return noKey, err
	}
	objs, _, err := context.FindObjects(session, 2)
	if err != nil {
		return noKey, err
	}
	if err = context.FindObjectsFinal(session); err != nil {
		return noKey, err
	}

	if len(objs) == 0 {
		err = fmt.Errorf("private key not found")
		return noKey, err
	}
	return objs[0], nil
}

type signer struct {
	context    *Ctx
	session    SessionHandle
	privateKey ObjectHandle
}

func makeSigner(context *Ctx) (*signer, error) {
	slot, err := getSlot(context, tokenLabel)
	if err != nil {
		return nil, err
	}
	session, err := context.OpenSession(slot, CKF_SERIAL_SESSION)
	if err != nil {
		return nil, err
	}

	if err = context.Login(session, CKU_USER, pin); err != nil {
		context.CloseSession(session)
		return nil, err
	}

	privateKey, err := getPrivateKey(context, session, privateKeyLabel)
	if err != nil {
		context.CloseSession(session)
		return nil, err
	}
	return &signer{context, session, privateKey}, nil
}

func (s *signer) sign(input []byte) ([]byte, error) {
	mechanism := []*Mechanism{NewMechanism(CKM_RSA_PKCS, nil)}
	if err := s.context.SignInit(s.session, mechanism, s.privateKey); err != nil {
		log.Fatalf("SignInit: %s", err)
	}

	signed, err := s.context.Sign(s.session, input)
	if err != nil {
		log.Fatalf("Sign: %s", err)
	}
	return signed, nil
}

type cache struct {
	signers []*signer
	// this variable signals the condition that there are signers available to be
	// used.
	cond *sync.Cond
}

func newCache(signers []*signer) cache {
	var mutex sync.Mutex
	return cache{
		signers: signers,
		cond:    sync.NewCond(&mutex),
	}
}

func (c *cache) get() *signer {
	c.cond.L.Lock()
	for len(c.signers) == 0 {
		c.cond.Wait()
	}

	instance := c.signers[len(c.signers)-1]
	c.signers = c.signers[:len(c.signers)-1]
	c.cond.L.Unlock()
	return instance
}

func (c *cache) put(instance *signer) {
	c.cond.L.Lock()
	c.signers = append(c.signers, instance)
	c.cond.Signal()
	c.cond.L.Unlock()
}

func (c *cache) sign(input []byte) ([]byte, error) {
	instance := c.get()
	defer c.put(instance)
	return instance.sign(input)
}

// TODO(miek): disabled for now. Fill out the correct values in hsm.db so we can use it.
func testParallel(t *testing.T) {
	if module == "" || tokenLabel == "" || pin == "" || privateKeyLabel == "" {
		t.Fatal("Must pass all flags: module, tokenLabel, pin, and privateKeyLabel")
		return
	}

	context, err := initPKCS11Context(module)
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		context.Finalize()
		context.Destroy()
	}()

	const nSigners = 100
	const nSignatures = 1000
	signers := make([]*signer, nSigners)
	for i := 0; i < nSigners; i++ {
		signers[i], err = makeSigner(context)
		if err != nil {
			t.Fatalf("Problem making signer: %s", err)
		}
	}
	pool := newCache(signers)

	output := make(chan []byte, nSignatures)
	for i := 0; i < nSignatures; i++ {
		go func() {
			result, err := pool.sign([]byte("hi"))
			if err != nil {
				t.Fatal(err)
			}
			output <- result
		}()
	}

	for i := 0; i < nSignatures; i++ {
		// Consume the output of the signers, but do nothing with it.
		<-output
	}

	for i := 0; i < nSigners; i++ {
		// Note: It is not necessary to call context.Logout. Closing the last
		// session will automatically log out, per PKCS#11 API.
		context.CloseSession(signers[i].session)
	}
}
<|endoftext|>