<commit_before>// Copyright 2012 Google, Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style license
// that can be found in the LICENSE file in the root of the source
// tree.

// This binary provides sample code for using the gopacket TCP assembler and TCP
// stream reader.  It reads packets off the wire and reconstructs HTTP requests
// it sees, logging them.
package main

import (
	"bufio"
	"flag"
	"io"
	"log"
	"net/http"
	"time"

	"github.com/google/gopacket"
	"github.com/google/gopacket/examples/util"
	"github.com/google/gopacket/layers"
	"github.com/google/gopacket/pcap"
	"github.com/google/gopacket/tcpassembly"
	"github.com/google/gopacket/tcpassembly/tcpreader"
)

var iface = flag.String("i", "eth0", "Interface to get packets from")
var fname = flag.String("r", "", "Filename to read from, overrides -i")
var snaplen = flag.Int("s", 1600, "SnapLen for pcap packet capture")
var filter = flag.String("f", "tcp and dst port 80", "BPF filter for pcap")
var logAllPackets = flag.Bool("v", false, "Logs every packet in great detail")

// Build a simple HTTP request parser using tcpassembly.StreamFactory and tcpassembly.Stream interfaces

// httpStreamFactory implements tcpassembly.StreamFactory
type httpStreamFactory struct{}

// httpStream will handle the actual decoding of http requests.
type httpStream struct {
	net, transport gopacket.Flow
	r              tcpreader.ReaderStream
}

func (h *httpStreamFactory) New(net, transport gopacket.Flow) tcpassembly.Stream {
	hstream := &httpStream{
		net:       net,
		transport: transport,
		r:         tcpreader.NewReaderStream(),
	}
	go hstream.run() // Important... we must guarantee that data from the reader stream is read.

	// ReaderStream implements tcpassembly.Stream, so we can return a pointer to it.
	return &hstream.r
}

func (h *httpStream) run() {
	buf := bufio.NewReader(&h.r)
	for {
		req, err := http.ReadResponse(buf, nil)
		if err == io.EOF {
			// We must read until we see an EOF... very important!
			return
		} else if err == io.ErrUnexpectedEOF {
			// TODO: Not sure what to do with these.
			return
		} else if err != nil {
			log.Println("Error reading stream", h.net, h.transport, ":", err)
		} else {

			contentType := req.Header["Content-Type"]

			log.Println("TYPE: ", contentType)

			bodyBytes := tcpreader.DiscardBytesToEOF(req.Body)
			req.Body.Close()
			log.Println("Received response from stream", h.net, h.transport, ":", req, "with", bodyBytes, "bytes in response body")

		}
	}
}

func main() {
	defer util.Run()()
	var handle *pcap.Handle
	var err error

	// Set up pcap packet capture
	if *fname != "" {
		log.Printf("Reading from pcap dump %q", *fname)
		handle, err = pcap.OpenOffline(*fname)
	} else {
		log.Printf("Starting capture on interface %q", *iface)
		handle, err = pcap.OpenLive(*iface, int32(*snaplen), true, pcap.BlockForever)
	}
	if err != nil {
		log.Fatal(err)
	}

	if err := handle.SetBPFFilter(*filter); err != nil {
		log.Fatal(err)
	}

	// Set up assembly
	streamFactory := &httpStreamFactory{}
	streamPool := tcpassembly.NewStreamPool(streamFactory)
	assembler := tcpassembly.NewAssembler(streamPool)

	log.Println("reading in packets")
	// Read in packets, pass to assembler.
	packetSource := gopacket.NewPacketSource(handle, handle.LinkType())
	packets := packetSource.Packets()
	ticker := time.Tick(time.Minute)
	for {
		select {
		case packet := <-packets:
			// A nil packet indicates the end of a pcap file.
			if packet == nil {
				return
			}
			if *logAllPackets {
				log.Println(packet)
			}
			if packet.NetworkLayer() == nil || packet.TransportLayer() == nil || packet.TransportLayer().LayerType() != layers.LayerTypeTCP {
				log.Println("Unusable packet")
				continue
			}
			tcp := packet.TransportLayer().(*layers.TCP)
			assembler.AssembleWithTimestamp(packet.NetworkLayer().NetworkFlow(), tcp, packet.Metadata().Timestamp)

		case <-ticker:
			// Every minute, flush connections that haven't seen activity in the past 2 minutes.
			assembler.FlushOlderThan(time.Now().Add(time.Minute * -2))
		}
	}
}
<commit_msg>Found problem with unexpected EOF<commit_after>// Copyright 2012 Google, Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style license
// that can be found in the LICENSE file in the root of the source
// tree.

// This binary provides sample code for using the gopacket TCP assembler and TCP
// stream reader.  It reads packets off the wire and reconstructs HTTP requests
// it sees, logging them.
package main

import (
	"bufio"
	"flag"
	"io"
	"log"
	"net/http"
	"time"

	"github.com/google/gopacket"
	"github.com/google/gopacket/examples/util"
	"github.com/google/gopacket/layers"
	"github.com/google/gopacket/pcap"
	"github.com/google/gopacket/tcpassembly"
	"github.com/google/gopacket/tcpassembly/tcpreader"
)

var iface = flag.String("i", "eth0", "Interface to get packets from")
var fname = flag.String("r", "", "Filename to read from, overrides -i")
var snaplen = flag.Int("s", 1600, "SnapLen for pcap packet capture")
var filter = flag.String("f", "tcp and dst port 80", "BPF filter for pcap")
var logAllPackets = flag.Bool("v", false, "Logs every packet in great detail")

// Build a simple HTTP request parser using tcpassembly.StreamFactory and tcpassembly.Stream interfaces

// httpStreamFactory implements tcpassembly.StreamFactory
type httpStreamFactory struct{}

// httpStream will handle the actual decoding of http requests.
type httpStream struct {
	net, transport gopacket.Flow
	r              tcpreader.ReaderStream
}

func (h *httpStreamFactory) New(net, transport gopacket.Flow) tcpassembly.Stream {
	hstream := &httpStream{
		net:       net,
		transport: transport,
		r:         tcpreader.NewReaderStream(),
	}
	go hstream.run() // Important... we must guarantee that data from the reader stream is read.

	// ReaderStream implements tcpassembly.Stream, so we can return a pointer to it.
	return &hstream.r
}

func (h *httpStream) run() {
	buf := bufio.NewReader(&h.r)
	for {
		req, err := http.ReadResponse(buf, nil)
		if err == io.EOF {
			// We must read until we see an EOF... very important!
			return
		} else if err == io.ErrUnexpectedEOF {
			// TODO: Not sure what to do with these.
			return
		} else if err != nil {
			log.Println("Error reading stream", h.net, h.transport, ":", err)
		} else {

			contentType := req.Header["Content-Type"]

			log.Println("TYPE: ", contentType)

			//bodyBytes := tcpreader.DiscardBytesToEOF(req.Body)

			for {
				_, err := tcpreader.DiscardBytesToFirstError(req.Body)
				if err == io.EOF {
					log.Println("EOF:", h.net, h.transport, ":", err)

				} else {
					log.Println("ERROR:", h.net, h.transport, ":", err)
				}
			}

			req.Body.Close()
			log.Println("Received response from stream", h.net, h.transport, ":", req)

		}
	}
}

func main() {
	defer util.Run()()
	var handle *pcap.Handle
	var err error

	// Set up pcap packet capture
	if *fname != "" {
		log.Printf("Reading from pcap dump %q", *fname)
		handle, err = pcap.OpenOffline(*fname)
	} else {
		log.Printf("Starting capture on interface %q", *iface)
		handle, err = pcap.OpenLive(*iface, int32(*snaplen), true, pcap.BlockForever)
	}
	if err != nil {
		log.Fatal(err)
	}

	if err := handle.SetBPFFilter(*filter); err != nil {
		log.Fatal(err)
	}

	// Set up assembly
	streamFactory := &httpStreamFactory{}
	streamPool := tcpassembly.NewStreamPool(streamFactory)
	assembler := tcpassembly.NewAssembler(streamPool)

	log.Println("reading in packets")
	// Read in packets, pass to assembler.
	packetSource := gopacket.NewPacketSource(handle, handle.LinkType())
	packets := packetSource.Packets()
	ticker := time.Tick(time.Minute)
	for {
		select {
		case packet := <-packets:
			// A nil packet indicates the end of a pcap file.
			if packet == nil {
				return
			}
			if *logAllPackets {
				log.Println(packet)
			}
			if packet.NetworkLayer() == nil || packet.TransportLayer() == nil || packet.TransportLayer().LayerType() != layers.LayerTypeTCP {
				log.Println("Unusable packet")
				continue
			}
			tcp := packet.TransportLayer().(*layers.TCP)
			assembler.AssembleWithTimestamp(packet.NetworkLayer().NetworkFlow(), tcp, packet.Metadata().Timestamp)

		case <-ticker:
			// Every minute, flush connections that haven't seen activity in the past 2 minutes.
			assembler.FlushOlderThan(time.Now().Add(time.Minute * -2))
		}
	}
}
<|endoftext|>


<commit_before>package tuple

import (
	"fmt"
	. "github.com/smartystreets/goconvey/convey"
	"math"
	"testing"
	"time"
)

type convTestInput struct {
	ValueDesc string
	Value     Value
	Expected  interface{}
}

func TestToBool(t *testing.T) {
	testCases := map[string][]convTestInput{
		"Null": {
			{"Null", Null{}, false},
		},
		"Bool": {
			{"true", Bool(true), true},
			{"false", Bool(false), false},
		},
		"Int": {
			{"positive", Int(2), true},
			{"negative", Int(-2), true},
			{"zero", Int(0), false},
		},
		"Float": {
			{"positive", Float(3.14), true},
			{"negative", Float(-3.14), true},
			{"zero", Float(0.0), false},
			{"NaN", Float(math.NaN()), false},
		},
		"String": {
			{"empty", String(""), false},
			{"non-empty", String("hoge"), true},
		},
		"Blob": {
			{"empty", Blob(""), false},
			{"nil", Blob(nil), false},
			{"non-empty", Blob("hoge"), true},
		},
		"Timestamp": {
			{"zero", Timestamp(time.Time{}), false},
			{"now", Timestamp(time.Now()), true},
		},
		"Array": {
			{"empty", Array{}, false},
			{"non-empty", Array{Int(2), String("foo")}, true},
		},
		"Map": {
			{"empty", Map{}, false},
			{"non-empty", Map{"a": Int(2), "b": String("foo")}, true},
		},
	}

	toFun := func(v Value) (interface{}, error) {
		val, err := ToBool(v)
		return val, err
	}
	runConversionTestCases(t, toFun, "ToBool", testCases)
}

func TestToInt(t *testing.T) {
	now := time.Now()
	negTime := time.Unix(-1, -1000)

	testCases := map[string][]convTestInput{
		"Null": {
			{"Null", Null{}, nil},
		},
		"Bool": {
			{"true", Bool(true), int64(1)},
			{"false", Bool(false), int64(0)},
		},
		"Int": {
			{"positive", Int(2), int64(2)},
			{"negative", Int(-2), int64(-2)},
			{"zero", Int(0), int64(0)},
		},
		"Float": {
			// normal conversion
			{"positive", Float(3.14), int64(3)},
			{"negative", Float(-3.14), int64(-3)},
			{"zero", Float(0.0), int64(0)},
			// we truncate and don't round
			{"positive (> x.5)", Float(2.71), int64(2)},
			{"negative (< x.5)", Float(-2.71), int64(-2)},
			// we cannot convert all numbers
			{"maximal positive", Float(math.MaxFloat64), nil},
			{"maximal negative", Float(-math.MaxFloat64), nil},
		},
		"String": {
			{"empty", String(""), nil},
			{"non-empty", String("hoge"), nil},
			{"numeric", String("123456"), int64(123456)},
		},
		"Blob": {
			{"empty", Blob(""), nil},
			{"non-empty", Blob("hoge"), nil},
		},
		"Timestamp": {
			// The zero value for a time.Time is *not* the timestamp
			// that has unix time zero!
			{"zero", Timestamp(time.Time{}), int64(-62135596800000000)},
			{"now", Timestamp(now), now.UnixNano() / 1000},
			{"negative", Timestamp(negTime), negTime.UnixNano() / 1000},
		},
		"Array": {
			{"empty", Array{}, nil},
			{"non-empty", Array{Int(2), String("foo")}, nil},
		},
		"Map": {
			{"empty", Map{}, nil},
			{"non-empty", Map{"a": Int(2), "b": String("foo")}, nil},
		},
	}

	toFun := func(v Value) (interface{}, error) {
		val, err := ToInt(v)
		return val, err
	}
	runConversionTestCases(t, toFun, "ToInt", testCases)
}

func TestToFloat(t *testing.T) {
	now := time.Now()
	// We would like to check whether ToFloat(timestamp) equals the
	// value of `float64(timestamp.UnixNano()) / 1e9` but actually
	// this occasionally fails due to numerical rounding. Therefore
	// we use a computation method closer to the actual computation,
	// method, but still based on UnixNano().
	nowSeconds := now.UnixNano() / 1e9
	nowNanoSeconds := now.UnixNano() % 1e9
	nowFloatSeconds := float64(nowSeconds) + float64(nowNanoSeconds)/1e9

	testCases := map[string][]convTestInput{
		"Null": {
			{"Null", Null{}, nil},
		},
		"Bool": {
			{"true", Bool(true), float64(1.0)},
			{"false", Bool(false), float64(0.0)},
		},
		"Int": {
			// normal conversion
			{"positive", Int(2), float64(2.0)},
			{"negative", Int(-2), float64(-2.0)},
			{"zero", Int(0), float64(0.0)},
			// float64 can represent all int64 numbers (with loss
			// of precision, though)
			{"maximal positive", Int(math.MaxInt64), float64(9.223372036854776e+18)},
			{"maximal negative", Int(math.MinInt64), float64(-9.223372036854776e+18)},
		},
		"Float": {
			{"positive", Float(3.14), float64(3.14)},
			{"negative", Float(-3.14), float64(-3.14)},
			{"zero", Float(0.0), float64(0)},
		},
		"String": {
			{"empty", String(""), nil},
			{"non-empty", String("hoge"), nil},
			{"numeric", String("123.456"), float64(123.456)},
		},
		"Blob": {
			{"empty", Blob(""), nil},
			{"non-empty", Blob("hoge"), nil},
		},
		"Timestamp": {
			// The zero value for a time.Time is *not* the timestamp
			// that has unix time zero!
			{"zero", Timestamp(time.Time{}), float64(-62135596800)},
			{"now", Timestamp(now), nowFloatSeconds},
		},
		"Array": {
			{"empty", Array{}, nil},
			{"non-empty", Array{Int(2), String("foo")}, nil},
		},
		"Map": {
			{"empty", Map{}, nil},
			{"non-empty", Map{"a": Int(2), "b": String("foo")}, nil},
		},
	}

	toFun := func(v Value) (interface{}, error) {
		val, err := ToFloat(v)
		return val, err
	}
	runConversionTestCases(t, toFun, "ToFloat", testCases)
}

func TestToString(t *testing.T) {
	now := time.Now()

	testCases := map[string][]convTestInput{
		"Null": {
			{"Null", Null{}, "null"},
		},
		"Bool": {
			{"true", Bool(true), "true"},
			{"false", Bool(false), "false"},
		},
		"Int": {
			{"positive", Int(2), "2"},
			{"negative", Int(-2), "-2"},
			{"zero", Int(0), "0"},
		},
		"Float": {
			{"positive", Float(3.14), "3.14"},
			{"negative", Float(-3.14), "-3.14"},
			{"zero", Float(0.0), "0"},
		},
		"String": {
			{"empty", String(""), ""},
			{"non-empty", String("hoge"), "hoge"},
		},
		"Blob": {
			{"empty", Blob(""), ""},
			{"non-empty", Blob("hoge"), "hoge"},
		},
		"Timestamp": {
			{"zero", Timestamp(time.Time{}), "0001-01-01T00:00:00Z"},
			{"now", Timestamp(now), now.Format(time.RFC3339Nano)},
		},
		"Array": {
			{"empty", Array{}, "tuple.Array{}"},
			{"non-empty", Array{Int(2), String("foo")}, `tuple.Array{2, "foo"}`},
		},
		"Map": {
			{"empty", Map{}, `tuple.Map{}`},
			// the following test would fail once in a while because
			// golang randomizes the keys for Maps, i.e., we cannot be sure
			// that we will always get the same string
			// convTestInput{"non-empty", Map{"a": Int(2), "b": String("foo")}, `tuple.Map{"a":2, "b":"foo"}`},
		},
	}

	toFun := func(v Value) (interface{}, error) {
		val, err := ToString(v)
		return val, err
	}
	runConversionTestCases(t, toFun, "ToString", testCases)
}

func TestToBlob(t *testing.T) {
	testCases := map[string][]convTestInput{
		"Null": {
			{"Null", Null{}, []byte(nil)},
		},
		"Bool": {
			{"true", Bool(true), nil},
			{"false", Bool(false), nil},
		},
		"Int": {
			{"positive", Int(2), nil},
			{"negative", Int(-2), nil},
			{"zero", Int(0), nil},
		},
		"Float": {
			{"positive", Float(3.14), nil},
			{"negative", Float(-3.14), nil},
			{"zero", Float(0.0), nil},
		},
		"String": {
			{"empty", String(""), []byte{}},
			{"non-empty", String("hoge"), []byte("hoge")},
			{"numeric", String("123.456"), []byte("123.456")},
		},
		"Blob": {
			{"nil", Blob(nil), []byte(nil)},
			{"empty", Blob{}, []byte{}},
			{"non-empty", Blob("hoge"), []byte("hoge")},
		},
		"Timestamp": {
			{"zero", Timestamp(time.Time{}), nil},
			{"now", Timestamp(time.Now()), nil},
		},
		"Array": {
			{"empty", Array{}, nil},
			{"non-empty", Array{Int(2), String("foo")}, nil},
		},
		"Map": {
			{"empty", Map{}, nil},
			{"non-empty", Map{"a": Int(2), "b": String("foo")}, nil},
		},
	}

	toFun := func(v Value) (interface{}, error) {
		val, err := ToBlob(v)
		return val, err
	}
	runConversionTestCases(t, toFun, "ToBlob", testCases)
}

func TestToTime(t *testing.T) {
	now := time.Now()
	jst, err := time.LoadLocation("JST")
	if err != nil {
		t.Fatal("Cannot load JST location:", err)
	}

	testCases := map[string][]convTestInput{
		"Null": {
			{"Null", Null{}, time.Time{}},
		},
		"Bool": {
			{"true", Bool(true), nil},
			{"false", Bool(false), nil},
		},
		"Int": {
			{"positive", Int(2), time.Unix(0, 2000)},
			{"negative", Int(-2), time.Unix(0, -2000)},
			{"large and positive", Int(2e6), time.Unix(2, 0)},
			{"large and negative", Int(-2e6), time.Unix(-2, 0)},
			{"zero", Int(0), time.Unix(0, 0)},
		},
		"Float": {
			{"positive", Float(3.14), time.Unix(3, 14e7)},
			{"negative", Float(-3.14), time.Unix(-3, -14e7)},
			{"negative (alternative)", Float(-3.14), time.Unix(-4, 86e7)},
			{"zero", Float(0.0), time.Unix(0, 0)},
		},
		"String": {
			{"empty", String(""), nil},
			{"non-empty", String("hoge"), nil},
			{"valid time string", String("1970-01-01T09:00:02+09:00"), time.Unix(2, 0).In(jst)},
			{"valid time string with ns", String(now.Format(time.RFC3339Nano)), now},
		},
		"Blob": {
			{"empty", Blob(""), nil},
			{"non-empty", Blob("hoge"), nil},
		},
		"Timestamp": {
			{"zero", Timestamp(time.Time{}), time.Time{}},
			{"now", Timestamp(now), now},
		},
		"Array": {
			{"empty", Array{}, nil},
			{"non-empty", Array{Int(2), String("foo")}, nil},
		},
		"Map": {
			{"empty", Map{}, nil},
			{"non-empty", Map{"a": Int(2), "b": String("foo")}, nil},
		},
	}

	toFun := func(v Value) (interface{}, error) {
		val, err := ToTime(v)
		return val, err
	}
	runConversionTestCases(t, toFun, "ToTime", testCases)
}

func runConversionTestCases(t *testing.T,
	toFun func(v Value) (interface{}, error),
	funcName string,
	testCases map[string][]convTestInput) {
	for valType, cases := range testCases {
		cases := cases
		Convey(fmt.Sprintf("Given a %s value", valType), t, func() {
			for _, testCase := range cases {
				tc := testCase
				Convey(fmt.Sprintf("When it is %s", tc.ValueDesc), func() {
					inVal := tc.Value
					exp := tc.Expected
					if exp == nil {
						Convey(fmt.Sprintf("Then %s returns an error", funcName), func() {
							_, err := toFun(inVal)
							So(err, ShouldNotBeNil)
						})
					} else {
						Convey(fmt.Sprintf("Then %s returns %v", funcName, exp), func() {
							val, err := toFun(inVal)
							So(err, ShouldBeNil)
							switch fval := val.(type) {
							case float64:
								switch fexp := exp.(type) {
								case float64:
									// equal up to machine precision
									So(fval-fexp, ShouldAlmostEqual, 0.0, 1e-15)
								default:
									So(val, ShouldResemble, exp)
								}
							default:
								So(val, ShouldResemble, exp)
							}
						})
					}
				})
			}
		})
	}
}
<commit_msg>Fix a failing test case due to time.Time's equivalence testing<commit_after>package tuple

import (
	"fmt"
	. "github.com/smartystreets/goconvey/convey"
	"math"
	"testing"
	"time"
)

type convTestInput struct {
	ValueDesc string
	Value     Value
	Expected  interface{}
}

func TestToBool(t *testing.T) {
	testCases := map[string][]convTestInput{
		"Null": {
			{"Null", Null{}, false},
		},
		"Bool": {
			{"true", Bool(true), true},
			{"false", Bool(false), false},
		},
		"Int": {
			{"positive", Int(2), true},
			{"negative", Int(-2), true},
			{"zero", Int(0), false},
		},
		"Float": {
			{"positive", Float(3.14), true},
			{"negative", Float(-3.14), true},
			{"zero", Float(0.0), false},
			{"NaN", Float(math.NaN()), false},
		},
		"String": {
			{"empty", String(""), false},
			{"non-empty", String("hoge"), true},
		},
		"Blob": {
			{"empty", Blob(""), false},
			{"nil", Blob(nil), false},
			{"non-empty", Blob("hoge"), true},
		},
		"Timestamp": {
			{"zero", Timestamp(time.Time{}), false},
			{"now", Timestamp(time.Now()), true},
		},
		"Array": {
			{"empty", Array{}, false},
			{"non-empty", Array{Int(2), String("foo")}, true},
		},
		"Map": {
			{"empty", Map{}, false},
			{"non-empty", Map{"a": Int(2), "b": String("foo")}, true},
		},
	}

	toFun := func(v Value) (interface{}, error) {
		val, err := ToBool(v)
		return val, err
	}
	runConversionTestCases(t, toFun, "ToBool", testCases)
}

func TestToInt(t *testing.T) {
	now := time.Now()
	negTime := time.Unix(-1, -1000)

	testCases := map[string][]convTestInput{
		"Null": {
			{"Null", Null{}, nil},
		},
		"Bool": {
			{"true", Bool(true), int64(1)},
			{"false", Bool(false), int64(0)},
		},
		"Int": {
			{"positive", Int(2), int64(2)},
			{"negative", Int(-2), int64(-2)},
			{"zero", Int(0), int64(0)},
		},
		"Float": {
			// normal conversion
			{"positive", Float(3.14), int64(3)},
			{"negative", Float(-3.14), int64(-3)},
			{"zero", Float(0.0), int64(0)},
			// we truncate and don't round
			{"positive (> x.5)", Float(2.71), int64(2)},
			{"negative (< x.5)", Float(-2.71), int64(-2)},
			// we cannot convert all numbers
			{"maximal positive", Float(math.MaxFloat64), nil},
			{"maximal negative", Float(-math.MaxFloat64), nil},
		},
		"String": {
			{"empty", String(""), nil},
			{"non-empty", String("hoge"), nil},
			{"numeric", String("123456"), int64(123456)},
		},
		"Blob": {
			{"empty", Blob(""), nil},
			{"non-empty", Blob("hoge"), nil},
		},
		"Timestamp": {
			// The zero value for a time.Time is *not* the timestamp
			// that has unix time zero!
			{"zero", Timestamp(time.Time{}), int64(-62135596800000000)},
			{"now", Timestamp(now), now.UnixNano() / 1000},
			{"negative", Timestamp(negTime), negTime.UnixNano() / 1000},
		},
		"Array": {
			{"empty", Array{}, nil},
			{"non-empty", Array{Int(2), String("foo")}, nil},
		},
		"Map": {
			{"empty", Map{}, nil},
			{"non-empty", Map{"a": Int(2), "b": String("foo")}, nil},
		},
	}

	toFun := func(v Value) (interface{}, error) {
		val, err := ToInt(v)
		return val, err
	}
	runConversionTestCases(t, toFun, "ToInt", testCases)
}

func TestToFloat(t *testing.T) {
	now := time.Now()
	// We would like to check whether ToFloat(timestamp) equals the
	// value of `float64(timestamp.UnixNano()) / 1e9` but actually
	// this occasionally fails due to numerical rounding. Therefore
	// we use a computation method closer to the actual computation,
	// method, but still based on UnixNano().
	nowSeconds := now.UnixNano() / 1e9
	nowNanoSeconds := now.UnixNano() % 1e9
	nowFloatSeconds := float64(nowSeconds) + float64(nowNanoSeconds)/1e9

	testCases := map[string][]convTestInput{
		"Null": {
			{"Null", Null{}, nil},
		},
		"Bool": {
			{"true", Bool(true), float64(1.0)},
			{"false", Bool(false), float64(0.0)},
		},
		"Int": {
			// normal conversion
			{"positive", Int(2), float64(2.0)},
			{"negative", Int(-2), float64(-2.0)},
			{"zero", Int(0), float64(0.0)},
			// float64 can represent all int64 numbers (with loss
			// of precision, though)
			{"maximal positive", Int(math.MaxInt64), float64(9.223372036854776e+18)},
			{"maximal negative", Int(math.MinInt64), float64(-9.223372036854776e+18)},
		},
		"Float": {
			{"positive", Float(3.14), float64(3.14)},
			{"negative", Float(-3.14), float64(-3.14)},
			{"zero", Float(0.0), float64(0)},
		},
		"String": {
			{"empty", String(""), nil},
			{"non-empty", String("hoge"), nil},
			{"numeric", String("123.456"), float64(123.456)},
		},
		"Blob": {
			{"empty", Blob(""), nil},
			{"non-empty", Blob("hoge"), nil},
		},
		"Timestamp": {
			// The zero value for a time.Time is *not* the timestamp
			// that has unix time zero!
			{"zero", Timestamp(time.Time{}), float64(-62135596800)},
			{"now", Timestamp(now), nowFloatSeconds},
		},
		"Array": {
			{"empty", Array{}, nil},
			{"non-empty", Array{Int(2), String("foo")}, nil},
		},
		"Map": {
			{"empty", Map{}, nil},
			{"non-empty", Map{"a": Int(2), "b": String("foo")}, nil},
		},
	}

	toFun := func(v Value) (interface{}, error) {
		val, err := ToFloat(v)
		return val, err
	}
	runConversionTestCases(t, toFun, "ToFloat", testCases)
}

func TestToString(t *testing.T) {
	now := time.Now()

	testCases := map[string][]convTestInput{
		"Null": {
			{"Null", Null{}, "null"},
		},
		"Bool": {
			{"true", Bool(true), "true"},
			{"false", Bool(false), "false"},
		},
		"Int": {
			{"positive", Int(2), "2"},
			{"negative", Int(-2), "-2"},
			{"zero", Int(0), "0"},
		},
		"Float": {
			{"positive", Float(3.14), "3.14"},
			{"negative", Float(-3.14), "-3.14"},
			{"zero", Float(0.0), "0"},
		},
		"String": {
			{"empty", String(""), ""},
			{"non-empty", String("hoge"), "hoge"},
		},
		"Blob": {
			{"empty", Blob(""), ""},
			{"non-empty", Blob("hoge"), "hoge"},
		},
		"Timestamp": {
			{"zero", Timestamp(time.Time{}), "0001-01-01T00:00:00Z"},
			{"now", Timestamp(now), now.Format(time.RFC3339Nano)},
		},
		"Array": {
			{"empty", Array{}, "tuple.Array{}"},
			{"non-empty", Array{Int(2), String("foo")}, `tuple.Array{2, "foo"}`},
		},
		"Map": {
			{"empty", Map{}, `tuple.Map{}`},
			// the following test would fail once in a while because
			// golang randomizes the keys for Maps, i.e., we cannot be sure
			// that we will always get the same string
			// convTestInput{"non-empty", Map{"a": Int(2), "b": String("foo")}, `tuple.Map{"a":2, "b":"foo"}`},
		},
	}

	toFun := func(v Value) (interface{}, error) {
		val, err := ToString(v)
		return val, err
	}
	runConversionTestCases(t, toFun, "ToString", testCases)
}

func TestToBlob(t *testing.T) {
	testCases := map[string][]convTestInput{
		"Null": {
			{"Null", Null{}, []byte(nil)},
		},
		"Bool": {
			{"true", Bool(true), nil},
			{"false", Bool(false), nil},
		},
		"Int": {
			{"positive", Int(2), nil},
			{"negative", Int(-2), nil},
			{"zero", Int(0), nil},
		},
		"Float": {
			{"positive", Float(3.14), nil},
			{"negative", Float(-3.14), nil},
			{"zero", Float(0.0), nil},
		},
		"String": {
			{"empty", String(""), []byte{}},
			{"non-empty", String("hoge"), []byte("hoge")},
			{"numeric", String("123.456"), []byte("123.456")},
		},
		"Blob": {
			{"nil", Blob(nil), []byte(nil)},
			{"empty", Blob{}, []byte{}},
			{"non-empty", Blob("hoge"), []byte("hoge")},
		},
		"Timestamp": {
			{"zero", Timestamp(time.Time{}), nil},
			{"now", Timestamp(time.Now()), nil},
		},
		"Array": {
			{"empty", Array{}, nil},
			{"non-empty", Array{Int(2), String("foo")}, nil},
		},
		"Map": {
			{"empty", Map{}, nil},
			{"non-empty", Map{"a": Int(2), "b": String("foo")}, nil},
		},
	}

	toFun := func(v Value) (interface{}, error) {
		val, err := ToBlob(v)
		return val, err
	}
	runConversionTestCases(t, toFun, "ToBlob", testCases)
}

func TestToTime(t *testing.T) {
	now := time.Now()

	testCases := map[string][]convTestInput{
		"Null": {
			{"Null", Null{}, time.Time{}},
		},
		"Bool": {
			{"true", Bool(true), nil},
			{"false", Bool(false), nil},
		},
		"Int": {
			{"positive", Int(2), time.Unix(0, 2000)},
			{"negative", Int(-2), time.Unix(0, -2000)},
			{"large and positive", Int(2e6), time.Unix(2, 0)},
			{"large and negative", Int(-2e6), time.Unix(-2, 0)},
			{"zero", Int(0), time.Unix(0, 0)},
		},
		"Float": {
			{"positive", Float(3.14), time.Unix(3, 14e7)},
			{"negative", Float(-3.14), time.Unix(-3, -14e7)},
			{"negative (alternative)", Float(-3.14), time.Unix(-4, 86e7)},
			{"zero", Float(0.0), time.Unix(0, 0)},
		},
		"String": {
			{"empty", String(""), nil},
			{"non-empty", String("hoge"), nil},
			{"valid time string", String("1970-01-01T09:00:02+09:00"), time.Unix(2, 0)},
			{"valid time string with ns", String(now.Format(time.RFC3339Nano)), now},
		},
		"Blob": {
			{"empty", Blob(""), nil},
			{"non-empty", Blob("hoge"), nil},
		},
		"Timestamp": {
			{"zero", Timestamp(time.Time{}), time.Time{}},
			{"now", Timestamp(now), now},
		},
		"Array": {
			{"empty", Array{}, nil},
			{"non-empty", Array{Int(2), String("foo")}, nil},
		},
		"Map": {
			{"empty", Map{}, nil},
			{"non-empty", Map{"a": Int(2), "b": String("foo")}, nil},
		},
	}

	toFun := func(v Value) (interface{}, error) {
		val, err := ToTime(v)
		return val, err
	}
	runConversionTestCases(t, toFun, "ToTime", testCases)
}

func runConversionTestCases(t *testing.T,
	toFun func(v Value) (interface{}, error),
	funcName string,
	testCases map[string][]convTestInput) {
	for valType, cases := range testCases {
		cases := cases
		Convey(fmt.Sprintf("Given a %s value", valType), t, func() {
			for _, testCase := range cases {
				tc := testCase
				Convey(fmt.Sprintf("When it is %s", tc.ValueDesc), func() {
					inVal := tc.Value
					exp := tc.Expected
					if exp == nil {
						Convey(fmt.Sprintf("Then %s returns an error", funcName), func() {
							_, err := toFun(inVal)
							So(err, ShouldNotBeNil)
						})
					} else {
						Convey(fmt.Sprintf("Then %s returns %v", funcName, exp), func() {
							val, err := toFun(inVal)
							So(err, ShouldBeNil)
							switch fval := val.(type) {
							case float64:
								switch fexp := exp.(type) {
								case float64:
									// equal up to machine precision
									So(fval-fexp, ShouldAlmostEqual, 0.0, 1e-15)
								default:
									So(val, ShouldResemble, exp)
								}
							case time.Time:
								So(val, ShouldHappenOnOrBetween, exp, exp)
							default:
								So(val, ShouldResemble, exp)
							}
						})
					}
				})
			}
		})
	}
}
<|endoftext|>


<commit_before>// Copyright © 2017 Circonus, Inc. <support@circonus.com>
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//

package plugins

import (
	"context"
	"encoding/json"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/circonus-labs/circonus-agent/api"
	"github.com/circonus-labs/circonus-agent/internal/config"
	cgm "github.com/circonus-labs/circonus-gometrics"
	"github.com/maier/go-appstats"
	"github.com/pkg/errors"
	"github.com/rs/zerolog/log"
	"github.com/spf13/viper"
)

// New returns a new instance of the plugins manager
func New(ctx context.Context) (*Plugins, error) {
	p := Plugins{
		ctx:           ctx,
		running:       false,
		logger:        log.With().Str("pkg", "plugins").Logger(),
		reservedNames: map[string]bool{"prom": true, "write": true, "statsd": true},
		active:        make(map[string]*plugin),
	}

	errMsg := "Invalid plugin directory"

	pluginDir := viper.GetString(config.KeyPluginDir)

	if pluginDir == "" {
		return nil, errors.New(errMsg + " (none)")
	}

	absDir, err := filepath.Abs(pluginDir)
	if err != nil {
		return nil, errors.Wrap(err, errMsg)
	}

	pluginDir = absDir

	fi, err := os.Stat(pluginDir)
	if err != nil {
		if os.IsNotExist(err) {
			p.logger.Warn().Err(err).Str("path", pluginDir).Msg("not found, ignoring")
			p.pluginDir = ""
			return &p, nil
		}
		return nil, errors.Wrap(err, errMsg)
	}

	if !fi.Mode().IsDir() {
		return nil, errors.Errorf(errMsg+" (%s) not a directory", pluginDir)
	}

	// also try opening, to verify permissions
	// if last dir on path is not accessible to user, stat doesn't return EPERM
	f, err := os.Open(pluginDir)
	if err != nil {
		return nil, errors.Wrap(err, errMsg)
	}
	f.Close()

	p.pluginDir = pluginDir

	return &p, nil
}

// Flush plugin metrics
func (p *Plugins) Flush(pluginName string) *cgm.Metrics {
	p.RLock()
	defer p.RUnlock()

	appstats.MapSet("plugins", "last_flush", time.Now())

	metrics := cgm.Metrics{}

	for pluginID, plug := range p.active {
		if pluginName == "" || // all plugins
			pluginID == pluginName || // specific plugin
			strings.HasPrefix(pluginID, pluginName+metricDelimiter) { // specific plugin with instances

			m := plug.drain()
			for mn, mv := range *m {
				metrics[pluginID+metricDelimiter+mn] = mv
			}
		}
	}

	return &metrics
}

// Stop any long running plugins
func (p *Plugins) Stop() error {
	p.logger.Info().Msg("Stopping plugins")
	return nil
}

// Run one or all plugins
func (p *Plugins) Run(pluginName string) error {
	p.Lock()

	if p.running {
		msg := "plugin run already in progress"
		p.logger.Info().Msg(msg)
		p.Unlock()
		return errors.Errorf(msg)
	}

	start := time.Now()
	appstats.MapSet("plugins", "last_run_start", start)

	p.running = true
	p.Unlock()

	var wg sync.WaitGroup

	if pluginName != "" {
		numFound := 0
		for pluginID, pluginRef := range p.active {
			if pluginID == pluginName || // specific plugin
				strings.HasPrefix(pluginID, pluginName+"`") { // specific plugin with instances
				numFound++
				wg.Add(1)
				go func(id string, plug *plugin) {
					plug.exec()
					wg.Done()
				}(pluginID, pluginRef)
			}
		}
		if numFound == 0 {
			p.logger.Error().
				Str("plugin", pluginName).
				Msg("Invalid/Unknown")
			p.running = false
			return errors.Errorf("invalid plugin (%s)", pluginName)
		}
	} else {
		for pluginID, pluginRef := range p.active {
			wg.Add(1)
			go func(id string, plug *plugin) {
				plug.exec()
				wg.Done()
			}(pluginID, pluginRef)
		}
	}

	wg.Wait()

	appstats.MapSet("plugins", "last_run_end", time.Now())
	appstats.MapSet("plugins", "last_run_duration", time.Since(start))

	p.Lock()
	p.running = false
	p.logger.Debug().Msg("all plugins done")
	p.Unlock()

	return nil
}

// IsValid determines if a specific plugin is valid
func (p *Plugins) IsValid(pluginName string) bool {
	if pluginName == "" {
		return false
	}

	p.RLock()
	defer p.RUnlock()

	for pluginID := range p.active {
		// specific plugin       plugin with instances
		if pluginID == pluginName || strings.HasPrefix(pluginID, pluginName+"`") {
			return true
		}
	}

	return false
}

// IsInternal checks to see if the plugin is one of the internal plugins (write|statsd)
func (p *Plugins) IsInternal(pluginName string) bool {
	if pluginName == "" {
		return false
	}
	_, reserved := p.reservedNames[pluginName]

	return reserved
}

// Inventory returns list of active plugins
func (p *Plugins) Inventory() []byte {
	p.Lock()
	defer p.Unlock()
	// inventory := make(map[string]*pluginDetails, len(p.active))
	inventory := make(api.Inventory, 0, len(p.active))
	for id, plug := range p.active {
		plug.Lock()
		p := api.Plugin{
			ID:              id,
			Name:            plug.id,
			Instance:        plug.instanceID,
			Command:         plug.command,
			Args:            plug.instanceArgs,
			LastRunStart:    plug.lastStart.Format(time.RFC3339Nano),
			LastRunEnd:      plug.lastEnd.Format(time.RFC3339Nano),
			LastRunDuration: plug.lastRunDuration.String(),
		}
		if plug.lastError != nil {
			p.LastError = plug.lastError.Error()
		}
		plug.Unlock()

		inventory = append(inventory, p)
	}
	data, err := json.Marshal(inventory)
	if err != nil {
		p.logger.Fatal().Err(err).Msg("inventory -> json")
	}
	return data
}
<commit_msg>upd: slight refactor of plugin inventory<commit_after>// Copyright © 2017 Circonus, Inc. <support@circonus.com>
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//

package plugins

import (
	"context"
	"encoding/json"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/circonus-labs/circonus-agent/api"
	"github.com/circonus-labs/circonus-agent/internal/config"
	cgm "github.com/circonus-labs/circonus-gometrics"
	"github.com/maier/go-appstats"
	"github.com/pkg/errors"
	"github.com/rs/zerolog/log"
	"github.com/spf13/viper"
)

// New returns a new instance of the plugins manager
func New(ctx context.Context) (*Plugins, error) {
	p := Plugins{
		ctx:           ctx,
		running:       false,
		logger:        log.With().Str("pkg", "plugins").Logger(),
		reservedNames: map[string]bool{"prom": true, "write": true, "statsd": true},
		active:        make(map[string]*plugin),
	}

	errMsg := "Invalid plugin directory"

	pluginDir := viper.GetString(config.KeyPluginDir)

	if pluginDir == "" {
		return nil, errors.New(errMsg + " (none)")
	}

	absDir, err := filepath.Abs(pluginDir)
	if err != nil {
		return nil, errors.Wrap(err, errMsg)
	}

	pluginDir = absDir

	fi, err := os.Stat(pluginDir)
	if err != nil {
		if os.IsNotExist(err) {
			p.logger.Warn().Err(err).Str("path", pluginDir).Msg("not found, ignoring")
			p.pluginDir = ""
			return &p, nil
		}
		return nil, errors.Wrap(err, errMsg)
	}

	if !fi.Mode().IsDir() {
		return nil, errors.Errorf(errMsg+" (%s) not a directory", pluginDir)
	}

	// also try opening, to verify permissions
	// if last dir on path is not accessible to user, stat doesn't return EPERM
	f, err := os.Open(pluginDir)
	if err != nil {
		return nil, errors.Wrap(err, errMsg)
	}
	f.Close()

	p.pluginDir = pluginDir

	return &p, nil
}

// Flush plugin metrics
func (p *Plugins) Flush(pluginName string) *cgm.Metrics {
	p.RLock()
	defer p.RUnlock()

	appstats.MapSet("plugins", "last_flush", time.Now())

	metrics := cgm.Metrics{}

	for pluginID, plug := range p.active {
		if pluginName == "" || // all plugins
			pluginID == pluginName || // specific plugin
			strings.HasPrefix(pluginID, pluginName+metricDelimiter) { // specific plugin with instances

			m := plug.drain()
			for mn, mv := range *m {
				metrics[pluginID+metricDelimiter+mn] = mv
			}
		}
	}

	return &metrics
}

// Stop any long running plugins
func (p *Plugins) Stop() error {
	p.logger.Info().Msg("Stopping plugins")
	return nil
}

// Run one or all plugins
func (p *Plugins) Run(pluginName string) error {
	p.Lock()

	if p.running {
		msg := "plugin run already in progress"
		p.logger.Info().Msg(msg)
		p.Unlock()
		return errors.Errorf(msg)
	}

	start := time.Now()
	appstats.MapSet("plugins", "last_run_start", start)

	p.running = true
	p.Unlock()

	var wg sync.WaitGroup

	if pluginName != "" {
		numFound := 0
		for pluginID, pluginRef := range p.active {
			if pluginID == pluginName || // specific plugin
				strings.HasPrefix(pluginID, pluginName+"`") { // specific plugin with instances
				numFound++
				wg.Add(1)
				go func(id string, plug *plugin) {
					plug.exec()
					wg.Done()
				}(pluginID, pluginRef)
			}
		}
		if numFound == 0 {
			p.logger.Error().
				Str("plugin", pluginName).
				Msg("Invalid/Unknown")
			p.running = false
			return errors.Errorf("invalid plugin (%s)", pluginName)
		}
	} else {
		for pluginID, pluginRef := range p.active {
			wg.Add(1)
			go func(id string, plug *plugin) {
				plug.exec()
				wg.Done()
			}(pluginID, pluginRef)
		}
	}

	wg.Wait()

	appstats.MapSet("plugins", "last_run_end", time.Now())
	appstats.MapSet("plugins", "last_run_duration", time.Since(start))

	p.Lock()
	p.running = false
	p.logger.Debug().Msg("all plugins done")
	p.Unlock()

	return nil
}

// IsValid determines if a specific plugin is valid
func (p *Plugins) IsValid(pluginName string) bool {
	if pluginName == "" {
		return false
	}

	p.RLock()
	defer p.RUnlock()

	for pluginID := range p.active {
		// specific plugin       plugin with instances
		if pluginID == pluginName || strings.HasPrefix(pluginID, pluginName+"`") {
			return true
		}
	}

	return false
}

// IsInternal checks to see if the plugin is one of the internal plugins (write|statsd)
func (p *Plugins) IsInternal(pluginName string) bool {
	if pluginName == "" {
		return false
	}
	_, reserved := p.reservedNames[pluginName]

	return reserved
}

// Inventory returns list of active plugins
func (p *Plugins) Inventory() []byte {
	p.Lock()
	defer p.Unlock()
	inventory := api.Inventory{}
	for id, plug := range p.active {
		plug.Lock()
		pinfo := api.Plugin{
			ID:              id,
			Name:            plug.id,
			Instance:        plug.instanceID,
			Command:         plug.command,
			Args:            plug.instanceArgs,
			LastRunStart:    plug.lastStart.Format(time.RFC3339Nano),
			LastRunEnd:      plug.lastEnd.Format(time.RFC3339Nano),
			LastRunDuration: plug.lastRunDuration.String(),
		}
		if plug.lastError != nil {
			pinfo.LastError = plug.lastError.Error()
		}
		plug.Unlock()
		inventory = append(inventory, pinfo)
	}
	data, err := json.Marshal(inventory)
	if err != nil {
		p.logger.Fatal().Err(err).Msg("inventory -> json")
	}
	return data
}
<|endoftext|>


<commit_before>package util

import (
	"reflect"
	"sync"
)

// A Map is a map that is safe for concurrent access and
// updates. A Map must be created with NewMap.
type Map struct {
	once   sync.Once
	mu     sync.RWMutex
	values map[interface{}]interface{}
}

func NewMap() *Map {
	return &Map{
		values: make(map[interface{}]interface{}),
	}
}

// Get retrieves a value from the Map. If the value is not
// present, ok will be false.
func (m *Map) Get(key interface{}) (val interface{}, ok bool) {
	m.mu.RLock()
	val, ok = m.values[key]
	m.mu.RUnlock()
	return val, ok
}

// Put stores a value in the map, overwriting any previous values
// stored under the key.
func (m *Map) Put(key, val interface{}) {
	m.mu.Lock()
	m.values[key] = val
	m.mu.Unlock()
}

// Add stores a value in the map under key. If there is already
// a value in the map for key, Add does not replace it, and
// returns false.
func (m *Map) Add(key, val interface{}) bool {
	m.mu.Lock()
	defer m.mu.Unlock()
	if _, ok := m.values[key]; ok {
		return false
	}
	m.values[key] = val
	return true
}

// Del deletes a value from a map. Subsequent Gets on the map
// will turn up empty.
func (m *Map) Del(key interface{}) {
	m.mu.Lock()
	delete(m.values, key)
	m.mu.Unlock()
}

// Fetch stores the value corresponding with key in the Map into v. v
// must be a pointer to the value's type, or a run-time panic will
// occur. If the key is not present in the Map, v is untouched and
// Fetch returns false.
func (m *Map) Fetch(key, v interface{}) bool {
	val, ok := m.Get(key)
	if ok {
		reflect.ValueOf(v).Elem().Set(reflect.ValueOf(val))
		return true
	}
	return false
}
<commit_msg>Add util.SafeMap.Do<commit_after>package util

import (
	"reflect"
	"sync"
)

// A Map is a map that is safe for concurrent access and
// updates. A Map must be created with NewMap.
type Map struct {
	once   sync.Once
	mu     sync.RWMutex
	values map[interface{}]interface{}
}

func NewMap() *Map {
	return &Map{
		values: make(map[interface{}]interface{}),
	}
}

// Get retrieves a value from the Map. If the value is not
// present, ok will be false.
func (m *Map) Get(key interface{}) (val interface{}, ok bool) {
	m.mu.RLock()
	val, ok = m.values[key]
	m.mu.RUnlock()
	return val, ok
}

// Put stores a value in the map, overwriting any previous values
// stored under the key.
func (m *Map) Put(key, val interface{}) {
	m.mu.Lock()
	m.values[key] = val
	m.mu.Unlock()
}

// Add stores a value in the map under key. If there is already
// a value in the map for key, Add does not replace it, and
// returns false.
func (m *Map) Add(key, val interface{}) bool {
	m.mu.Lock()
	defer m.mu.Unlock()
	if _, ok := m.values[key]; ok {
		return false
	}
	m.values[key] = val
	return true
}

// Del deletes a value from a map. Subsequent Gets on the map
// will turn up empty.
func (m *Map) Del(key interface{}) {
	m.mu.Lock()
	delete(m.values, key)
	m.mu.Unlock()
}

// Fetch stores the value corresponding with key in the Map into v. v
// must be a pointer to the value's type, or a run-time panic will
// occur. If the key is not present in the Map, v is untouched and
// Fetch returns false.
func (m *Map) Fetch(key, v interface{}) bool {
	val, ok := m.Get(key)
	if ok {
		reflect.ValueOf(v).Elem().Set(reflect.ValueOf(val))
		return true
	}
	return false
}

// Do calls f while holding the write lock for a Map.
func (m *Map) Do(f func(map[interface{}]interface{})) {
	m.mu.Lock()
	f(m.values)
	m.mu.Unlock()
}
<|endoftext|>


<commit_before>// Copyright 2016 The nvim-go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package command

import (
	"context"

	buildctx "nvim-go/ctx"
	"nvim-go/logger"

	"github.com/neovim/go-client/nvim"
	"github.com/neovim/go-client/nvim/plugin"
	"go.uber.org/zap"
	"golang.org/x/sync/syncmap"
)

// Command represents a nvim-go plugins commands.
type Command struct {
	ctx context.Context
	log *zap.Logger

	Nvim      *nvim.Nvim
	buildctxt *buildctx.Context
	errs      *syncmap.Map
}

// NewCommand return the new Command type with initialize some variables.
func NewCommand(ctx context.Context, v *nvim.Nvim, buildctxt *buildctx.Context) *Command {
	return &Command{
		Nvim:      v,
		ctx:       ctx,
		log:       logger.FromContext(ctx).Named("command"),
		buildctxt: buildctxt,
		errs:      new(syncmap.Map),
	}
}

// Register register nvim-go command or function to Neovim over the msgpack-rpc plugin interface.
func Register(ctx context.Context, p *plugin.Plugin, buildctxt *buildctx.Context) *Command {
	c := &Command{
		ctx:       ctx,
		log:       logger.FromContext(ctx).Named("command"),
		Nvim:      p.Nvim,
		buildctxt: buildctxt,
		errs:      new(syncmap.Map),
	}

	// Register command and function
	// CommandOptions order: Name, NArgs, Range, Count, Addr, Bang, Register, Eval, Bar, Complete
	p.HandleCommand(&plugin.CommandOptions{Name: "Gobuild", Bang: true, Eval: "[getcwd(), expand('%:p')]"}, c.cmdBuild)
	p.HandleCommand(&plugin.CommandOptions{Name: "GoCover", Eval: "[getcwd(), expand('%:p')]"}, c.cmdCover)
	p.HandleCommand(&plugin.CommandOptions{Name: "Gofmt", Eval: "expand('%:p:h')"}, c.cmdFmt)
	p.HandleCommand(&plugin.CommandOptions{Name: "GoGenerateTest", NArgs: "*", Range: "%", Addr: "line", Bang: true, Eval: "expand('%:p:h')", Complete: "file"}, c.cmdGenerateTest)
	p.HandleFunction(&plugin.FunctionOptions{Name: "GoGuru", Eval: "[getcwd(), expand('%:p'), &modified, line2byte(line('.')) + (col('.')-2)]"}, c.funcGuru)
	p.HandleCommand(&plugin.CommandOptions{Name: "GoIferr", Eval: "expand('%:p')"}, c.cmdIferr)
	p.HandleCommand(&plugin.CommandOptions{Name: "Golint", NArgs: "?", Eval: "expand('%:p')", Complete: "customlist,GoLintCompletion"}, c.cmdLint)
	p.HandleCommand(&plugin.CommandOptions{Name: "Gometalinter", Eval: "getcwd()"}, c.cmdMetalinter)
	p.HandleCommand(&plugin.CommandOptions{Name: "Gorename", NArgs: "?", Bang: true, Eval: "[getcwd(), expand('%:p'), expand('<cword>')]"}, c.cmdRename)
	p.HandleCommand(&plugin.CommandOptions{Name: "Gorun", NArgs: "*", Eval: "expand('%:p')"}, c.cmdRun)
	p.HandleCommand(&plugin.CommandOptions{Name: "GorunLast", Eval: "expand('%:p')"}, c.cmdRunLast)
	p.HandleCommand(&plugin.CommandOptions{Name: "Gotest", NArgs: "*", Eval: "expand('%:p:h')"}, c.cmdTest)
	p.HandleCommand(&plugin.CommandOptions{Name: "GoSwitchTest", Eval: "[getcwd(), expand('%:p'), line2byte(line('.')) + (col('.')-2)]"}, c.cmdSwitchTest)
	p.HandleCommand(&plugin.CommandOptions{Name: "Govet", NArgs: "*", Eval: "[getcwd(), expand('%:p')]", Complete: "customlist,GoVetCompletion"}, c.cmdVet)

	// Commnad completion
	p.HandleFunction(&plugin.FunctionOptions{Name: "GoLintCompletion", Eval: "getcwd()"}, c.cmdLintComplete) // list the file, directory and go packages
	p.HandleFunction(&plugin.FunctionOptions{Name: "GoVetCompletion", Eval: "getcwd()"}, c.cmdVetComplete)   // flag for go tool vet

	// for debug
	p.HandleCommand(&plugin.CommandOptions{Name: "GoByteOffset", Range: "%", Eval: "expand('%:p')"}, c.cmdByteOffset)
	p.HandleCommand(&plugin.CommandOptions{Name: "GoBuffers"}, c.cmdBuffers)
	p.HandleCommand(&plugin.CommandOptions{Name: "GoWindows"}, c.cmdWindows)
	p.HandleCommand(&plugin.CommandOptions{Name: "GoTabpages"}, c.cmdTabpagas)

	return c
}
<commit_msg>nvim-go/command: use NewCommand on Register<commit_after>// Copyright 2016 The nvim-go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package command

import (
	"context"

	buildctx "nvim-go/ctx"
	"nvim-go/logger"

	"github.com/neovim/go-client/nvim"
	"github.com/neovim/go-client/nvim/plugin"
	"go.uber.org/zap"
	"golang.org/x/sync/syncmap"
)

// Command represents a nvim-go plugins commands.
type Command struct {
	ctx context.Context
	log *zap.Logger

	Nvim      *nvim.Nvim
	buildctxt *buildctx.Context
	errs      *syncmap.Map
}

// NewCommand return the new Command type with initialize some variables.
func NewCommand(ctx context.Context, v *nvim.Nvim, buildctxt *buildctx.Context) *Command {
	return &Command{
		Nvim:      v,
		ctx:       ctx,
		log:       logger.FromContext(ctx).Named("command"),
		buildctxt: buildctxt,
		errs:      new(syncmap.Map),
	}
}

// Register register nvim-go command or function to Neovim over the msgpack-rpc plugin interface.
func Register(ctx context.Context, p *plugin.Plugin, buildctxt *buildctx.Context) *Command {
	c := NewCommand(ctx, p.Nvim, buildctxt)

	// Register command and function
	// CommandOptions order: Name, NArgs, Range, Count, Addr, Bang, Register, Eval, Bar, Complete
	p.HandleCommand(&plugin.CommandOptions{Name: "Gobuild", Bang: true, Eval: "[getcwd(), expand('%:p')]"}, c.cmdBuild)
	p.HandleCommand(&plugin.CommandOptions{Name: "GoCover", Eval: "[getcwd(), expand('%:p')]"}, c.cmdCover)
	p.HandleCommand(&plugin.CommandOptions{Name: "Gofmt", Eval: "expand('%:p:h')"}, c.cmdFmt)
	p.HandleCommand(&plugin.CommandOptions{Name: "GoGenerateTest", NArgs: "*", Range: "%", Addr: "line", Bang: true, Eval: "expand('%:p:h')", Complete: "file"}, c.cmdGenerateTest)
	p.HandleFunction(&plugin.FunctionOptions{Name: "GoGuru", Eval: "[getcwd(), expand('%:p'), &modified, line2byte(line('.')) + (col('.')-2)]"}, c.funcGuru)
	p.HandleCommand(&plugin.CommandOptions{Name: "GoIferr", Eval: "expand('%:p')"}, c.cmdIferr)
	p.HandleCommand(&plugin.CommandOptions{Name: "Golint", NArgs: "?", Eval: "expand('%:p')", Complete: "customlist,GoLintCompletion"}, c.cmdLint)
	p.HandleCommand(&plugin.CommandOptions{Name: "Gometalinter", Eval: "getcwd()"}, c.cmdMetalinter)
	p.HandleCommand(&plugin.CommandOptions{Name: "Gorename", NArgs: "?", Bang: true, Eval: "[getcwd(), expand('%:p'), expand('<cword>')]"}, c.cmdRename)
	p.HandleCommand(&plugin.CommandOptions{Name: "Gorun", NArgs: "*", Eval: "expand('%:p')"}, c.cmdRun)
	p.HandleCommand(&plugin.CommandOptions{Name: "GorunLast", Eval: "expand('%:p')"}, c.cmdRunLast)
	p.HandleCommand(&plugin.CommandOptions{Name: "Gotest", NArgs: "*", Eval: "expand('%:p:h')"}, c.cmdTest)
	p.HandleCommand(&plugin.CommandOptions{Name: "GoSwitchTest", Eval: "[getcwd(), expand('%:p'), line2byte(line('.')) + (col('.')-2)]"}, c.cmdSwitchTest)
	p.HandleCommand(&plugin.CommandOptions{Name: "Govet", NArgs: "*", Eval: "[getcwd(), expand('%:p')]", Complete: "customlist,GoVetCompletion"}, c.cmdVet)

	// Commnad completion
	p.HandleFunction(&plugin.FunctionOptions{Name: "GoLintCompletion", Eval: "getcwd()"}, c.cmdLintComplete) // list the file, directory and go packages
	p.HandleFunction(&plugin.FunctionOptions{Name: "GoVetCompletion", Eval: "getcwd()"}, c.cmdVetComplete)   // flag for go tool vet

	// for debug
	p.HandleCommand(&plugin.CommandOptions{Name: "GoByteOffset", Range: "%", Eval: "expand('%:p')"}, c.cmdByteOffset)
	p.HandleCommand(&plugin.CommandOptions{Name: "GoBuffers"}, c.cmdBuffers)
	p.HandleCommand(&plugin.CommandOptions{Name: "GoWindows"}, c.cmdWindows)
	p.HandleCommand(&plugin.CommandOptions{Name: "GoTabpages"}, c.cmdTabpagas)

	return c
}
<|endoftext|>


<commit_before>// Package reference provides a general type to represent any way of referencing images within the registry.
// Its main purpose is to abstract tags and digests (content-addressable hash).
//
// Grammar
//
// 	reference                       := repository [ ":" tag ] [ "@" digest ]
//
//	// repository.go
//	repository			:= hostname ['/' component]+
//	hostname 			:= hostcomponent [':' port-number]
//	component			:= subcomponent [separator subcomponent]*
//	subcomponent			:= alpha-numeric ['-'* alpha-numeric]*
//	hostcomponent                   := [hostpart '.']* hostpart
// 	alpha-numeric			:= /[a-z0-9]+/
//	separator			:= /([_.]|__)/
//	port-number			:= /[0-9]+/
//	hostpart                        := /([a-z0-9]|[a-z0-9][a-z0-9-]*[a-z0-9])/
//
//	// tag.go
//	tag                             := /[\w][\w.-]{0,127}/
//
//	// from the digest package
//	digest                          := digest-algorithm ":" digest-hex
//	digest-algorithm                := digest-algorithm-component [ digest-algorithm-separator digest-algorithm-component ]
//	digest-algorithm-separator      := /[+.-_]/
//	digest-algorithm-component      := /[A-Za-z][A-Za-z0-9]*/
//	digest-hex                      := /[0-9a-fA-F]{32,}/ ; Atleast 128 bit digest value
package reference

import (
	"errors"
	"fmt"

	"github.com/docker/distribution/digest"
)

const (
	// NameTotalLengthMax is the maximum total number of characters in a repository name.
	NameTotalLengthMax = 255
)

var (
	// ErrReferenceInvalidFormat represents an error while trying to parse a string as a reference.
	ErrReferenceInvalidFormat = errors.New("invalid reference format")

	// ErrNameEmpty is returned for empty, invalid repository names.
	ErrNameEmpty = errors.New("repository name must have at least one component")

	// ErrNameTooLong is returned when a repository name is longer than
	// RepositoryNameTotalLengthMax
	ErrNameTooLong = fmt.Errorf("repository name must not be more than %v characters", NameTotalLengthMax)
)

// Reference is an opaque object reference identifier that may include
// modifiers such as a hostname, name, tag, and digest.
type Reference interface {
	// String returns the full reference
	String() string
}

// Field provides a wrapper type for resolving correct reference types when
// working with encoding.
type Field struct {
	reference Reference
}

// AsField wraps a reference in a Field for encoding.
func AsField(reference Reference) Field {
	return Field{reference}
}

// Reference unwraps the reference type from the field to
// return the Reference object. This object should be
// of the appropriate type to further check for different
// reference types.
func (f Field) Reference() Reference {
	return f.reference
}

// MarshalText serializes the field to byte text which
// is the string of the reference.
func (f Field) MarshalText() (p []byte, err error) {
	return []byte(f.reference.String()), nil
}

// UnmarshalText parses text bytes by invoking the
// reference parser to ensure the appropriately
// typed reference object is wrapped by field.
func (f *Field) UnmarshalText(p []byte) error {
	r, err := Parse(string(p))
	if err != nil {
		return err
	}

	f.reference = r
	return nil
}

// Named is an object with a full name
type Named interface {
	Name() string
}

// Tagged is an object which has a tag
type Tagged interface {
	Tag() string
}

// Digested is an object which has a digest
// in which it can be referenced by
type Digested interface {
	Digest() digest.Digest
}

// Canonical reference is an object with a fully unique
// name including a name with hostname and digest
type Canonical interface {
	Reference
	Named
	Digested
}

// SplitHostname splits a named reference into a
// hostname and name string. If no valid hostname is
// found, the hostname is empty and the full value
// is returned as name
func SplitHostname(named Named) (string, string) {
	name := named.Name()
	match := anchoredNameRegexp.FindStringSubmatch(name)
	if match == nil || len(match) != 3 {
		return "", name
	}
	return match[1], match[2]
}

// Parse parses s and returns a syntactically valid Reference.
// If an error was encountered it is returned, along with a nil Reference.
// NOTE: Parse will not handle short digests.
func Parse(s string) (Reference, error) {
	matches := ReferenceRegexp.FindStringSubmatch(s)
	if matches == nil {
		if s == "" {
			return nil, ErrNameEmpty
		}
		// TODO(dmcgowan): Provide more specific and helpful error
		return nil, ErrReferenceInvalidFormat
	}

	if len(matches[1]) > NameTotalLengthMax {
		return nil, ErrNameTooLong
	}

	ref := reference{
		name: matches[1],
		tag:  matches[2],
	}
	if matches[3] != "" {
		var err error
		ref.digest, err = digest.ParseDigest(matches[3])
		if err != nil {
			return nil, err
		}
	}

	r := getBestReferenceType(ref)
	if r == nil {
		return nil, ErrNameEmpty
	}

	return r, nil
}

// ParseNamed parses the input string and returns a named
// object representing the given string. If the input is
// invalid ErrReferenceInvalidFormat will be returned.
func ParseNamed(name string) (Named, error) {
	if !anchoredNameRegexp.MatchString(name) {
		return nil, ErrReferenceInvalidFormat
	}
	return repository(name), nil
}

func getBestReferenceType(ref reference) Reference {
	if ref.name == "" {
		// Allow digest only references
		if ref.digest != "" {
			return digestReference(ref.digest)
		}
		return nil
	}
	if ref.tag == "" {
		if ref.digest != "" {
			return canonicalReference{
				name:   ref.name,
				digest: ref.digest,
			}
		}
		return repository(ref.name)
	}
	if ref.digest == "" {
		return taggedReference{
			name: ref.name,
			tag:  ref.tag,
		}
	}

	return ref
}

type reference struct {
	name   string
	tag    string
	digest digest.Digest
}

func (r reference) String() string {
	return r.name + ":" + r.tag + "@" + r.digest.String()
}

func (r reference) Name() string {
	return r.name
}

func (r reference) Tag() string {
	return r.tag
}

func (r reference) Digest() digest.Digest {
	return r.digest
}

type repository string

func (r repository) String() string {
	return string(r)
}

func (r repository) Name() string {
	return string(r)
}

type digestReference digest.Digest

func (d digestReference) String() string {
	return d.String()
}

func (d digestReference) Digest() digest.Digest {
	return digest.Digest(d)
}

type taggedReference struct {
	name string
	tag  string
}

func (t taggedReference) String() string {
	return t.name + ":" + t.tag
}

func (t taggedReference) Name() string {
	return t.name
}

func (t taggedReference) Tag() string {
	return t.tag
}

type canonicalReference struct {
	name   string
	digest digest.Digest
}

func (c canonicalReference) String() string {
	return c.name + "@" + c.digest.String()
}

func (c canonicalReference) Name() string {
	return c.name
}

func (c canonicalReference) Digest() digest.Digest {
	return c.digest
}
<commit_msg>Make Named,Tagged, and Digested implement Reference<commit_after>// Package reference provides a general type to represent any way of referencing images within the registry.
// Its main purpose is to abstract tags and digests (content-addressable hash).
//
// Grammar
//
// 	reference                       := repository [ ":" tag ] [ "@" digest ]
//
//	// repository.go
//	repository			:= hostname ['/' component]+
//	hostname 			:= hostcomponent [':' port-number]
//	component			:= subcomponent [separator subcomponent]*
//	subcomponent			:= alpha-numeric ['-'* alpha-numeric]*
//	hostcomponent                   := [hostpart '.']* hostpart
// 	alpha-numeric			:= /[a-z0-9]+/
//	separator			:= /([_.]|__)/
//	port-number			:= /[0-9]+/
//	hostpart                        := /([a-z0-9]|[a-z0-9][a-z0-9-]*[a-z0-9])/
//
//	// tag.go
//	tag                             := /[\w][\w.-]{0,127}/
//
//	// from the digest package
//	digest                          := digest-algorithm ":" digest-hex
//	digest-algorithm                := digest-algorithm-component [ digest-algorithm-separator digest-algorithm-component ]
//	digest-algorithm-separator      := /[+.-_]/
//	digest-algorithm-component      := /[A-Za-z][A-Za-z0-9]*/
//	digest-hex                      := /[0-9a-fA-F]{32,}/ ; Atleast 128 bit digest value
package reference

import (
	"errors"
	"fmt"

	"github.com/docker/distribution/digest"
)

const (
	// NameTotalLengthMax is the maximum total number of characters in a repository name.
	NameTotalLengthMax = 255
)

var (
	// ErrReferenceInvalidFormat represents an error while trying to parse a string as a reference.
	ErrReferenceInvalidFormat = errors.New("invalid reference format")

	// ErrNameEmpty is returned for empty, invalid repository names.
	ErrNameEmpty = errors.New("repository name must have at least one component")

	// ErrNameTooLong is returned when a repository name is longer than
	// RepositoryNameTotalLengthMax
	ErrNameTooLong = fmt.Errorf("repository name must not be more than %v characters", NameTotalLengthMax)
)

// Reference is an opaque object reference identifier that may include
// modifiers such as a hostname, name, tag, and digest.
type Reference interface {
	// String returns the full reference
	String() string
}

// Field provides a wrapper type for resolving correct reference types when
// working with encoding.
type Field struct {
	reference Reference
}

// AsField wraps a reference in a Field for encoding.
func AsField(reference Reference) Field {
	return Field{reference}
}

// Reference unwraps the reference type from the field to
// return the Reference object. This object should be
// of the appropriate type to further check for different
// reference types.
func (f Field) Reference() Reference {
	return f.reference
}

// MarshalText serializes the field to byte text which
// is the string of the reference.
func (f Field) MarshalText() (p []byte, err error) {
	return []byte(f.reference.String()), nil
}

// UnmarshalText parses text bytes by invoking the
// reference parser to ensure the appropriately
// typed reference object is wrapped by field.
func (f *Field) UnmarshalText(p []byte) error {
	r, err := Parse(string(p))
	if err != nil {
		return err
	}

	f.reference = r
	return nil
}

// Named is an object with a full name
type Named interface {
	Reference
	Name() string
}

// Tagged is an object which has a tag
type Tagged interface {
	Reference
	Tag() string
}

// Digested is an object which has a digest
// in which it can be referenced by
type Digested interface {
	Reference
	Digest() digest.Digest
}

// Canonical reference is an object with a fully unique
// name including a name with hostname and digest
type Canonical interface {
	Named
	Digest() digest.Digest
}

// SplitHostname splits a named reference into a
// hostname and name string. If no valid hostname is
// found, the hostname is empty and the full value
// is returned as name
func SplitHostname(named Named) (string, string) {
	name := named.Name()
	match := anchoredNameRegexp.FindStringSubmatch(name)
	if match == nil || len(match) != 3 {
		return "", name
	}
	return match[1], match[2]
}

// Parse parses s and returns a syntactically valid Reference.
// If an error was encountered it is returned, along with a nil Reference.
// NOTE: Parse will not handle short digests.
func Parse(s string) (Reference, error) {
	matches := ReferenceRegexp.FindStringSubmatch(s)
	if matches == nil {
		if s == "" {
			return nil, ErrNameEmpty
		}
		// TODO(dmcgowan): Provide more specific and helpful error
		return nil, ErrReferenceInvalidFormat
	}

	if len(matches[1]) > NameTotalLengthMax {
		return nil, ErrNameTooLong
	}

	ref := reference{
		name: matches[1],
		tag:  matches[2],
	}
	if matches[3] != "" {
		var err error
		ref.digest, err = digest.ParseDigest(matches[3])
		if err != nil {
			return nil, err
		}
	}

	r := getBestReferenceType(ref)
	if r == nil {
		return nil, ErrNameEmpty
	}

	return r, nil
}

// ParseNamed parses the input string and returns a named
// object representing the given string. If the input is
// invalid ErrReferenceInvalidFormat will be returned.
func ParseNamed(name string) (Named, error) {
	if !anchoredNameRegexp.MatchString(name) {
		return nil, ErrReferenceInvalidFormat
	}
	return repository(name), nil
}

func getBestReferenceType(ref reference) Reference {
	if ref.name == "" {
		// Allow digest only references
		if ref.digest != "" {
			return digestReference(ref.digest)
		}
		return nil
	}
	if ref.tag == "" {
		if ref.digest != "" {
			return canonicalReference{
				name:   ref.name,
				digest: ref.digest,
			}
		}
		return repository(ref.name)
	}
	if ref.digest == "" {
		return taggedReference{
			name: ref.name,
			tag:  ref.tag,
		}
	}

	return ref
}

type reference struct {
	name   string
	tag    string
	digest digest.Digest
}

func (r reference) String() string {
	return r.name + ":" + r.tag + "@" + r.digest.String()
}

func (r reference) Name() string {
	return r.name
}

func (r reference) Tag() string {
	return r.tag
}

func (r reference) Digest() digest.Digest {
	return r.digest
}

type repository string

func (r repository) String() string {
	return string(r)
}

func (r repository) Name() string {
	return string(r)
}

type digestReference digest.Digest

func (d digestReference) String() string {
	return d.String()
}

func (d digestReference) Digest() digest.Digest {
	return digest.Digest(d)
}

type taggedReference struct {
	name string
	tag  string
}

func (t taggedReference) String() string {
	return t.name + ":" + t.tag
}

func (t taggedReference) Name() string {
	return t.name
}

func (t taggedReference) Tag() string {
	return t.tag
}

type canonicalReference struct {
	name   string
	digest digest.Digest
}

func (c canonicalReference) String() string {
	return c.name + "@" + c.digest.String()
}

func (c canonicalReference) Name() string {
	return c.name
}

func (c canonicalReference) Digest() digest.Digest {
	return c.digest
}
<|endoftext|>


<commit_before>package reflector

import (
	"fmt"
	"reflect"
	"strconv"
	"strings"
)

type Obj struct {
	iface interface{}

	isStruct      bool
	isPtrToStruct bool

	// If ptr to struct, this field will contain the type of that struct
	underlyingType reflect.Type

	objType reflect.Type
	objKind reflect.Kind
}

func NewFromType(ty reflect.Type) *Obj {
	return New(reflect.New(ty).Interface())
}

func New(obj interface{}) *Obj {
	o := &Obj{iface: obj}
	o.objType = reflect.TypeOf(obj)
	o.objKind = o.objType.Kind()

	ty := o.Type()
	if ty.Kind() == reflect.Struct {
		o.isStruct = true
	}
	if ty.Kind() == reflect.Ptr && ty.Elem().Kind() == reflect.Struct {
		ty = ty.Elem()
		o.isPtrToStruct = true
	}
	o.underlyingType = ty
	return o
}

func (o *Obj) Fields() []ObjField {
	return o.fields(reflect.TypeOf(o.iface), false)
}

func (o Obj) FieldsFlattened() []ObjField {
	return o.fields(reflect.TypeOf(o.iface), true)
}

func (o *Obj) fields(ty reflect.Type, flatten bool) []ObjField {
	fields := make([]ObjField, 0)

	if ty.Kind() == reflect.Ptr {
		ty = ty.Elem()
	}

	if ty.Kind() != reflect.Struct {
		return fields // No need to populate nonstructs
	}

	for i := 0; i < ty.NumField(); i++ {
		field := ty.Field(i)

		k := field.Type.Kind()
		if string(field.Name[0]) == strings.ToUpper(string(field.Name[0])) {
			if flatten && k == reflect.Struct && field.Anonymous {
				fields = append(fields, o.fields(field.Type, flatten)...)
			} else {
				fields = append(fields, *newObjField(o, field.Name))
			}
		}
	}

	return fields
}

func (o Obj) IsPtr() bool {
	return o.objKind == reflect.Ptr
}

func (o Obj) IsStructOrPtrToStruct() bool {
	return o.isStruct || o.isPtrToStruct
}

func (o *Obj) Field(name string) *ObjField {
	return &ObjField{
		obj:  o,
		name: name,
	}
}

func (o Obj) Type() reflect.Type {
	return o.objType
}

func (o Obj) Kind() reflect.Kind {
	return o.objKind
}

func (o *Obj) Method(name string) *ObjMethod {
	return newObjMethod(o, name)
}

func (o *Obj) Methods() []ObjMethod {
	res := []ObjMethod{}
	ty := o.Type()
	for i := 0; i < ty.NumMethod(); i++ {
		method := ty.Method(i)
		res = append(res, *newObjMethod(o, method.Name))
	}
	return res
}

type ObjField struct {
	obj  *Obj
	name string
}

func newObjField(obj *Obj, name string) *ObjField {
	return &ObjField{
		obj:  obj,
		name: name,
	}
}

func (of *ObjField) Name() string {
	return of.name
}

func (of *ObjField) Kind() reflect.Kind {
	ty, err := of.Type()
	if err != nil {
		return reflect.Invalid
	}
	return ty.Kind()
}

func (of *ObjField) Type() (reflect.Type, error) {
	value, err := of.Get()
	if err != nil {
		return nil, fmt.Errorf("Invalid field %s in %T", of.name, of.obj.iface)
	}
	return reflect.TypeOf(value), nil
}

func (of *ObjField) Tag(tag string) (string, error) {
	_, field, err := of.field()
	if err != nil {
		return "", err
	}
	return (*field).Tag.Get(tag), nil
}

func (of *ObjField) Tags() (map[string]string, error) {
	_, field, err := of.field()
	if err != nil {
		return nil, err
	}

	res := map[string]string{}
	tag := (*field).Tag

	// This code is copied/modified from: reflect/type.go:
	for tag != "" {
		// Skip leading space.
		i := 0
		for i < len(tag) && tag[i] == ' ' {
			i++
		}
		tag = tag[i:]
		if tag == "" {
			break
		}

		// Scan to colon. A space, a quote or a control character is a syntax error.
		// Strictly speaking, control chars include the range [0x7f, 0x9f], not just
		// [0x00, 0x1f], but in practice, we ignore the multi-byte control characters
		// as it is simpler to inspect the tag's bytes than the tag's runes.
		i = 0
		for i < len(tag) && tag[i] > ' ' && tag[i] != ':' && tag[i] != '"' && tag[i] != 0x7f {
			i++
		}
		if i == 0 || i+1 >= len(tag) || tag[i] != ':' || tag[i+1] != '"' {
			break
		}
		name := string(tag[:i])
		tag = tag[i+1:]

		// Scan quoted string to find value.
		i = 1
		for i < len(tag) && tag[i] != '"' {
			if tag[i] == '\\' {
				i++
			}
			i++
		}
		if i >= len(tag) {
			break
		}
		qvalue := string(tag[:i+1])
		tag = tag[i+1:]

		value, err := strconv.Unquote(qvalue)
		if err != nil {
			return nil, fmt.Errorf("Cannot unquote tag %s in %T.%s: %s", name, of.obj.iface, of.name, err.Error())
		}
		res[name] = value
		/*
			if key == name {
				value, err := strconv.Unquote(qvalue)
				if err != nil {
					break
				}
				return value
			}
		*/
	}

	return res, nil
}

// TagExpanded returns the tag value "expanded" with commas
func (of *ObjField) TagExpanded(tag string) ([]string, error) {
	_, field, err := of.field()
	if err != nil {
		return nil, err
	}
	return strings.Split((*field).Tag.Get(tag), ","), nil
}

func (of *ObjField) Valid() bool {
	if !of.obj.IsStructOrPtrToStruct() {
		return false
	}
	_, found := of.obj.underlyingType.FieldByName(of.name)
	return found
}

func (of *ObjField) Anonymous() bool {
	field, found := of.obj.underlyingType.FieldByName(of.name)
	if !found {
		return false
	}
	return field.Anonymous
}

func (of *ObjField) Set(value interface{}) error {
	if !of.obj.IsStructOrPtrToStruct() {
		return fmt.Errorf("Cannot set %s in %T because obj is not a pointer to struct", of.name, of.obj.iface)
	}

	v := reflect.ValueOf(value)
	var field reflect.Value
	if of.obj.isPtrToStruct {
		field = reflect.ValueOf(of.obj.iface).Elem().FieldByName(of.name)
	} else {
		field = reflect.ValueOf(of.obj.iface).FieldByName(of.name)
	}

	if !field.IsValid() {
		return fmt.Errorf("Field %s in %T not valid", of.name, of.obj.iface)
	}
	if !field.CanSet() {
		return fmt.Errorf("Field %s in %T not settable", of.name, of.obj.iface)
	}

	field.Set(v)

	return nil
}

func (of *ObjField) field() (*reflect.Value, *reflect.StructField, error) {
	if !of.obj.IsStructOrPtrToStruct() {
		return nil, nil, fmt.Errorf("Cannot get %s in %T because underlying obj is not a struct", of.name, of.obj.iface)
	}

	var valueField reflect.Value
	var structField reflect.StructField
	var found bool
	if of.obj.isPtrToStruct {
		valueField = reflect.ValueOf(of.obj.iface).Elem().FieldByName(of.name)
		structField, found = reflect.TypeOf(of.obj.iface).Elem().FieldByName(of.name)
	} else {
		valueField = reflect.ValueOf(of.obj.iface).FieldByName(of.name)
		structField, found = reflect.TypeOf(of.obj.iface).FieldByName(of.name)
	}

	if !found {
		return nil, nil, fmt.Errorf("Field not found %s in %T", of.name, of.obj.iface)
	}

	return &valueField, &structField, nil
}

func (of *ObjField) Get() (interface{}, error) {
	fptr, _, err := of.field()
	if err != nil {
		return nil, err
	}
	field := *fptr

	if !field.IsValid() {
		return nil, fmt.Errorf("Invalid field %s in %T", of.name, of.obj.iface)
	}

	value := field.Interface()
	return value, nil
}

type ObjMethod struct {
	obj    *Obj
	name   string
	method reflect.Value
}

func newObjMethod(obj *Obj, name string) *ObjMethod {
	return &ObjMethod{
		obj:    obj,
		name:   name,
		method: reflect.ValueOf(obj.iface).MethodByName(name),
	}
}

func (om *ObjMethod) InTypes() []reflect.Type {
	method := reflect.ValueOf(om.obj.iface).MethodByName(om.name)
	if !method.IsValid() {
		return []reflect.Type{}
	}
	ty := method.Type()
	out := make([]reflect.Type, ty.NumIn())
	for i := 0; i < ty.NumIn(); i++ {
		out[i] = ty.In(i)
	}
	return out
}

func (om *ObjMethod) OutTypes() []reflect.Type {
	method := reflect.ValueOf(om.obj.iface).MethodByName(om.name)
	if !method.IsValid() {
		return []reflect.Type{}
	}
	ty := method.Type()
	out := make([]reflect.Type, ty.NumOut())
	for i := 0; i < ty.NumOut(); i++ {
		out[i] = ty.Out(i)
	}
	return out
}

func (om *ObjMethod) IsValid() bool {
	return om.method.IsValid()
}

// Call calls this method. Note that in the error returning value is not the error from the method call
func (om *ObjMethod) Call(args ...interface{}) (*CallResult, error) {
	if !om.method.IsValid() {
		return nil, fmt.Errorf("Invalid method %s in %T", om.name, om.obj.iface)
	}
	in := make([]reflect.Value, len(args))
	for n := range args {
		in[n] = reflect.ValueOf(args[n])
	}
	out := om.method.Call(in)
	res := make([]interface{}, len(out))
	for n := range out {
		res[n] = out[n].Interface()
	}
	return newCallResult(res), nil
}

type CallResult struct {
	Result []interface{}
	Error  error
}

func newCallResult(res []interface{}) *CallResult {
	cr := &CallResult{Result: res}
	if len(res) == 0 {
		return cr
	}
	errorCandidate := res[len(res)-1]
	if errorCandidate != nil {
		if err, is := errorCandidate.(error); is {
			cr.Error = err
		}
	}
	return cr
}

// IsError checks if the last value is a non-nil error
func (cr *CallResult) IsError() bool {
	return cr.Error != nil
}
<commit_msg>Better check for exportable<commit_after>package reflector

import (
	"fmt"
	"reflect"
	"strconv"
	"strings"
)

type Obj struct {
	iface interface{}

	isStruct      bool
	isPtrToStruct bool

	// If ptr to struct, this field will contain the type of that struct
	underlyingType reflect.Type

	objType reflect.Type
	objKind reflect.Kind
}

func NewFromType(ty reflect.Type) *Obj {
	return New(reflect.New(ty).Interface())
}

func New(obj interface{}) *Obj {
	o := &Obj{iface: obj}
	o.objType = reflect.TypeOf(obj)
	o.objKind = o.objType.Kind()

	ty := o.Type()
	if ty.Kind() == reflect.Struct {
		o.isStruct = true
	}
	if ty.Kind() == reflect.Ptr && ty.Elem().Kind() == reflect.Struct {
		ty = ty.Elem()
		o.isPtrToStruct = true
	}
	o.underlyingType = ty
	return o
}

func (o *Obj) Fields() []ObjField {
	return o.fields(reflect.TypeOf(o.iface), false)
}

func (o Obj) FieldsFlattened() []ObjField {
	return o.fields(reflect.TypeOf(o.iface), true)
}

func (o *Obj) fields(ty reflect.Type, flatten bool) []ObjField {
	fields := make([]ObjField, 0)

	if ty.Kind() == reflect.Ptr {
		ty = ty.Elem()
	}

	if ty.Kind() != reflect.Struct {
		return fields // No need to populate nonstructs
	}

	for i := 0; i < ty.NumField(); i++ {
		field := ty.Field(i)

		k := field.Type.Kind()
		if isExportable(field) {
			if flatten && k == reflect.Struct && field.Anonymous {
				fields = append(fields, o.fields(field.Type, flatten)...)
			} else {
				fields = append(fields, *newObjField(o, field.Name))
			}
		}
	}

	return fields
}

func (o Obj) IsPtr() bool {
	return o.objKind == reflect.Ptr
}

func (o Obj) IsStructOrPtrToStruct() bool {
	return o.isStruct || o.isPtrToStruct
}

func (o *Obj) Field(name string) *ObjField {
	return &ObjField{
		obj:  o,
		name: name,
	}
}

func (o Obj) Type() reflect.Type {
	return o.objType
}

func (o Obj) Kind() reflect.Kind {
	return o.objKind
}

func (o *Obj) Method(name string) *ObjMethod {
	return newObjMethod(o, name)
}

func (o *Obj) Methods() []ObjMethod {
	res := []ObjMethod{}
	ty := o.Type()
	for i := 0; i < ty.NumMethod(); i++ {
		method := ty.Method(i)
		res = append(res, *newObjMethod(o, method.Name))
	}
	return res
}

type ObjField struct {
	obj  *Obj
	name string
}

func newObjField(obj *Obj, name string) *ObjField {
	return &ObjField{
		obj:  obj,
		name: name,
	}
}

func (of *ObjField) Name() string {
	return of.name
}

func (of *ObjField) Kind() reflect.Kind {
	ty, err := of.Type()
	if err != nil {
		return reflect.Invalid
	}
	return ty.Kind()
}

func (of *ObjField) Type() (reflect.Type, error) {
	value, err := of.Get()
	if err != nil {
		return nil, fmt.Errorf("Invalid field %s in %T", of.name, of.obj.iface)
	}
	return reflect.TypeOf(value), nil
}

func (of *ObjField) Tag(tag string) (string, error) {
	_, field, err := of.field()
	if err != nil {
		return "", err
	}
	return (*field).Tag.Get(tag), nil
}

func (of *ObjField) Tags() (map[string]string, error) {
	_, field, err := of.field()
	if err != nil {
		return nil, err
	}

	res := map[string]string{}
	tag := (*field).Tag

	// This code is copied/modified from: reflect/type.go:
	for tag != "" {
		// Skip leading space.
		i := 0
		for i < len(tag) && tag[i] == ' ' {
			i++
		}
		tag = tag[i:]
		if tag == "" {
			break
		}

		// Scan to colon. A space, a quote or a control character is a syntax error.
		// Strictly speaking, control chars include the range [0x7f, 0x9f], not just
		// [0x00, 0x1f], but in practice, we ignore the multi-byte control characters
		// as it is simpler to inspect the tag's bytes than the tag's runes.
		i = 0
		for i < len(tag) && tag[i] > ' ' && tag[i] != ':' && tag[i] != '"' && tag[i] != 0x7f {
			i++
		}
		if i == 0 || i+1 >= len(tag) || tag[i] != ':' || tag[i+1] != '"' {
			break
		}
		name := string(tag[:i])
		tag = tag[i+1:]

		// Scan quoted string to find value.
		i = 1
		for i < len(tag) && tag[i] != '"' {
			if tag[i] == '\\' {
				i++
			}
			i++
		}
		if i >= len(tag) {
			break
		}
		qvalue := string(tag[:i+1])
		tag = tag[i+1:]

		value, err := strconv.Unquote(qvalue)
		if err != nil {
			return nil, fmt.Errorf("Cannot unquote tag %s in %T.%s: %s", name, of.obj.iface, of.name, err.Error())
		}
		res[name] = value
		/*
			if key == name {
				value, err := strconv.Unquote(qvalue)
				if err != nil {
					break
				}
				return value
			}
		*/
	}

	return res, nil
}

// TagExpanded returns the tag value "expanded" with commas
func (of *ObjField) TagExpanded(tag string) ([]string, error) {
	_, field, err := of.field()
	if err != nil {
		return nil, err
	}
	return strings.Split((*field).Tag.Get(tag), ","), nil
}

func (of *ObjField) Valid() bool {
	if !of.obj.IsStructOrPtrToStruct() {
		return false
	}
	_, found := of.obj.underlyingType.FieldByName(of.name)
	return found
}

func (of *ObjField) Anonymous() bool {
	field, found := of.obj.underlyingType.FieldByName(of.name)
	if !found {
		return false
	}
	return field.Anonymous
}

func (of *ObjField) Set(value interface{}) error {
	if !of.obj.IsStructOrPtrToStruct() {
		return fmt.Errorf("Cannot set %s in %T because obj is not a pointer to struct", of.name, of.obj.iface)
	}

	v := reflect.ValueOf(value)
	var field reflect.Value
	if of.obj.isPtrToStruct {
		field = reflect.ValueOf(of.obj.iface).Elem().FieldByName(of.name)
	} else {
		field = reflect.ValueOf(of.obj.iface).FieldByName(of.name)
	}

	if !field.IsValid() {
		return fmt.Errorf("Field %s in %T not valid", of.name, of.obj.iface)
	}
	if !field.CanSet() {
		return fmt.Errorf("Field %s in %T not settable", of.name, of.obj.iface)
	}

	field.Set(v)

	return nil
}

func (of *ObjField) field() (*reflect.Value, *reflect.StructField, error) {
	if !of.obj.IsStructOrPtrToStruct() {
		return nil, nil, fmt.Errorf("Cannot get %s in %T because underlying obj is not a struct", of.name, of.obj.iface)
	}

	var valueField reflect.Value
	var structField reflect.StructField
	var found bool
	if of.obj.isPtrToStruct {
		valueField = reflect.ValueOf(of.obj.iface).Elem().FieldByName(of.name)
		structField, found = reflect.TypeOf(of.obj.iface).Elem().FieldByName(of.name)
	} else {
		valueField = reflect.ValueOf(of.obj.iface).FieldByName(of.name)
		structField, found = reflect.TypeOf(of.obj.iface).FieldByName(of.name)
	}

	if !found {
		return nil, nil, fmt.Errorf("Field not found %s in %T", of.name, of.obj.iface)
	}

	return &valueField, &structField, nil
}

func (of *ObjField) Get() (interface{}, error) {
	fptr, _, err := of.field()
	if err != nil {
		return nil, err
	}
	field := *fptr

	if !field.IsValid() {
		return nil, fmt.Errorf("Invalid field %s in %T", of.name, of.obj.iface)
	}

	value := field.Interface()
	return value, nil
}

type ObjMethod struct {
	obj    *Obj
	name   string
	method reflect.Value
}

func newObjMethod(obj *Obj, name string) *ObjMethod {
	return &ObjMethod{
		obj:    obj,
		name:   name,
		method: reflect.ValueOf(obj.iface).MethodByName(name),
	}
}

func (om *ObjMethod) InTypes() []reflect.Type {
	method := reflect.ValueOf(om.obj.iface).MethodByName(om.name)
	if !method.IsValid() {
		return []reflect.Type{}
	}
	ty := method.Type()
	out := make([]reflect.Type, ty.NumIn())
	for i := 0; i < ty.NumIn(); i++ {
		out[i] = ty.In(i)
	}
	return out
}

func (om *ObjMethod) OutTypes() []reflect.Type {
	method := reflect.ValueOf(om.obj.iface).MethodByName(om.name)
	if !method.IsValid() {
		return []reflect.Type{}
	}
	ty := method.Type()
	out := make([]reflect.Type, ty.NumOut())
	for i := 0; i < ty.NumOut(); i++ {
		out[i] = ty.Out(i)
	}
	return out
}

func (om *ObjMethod) IsValid() bool {
	return om.method.IsValid()
}

// Call calls this method. Note that in the error returning value is not the error from the method call
func (om *ObjMethod) Call(args ...interface{}) (*CallResult, error) {
	if !om.method.IsValid() {
		return nil, fmt.Errorf("Invalid method %s in %T", om.name, om.obj.iface)
	}
	in := make([]reflect.Value, len(args))
	for n := range args {
		in[n] = reflect.ValueOf(args[n])
	}
	out := om.method.Call(in)
	res := make([]interface{}, len(out))
	for n := range out {
		res[n] = out[n].Interface()
	}
	return newCallResult(res), nil
}

type CallResult struct {
	Result []interface{}
	Error  error
}

func newCallResult(res []interface{}) *CallResult {
	cr := &CallResult{Result: res}
	if len(res) == 0 {
		return cr
	}
	errorCandidate := res[len(res)-1]
	if errorCandidate != nil {
		if err, is := errorCandidate.(error); is {
			cr.Error = err
		}
	}
	return cr
}

// IsError checks if the last value is a non-nil error
func (cr *CallResult) IsError() bool {
	return cr.Error != nil
}

func isExportable(field reflect.StructField) bool {
	return field.PkgPath == ""
}
<|endoftext|>


<commit_before>//  Copyright 2016 Walter Schulze
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package mem

import (
	"github.com/katydid/katydid/relapse/ast"
)

//Compile memoizes the full state space, all possible things that can be memoized.
func Compile(g *ast.Grammar) (*Mem, error) {
	mem, err := New(g)
	if err != nil {
		return nil, err
	}
	changed := true
	visited := make(map[int]bool)
	for changed {
		changed = false
		for state := range mem.patterns {
			if visited[state] {
				continue
			}
			visited[state] = true
			if err := compile(mem, state); err != nil {
				return nil, err
			}
			changed = true
		}
	}
	return mem, nil
}

func compile(mem *Mem, current int) error {
	mem.getNullable(current)
	mem.accept(current)
	mem.escapable(current)

	callTree, err := mem.getCallTree(current)
	if err != nil {
		return err
	}
	leafs := getLeafs(callTree)
	for _, leaf := range leafs {
		childlen := len(mem.patterns[leaf.child])
		max := newBitSet(childlen)
		for i := 0; i < childlen; i++ {
			max.set(i, true)
		}
		current := newBitSet(childlen)
		for {
			nullIndex := mem.nullables.add(current)
			mem.getReturnn(leaf.stackIndex, nullIndex)
			if current.equal(max) {
				break
			}
			current = current.inc()
		}
	}
	return nil
}

func getLeafs(callTree *CallNode) []*CallNode {
	if callTree.cond == nil {
		return []*CallNode{callTree}
	}
	then := getLeafs(callTree.then)
	els := getLeafs(callTree.els)
	return append(then, els...)
}
<commit_msg>state explosion error<commit_after>//  Copyright 2016 Walter Schulze
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package mem

import (
	"errors"
	"github.com/katydid/katydid/relapse/ast"
)

var ErrTooManyStates = errors.New("a state explosion has occured")

//Compile memoizes the full state space, all possible things that can be memoized.
func Compile(g *ast.Grammar) (*Mem, error) {
	mem, err := New(g)
	if err != nil {
		return nil, err
	}
	changed := true
	visited := make(map[int]bool)
	for changed {
		changed = false
		for state := range mem.patterns {
			if visited[state] {
				continue
			}
			visited[state] = true
			if err := compile(mem, state); err != nil {
				return nil, err
			}
			changed = true
		}
	}
	return mem, nil
}

func compile(mem *Mem, current int) error {
	mem.getNullable(current)
	mem.accept(current)
	mem.escapable(current)

	callTree, err := mem.getCallTree(current)
	if err != nil {
		return err
	}
	leafs := getLeafs(callTree)
	for _, leaf := range leafs {
		childlen := len(mem.patterns[leaf.child])
		if childlen > 64 {
			return ErrTooManyStates
		}
		max := newBitSet(childlen)
		for i := 0; i < childlen; i++ {
			max.set(i, true)
		}
		current := newBitSet(childlen)
		for {
			nullIndex := mem.nullables.add(current)
			mem.getReturnn(leaf.stackIndex, nullIndex)
			if current.equal(max) {
				break
			}
			current = current.inc()
		}
	}
	return nil
}

func getLeafs(callTree *CallNode) []*CallNode {
	if callTree.cond == nil {
		return []*CallNode{callTree}
	}
	then := getLeafs(callTree.then)
	els := getLeafs(callTree.els)
	return append(then, els...)
}
<|endoftext|>