<commit_before>package typescriptify

import (
	"fmt"
	"os"
	"reflect"
	"strings"
	"testing"
)

type Address struct {
	// Used in html
	Duration float64 `json:"duration"`
	Text1    string  `json:"text,omitempty"`
	// Ignored:
	Text2 string `json:",omitempty"`
	Text3 string `json:"-"`
}

type Dummy struct {
	Something string `json:"something"`
}

type Person struct {
	Name      string    `json:"name"`
	Nicknames []string  `json:"nicknames"`
	Addresses []Address `json:"addresses"`
	Dummy     Dummy     `json:"a"`
}

func TestTypescriptifyWithTypes(t *testing.T) {
	converter := New()

	converter.AddType(reflect.TypeOf(Person{}))

	desiredResult := `class Dummy {
        something : string;
}
class Address {
        duration : number;
        text : string;
}
class Person {
        name : string;
        nicknames : string[];
        addresses : Address[];
        a : Dummy;
}`
	testConverter(t, converter, desiredResult)
}

func TestTypescriptifyWithInstances(t *testing.T) {
	converter := New()

	converter.Add(Person{})
	converter.Add(Dummy{})

	desiredResult := `class Dummy {
        something : string;
}
class Address {
        duration : number;
        text : string;
}
class Person {
        name : string;
        nicknames : string[];
        addresses : Address[];
        a : Dummy;
}`
	testConverter(t, converter, desiredResult)
}

func TestTypescriptifyWithDoubleClasses(t *testing.T) {
	converter := New()

	converter.AddType(reflect.TypeOf(Person{}))
	converter.AddType(reflect.TypeOf(Person{}))

	desiredResult := `class Dummy {
        something : string;
}
class Address {
        duration : number;
        text : string;
}
class Person {
        name : string;
        nicknames : string[];
        addresses : Address[];
        a : Dummy;
}`
	testConverter(t, converter, desiredResult)
}

func TestWithPrefixes(t *testing.T) {
	converter := New()

	converter.Prefix("test_")

	converter.Add(Address{})
	converter.Add(Dummy{})

	desiredResult := `class test_Address {
        duration : number;
        text : string;
}
class test_Dummy {
        something : string;
}`
	testConverter(t, converter, desiredResult)
}

func testConverter(t *testing.T, converter *TypeScriptify, desiredResult string) {
	typeScriptCode, err := converter.Convert(nil)
	if err != nil {
		panic(err.Error())
	}

	typeScriptCode = strings.Trim(typeScriptCode, " \t\n\r")
	if typeScriptCode != desiredResult {
		lines1 := strings.Split(typeScriptCode, "\n")
		lines2 := strings.Split(desiredResult, "\n")

		if len(lines1) != len(lines2) {
			os.Stderr.WriteString(fmt.Sprintf("Lines: %d != %d\n", len(lines1), len(lines2)))
			os.Stderr.WriteString(fmt.Sprintf("Expected:\n%s\n\nGot:\n%s\n", desiredResult, typeScriptCode))
			t.Fail()
		} else {
			for i := 0; i < len(lines1); i++ {
				line1 := strings.Trim(lines1[i], " \t\r\n")
				line2 := strings.Trim(lines2[i], " \t\r\n")
				if line1 != line2 {
					os.Stderr.WriteString(fmt.Sprintf("%d. line don't match: `%s` != `%s`\n", line1, line2))
					os.Stderr.WriteString(fmt.Sprintf("Expected:\n%s\n\nGot:\n%s\n", desiredResult, typeScriptCode))
					t.Fail()
				}
			}
		}
	}
}
<commit_msg>Tests fix<commit_after>package typescriptify

import (
	"fmt"
	"os"
	"reflect"
	"strings"
	"testing"
)

type Address struct {
	// Used in html
	Duration float64 `json:"duration"`
	Text1    string  `json:"text,omitempty"`
	// Ignored:
	Text2 string `json:",omitempty"`
	Text3 string `json:"-"`
}

type Dummy struct {
	Something string `json:"something"`
}

type Person struct {
	Name      string    `json:"name"`
	Nicknames []string  `json:"nicknames"`
	Addresses []Address `json:"addresses"`
	Dummy     Dummy     `json:"a"`
}

func TestTypescriptifyWithTypes(t *testing.T) {
	converter := New()

	converter.AddType(reflect.TypeOf(Person{}))

	desiredResult := `class Dummy {
        something: string;
}
class Address {
        duration: number;
        text: string;
}
class Person {
        name: string;
        nicknames: string[];
        addresses: Address[];
        a: Dummy;
}`
	testConverter(t, converter, desiredResult)
}

func TestTypescriptifyWithInstances(t *testing.T) {
	converter := New()

	converter.Add(Person{})
	converter.Add(Dummy{})

	desiredResult := `class Dummy {
        something: string;
}
class Address {
        duration: number;
        text: string;
}
class Person {
        name: string;
        nicknames: string[];
        addresses: Address[];
        a: Dummy;
}`
	testConverter(t, converter, desiredResult)
}

func TestTypescriptifyWithDoubleClasses(t *testing.T) {
	converter := New()

	converter.AddType(reflect.TypeOf(Person{}))
	converter.AddType(reflect.TypeOf(Person{}))

	desiredResult := `class Dummy {
        something: string;
}
class Address {
        duration: number;
        text: string;
}
class Person {
        name: string;
        nicknames: string[];
        addresses: Address[];
        a: Dummy;
}`
	testConverter(t, converter, desiredResult)
}

func TestWithPrefixes(t *testing.T) {
	converter := New()

	converter.Prefix("test_")

	converter.Add(Address{})
	converter.Add(Dummy{})

	desiredResult := `class test_Address {
        duration: number;
        text: string;
}
class test_Dummy {
        something: string;
}`
	testConverter(t, converter, desiredResult)
}

func testConverter(t *testing.T, converter *TypeScriptify, desiredResult string) {
	typeScriptCode, err := converter.Convert(nil)
	if err != nil {
		panic(err.Error())
	}

	typeScriptCode = strings.Trim(typeScriptCode, " \t\n\r")
	if typeScriptCode != desiredResult {
		lines1 := strings.Split(typeScriptCode, "\n")
		lines2 := strings.Split(desiredResult, "\n")

		if len(lines1) != len(lines2) {
			os.Stderr.WriteString(fmt.Sprintf("Lines: %d != %d\n", len(lines1), len(lines2)))
			os.Stderr.WriteString(fmt.Sprintf("Expected:\n%s\n\nGot:\n%s\n", desiredResult, typeScriptCode))
			t.Fail()
		} else {
			for i := 0; i < len(lines1); i++ {
				line1 := strings.Trim(lines1[i], " \t\r\n")
				line2 := strings.Trim(lines2[i], " \t\r\n")
				if line1 != line2 {
					os.Stderr.WriteString(fmt.Sprintf("%d. line don't match: `%s` != `%s`\n", i+1, line1, line2))
					os.Stderr.WriteString(fmt.Sprintf("Expected:\n%s\n\nGot:\n%s\n", desiredResult, typeScriptCode))
					t.Fail()
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package api

import (
	"testing"
)

// TestApiClient tests that the API client connects to the server tester and
// can call and decode routes correctly.
func TestApiClient(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	st, err := createServerTester("TestApiClient")
	if err != nil {
		t.Fatal(err)
	}
	defer st.server.Close()

	c := NewClient("localhost:9980", "")
	var gatewayInfo GatewayGET
	err = c.Get("/gateway", &gatewayInfo)
	if err != nil {
		t.Fatal(err)
	}
}

// TestAuthenticatedApiClient tests that the API client connects to an
// authenticated server tester and can call and decode routes correctly, using
// the correct password.
func TestAuthenticatedApiClient(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	testpass := "testPassword"
	st, err := createAuthenticatedServerTester("TestAuthenticatedApiClient", testpass)
	if err != nil {
		t.Fatal(err)
	}
	defer st.server.Close()

	c := NewClient("localhost:9980", "")
	var gatewayInfo GatewayGET
	err = c.Get("/gateway", &gatewayInfo)
	if err == nil {
		t.Fatal("api.Client did not return an error when requesting an authenticated resource without a password")
	}
	c = NewClient("localhost:9980", testpass)
	err = c.Get("/gateway", &gatewayInfo)
	if err != nil {
		t.Fatal(err)
	}
}
<commit_msg>fix client tests to use st.server.listener.Addr.String() instead of hardcoded address<commit_after>package api

import (
	"testing"
)

// TestApiClient tests that the API client connects to the server tester and
// can call and decode routes correctly.
func TestApiClient(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	st, err := createServerTester("TestApiClient")
	if err != nil {
		t.Fatal(err)
	}
	defer st.server.Close()

	c := NewClient(st.server.listener.Addr().String(), "")
	var gatewayInfo GatewayGET
	err = c.Get("/gateway", &gatewayInfo)
	if err != nil {
		t.Fatal(err)
	}
}

// TestAuthenticatedApiClient tests that the API client connects to an
// authenticated server tester and can call and decode routes correctly, using
// the correct password.
func TestAuthenticatedApiClient(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	testpass := "testPassword"
	st, err := createAuthenticatedServerTester("TestAuthenticatedApiClient", testpass)
	if err != nil {
		t.Fatal(err)
	}
	defer st.server.Close()

	c := NewClient(st.server.listener.Addr().String(), "")
	var walletAddress WalletAddressGET
	err = c.Get("/wallet/address", &walletAddress)
	if err == nil {
		t.Fatal("api.Client did not return an error when requesting an authenticated resource without a password")
	}
	c = NewClient(st.server.listener.Addr().String(), testpass)
	err = c.Get("/wallet/address", &walletAddress)
	if err != nil {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package vecty

import (
	"fmt"
	"strings"

	"github.com/gopherjs/gopherjs/js"
)

// EventListener is markup that specifies a callback function to be invoked when
// the named DOM event is fired.
type EventListener struct {
	Name                string
	Listener            func(*Event)
	callPreventDefault  bool
	callStopPropagation bool
	wrapper             func(jsEvent *js.Object)
}

// PreventDefault prevents the default behavior of the event from occuring.
//
// See https://developer.mozilla.org/en-US/docs/Web/API/Event/preventDefault.
func (l *EventListener) PreventDefault() *EventListener {
	l.callPreventDefault = true
	return l
}

// StopPropagation prevents further propagation of the current event in the
// capturing and bubbling phases.
//
// See https://developer.mozilla.org/en-US/docs/Web/API/Event/stopPropagation.
func (l *EventListener) StopPropagation() *EventListener {
	l.callStopPropagation = true
	return l
}

// Apply implements the Markup interface.
func (l *EventListener) Apply(h *HTML) {
	h.eventListeners = append(h.eventListeners, l)
}

// Event represents a DOM event.
type Event struct {
	Target *js.Object
}

// MarkupOrComponentOrHTML represents one of:
//
//  Markup
//  Component
//  *HTML
//
// If the underlying value is not one of these types, the code handling the
// value is expected to panic.
type MarkupOrComponentOrHTML interface{}

func apply(m MarkupOrComponentOrHTML, h *HTML) {
	if m == nil {
		return
	}
	switch m := m.(type) {
	case Markup:
		m.Apply(h)
	case Component:
		h.children = append(h.children, m)
	case *HTML:
		h.children = append(h.children, m)
	default:
		panic(fmt.Sprintf("vecty: invalid type %T does not match MarkupOrComponent interface", m))
	}
}

// Markup represents some type of markup (a style, property, data, etc) which
// can be applied to a given HTML element or text node.
type Markup interface {
	// Apply applies the markup to the given HTML element or text node.
	Apply(h *HTML)
}

type markupFunc func(h *HTML)

func (m markupFunc) Apply(h *HTML) { m(h) }

// Style returns Markup which applies the given CSS style. Generally, this
// function is not used directly but rather the style subpackage (which is type
// safe) is used instead.
func Style(key, value string) Markup {
	return markupFunc(func(h *HTML) {
		if h.styles == nil {
			h.styles = make(map[string]string)
		}
		h.styles[key] = value
	})
}

// Property returns Markup which applies the given JavaScript property to an
// HTML element or text node. Generally, this function is not used directly but
// rather the style subpackage (which is type safe) is used instead.
func Property(key string, value interface{}) Markup {
	return markupFunc(func(h *HTML) {
		if h.properties == nil {
			h.properties = make(map[string]interface{})
		}
		h.properties[key] = value
	})
}

// Data returns Markup which applies the given data attribute.
func Data(key, value string) Markup {
	return markupFunc(func(h *HTML) {
		if h.dataset == nil {
			h.dataset = make(map[string]string)
		}
		h.dataset[key] = value
	})
}

// ClassMap is markup that specifies classes to be applied to an element if
// their boolean value are true.
type ClassMap map[string]bool

// Apply implements the Markup interface.
func (m ClassMap) Apply(h *HTML) {
	var classes []string
	for name, active := range m {
		if active {
			classes = append(classes, name)
		}
	}
	Property("className", strings.Join(classes, " ")).Apply(h)
}

// List represents a list of Markup, Component, or HTML which is individually
// applied to an HTML element or text node.
type List []MarkupOrComponentOrHTML

// Apply implements the Markup interface.
func (l List) Apply(h *HTML) {
	for _, m := range l {
		apply(m, h)
	}
}

// If returns nil if cond is false, otherwise it returns the given markup.
func If(cond bool, markup ...MarkupOrComponentOrHTML) MarkupOrComponentOrHTML {
	if cond {
		return List(markup)
	}
	return nil
}
<commit_msg>Better support for nil in apply<commit_after>package vecty

import (
	"fmt"
	"strings"

	"github.com/gopherjs/gopherjs/js"
)

// EventListener is markup that specifies a callback function to be invoked when
// the named DOM event is fired.
type EventListener struct {
	Name                string
	Listener            func(*Event)
	callPreventDefault  bool
	callStopPropagation bool
	wrapper             func(jsEvent *js.Object)
}

// PreventDefault prevents the default behavior of the event from occuring.
//
// See https://developer.mozilla.org/en-US/docs/Web/API/Event/preventDefault.
func (l *EventListener) PreventDefault() *EventListener {
	l.callPreventDefault = true
	return l
}

// StopPropagation prevents further propagation of the current event in the
// capturing and bubbling phases.
//
// See https://developer.mozilla.org/en-US/docs/Web/API/Event/stopPropagation.
func (l *EventListener) StopPropagation() *EventListener {
	l.callStopPropagation = true
	return l
}

// Apply implements the Markup interface.
func (l *EventListener) Apply(h *HTML) {
	h.eventListeners = append(h.eventListeners, l)
}

// Event represents a DOM event.
type Event struct {
	Target *js.Object
}

// MarkupOrComponentOrHTML represents one of:
//
//  Markup
//  Component
//  *HTML
//
// If the underlying value is not one of these types, the code handling the
// value is expected to panic.
type MarkupOrComponentOrHTML interface{}

func apply(m MarkupOrComponentOrHTML, h *HTML) {
	if m == nil {
		return
	}
	switch m := m.(type) {
	case Markup:
		m.Apply(h)
	case Component:
		h.children = append(h.children, m)
	case *HTML:
		if m == nil {
			return
		}
		h.children = append(h.children, m)
	default:
		panic(fmt.Sprintf("vecty: invalid type %T does not match MarkupOrComponent interface", m))
	}
}

// Markup represents some type of markup (a style, property, data, etc) which
// can be applied to a given HTML element or text node.
type Markup interface {
	// Apply applies the markup to the given HTML element or text node.
	Apply(h *HTML)
}

type markupFunc func(h *HTML)

func (m markupFunc) Apply(h *HTML) { m(h) }

// Style returns Markup which applies the given CSS style. Generally, this
// function is not used directly but rather the style subpackage (which is type
// safe) is used instead.
func Style(key, value string) Markup {
	return markupFunc(func(h *HTML) {
		if h.styles == nil {
			h.styles = make(map[string]string)
		}
		h.styles[key] = value
	})
}

// Property returns Markup which applies the given JavaScript property to an
// HTML element or text node. Generally, this function is not used directly but
// rather the style subpackage (which is type safe) is used instead.
func Property(key string, value interface{}) Markup {
	return markupFunc(func(h *HTML) {
		if h.properties == nil {
			h.properties = make(map[string]interface{})
		}
		h.properties[key] = value
	})
}

// Data returns Markup which applies the given data attribute.
func Data(key, value string) Markup {
	return markupFunc(func(h *HTML) {
		if h.dataset == nil {
			h.dataset = make(map[string]string)
		}
		h.dataset[key] = value
	})
}

// ClassMap is markup that specifies classes to be applied to an element if
// their boolean value are true.
type ClassMap map[string]bool

// Apply implements the Markup interface.
func (m ClassMap) Apply(h *HTML) {
	var classes []string
	for name, active := range m {
		if active {
			classes = append(classes, name)
		}
	}
	Property("className", strings.Join(classes, " ")).Apply(h)
}

// List represents a list of Markup, Component, or HTML which is individually
// applied to an HTML element or text node.
type List []MarkupOrComponentOrHTML

// Apply implements the Markup interface.
func (l List) Apply(h *HTML) {
	for _, m := range l {
		apply(m, h)
	}
}

// If returns nil if cond is false, otherwise it returns the given markup.
func If(cond bool, markup ...MarkupOrComponentOrHTML) MarkupOrComponentOrHTML {
	if cond {
		return List(markup)
	}
	return nil
}
<|endoftext|>


<commit_before>package metrics

import (
	"reflect"
	"strconv"

	log "github.com/cihub/seelog"
	"github.com/fatih/structs"
	serverprotocol "github.com/stampzilla/stampzilla-go/nodes/stampzilla-server/protocol"
)

type Logger interface {
	Log(key string, value interface{})
	Commit(node interface{})
}

type Metrics struct {
	previous map[string]interface{}
	loggers  []Logger
	queue    chan interface{}
}

func New() *Metrics {
	return &Metrics{
		make(map[string]interface{}),
		nil,
		make(chan interface{}, 100),
	}
}
func (m *Metrics) AddLogger(l Logger) {
	log.Infof("Adding logger: %T\n", l)
	m.loggers = append(m.loggers, l)
}
func (m *Metrics) Start() {
	go m.worker()
}
func (m *Metrics) worker() {
	for s := range m.queue {
		m.update(s)
	}
}
func (m *Metrics) Update(s interface{}) {
	m.queue <- s
}

func (m *Metrics) update(s interface{}) {
	if len(m.loggers) == 0 {
		return
	}

	current := structToMetrics(s)
	if len(m.previous) == 0 {
		m.previous = current
		return
	}

	changed := false
	for k, v := range current {
		if m.isDiff(k, v) {
			log.Info("found diff. logging!")
			m.log(k, v)
			changed = true
		}
	}

	if changed {
		for _, l := range m.loggers {
			l.Commit(s)
		}
	}
	m.updatePrevious(current)
}

func (m *Metrics) updatePrevious(s map[string]interface{}) {
	for k, v := range s {
		m.previous[k] = v
	}
}

func (m *Metrics) log(key string, value interface{}) {
	for _, l := range m.loggers {
		l.Log(key, value)
	}
}
func (m *Metrics) isDiff(k string, v interface{}) bool {
	if oldValue, ok := m.previous[k]; ok {
		if oldValue != v {
			return true
		}
	}
	return false
}

func structToMetrics(s interface{}) map[string]interface{} {
	flattened := make(map[string]interface{})
	baseName := ""
	if node, ok := s.(serverprotocol.Node); ok {
		//st := structs.New(node)
		baseName = node.Uuid
	}
	flatten(structs.Map(s), baseName, &flattened)
	return flattened
}

func flatten(inputJSON map[string]interface{}, lkey string, flattened *map[string]interface{}) {
	for rkey, value := range inputJSON {
		key := lkey + "_" + rkey
		if lkey == "" {
			key = rkey
		}

		if value == nil {
			continue
		}

		if structs.IsStruct(value) {
			//fmt.Println("Its a struct: ", value)
			value = structs.Map(value)
		}
		reflectValue := reflect.ValueOf(value)
		if reflectValue.Type().Kind() == reflect.Map {
			//fmt.Println("its a map: ", value)
			out := make(map[string]interface{})
			for _, b := range reflectValue.MapKeys() {
				out[b.String()] = reflectValue.MapIndex(b).Interface()
			}
			value = out
		}

		switch v := value.(type) {
		case map[string]interface{}:
			flatten(v, key, flattened)
		default:
			(*flattened)[key] = cast(v)
		}

	}
}

func cast(s interface{}) interface{} {
	switch v := s.(type) {
	case int:
		return v
		//return strconv.Itoa(v)
	case float64:
		//return strconv.FormatFloat(v, 'f', -1, 64)
		return v
	case string:
		if n, err := strconv.Atoi(v); err == nil {
			return n
		}
		return v
	case bool:
		if v {
			return 1
		}
		return 0
	}
	return ""
}
<commit_msg>Added so all values are commited the first time a state update is recived<commit_after>package metrics

import (
	"reflect"
	"strconv"

	log "github.com/cihub/seelog"
	"github.com/fatih/structs"
	serverprotocol "github.com/stampzilla/stampzilla-go/nodes/stampzilla-server/protocol"
)

type Logger interface {
	Log(key string, value interface{})
	Commit(node interface{})
}

type Metrics struct {
	previous map[string]interface{}
	loggers  []Logger
	queue    chan interface{}
}

func New() *Metrics {
	return &Metrics{
		make(map[string]interface{}),
		nil,
		make(chan interface{}, 100),
	}
}
func (m *Metrics) AddLogger(l Logger) {
	log.Infof("Adding logger: %T\n", l)
	m.loggers = append(m.loggers, l)
}
func (m *Metrics) Start() {
	go m.worker()
}
func (m *Metrics) worker() {
	for s := range m.queue {
		m.update(s)
	}
}
func (m *Metrics) Update(s interface{}) {
	m.queue <- s
}

func (m *Metrics) update(s interface{}) {
	if len(m.loggers) == 0 {
		return
	}

	current := structToMetrics(s)
	if len(m.previous) == 0 { // No previous values exists, then use this one and commit all values
		m.previous = current

		// Force commit the first set off values
		for k, v := range current {
			m.log(k, v)
		}
		for _, l := range m.loggers {
			l.Commit(s)
		}
		return
	}

	changed := false
	for k, v := range current {
		if m.isDiff(k, v) {
			log.Info("found diff. logging!")
			m.log(k, v)
			changed = true
		}
	}

	if changed {
		for _, l := range m.loggers {
			l.Commit(s)
		}
	}
	m.updatePrevious(current)
}

func (m *Metrics) updatePrevious(s map[string]interface{}) {
	for k, v := range s {
		m.previous[k] = v
	}
}

func (m *Metrics) log(key string, value interface{}) {
	for _, l := range m.loggers {
		l.Log(key, value)
	}
}
func (m *Metrics) isDiff(k string, v interface{}) bool {
	if oldValue, ok := m.previous[k]; ok {
		if oldValue != v {
			return true
		}
	}
	return false
}

func structToMetrics(s interface{}) map[string]interface{} {
	flattened := make(map[string]interface{})
	baseName := ""
	if node, ok := s.(serverprotocol.Node); ok {
		//st := structs.New(node)
		baseName = node.Uuid
	}
	flatten(structs.Map(s), baseName, &flattened)
	return flattened
}

func flatten(inputJSON map[string]interface{}, lkey string, flattened *map[string]interface{}) {
	for rkey, value := range inputJSON {
		key := lkey + "_" + rkey
		if lkey == "" {
			key = rkey
		}

		if value == nil {
			continue
		}

		if structs.IsStruct(value) {
			//fmt.Println("Its a struct: ", value)
			value = structs.Map(value)
		}
		reflectValue := reflect.ValueOf(value)
		if reflectValue.Type().Kind() == reflect.Map {
			//fmt.Println("its a map: ", value)
			out := make(map[string]interface{})
			for _, b := range reflectValue.MapKeys() {
				out[b.String()] = reflectValue.MapIndex(b).Interface()
			}
			value = out
		}

		switch v := value.(type) {
		case map[string]interface{}:
			flatten(v, key, flattened)
		default:
			(*flattened)[key] = cast(v)
		}

	}
}

func cast(s interface{}) interface{} {
	switch v := s.(type) {
	case int:
		return v
		//return strconv.Itoa(v)
	case float64:
		//return strconv.FormatFloat(v, 'f', -1, 64)
		return v
	case string:
		if n, err := strconv.Atoi(v); err == nil {
			return n
		}
		return v
	case bool:
		if v {
			return 1
		}
		return 0
	}
	return ""
}
<|endoftext|>


<commit_before>// Copyright 2014 The lime Authors.
// Use of this source code is governed by a 2-clause
// BSD-style license that can be found in the LICENSE file.

package backend

import (
	"fmt"
	"testing"
)

type (
	DummyApplicationCommand struct {
		DefaultCommand
	}

	DummyWindowCommand struct {
		DefaultCommand
	}

	DummyTextCommand struct {
		DefaultCommand
	}
)

func (c *DummyApplicationCommand) Run() error {
	return fmt.Errorf("Ran")
}

func (c *DummyApplicationCommand) IsChecked() bool {
	return false
}

func (c *DummyWindowCommand) Run(w *Window) error {
	return fmt.Errorf("Ran")
}

func (c *DummyTextCommand) Run(v *View, e *Edit) error {
	return fmt.Errorf("Ran")
}

func TestPascalCaseToSnakeCase(t *testing.T) {
	tests := []struct {
		in  string
		out string
	}{
		{
			"TestString",
			"test_string",
		},
		{
			"Teststring",
			"teststring",
		},
	}

	for i, test := range tests {
		out := PascalCaseToSnakeCase(test.in)

		if out != test.out {
			t.Errorf("Test %d: Expected %s, but got %s", i, test.out, out)
		}
	}
}

func TestRegisterApplicationCommand(t *testing.T) {
	name := "app_test_command"
	ac := DummyApplicationCommand{}
	ch := GetEditor().CommandHandler()

	err := ch.Register(name, &ac)

	if err != nil {
		t.Errorf("Got error while registering: %s", err)
	}

	err = ch.RunApplicationCommand(name, Args{})

	if err == nil {
		t.Errorf("Expected %s to run, but it didn't", name)
	} else if err.Error() != "Ran" {
		t.Errorf("Expected %s to run, but it got an error: %v", name, err)
	}
}

func TestRegisterWindowCommand(t *testing.T) {
	var w Window

	name := "wnd_test_command"
	wc := DummyWindowCommand{}
	ch := GetEditor().CommandHandler()

	err := ch.Register(name, &wc)

	if err != nil {
		t.Errorf("Got error while registering: %s", err)
	}

	err = ch.RunWindowCommand(&w, name, Args{})

	if err == nil {
		t.Errorf("Expected %s to run, but it didn't", name)
	} else if err.Error() != "Ran" {
		t.Errorf("Expected %s to run, but it got an error: %v", name, err)
	}
}

func TestRegisterTextCommand(t *testing.T) {
	ed := GetEditor()

	name := "text_test_command"
	tc := DummyTextCommand{}
	ch := ed.CommandHandler()

	err := ch.Register(name, &tc)

	if err != nil {
		t.Errorf("Got error while registering: %s", err)
	}

	v := ed.NewWindow().NewFile()
	err = ch.RunTextCommand(v, name, Args{})

	if err == nil {
		t.Errorf("Expected %s to run, but it didn't", name)
	} else if err.Error() != "Ran" {
		t.Errorf("Expected %s to run, but it got an error: %v", name, err)
	}
}
<commit_msg>Rename the tests to better represent what they do.<commit_after>// Copyright 2014 The lime Authors.
// Use of this source code is governed by a 2-clause
// BSD-style license that can be found in the LICENSE file.

package backend

import (
	"fmt"
	"testing"
)

type (
	DummyApplicationCommand struct {
		DefaultCommand
	}

	DummyWindowCommand struct {
		DefaultCommand
	}

	DummyTextCommand struct {
		DefaultCommand
	}
)

func (c *DummyApplicationCommand) Run() error {
	return fmt.Errorf("Ran")
}

func (c *DummyApplicationCommand) IsChecked() bool {
	return false
}

func (c *DummyWindowCommand) Run(w *Window) error {
	return fmt.Errorf("Ran")
}

func (c *DummyTextCommand) Run(v *View, e *Edit) error {
	return fmt.Errorf("Ran")
}

func TestPascalCaseToSnakeCase(t *testing.T) {
	tests := []struct {
		in  string
		out string
	}{
		{
			"TestString",
			"test_string",
		},
		{
			"Teststring",
			"teststring",
		},
	}

	for i, test := range tests {
		out := PascalCaseToSnakeCase(test.in)

		if out != test.out {
			t.Errorf("Test %d: Expected %s, but got %s", i, test.out, out)
		}
	}
}

func TestRegisterAndRunApplicationCommand(t *testing.T) {
	name := "app_test_command"
	ac := DummyApplicationCommand{}
	ch := GetEditor().CommandHandler()

	err := ch.Register(name, &ac)

	if err != nil {
		t.Errorf("Got error while registering: %s", err)
	}

	err = ch.RunApplicationCommand(name, Args{})

	if err == nil {
		t.Errorf("Expected %s to run, but it didn't", name)
	} else if err.Error() != "Ran" {
		t.Errorf("Expected %s to run, but it got an error: %v", name, err)
	}
}

func TestRegisterAndRunWindowCommand(t *testing.T) {
	var w Window

	name := "wnd_test_command"
	wc := DummyWindowCommand{}
	ch := GetEditor().CommandHandler()

	err := ch.Register(name, &wc)

	if err != nil {
		t.Errorf("Got error while registering: %s", err)
	}

	err = ch.RunWindowCommand(&w, name, Args{})

	if err == nil {
		t.Errorf("Expected %s to run, but it didn't", name)
	} else if err.Error() != "Ran" {
		t.Errorf("Expected %s to run, but it got an error: %v", name, err)
	}
}

func TestRegisterAndRunTextCommand(t *testing.T) {
	ed := GetEditor()

	name := "text_test_command"
	tc := DummyTextCommand{}
	ch := ed.CommandHandler()

	err := ch.Register(name, &tc)

	if err != nil {
		t.Errorf("Got error while registering: %s", err)
	}

	v := ed.NewWindow().NewFile()
	err = ch.RunTextCommand(v, name, Args{})

	if err == nil {
		t.Errorf("Expected %s to run, but it didn't", name)
	} else if err.Error() != "Ran" {
		t.Errorf("Expected %s to run, but it got an error: %v", name, err)
	}
}
<|endoftext|>


<commit_before>package config

import "github.com/concourse/atc"

// these are expressly tucked away so as to avoid accidental use in public API
// endpoints as that could leak credentials

type JobInput struct {
	Name     string
	Resource string
	Passed   []string
	Trigger  bool
	Version  *atc.VersionConfig
	Params   atc.Params
	Tags     atc.Tags
}

type JobOutput struct {
	Name     string
	Resource string
}

func JobInputs(config atc.JobConfig) []JobInput {
	return collectInputs(atc.PlanConfig{
		Do:      &config.Plan,
		Ensure:  config.Ensure,
		Failure: config.Failure,
		Success: config.Success,
	})
}

func JobOutputs(config atc.JobConfig) []JobOutput {
	return collectOutputs(atc.PlanConfig{
		Do:      &config.Plan,
		Ensure:  config.Ensure,
		Failure: config.Failure,
		Success: config.Success,
	})
}

func collectInputs(plan atc.PlanConfig) []JobInput {
	var inputs []JobInput

	if plan.Success != nil {
		inputs = append(inputs, collectInputs(*plan.Success)...)
	}

	if plan.Failure != nil {
		inputs = append(inputs, collectInputs(*plan.Failure)...)
	}

	if plan.Ensure != nil {
		inputs = append(inputs, collectInputs(*plan.Ensure)...)
	}

	if plan.Try != nil {
		inputs = append(inputs, collectInputs(*plan.Try)...)
	}

	if plan.Do != nil {
		for _, p := range *plan.Do {
			inputs = append(inputs, collectInputs(p)...)
		}
	}

	if plan.Aggregate != nil {
		for _, p := range *plan.Aggregate {
			inputs = append(inputs, collectInputs(p)...)
		}
	}

	if plan.Get != "" {
		get := plan.Get

		resource := get
		if plan.Resource != "" {
			resource = plan.Resource
		}

		inputs = append(inputs, JobInput{
			Name:     get,
			Resource: resource,
			Passed:   plan.Passed,
			Version:  plan.Version,
			Trigger:  plan.Trigger,
			Params:   plan.Params,
			Tags:     plan.Tags,
		})
	}

	return inputs
}

func collectOutputs(plan atc.PlanConfig) []JobOutput {
	var outputs []JobOutput

	if plan.Success != nil {
		outputs = append(outputs, collectOutputs(*plan.Success)...)
	}

	if plan.Failure != nil {
		outputs = append(outputs, collectOutputs(*plan.Failure)...)
	}

	if plan.Ensure != nil {
		outputs = append(outputs, collectOutputs(*plan.Ensure)...)
	}

	if plan.Try != nil {
		outputs = append(outputs, collectOutputs(*plan.Try)...)
	}

	if plan.Do != nil {
		for _, p := range *plan.Do {
			outputs = append(outputs, collectOutputs(p)...)
		}
	}

	if plan.Aggregate != nil {
		var outputs []JobOutput

		for _, p := range *plan.Aggregate {
			outputs = append(outputs, collectOutputs(p)...)
		}

		return outputs
	}

	if plan.Put != "" {
		put := plan.Put

		resource := put
		if plan.Resource != "" {
			resource = plan.Resource
		}

		outputs = append(outputs, JobOutput{
			Name:     put,
			Resource: resource,
		})
	}

	return outputs
}
<commit_msg>fix skipping hooks on aggregates<commit_after>package config

import "github.com/concourse/atc"

// these are expressly tucked away so as to avoid accidental use in public API
// endpoints as that could leak credentials

type JobInput struct {
	Name     string
	Resource string
	Passed   []string
	Trigger  bool
	Version  *atc.VersionConfig
	Params   atc.Params
	Tags     atc.Tags
}

type JobOutput struct {
	Name     string
	Resource string
}

func JobInputs(config atc.JobConfig) []JobInput {
	return collectInputs(atc.PlanConfig{
		Do:      &config.Plan,
		Ensure:  config.Ensure,
		Failure: config.Failure,
		Success: config.Success,
	})
}

func JobOutputs(config atc.JobConfig) []JobOutput {
	return collectOutputs(atc.PlanConfig{
		Do:      &config.Plan,
		Ensure:  config.Ensure,
		Failure: config.Failure,
		Success: config.Success,
	})
}

func collectInputs(plan atc.PlanConfig) []JobInput {
	var inputs []JobInput

	if plan.Success != nil {
		inputs = append(inputs, collectInputs(*plan.Success)...)
	}

	if plan.Failure != nil {
		inputs = append(inputs, collectInputs(*plan.Failure)...)
	}

	if plan.Ensure != nil {
		inputs = append(inputs, collectInputs(*plan.Ensure)...)
	}

	if plan.Try != nil {
		inputs = append(inputs, collectInputs(*plan.Try)...)
	}

	if plan.Do != nil {
		for _, p := range *plan.Do {
			inputs = append(inputs, collectInputs(p)...)
		}
	}

	if plan.Aggregate != nil {
		for _, p := range *plan.Aggregate {
			inputs = append(inputs, collectInputs(p)...)
		}
	}

	if plan.Get != "" {
		get := plan.Get

		resource := get
		if plan.Resource != "" {
			resource = plan.Resource
		}

		inputs = append(inputs, JobInput{
			Name:     get,
			Resource: resource,
			Passed:   plan.Passed,
			Version:  plan.Version,
			Trigger:  plan.Trigger,
			Params:   plan.Params,
			Tags:     plan.Tags,
		})
	}

	return inputs
}

func collectOutputs(plan atc.PlanConfig) []JobOutput {
	var outputs []JobOutput

	if plan.Success != nil {
		outputs = append(outputs, collectOutputs(*plan.Success)...)
	}

	if plan.Failure != nil {
		outputs = append(outputs, collectOutputs(*plan.Failure)...)
	}

	if plan.Ensure != nil {
		outputs = append(outputs, collectOutputs(*plan.Ensure)...)
	}

	if plan.Try != nil {
		outputs = append(outputs, collectOutputs(*plan.Try)...)
	}

	if plan.Do != nil {
		for _, p := range *plan.Do {
			outputs = append(outputs, collectOutputs(p)...)
		}
	}

	if plan.Aggregate != nil {
		for _, p := range *plan.Aggregate {
			outputs = append(outputs, collectOutputs(p)...)
		}
	}

	if plan.Put != "" {
		put := plan.Put

		resource := put
		if plan.Resource != "" {
			resource = plan.Resource
		}

		outputs = append(outputs, JobOutput{
			Name:     put,
			Resource: resource,
		})
	}

	return outputs
}
<|endoftext|>


<commit_before>package xpc

/*
#include "xpc_wrapper.h"
*/
import "C"

import (
	"encoding/hex"
	"errors"
	"fmt"
	"log"
	"reflect"
	"strings"
	"unsafe"
)

type XPC struct {
	conn C.xpc_connection_t
}

func (x *XPC) Send(msg interface{}, verbose bool) {
	// verbose == true converts the type from bool to C._Bool
	C.XpcSendMessage(x.conn, goToXpc(msg), true, verbose == true)
}

//
// minimal XPC support required for BLE
//

// a dictionary of things
type Dict map[string]interface{}

func (d Dict) Contains(k string) bool {
	_, ok := d[k]
	return ok
}

func (d Dict) MustGetDict(k string) Dict {
	return d[k].(Dict)
}

func (d Dict) MustGetArray(k string) Array {
	return d[k].(Array)
}

func (d Dict) MustGetBytes(k string) []byte {
	return d[k].([]byte)
}

func (d Dict) MustGetHexBytes(k string) string {
	return hex.EncodeToString(d[k].([]byte))
}

func (d Dict) MustGetInt(k string) int {
	return int(d[k].(int64))
}

func (d Dict) MustGetUUID(k string) UUID {
	return d[k].(UUID)
}

func (d Dict) GetString(k, defv string) string {
	if v, ok := d[k]; ok {
		//log.Printf("GetString %s %#v\n", k, v)
		return v.(string)
	}
	//log.Printf("GetString %s default %#v\n", k, defv)
	return defv
}

func (d Dict) GetBytes(k string, defv []byte) []byte {
	if v, ok := d[k]; ok {
		//log.Printf("GetBytes %s %#v\n", k, v)
		return v.([]byte)
	}
	//log.Printf("GetBytes %s default %#v\n", k, defv)
	return defv
}

func (d Dict) GetInt(k string, defv int) int {
	if v, ok := d[k]; ok {
		//log.Printf("GetString %s %#v\n", k, v)
		return int(v.(int64))
	}
	//log.Printf("GetString %s default %#v\n", k, defv)
	return defv
}

func (d Dict) GetUUID(k string) UUID {
	return GetUUID(d[k])
}

// an Array of things
type Array []interface{}

func (a Array) GetUUID(k int) UUID {
	return GetUUID(a[k])
}

// a UUID
type UUID [16]byte

func NewUUID(b []byte) (uuid UUID) {
	copy(uuid[:], b)
	return uuid
}

func MakeUUID(s string) UUID {
	s = strings.Replace(s, "-", "", -1)
	sl, _ := hex.DecodeString(s)
	return NewUUID(sl)
}

func MustUUID(s string) UUID {
	s = strings.Replace(s, "-", "", -1)
	if len(s) != 32 {
		log.Fatal("invalid UUID")
	}
	sl, err := hex.DecodeString(s)
	if err != nil {
		log.Fatalf("invalid UUID %q: %v", s, err)
	}
	return NewUUID(sl)
}

func (uuid UUID) Bytes() []byte {
	return uuid[:]
}

func (uuid UUID) String() string {
	return hex.EncodeToString(uuid[:])
}

func GetUUID(v interface{}) UUID {
	if v == nil {
		return UUID{}
	}

	if uuid, ok := v.(UUID); ok {
		return uuid
	}

	if bytes, ok := v.([]byte); ok {
		uuid := UUID{}

		for i, b := range bytes {
			uuid[i] = b
		}

		return uuid
	}

	if bytes, ok := v.([]uint8); ok {
		uuid := UUID{}

		for i, b := range bytes {
			uuid[i] = b
		}

		return uuid
	}

	log.Fatalf("invalid type for UUID: %#v", v)
	return UUID{}
}

var (
	CONNECTION_INVALID     = errors.New("connection invalid")
	CONNECTION_INTERRUPTED = errors.New("connection interrupted")
	CONNECTION_TERMINATED  = errors.New("connection terminated")

	TYPE_OF_UUID  = reflect.TypeOf(UUID{})
	TYPE_OF_BYTES = reflect.TypeOf([]byte{})

	handlers = map[uintptr]XpcEventHandler{}
)

type XpcEventHandler interface {
	HandleXpcEvent(event Dict, err error)
}

func XpcConnect(service string, eh XpcEventHandler) XPC {
	// func XpcConnect(service string, eh XpcEventHandler) C.xpc_connection_t {
	ctx := uintptr(unsafe.Pointer(&eh))
	handlers[ctx] = eh

	cservice := C.CString(service)
	defer C.free(unsafe.Pointer(cservice))
	// return C.XpcConnect(cservice, C.uintptr_t(ctx))
	return XPC{conn: C.XpcConnect(cservice, C.uintptr_t(ctx))}
}

//export handleXpcEvent
func handleXpcEvent(event C.xpc_object_t, p C.ulong) {
	//log.Printf("handleXpcEvent %#v %#v\n", event, p)

	t := C.xpc_get_type(event)

	eh := handlers[uintptr(p)]
	if eh == nil {
		//log.Println("no handler for", p)
		return
	}

	if t == C.TYPE_ERROR {
		switch event {
		case C.ERROR_CONNECTION_INVALID:
			// The client process on the other end of the connection has either
			// crashed or cancelled the connection. After receiving this error,
			// the connection is in an invalid state, and you do not need to
			// call xpc_connection_cancel(). Just tear down any associated state
			// here.
			//log.Println("connection invalid")
			eh.HandleXpcEvent(nil, CONNECTION_INVALID)
		case C.ERROR_CONNECTION_INTERRUPTED:
			//log.Println("connection interrupted")
			eh.HandleXpcEvent(nil, CONNECTION_INTERRUPTED)
		case C.ERROR_CONNECTION_TERMINATED:
			// Handle per-connection termination cleanup.
			//log.Println("connection terminated")
			eh.HandleXpcEvent(nil, CONNECTION_TERMINATED)
		default:
			//log.Println("got some error", event)
			eh.HandleXpcEvent(nil, fmt.Errorf("%v", event))
		}
	} else {
		eh.HandleXpcEvent(xpcToGo(event).(Dict), nil)
	}
}

// goToXpc converts a go object to an xpc object
func goToXpc(o interface{}) C.xpc_object_t {
	return valueToXpc(reflect.ValueOf(o))
}

// valueToXpc converts a go Value to an xpc object
//
// note that not all the types are supported, but only the subset required for Blued
func valueToXpc(val reflect.Value) C.xpc_object_t {
	if !val.IsValid() {
		return nil
	}

	var xv C.xpc_object_t

	switch val.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		xv = C.xpc_int64_create(C.int64_t(val.Int()))

	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32:
		xv = C.xpc_int64_create(C.int64_t(val.Uint()))

	case reflect.String:
		xv = C.xpc_string_create(C.CString(val.String()))

	case reflect.Map:
		xv = C.xpc_dictionary_create(nil, nil, 0)
		for _, k := range val.MapKeys() {
			v := valueToXpc(val.MapIndex(k))
			C.xpc_dictionary_set_value(xv, C.CString(k.String()), v)
			if v != nil {
				C.xpc_release(v)
			}
		}

	case reflect.Array, reflect.Slice:
		if val.Type() == TYPE_OF_UUID {
			// Array of bytes
			var uuid [16]byte
			reflect.Copy(reflect.ValueOf(uuid[:]), val)
			xv = C.xpc_uuid_create(C.ptr_to_uuid(unsafe.Pointer(&uuid[0])))
		} else if val.Type() == TYPE_OF_BYTES {
			// slice of bytes
			xv = C.xpc_data_create(unsafe.Pointer(val.Pointer()), C.size_t(val.Len()))
		} else {
			xv = C.xpc_array_create(nil, 0)
			l := val.Len()

			for i := 0; i < l; i++ {
				v := valueToXpc(val.Index(i))
				C.xpc_array_append_value(xv, v)
				if v != nil {
					C.xpc_release(v)
				}
			}
		}

	case reflect.Interface, reflect.Ptr:
		xv = valueToXpc(val.Elem())

	default:
		log.Fatalf("unsupported %#v", val.String())
	}

	return xv
}

//export arraySet
func arraySet(u C.uintptr_t, i C.int, v C.xpc_object_t) {
	a := *(*Array)(unsafe.Pointer(uintptr(u)))
	a[i] = xpcToGo(v)
}

//export dictSet
func dictSet(u C.uintptr_t, k *C.char, v C.xpc_object_t) {
	d := *(*Dict)(unsafe.Pointer(uintptr(u)))
	d[C.GoString(k)] = xpcToGo(v)
}

// xpcToGo converts an xpc object to a go object
//
// note that not all the types are supported, but only the subset required for Blued
func xpcToGo(v C.xpc_object_t) interface{} {
	t := C.xpc_get_type(v)

	switch t {
	case C.TYPE_ARRAY:
		a := make(Array, C.int(C.xpc_array_get_count(v)))
		p := uintptr(unsafe.Pointer(&a))
		C.XpcArrayApply(C.uintptr_t(p), v)
		return a

	case C.TYPE_DATA:
		return C.GoBytes(C.xpc_data_get_bytes_ptr(v), C.int(C.xpc_data_get_length(v)))

	case C.TYPE_DICT:
		d := make(Dict)
		p := uintptr(unsafe.Pointer(&d))
		C.XpcDictApply(C.uintptr_t(p), v)
		return d

	case C.TYPE_INT64:
		return int64(C.xpc_int64_get_value(v))

	case C.TYPE_STRING:
		return C.GoString(C.xpc_string_get_string_ptr(v))

	case C.TYPE_UUID:
		a := [16]byte{}
		C.XpcUUIDGetBytes(unsafe.Pointer(&a), v)
		return UUID(a)

	default:
		log.Fatalf("unexpected type %#v, value %#v", t, v)
	}

	return nil
}

// xpc_release is needed by tests, since they can't use CGO
func xpc_release(xv C.xpc_object_t) {
	C.xpc_release(xv)
}

// this is used to check the OS version

type Utsname struct {
	Sysname  string
	Nodename string
	Release  string
	Version  string
	Machine  string
}

func Uname(utsname *Utsname) error {
	var cstruct C.struct_utsname
	if err := C.uname(&cstruct); err != 0 {
		return errors.New("utsname error")
	}

	// XXX: this may crash if any value is exactly 256 characters (no 0 terminator)
	utsname.Sysname = C.GoString(&cstruct.sysname[0])
	utsname.Nodename = C.GoString(&cstruct.nodename[0])
	utsname.Release = C.GoString(&cstruct.release[0])
	utsname.Version = C.GoString(&cstruct.version[0])
	utsname.Machine = C.GoString(&cstruct.machine[0])

	return nil
}
<commit_msg>cleanup<commit_after>package xpc

/*
#include "xpc_wrapper.h"
*/
import "C"

import (
	"encoding/hex"
	"errors"
	"fmt"
	"log"
	"reflect"
	"strings"
	"unsafe"
)

type XPC struct {
	conn C.xpc_connection_t
}

func (x *XPC) Send(msg interface{}, verbose bool) {
	// verbose == true converts the type from bool to C._Bool
	C.XpcSendMessage(x.conn, goToXpc(msg), true, verbose == true)
}

//
// minimal XPC support required for BLE
//

// a dictionary of things
type Dict map[string]interface{}

func (d Dict) Contains(k string) bool {
	_, ok := d[k]
	return ok
}

func (d Dict) MustGetDict(k string) Dict       { return d[k].(Dict) }
func (d Dict) MustGetArray(k string) Array     { return d[k].(Array) }
func (d Dict) MustGetBytes(k string) []byte    { return d[k].([]byte) }
func (d Dict) MustGetHexBytes(k string) string { return hex.EncodeToString(d[k].([]byte)) }
func (d Dict) MustGetInt(k string) int         { return int(d[k].(int64)) }
func (d Dict) MustGetUUID(k string) UUID       { return d[k].(UUID) }

func (d Dict) GetString(k, defv string) string {
	if v, ok := d[k]; ok {
		//log.Printf("GetString %s %#v\n", k, v)
		return v.(string)
	}
	//log.Printf("GetString %s default %#v\n", k, defv)
	return defv
}

func (d Dict) GetBytes(k string, defv []byte) []byte {
	if v, ok := d[k]; ok {
		//log.Printf("GetBytes %s %#v\n", k, v)
		return v.([]byte)
	}
	//log.Printf("GetBytes %s default %#v\n", k, defv)
	return defv
}

func (d Dict) GetInt(k string, defv int) int {
	if v, ok := d[k]; ok {
		//log.Printf("GetString %s %#v\n", k, v)
		return int(v.(int64))
	}
	//log.Printf("GetString %s default %#v\n", k, defv)
	return defv
}

func (d Dict) GetUUID(k string) UUID {
	return GetUUID(d[k])
}

// an Array of things
type Array []interface{}

func (a Array) GetUUID(k int) UUID {
	return GetUUID(a[k])
}

// a UUID
type UUID [16]byte

func NewUUID(b []byte) (uuid UUID) {
	copy(uuid[:], b)
	return uuid
}

func MakeUUID(s string) UUID {
	s = strings.Replace(s, "-", "", -1)
	sl, _ := hex.DecodeString(s)
	return NewUUID(sl)
}

func MustUUID(s string) UUID {
	s = strings.Replace(s, "-", "", -1)
	if len(s) != 32 {
		log.Fatal("invalid UUID")
	}
	sl, err := hex.DecodeString(s)
	if err != nil {
		log.Fatalf("invalid UUID %q: %v", s, err)
	}
	return NewUUID(sl)
}

func (uuid UUID) Bytes() []byte {
	return uuid[:]
}

func (uuid UUID) String() string {
	return hex.EncodeToString(uuid[:])
}

func GetUUID(v interface{}) UUID {
	if v == nil {
		return UUID{}
	}

	if uuid, ok := v.(UUID); ok {
		return uuid
	}

	if bytes, ok := v.([]byte); ok {
		uuid := UUID{}

		for i, b := range bytes {
			uuid[i] = b
		}

		return uuid
	}

	if bytes, ok := v.([]uint8); ok {
		uuid := UUID{}

		for i, b := range bytes {
			uuid[i] = b
		}

		return uuid
	}

	log.Fatalf("invalid type for UUID: %#v", v)
	return UUID{}
}

var (
	ErrConnectionInvalid     = errors.New("connection invalid")
	ErrConnectionInterrupted = errors.New("connection interrupted")
	ErrConnectionTerminated  = errors.New("connection terminated")

	typeOfUUID  = reflect.TypeOf(UUID{})
	typeOfBytes = reflect.TypeOf([]byte{})

	handlers = map[uintptr]XpcEventHandler{}
)

type XpcEventHandler interface {
	HandleXpcEvent(event Dict, err error)
}

func XpcConnect(service string, eh XpcEventHandler) XPC {
	// func XpcConnect(service string, eh XpcEventHandler) C.xpc_connection_t {
	ctx := uintptr(unsafe.Pointer(&eh))
	handlers[ctx] = eh

	cservice := C.CString(service)
	defer C.free(unsafe.Pointer(cservice))
	// return C.XpcConnect(cservice, C.uintptr_t(ctx))
	return XPC{conn: C.XpcConnect(cservice, C.uintptr_t(ctx))}
}

//export handleXpcEvent
func handleXpcEvent(event C.xpc_object_t, p C.ulong) {
	//log.Printf("handleXpcEvent %#v %#v\n", event, p)

	t := C.xpc_get_type(event)

	eh := handlers[uintptr(p)]
	if eh == nil {
		//log.Println("no handler for", p)
		return
	}

	if t == C.TYPE_ERROR {
		switch event {
		case C.ERROR_CONNECTION_INVALID:
			// The client process on the other end of the connection has either
			// crashed or cancelled the connection. After receiving this error,
			// the connection is in an invalid state, and you do not need to
			// call xpc_connection_cancel(). Just tear down any associated state
			// here.
			//log.Println("connection invalid")
			eh.HandleXpcEvent(nil, ErrConnectionInvalid)
		case C.ERROR_CONNECTION_INTERRUPTED:
			//log.Println("connection interrupted")
			eh.HandleXpcEvent(nil, ErrConnectionInterrupted)
		case C.ERROR_CONNECTION_TERMINATED:
			// Handle per-connection termination cleanup.
			//log.Println("connection terminated")
			eh.HandleXpcEvent(nil, ErrConnectionTerminated)
		default:
			//log.Println("got some error", event)
			eh.HandleXpcEvent(nil, fmt.Errorf("%v", event))
		}
	} else {
		eh.HandleXpcEvent(xpcToGo(event).(Dict), nil)
	}
}

// goToXpc converts a go object to an xpc object
func goToXpc(o interface{}) C.xpc_object_t {
	return valueToXpc(reflect.ValueOf(o))
}

// valueToXpc converts a go Value to an xpc object
//
// note that not all the types are supported, but only the subset required for Blued
func valueToXpc(val reflect.Value) C.xpc_object_t {
	if !val.IsValid() {
		return nil
	}

	var xv C.xpc_object_t

	switch val.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		xv = C.xpc_int64_create(C.int64_t(val.Int()))

	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32:
		xv = C.xpc_int64_create(C.int64_t(val.Uint()))

	case reflect.String:
		xv = C.xpc_string_create(C.CString(val.String()))

	case reflect.Map:
		xv = C.xpc_dictionary_create(nil, nil, 0)
		for _, k := range val.MapKeys() {
			v := valueToXpc(val.MapIndex(k))
			C.xpc_dictionary_set_value(xv, C.CString(k.String()), v)
			if v != nil {
				C.xpc_release(v)
			}
		}

	case reflect.Array, reflect.Slice:
		if val.Type() == typeOfUUID {
			// Array of bytes
			var uuid [16]byte
			reflect.Copy(reflect.ValueOf(uuid[:]), val)
			xv = C.xpc_uuid_create(C.ptr_to_uuid(unsafe.Pointer(&uuid[0])))
		} else if val.Type() == typeOfBytes {
			// slice of bytes
			xv = C.xpc_data_create(unsafe.Pointer(val.Pointer()), C.size_t(val.Len()))
		} else {
			xv = C.xpc_array_create(nil, 0)
			l := val.Len()

			for i := 0; i < l; i++ {
				v := valueToXpc(val.Index(i))
				C.xpc_array_append_value(xv, v)
				if v != nil {
					C.xpc_release(v)
				}
			}
		}

	case reflect.Interface, reflect.Ptr:
		xv = valueToXpc(val.Elem())

	default:
		log.Fatalf("unsupported %#v", val.String())
	}

	return xv
}

//export arraySet
func arraySet(u C.uintptr_t, i C.int, v C.xpc_object_t) {
	a := *(*Array)(unsafe.Pointer(uintptr(u)))
	a[i] = xpcToGo(v)
}

//export dictSet
func dictSet(u C.uintptr_t, k *C.char, v C.xpc_object_t) {
	d := *(*Dict)(unsafe.Pointer(uintptr(u)))
	d[C.GoString(k)] = xpcToGo(v)
}

// xpcToGo converts an xpc object to a go object
//
// note that not all the types are supported, but only the subset required for Blued
func xpcToGo(v C.xpc_object_t) interface{} {
	t := C.xpc_get_type(v)

	switch t {
	case C.TYPE_ARRAY:
		a := make(Array, C.int(C.xpc_array_get_count(v)))
		p := uintptr(unsafe.Pointer(&a))
		C.XpcArrayApply(C.uintptr_t(p), v)
		return a

	case C.TYPE_DATA:
		return C.GoBytes(C.xpc_data_get_bytes_ptr(v), C.int(C.xpc_data_get_length(v)))

	case C.TYPE_DICT:
		d := make(Dict)
		p := uintptr(unsafe.Pointer(&d))
		C.XpcDictApply(C.uintptr_t(p), v)
		return d

	case C.TYPE_INT64:
		return int64(C.xpc_int64_get_value(v))

	case C.TYPE_STRING:
		return C.GoString(C.xpc_string_get_string_ptr(v))

	case C.TYPE_UUID:
		a := [16]byte{}
		C.XpcUUIDGetBytes(unsafe.Pointer(&a), v)
		return UUID(a)

	default:
		log.Fatalf("unexpected type %#v, value %#v", t, v)
	}

	return nil
}

// xpc_release is needed by tests, since they can't use CGO
func xpc_release(xv C.xpc_object_t) {
	C.xpc_release(xv)
}

// this is used to check the OS version

type Utsname struct {
	Sysname  string
	Nodename string
	Release  string
	Version  string
	Machine  string
}

func Uname(utsname *Utsname) error {
	var cstruct C.struct_utsname
	if err := C.uname(&cstruct); err != 0 {
		return errors.New("utsname error")
	}

	// XXX: this may crash if any value is exactly 256 characters (no 0 terminator)
	utsname.Sysname = C.GoString(&cstruct.sysname[0])
	utsname.Nodename = C.GoString(&cstruct.nodename[0])
	utsname.Release = C.GoString(&cstruct.release[0])
	utsname.Version = C.GoString(&cstruct.version[0])
	utsname.Machine = C.GoString(&cstruct.machine[0])

	return nil
}
<|endoftext|>


<commit_before>package notifierKafka

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"sync"
	"time"

	"github.com/raintank/schema"

	"github.com/Shopify/sarama"
	"github.com/grafana/metrictank/mdata"
	"github.com/grafana/metrictank/util"
	log "github.com/sirupsen/logrus"
)

type NotifierKafka struct {
	instance string
	in       chan mdata.SavedChunk
	buf      []mdata.SavedChunk
	wg       sync.WaitGroup
	bPool    *util.BufferPool
	handler  mdata.NotifierHandler
	client   sarama.Client
	consumer sarama.Consumer
	producer sarama.SyncProducer
	StopChan chan int

	// signal to PartitionConsumers to shutdown
	stopConsuming chan struct{}
}

func New(instance string, handler mdata.NotifierHandler) *NotifierKafka {
	client, err := sarama.NewClient(brokers, config)
	if err != nil {
		log.Fatalf("kafka-cluster: failed to start client: %s", err)
	}
	consumer, err := sarama.NewConsumerFromClient(client)
	if err != nil {
		log.Fatalf("kafka-cluster: failed to initialize consumer: %s", err)
	}
	log.Info("kafka-cluster: consumer initialized without error")

	producer, err := sarama.NewSyncProducerFromClient(client)
	if err != nil {
		log.Fatalf("kafka-cluster: failed to initialize producer: %s", err)
	}

	c := NotifierKafka{
		instance: instance,
		in:       make(chan mdata.SavedChunk),
		bPool:    util.NewBufferPool(),
		handler:  handler,
		client:   client,
		consumer: consumer,
		producer: producer,

		StopChan:      make(chan int),
		stopConsuming: make(chan struct{}),
	}
	c.start()
	go c.produce()

	return &c
}

func (c *NotifierKafka) start() {
	pre := time.Now()
	processBacklog := new(sync.WaitGroup)
	for _, partition := range partitions {
		type offset struct {
			offsetStart int64
			offsetTime  int64
			offsetName  string
			offsetError error
		}
		var startOffset int64
		var validOffset bool
		var offsetFromDuration int64

		offsets := make([]offset, 0, 3)

		if offsetStr != "oldest" && offsetStr != "newest" {
			offsetFromDuration = time.Now().Add(-1*offsetDuration).UnixNano() / int64(time.Millisecond)
		}

		offsets[0].offsetTime = sarama.OffsetOldest
		offsets[0].offsetName = "oldest"
		offsets[1].offsetTime = sarama.OffsetNewest
		offsets[1].offsetName = "newest"
		offsets[2].offsetTime = offsetFromDuration
		offsets[2].offsetName = "custom"

		// get all of the offsets
		for i := 0; i < 2; i++ {
			tmpOffset, err := c.client.GetOffset(topic, partition, offsets[i].offsetTime)
			offsets[i].offsetStart = tmpOffset
			offsets[i].offsetError = err
		}

		switch offsetStr {
		case "oldest":
			if offsets[0].offsetError == nil {
				validOffset = true
				startOffset = offsets[0].offsetStart
			}
		case "newest":
			if offsets[1].offsetError == nil {
				validOffset = true
				startOffset = offsets[1].offsetStart
			}
		default:
			if offsets[2].offsetError == nil {
				validOffset = true
				startOffset = offsets[2].offsetStart
			}
		}

		// try to find a valid offset with priority for custom time, oldest, and then newest
		if !validOffset {
			log.Warnf("kafka-cluster: failed to find a valid offset for topic: %s using: %s -> attempting to find a valid offset\n", topic, offsetStr)
			for i := 2; i >= 0; i-- {
				if offsets[i].offsetError == nil {
					validOffset = true
					startOffset = offsets[i].offsetStart
					log.Warnf("kafka-cluster: using fallback offset for topic: %s fallback: %s offset: %d\n", topic, offsets[i].offsetName, offsets[i].offsetStart)
					break
				}
			}
		}

		// in case we did not originally have a valid offset, we need to re-check here
		if validOffset {
			processBacklog.Add(1)
			go c.consumePartition(topic, partition, startOffset, processBacklog)
			// move on to the next partition
			continue
		}

		// no valid offsets found
		log.Fatalf("kafka-cluster: tried all fallbacks, could not find a valid offset for topic: %s using %s\n", topic, offsetStr)
	}
	// wait for our backlog to be processed before returning.  This will block metrictank from consuming metrics until
	// we have processed old metricPersist messages. The end result is that we wont overwrite chunks in cassandra that
	// have already been previously written.
	// We don't wait more than backlogProcessTimeout for the backlog to be processed.
	log.Info("kafka-cluster: waiting for metricPersist backlog to be processed.")
	backlogProcessed := make(chan struct{}, 1)
	go func() {
		processBacklog.Wait()
		backlogProcessed <- struct{}{}
	}()

	select {
	case <-time.After(backlogProcessTimeout):
		log.Warnf("kafka-cluster: Processing metricPersist backlog has taken too long, giving up lock after %s.", backlogProcessTimeout)
	case <-backlogProcessed:
		log.Infof("kafka-cluster: metricPersist backlog processed in %s.", time.Since(pre))
	}

}

func (c *NotifierKafka) updateProcessBacklog(lastReadOffset int64, lastAvailableOffsetAtStartup int64, processBacklog *sync.WaitGroup) bool {
	if lastReadOffset >= lastAvailableOffsetAtStartup {
		processBacklog.Done()
		return true
	}
	return false
}

func (c *NotifierKafka) getLastAvailableOffset(topic string, partition int32) (lastAvailableOffset int64, err error) {
	nextOffset, err := c.client.GetOffset(topic, partition, sarama.OffsetNewest)

	if err != nil {
		log.Errorf("kafka-cluster failed to get offset of last available message in partition %s:%d. %s", topic, partition, err)
		lastAvailableOffset = -1
	} else {
		// nextOffset is the offset of the message that will be produced next. There is no
		// message with that offset that we can consume yet
		lastAvailableOffset = nextOffset - 1
	}

	return
}

func (c *NotifierKafka) updateMetrics(topic string, partition int32, lastReadOffset int64) {
	lastAvailableOffset, err := c.getLastAvailableOffset(topic, partition)
	if err == nil {
		partitionLogSize[partition].Set(int(lastAvailableOffset + 1))
		partitionLag[partition].Set(int(lastAvailableOffset - lastReadOffset))
	}
	partitionOffset[partition].Set(int(lastReadOffset))
}

func (c *NotifierKafka) consumePartition(topic string, partition int32, startOffset int64, processBacklog *sync.WaitGroup) {
	c.wg.Add(1)
	defer c.wg.Done()

	pc, err := c.consumer.ConsumePartition(topic, partition, startOffset)
	if err != nil {
		log.Fatalf("kafka-cluster: failed to start partitionConsumer for %s:%d. %s", topic, partition, err)
	}
	log.Infof("kafka-cluster: consuming from %s:%d from offset %d", topic, partition, startOffset)

	messages := pc.Messages()
	ticker := time.NewTicker(5 * time.Second)

	lastReadOffset := startOffset - 1
	lastAvailableOffsetAtStartup, err := c.getLastAvailableOffset(topic, partition)
	if err != nil {
		log.Fatalf("kafka-cluster: failed to get newest offset for topic %s part %d: %s", topic, partition, err)
	}
	backlogProcessed := c.updateProcessBacklog(lastReadOffset, lastAvailableOffsetAtStartup, processBacklog)
	c.updateMetrics(topic, partition, lastReadOffset)

	for {
		select {
		case msg := <-messages:
			log.Debugf("kafka-cluster: received message: Topic %s, Partition: %d, Offset: %d, Key: %x", msg.Topic, msg.Partition, msg.Offset, msg.Key)
			c.handler.Handle(msg.Value)
			lastReadOffset = msg.Offset
		case <-ticker.C:
			if !backlogProcessed {
				backlogProcessed = c.updateProcessBacklog(lastReadOffset, lastAvailableOffsetAtStartup, processBacklog)
			}
			c.updateMetrics(topic, partition, lastReadOffset)
		case <-c.stopConsuming:
			pc.Close()
			log.Infof("kafka-cluster: consumer for %s:%d ended.", topic, partition)
			return
		}
	}
}

// Stop will initiate a graceful stop of the Consumer (permanent)
//
// NOTE: receive on StopChan to block until this process completes
func (c *NotifierKafka) Stop() {
	// closes notifications and messages channels, amongst others
	close(c.stopConsuming)
	c.producer.Close()

	go func() {
		c.wg.Wait()
		close(c.StopChan)
	}()
}

func (c *NotifierKafka) Send(sc mdata.SavedChunk) {
	c.in <- sc
}

func (c *NotifierKafka) produce() {
	ticker := time.NewTicker(time.Second)
	max := 5000
	for {
		select {
		case chunk := <-c.in:
			c.buf = append(c.buf, chunk)
			if len(c.buf) == max {
				c.flush()
			}
		case <-ticker.C:
			c.flush()
		}
	}
}

// flush makes sure the batch gets sent, asynchronously.
func (c *NotifierKafka) flush() {
	if len(c.buf) == 0 {
		return
	}

	// In order to correctly route the saveMessages to the correct partition,
	// we can't send them in batches anymore.
	payload := make([]*sarama.ProducerMessage, 0, len(c.buf))
	var pMsg mdata.PersistMessageBatch
	for i, msg := range c.buf {
		amkey, err := schema.AMKeyFromString(msg.Key)
		if err != nil {
			log.Errorf("kafka-cluster: failed to parse key %q", msg.Key)
			continue
		}

		partition, ok := c.handler.PartitionOf(amkey.MKey)
		if !ok {
			log.Errorf("kafka-cluster: failed to lookup metricDef with id %s", msg.Key)
			continue
		}
		buf := bytes.NewBuffer(c.bPool.Get())
		binary.Write(buf, binary.LittleEndian, uint8(mdata.PersistMessageBatchV1))
		encoder := json.NewEncoder(buf)
		pMsg = mdata.PersistMessageBatch{Instance: c.instance, SavedChunks: c.buf[i : i+1]}
		err = encoder.Encode(&pMsg)
		if err != nil {
			log.Fatalf("kafka-cluster: failed to marshal persistMessage to json.")
		}
		messagesSize.Value(buf.Len())
		kafkaMsg := &sarama.ProducerMessage{
			Topic:     topic,
			Value:     sarama.ByteEncoder(buf.Bytes()),
			Partition: partition,
		}
		payload = append(payload, kafkaMsg)
	}

	c.buf = nil

	go func() {
		log.Debugf("kafka-cluster: sending %d batch metricPersist messages", len(payload))
		sent := false
		for !sent {
			err := c.producer.SendMessages(payload)
			if err != nil {
				log.Warnf("kafka-cluster: publisher %s", err)
			} else {
				sent = true
			}
			time.Sleep(time.Second)
		}
		messagesPublished.Add(len(payload))
		// put our buffers back in the bufferPool
		for _, msg := range payload {
			c.bPool.Put([]byte(msg.Value.(sarama.ByteEncoder)))
		}
	}()
}
<commit_msg>Replace slices with maps Add block comment to describe scenarios Add more error messages<commit_after>package notifierKafka

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"sync"
	"time"

	"github.com/raintank/schema"

	"github.com/Shopify/sarama"
	"github.com/grafana/metrictank/mdata"
	"github.com/grafana/metrictank/util"
	log "github.com/sirupsen/logrus"
)

type NotifierKafka struct {
	instance string
	in       chan mdata.SavedChunk
	buf      []mdata.SavedChunk
	wg       sync.WaitGroup
	bPool    *util.BufferPool
	handler  mdata.NotifierHandler
	client   sarama.Client
	consumer sarama.Consumer
	producer sarama.SyncProducer
	StopChan chan int

	// signal to PartitionConsumers to shutdown
	stopConsuming chan struct{}
}

func New(instance string, handler mdata.NotifierHandler) *NotifierKafka {
	client, err := sarama.NewClient(brokers, config)
	if err != nil {
		log.Fatalf("kafka-cluster: failed to start client: %s", err)
	}
	consumer, err := sarama.NewConsumerFromClient(client)
	if err != nil {
		log.Fatalf("kafka-cluster: failed to initialize consumer: %s", err)
	}
	log.Info("kafka-cluster: consumer initialized without error")

	producer, err := sarama.NewSyncProducerFromClient(client)
	if err != nil {
		log.Fatalf("kafka-cluster: failed to initialize producer: %s", err)
	}

	c := NotifierKafka{
		instance: instance,
		in:       make(chan mdata.SavedChunk),
		bPool:    util.NewBufferPool(),
		handler:  handler,
		client:   client,
		consumer: consumer,
		producer: producer,

		StopChan:      make(chan int),
		stopConsuming: make(chan struct{}),
	}
	c.start()
	go c.produce()

	return &c
}

func (c *NotifierKafka) start() {
	// offset is defined here because it is not used anywhere else
	type offset struct {
		// the actual kafka offset to use
		offsetStart int64
		// the time we use to attempt to get an offset (i.e. sarama.OffsetOldest)
		offsetTime int64
		// any errors returned from attempting to get an offset
		offsetError error
	}

	pre := time.Now()
	processBacklog := new(sync.WaitGroup)

	// | scenario					| offsetOldest  | offsetNewest | offsetTime
	// -------------------------------------------------------------------------------------------------------------------------------
	// | new empty partition		| error	   		| 0			   | error
	// | new with messages			| 0		   		| validOffset  | validOffset or error if offsetTime is earlier than first message
	// | existing with messages		| validOffset	| validOffset  | validOffset
	// | existing with no messages	| error			| validOffset  | error

	for _, partition := range partitions {
		var startOffset int64
		var validOffset bool
		var offsetFromDuration int64

		// make sure we are using a custom time offset, otherwise leave
		// offsetFromDuration at 0
		if offsetStr != "oldest" && offsetStr != "newest" {
			offsetFromDuration = time.Now().Add(-1*offsetDuration).UnixNano() / int64(time.Millisecond)
		}

		offsets := map[string]*offset{
			"oldest": {offsetTime: sarama.OffsetOldest},
			"newest": {offsetTime: sarama.OffsetNewest},
			"custom": {offsetTime: offsetFromDuration},
		}

		// get all of the offsets
		for _, name := range []string{"newest", "custom", "oldest"} {
			tmpOffset, err := c.client.GetOffset(topic, partition, offsets[name].offsetTime)
			offsets[name].offsetStart = tmpOffset
			offsets[name].offsetError = err
		}

		switch offsetStr {
		case "oldest":
			if offsets[offsetStr].offsetError == nil {
				validOffset = true
				startOffset = offsets[offsetStr].offsetStart
				break
			}
			log.Errorf("kafka-cluster: failed to find a valid offset for topic: %s with offset selector: %s for partition: %d -- %s\n",
				topic, offsetStr, partition, offsets[offsetStr].offsetError)
		case "newest":
			if offsets[offsetStr].offsetError == nil {
				validOffset = true
				startOffset = offsets[offsetStr].offsetStart
				break
			}
			log.Errorf("kafka-cluster: failed to find a valid offset for topic: %s with offset selector: %s for partition: %d -- %s\n",
				topic, offsetStr, partition, offsets[offsetStr].offsetError)
		default:
			if offsets["custom"].offsetError == nil {
				validOffset = true
				startOffset = offsets["custom"].offsetStart
				break
			}
			log.Errorf("kafka-cluster: failed to find a valid offset for topic: %s with offset selector: %s for partition: %d -- %s\n",
				topic, offsetStr, partition, offsets["custom"].offsetError)
		}

		// try to find a valid offset with priority for custom time, oldest, and then newest
		if !validOffset {
			for _, name := range []string{"custom", "oldest", "newest"} {
				if offsets[name].offsetError == nil {
					validOffset = true
					startOffset = offsets[name].offsetStart
					log.Warnf("kafka-cluster: using fallback offset for topic: %s fallback: %s offset: %d partition: %d\n",
						topic, name, startOffset, partition)
					break
				}
				log.Warnf("kafka-cluster: fallback offset %s is invalid: %s\n", name, offsets[name].offsetError)
			}
		}

		// re-check to see if we have a valid offset now
		if !validOffset {
			log.Fatalf("kafka-cluster: tried all fallbacks, could not find a valid offset for topic: %s with offset selector: %s for partition: %d\n",
				topic, offsetStr, partition)
		}

		processBacklog.Add(1)
		go c.consumePartition(topic, partition, startOffset, processBacklog)
	}
	// wait for our backlog to be processed before returning.  This will block metrictank from consuming metrics until
	// we have processed old metricPersist messages. The end result is that we wont overwrite chunks in cassandra that
	// have already been previously written.
	// We don't wait more than backlogProcessTimeout for the backlog to be processed.
	log.Info("kafka-cluster: waiting for metricPersist backlog to be processed.")
	backlogProcessed := make(chan struct{}, 1)
	go func() {
		processBacklog.Wait()
		backlogProcessed <- struct{}{}
	}()

	select {
	case <-time.After(backlogProcessTimeout):
		log.Warnf("kafka-cluster: Processing metricPersist backlog has taken too long, giving up lock after %s.", backlogProcessTimeout)
	case <-backlogProcessed:
		log.Infof("kafka-cluster: metricPersist backlog processed in %s.", time.Since(pre))
	}

}

func (c *NotifierKafka) updateProcessBacklog(lastReadOffset int64, lastAvailableOffsetAtStartup int64, processBacklog *sync.WaitGroup) bool {
	if lastReadOffset >= lastAvailableOffsetAtStartup {
		processBacklog.Done()
		return true
	}
	return false
}

func (c *NotifierKafka) getLastAvailableOffset(topic string, partition int32) (lastAvailableOffset int64, err error) {
	nextOffset, err := c.client.GetOffset(topic, partition, sarama.OffsetNewest)

	if err != nil {
		log.Errorf("kafka-cluster failed to get offset of last available message in partition %s:%d. %s", topic, partition, err)
		lastAvailableOffset = -1
	} else {
		// nextOffset is the offset of the message that will be produced next. There is no
		// message with that offset that we can consume yet
		lastAvailableOffset = nextOffset - 1
	}

	return
}

func (c *NotifierKafka) updateMetrics(topic string, partition int32, lastReadOffset int64) {
	lastAvailableOffset, err := c.getLastAvailableOffset(topic, partition)
	if err == nil {
		partitionLogSize[partition].Set(int(lastAvailableOffset + 1))
		partitionLag[partition].Set(int(lastAvailableOffset - lastReadOffset))
	}
	partitionOffset[partition].Set(int(lastReadOffset))
}

func (c *NotifierKafka) consumePartition(topic string, partition int32, startOffset int64, processBacklog *sync.WaitGroup) {
	c.wg.Add(1)
	defer c.wg.Done()

	pc, err := c.consumer.ConsumePartition(topic, partition, startOffset)
	if err != nil {
		log.Fatalf("kafka-cluster: failed to start partitionConsumer for %s:%d. %s", topic, partition, err)
	}
	log.Infof("kafka-cluster: consuming from %s:%d from offset %d", topic, partition, startOffset)

	messages := pc.Messages()
	ticker := time.NewTicker(5 * time.Second)

	lastReadOffset := startOffset - 1
	lastAvailableOffsetAtStartup, err := c.getLastAvailableOffset(topic, partition)
	if err != nil {
		log.Fatalf("kafka-cluster: failed to get newest offset for topic %s part %d: %s", topic, partition, err)
	}
	backlogProcessed := c.updateProcessBacklog(lastReadOffset, lastAvailableOffsetAtStartup, processBacklog)
	c.updateMetrics(topic, partition, lastReadOffset)

	for {
		select {
		case msg := <-messages:
			log.Debugf("kafka-cluster: received message: Topic %s, Partition: %d, Offset: %d, Key: %x", msg.Topic, msg.Partition, msg.Offset, msg.Key)
			c.handler.Handle(msg.Value)
			lastReadOffset = msg.Offset
		case <-ticker.C:
			if !backlogProcessed {
				backlogProcessed = c.updateProcessBacklog(lastReadOffset, lastAvailableOffsetAtStartup, processBacklog)
			}
			c.updateMetrics(topic, partition, lastReadOffset)
		case <-c.stopConsuming:
			pc.Close()
			log.Infof("kafka-cluster: consumer for %s:%d ended.", topic, partition)
			return
		}
	}
}

// Stop will initiate a graceful stop of the Consumer (permanent)
//
// NOTE: receive on StopChan to block until this process completes
func (c *NotifierKafka) Stop() {
	// closes notifications and messages channels, amongst others
	close(c.stopConsuming)
	c.producer.Close()

	go func() {
		c.wg.Wait()
		close(c.StopChan)
	}()
}

func (c *NotifierKafka) Send(sc mdata.SavedChunk) {
	c.in <- sc
}

func (c *NotifierKafka) produce() {
	ticker := time.NewTicker(time.Second)
	max := 5000
	for {
		select {
		case chunk := <-c.in:
			c.buf = append(c.buf, chunk)
			if len(c.buf) == max {
				c.flush()
			}
		case <-ticker.C:
			c.flush()
		}
	}
}

// flush makes sure the batch gets sent, asynchronously.
func (c *NotifierKafka) flush() {
	if len(c.buf) == 0 {
		return
	}

	// In order to correctly route the saveMessages to the correct partition,
	// we can't send them in batches anymore.
	payload := make([]*sarama.ProducerMessage, 0, len(c.buf))
	var pMsg mdata.PersistMessageBatch
	for i, msg := range c.buf {
		amkey, err := schema.AMKeyFromString(msg.Key)
		if err != nil {
			log.Errorf("kafka-cluster: failed to parse key %q", msg.Key)
			continue
		}

		partition, ok := c.handler.PartitionOf(amkey.MKey)
		if !ok {
			log.Errorf("kafka-cluster: failed to lookup metricDef with id %s", msg.Key)
			continue
		}
		buf := bytes.NewBuffer(c.bPool.Get())
		binary.Write(buf, binary.LittleEndian, uint8(mdata.PersistMessageBatchV1))
		encoder := json.NewEncoder(buf)
		pMsg = mdata.PersistMessageBatch{Instance: c.instance, SavedChunks: c.buf[i : i+1]}
		err = encoder.Encode(&pMsg)
		if err != nil {
			log.Fatalf("kafka-cluster: failed to marshal persistMessage to json.")
		}
		messagesSize.Value(buf.Len())
		kafkaMsg := &sarama.ProducerMessage{
			Topic:     topic,
			Value:     sarama.ByteEncoder(buf.Bytes()),
			Partition: partition,
		}
		payload = append(payload, kafkaMsg)
	}

	c.buf = nil

	go func() {
		log.Debugf("kafka-cluster: sending %d batch metricPersist messages", len(payload))
		sent := false
		for !sent {
			err := c.producer.SendMessages(payload)
			if err != nil {
				log.Warnf("kafka-cluster: publisher %s", err)
			} else {
				sent = true
			}
			time.Sleep(time.Second)
		}
		messagesPublished.Add(len(payload))
		// put our buffers back in the bufferPool
		for _, msg := range payload {
			c.bPool.Put([]byte(msg.Value.(sarama.ByteEncoder)))
		}
	}()
}
<|endoftext|>


<commit_before>package image

import "testing"

func TestGetCompressionMethod(t *testing.T) {

	assertCompressionMethod := func(code uint8, method int, direction int, trans int, palette uint8) {

		stripeType, err := getCompressionMethod(0, code)
		if err != nil {
			t.Errorf("Method returns error %v for %x", err.Error(), code)
		}
		if stripeType.method != method {
			t.Errorf("Method doesn't match for code %x", code)
		}
		if stripeType.direction != direction {
			t.Errorf("Direction doesn't match for code %x", code)
		}
		if stripeType.transparent != trans {
			t.Errorf("Transparency doesn't match for code %x", code)
		}
		if stripeType.paletteLength != palette {
			t.Errorf("Palette size %x doesn't match for code %x", code)
		}
	}

	assertCompressionMethod(0x01, MethodUncompressed, Horizontal, NoTransp, 255)
	assertCompressionMethod(0x0e, MethodOne, Vertical, NoTransp, 4)
	assertCompressionMethod(0x12, MethodOne, Vertical, NoTransp, 8)

	assertCompressionMethod(0x22, MethodOne, Vertical, Transp, 4)
	assertCompressionMethod(0x26, MethodOne, Vertical, Transp, 8)

	assertCompressionMethod(0x2c, MethodOne, Horizontal, Transp, 4)
	assertCompressionMethod(0x30, MethodOne, Horizontal, Transp, 8)

	assertCompressionMethod(0x40, MethodTwo, Horizontal, NoTransp, 4)
	assertCompressionMethod(0x44, MethodTwo, Horizontal, NoTransp, 8)

	assertCompressionMethod(0x54, MethodTwo, Horizontal, Transp, 4)
	assertCompressionMethod(0x58, MethodTwo, Horizontal, Transp, 8)

	assertCompressionMethod(0x68, MethodTwo, Horizontal, Transp, 4)
	assertCompressionMethod(0x6c, MethodTwo, Horizontal, Transp, 8)

	assertCompressionMethod(0x7c, MethodTwo, Horizontal, NoTransp, 4)
	assertCompressionMethod(0x80, MethodTwo, Horizontal, NoTransp, 8)
}
<commit_msg>Add todo test<commit_after>package image

import "testing"

/*
// TODO
func TestPalette(t *testing.T) {
	t.Errorf("Not implemented")
}
*/

func TestGetCompressionMethod(t *testing.T) {

	assertCompressionMethod := func(code uint8, method int, direction int, trans int, palette uint8) {

		stripeType, err := getCompressionMethod(0, code)
		if err != nil {
			t.Errorf("Method returns error %v for %x", err.Error(), code)
		}
		if stripeType.method != method {
			t.Errorf("Method doesn't match for code %x", code)
		}
		if stripeType.direction != direction {
			t.Errorf("Direction doesn't match for code %x", code)
		}
		if stripeType.transparent != trans {
			t.Errorf("Transparency doesn't match for code %x", code)
		}
		if stripeType.paletteLength != palette {
			t.Errorf("Palette size %x doesn't match for code %x", code)
		}
	}

	assertCompressionMethod(0x01, MethodUncompressed, Horizontal, NoTransp, 255)
	assertCompressionMethod(0x0e, MethodOne, Vertical, NoTransp, 4)
	assertCompressionMethod(0x12, MethodOne, Vertical, NoTransp, 8)

	assertCompressionMethod(0x22, MethodOne, Vertical, Transp, 4)
	assertCompressionMethod(0x26, MethodOne, Vertical, Transp, 8)

	assertCompressionMethod(0x2c, MethodOne, Horizontal, Transp, 4)
	assertCompressionMethod(0x30, MethodOne, Horizontal, Transp, 8)

	assertCompressionMethod(0x40, MethodTwo, Horizontal, NoTransp, 4)
	assertCompressionMethod(0x44, MethodTwo, Horizontal, NoTransp, 8)

	assertCompressionMethod(0x54, MethodTwo, Horizontal, Transp, 4)
	assertCompressionMethod(0x58, MethodTwo, Horizontal, Transp, 8)

	assertCompressionMethod(0x68, MethodTwo, Horizontal, Transp, 4)
	assertCompressionMethod(0x6c, MethodTwo, Horizontal, Transp, 8)

	assertCompressionMethod(0x7c, MethodTwo, Horizontal, NoTransp, 4)
	assertCompressionMethod(0x80, MethodTwo, Horizontal, NoTransp, 8)
}
<|endoftext|>


<commit_before>package host

import (
	"crypto/tls"
	"net/http"
	"net/http/httputil"
	"net/url"
	"path"
	"time"

	"github.com/kataras/iris/v12/core/netutil"
)

// ProxyHandler returns a new ReverseProxy that rewrites
// URLs to the scheme, host, and base path provided in target. If the
// target's path is "/base" and the incoming request was for "/dir",
// the target request will be for /base/dir.
//
// Relative to httputil.NewSingleHostReverseProxy with some additions.
//
// Look `ProxyHandlerRemote` too.
func ProxyHandler(target *url.URL) *httputil.ReverseProxy {
	director := func(req *http.Request) {
		modifyProxiedRequest(req, target)
		req.URL.Path = path.Join(target.Path, req.URL.Path)
	}

	p := &httputil.ReverseProxy{Director: director}

	if netutil.IsLoopbackHost(target.Host) {
		transport := &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // lint:ignore
		}
		p.Transport = transport
	}

	return p
}

func modifyProxiedRequest(req *http.Request, target *url.URL) {
	req.URL.Scheme = target.Scheme
	req.URL.Host = target.Host
	req.Host = target.Host

	if target.RawQuery == "" || req.URL.RawQuery == "" {
		req.URL.RawQuery = target.RawQuery + req.URL.RawQuery
	} else {
		req.URL.RawQuery = target.RawQuery + "&" + req.URL.RawQuery
	}

	if _, ok := req.Header["User-Agent"]; !ok {
		// explicitly disable User-Agent so it's not set to default value
		req.Header.Set("User-Agent", "")
	}
}

// ProxyHandlerRemote returns a new ReverseProxy that rewrites
// URLs to the scheme, host, and path provided in target.
// Case 1: req.Host == target.Host
// behavior same as ProxyHandler
// Case 2: req.Host != target.Host
// the target request will be forwarded to the target's url
// insecureSkipVerify indicates enable ssl certificate verification or not.
//
// Look `ProxyHandler` too.
func ProxyHandlerRemote(target *url.URL, insecureSkipVerify bool) *httputil.ReverseProxy {
	director := func(req *http.Request) {
		modifyProxiedRequest(req, target)

		if req.Host != target.Host {
			req.URL.Path = target.Path
		} else {
			req.URL.Path = path.Join(target.Path, req.URL.Path)
		}
	}
	p := &httputil.ReverseProxy{Director: director}

	if netutil.IsLoopbackHost(target.Host) {
		insecureSkipVerify = true
	}

	transport := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: insecureSkipVerify}, // lint:ignore
	}
	p.Transport = transport
	return p
}

// NewProxy returns a new host (server supervisor) which
// proxies all requests to the target.
// It uses the httputil.NewSingleHostReverseProxy.
//
// Usage:
// target, _ := url.Parse("https://mydomain.com")
// proxy := NewProxy("mydomain.com:80", target)
// proxy.ListenAndServe() // use of `proxy.Shutdown` to close the proxy server.
func NewProxy(hostAddr string, target *url.URL) *Supervisor {
	proxyHandler := ProxyHandler(target)
	proxy := New(&http.Server{
		Addr:    hostAddr,
		Handler: proxyHandler,
	})

	return proxy
}

// NewProxyRemote returns a new host (server supervisor) which
// proxies all requests to the target.
// It uses the httputil.NewSingleHostReverseProxy.
//
// Usage:
// target, _ := url.Parse("https://anotherdomain.com/abc")
// proxy := NewProxyRemote("mydomain.com", target, false)
// proxy.ListenAndServe() // use of `proxy.Shutdown` to close the proxy server.
func NewProxyRemote(hostAddr string, target *url.URL, insecureSkipVerify bool) *Supervisor {
	proxyHandler := ProxyHandlerRemote(target, insecureSkipVerify)
	proxy := New(&http.Server{
		Addr:    hostAddr,
		Handler: proxyHandler,
	})

	return proxy
}

// NewRedirection returns a new host (server supervisor) which
// redirects all requests to the target.
// Usage:
// target, _ := url.Parse("https://mydomain.com")
// r := NewRedirection(":80", target, 307)
// r.ListenAndServe() // use of `r.Shutdown` to close this server.
func NewRedirection(hostAddr string, target *url.URL, redirectStatus int) *Supervisor {
	redirectSrv := &http.Server{
		ReadTimeout:  30 * time.Second,
		WriteTimeout: 60 * time.Second,
		Addr:         hostAddr,
		Handler:      RedirectHandler(target, redirectStatus),
	}

	return New(redirectSrv)
}

// RedirectHandler returns a simple redirect handler.
// See `NewProxy` or `ProxyHandler` for more features.
func RedirectHandler(target *url.URL, redirectStatus int) http.Handler {
	targetURI := target.String()
	if redirectStatus <= 300 {
		// here we should use StatusPermanentRedirect but
		// that may result on unexpected behavior
		// for end-developers who might change their minds
		// after a while, so keep status temporary.
		// Note thatwe could also use StatusFound
		// as we do on the `Context#Redirect`.
		// It will also help us to prevent any post data issues.
		redirectStatus = http.StatusTemporaryRedirect
	}

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		redirectTo := path.Join(targetURI, r.URL.Path)
		if len(r.URL.RawQuery) > 0 {
			redirectTo += "?" + r.URL.RawQuery
		}
		http.Redirect(w, r, redirectTo, redirectStatus)
	})
}
<commit_msg>udpate modifyProxiedRequest<commit_after>package host

import (
	"crypto/tls"
	"net/http"
	"net/http/httputil"
	"net/url"
	"path"
	"time"

	"github.com/kataras/iris/v12/core/netutil"
)

// ProxyHandler returns a new ReverseProxy that rewrites
// URLs to the scheme, host, and base path provided in target. If the
// target's path is "/base" and the incoming request was for "/dir",
// the target request will be for /base/dir.
//
// Relative to httputil.NewSingleHostReverseProxy with some additions.
//
// Look `ProxyHandlerRemote` too.
func ProxyHandler(target *url.URL) *httputil.ReverseProxy {
	director := func(req *http.Request) {
		modifyProxiedRequest(req, target)
		req.Host = target.Host
		req.URL.Path = path.Join(target.Path, req.URL.Path)
	}

	p := &httputil.ReverseProxy{Director: director}

	if netutil.IsLoopbackHost(target.Host) {
		transport := &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // lint:ignore
		}
		p.Transport = transport
	}

	return p
}

func modifyProxiedRequest(req *http.Request, target *url.URL) {
	req.URL.Scheme = target.Scheme
	req.URL.Host = target.Host

	if target.RawQuery == "" || req.URL.RawQuery == "" {
		req.URL.RawQuery = target.RawQuery + req.URL.RawQuery
	} else {
		req.URL.RawQuery = target.RawQuery + "&" + req.URL.RawQuery
	}

	if _, ok := req.Header["User-Agent"]; !ok {
		// explicitly disable User-Agent so it's not set to default value
		req.Header.Set("User-Agent", "")
	}
}

// ProxyHandlerRemote returns a new ReverseProxy that rewrites
// URLs to the scheme, host, and path provided in target.
// Case 1: req.Host == target.Host
// behavior same as ProxyHandler
// Case 2: req.Host != target.Host
// the target request will be forwarded to the target's url
// insecureSkipVerify indicates enable ssl certificate verification or not.
//
// Look `ProxyHandler` too.
func ProxyHandlerRemote(target *url.URL, insecureSkipVerify bool) *httputil.ReverseProxy {
	director := func(req *http.Request) {
		modifyProxiedRequest(req, target)

		if req.Host != target.Host {
			req.URL.Path = target.Path
		} else {
			req.URL.Path = path.Join(target.Path, req.URL.Path)
		}

		req.Host = target.Host
	}
	p := &httputil.ReverseProxy{Director: director}

	if netutil.IsLoopbackHost(target.Host) {
		insecureSkipVerify = true
	}

	transport := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: insecureSkipVerify}, // lint:ignore
	}
	p.Transport = transport
	return p
}

// NewProxy returns a new host (server supervisor) which
// proxies all requests to the target.
// It uses the httputil.NewSingleHostReverseProxy.
//
// Usage:
// target, _ := url.Parse("https://mydomain.com")
// proxy := NewProxy("mydomain.com:80", target)
// proxy.ListenAndServe() // use of `proxy.Shutdown` to close the proxy server.
func NewProxy(hostAddr string, target *url.URL) *Supervisor {
	proxyHandler := ProxyHandler(target)
	proxy := New(&http.Server{
		Addr:    hostAddr,
		Handler: proxyHandler,
	})

	return proxy
}

// NewProxyRemote returns a new host (server supervisor) which
// proxies all requests to the target.
// It uses the httputil.NewSingleHostReverseProxy.
//
// Usage:
// target, _ := url.Parse("https://anotherdomain.com/abc")
// proxy := NewProxyRemote("mydomain.com", target, false)
// proxy.ListenAndServe() // use of `proxy.Shutdown` to close the proxy server.
func NewProxyRemote(hostAddr string, target *url.URL, insecureSkipVerify bool) *Supervisor {
	proxyHandler := ProxyHandlerRemote(target, insecureSkipVerify)
	proxy := New(&http.Server{
		Addr:    hostAddr,
		Handler: proxyHandler,
	})

	return proxy
}

// NewRedirection returns a new host (server supervisor) which
// redirects all requests to the target.
// Usage:
// target, _ := url.Parse("https://mydomain.com")
// r := NewRedirection(":80", target, 307)
// r.ListenAndServe() // use of `r.Shutdown` to close this server.
func NewRedirection(hostAddr string, target *url.URL, redirectStatus int) *Supervisor {
	redirectSrv := &http.Server{
		ReadTimeout:  30 * time.Second,
		WriteTimeout: 60 * time.Second,
		Addr:         hostAddr,
		Handler:      RedirectHandler(target, redirectStatus),
	}

	return New(redirectSrv)
}

// RedirectHandler returns a simple redirect handler.
// See `NewProxy` or `ProxyHandler` for more features.
func RedirectHandler(target *url.URL, redirectStatus int) http.Handler {
	targetURI := target.String()
	if redirectStatus <= 300 {
		// here we should use StatusPermanentRedirect but
		// that may result on unexpected behavior
		// for end-developers who might change their minds
		// after a while, so keep status temporary.
		// Note thatwe could also use StatusFound
		// as we do on the `Context#Redirect`.
		// It will also help us to prevent any post data issues.
		redirectStatus = http.StatusTemporaryRedirect
	}

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		redirectTo := path.Join(targetURI, r.URL.Path)
		if len(r.URL.RawQuery) > 0 {
			redirectTo += "?" + r.URL.RawQuery
		}
		http.Redirect(w, r, redirectTo, redirectStatus)
	})
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package filepath

import (
	"os"
	"sort"
	"strings"
	"utf8"
)

var ErrBadPattern = os.NewError("syntax error in pattern")

// Match returns true if name matches the shell file name pattern.
// The pattern syntax is:
//
//	pattern:
//		{ term }
//	term:
//		'*'         matches any sequence of non-Separator characters
//		'?'         matches any single non-Separator character
//		'[' [ '^' ] { character-range } ']'
//		            character class (must be non-empty)
//		c           matches character c (c != '*', '?', '\\', '[')
//		'\\' c      matches character c
//
//	character-range:
//		c           matches character c (c != '\\', '-', ']')
//		'\\' c      matches character c
//		lo '-' hi   matches character c for lo <= c <= hi
//
// Match requires pattern to match all of name, not just a substring.
// The only possible error return is when pattern is malformed.
//
func Match(pattern, name string) (matched bool, err os.Error) {
Pattern:
	for len(pattern) > 0 {
		var star bool
		var chunk string
		star, chunk, pattern = scanChunk(pattern)
		if star && chunk == "" {
			// Trailing * matches rest of string unless it has a /.
			return strings.Index(name, string(Separator)) < 0, nil
		}
		// Look for match at current position.
		t, ok, err := matchChunk(chunk, name)
		// if we're the last chunk, make sure we've exhausted the name
		// otherwise we'll give a false result even if we could still match
		// using the star
		if ok && (len(t) == 0 || len(pattern) > 0) {
			name = t
			continue
		}
		if err != nil {
			return false, err
		}
		if star {
			// Look for match skipping i+1 bytes.
			// Cannot skip /.
			for i := 0; i < len(name) && name[i] != Separator; i++ {
				t, ok, err := matchChunk(chunk, name[i+1:])
				if ok {
					// if we're the last chunk, make sure we exhausted the name
					if len(pattern) == 0 && len(t) > 0 {
						continue
					}
					name = t
					continue Pattern
				}
				if err != nil {
					return false, err
				}
			}
		}
		return false, nil
	}
	return len(name) == 0, nil
}

// scanChunk gets the next segment of pattern, which is a non-star string
// possibly preceded by a star.
func scanChunk(pattern string) (star bool, chunk, rest string) {
	for len(pattern) > 0 && pattern[0] == '*' {
		pattern = pattern[1:]
		star = true
	}
	inrange := false
	var i int
Scan:
	for i = 0; i < len(pattern); i++ {
		switch pattern[i] {
		case '\\':
			// error check handled in matchChunk: bad pattern.
			if i+1 < len(pattern) {
				i++
			}
		case '[':
			inrange = true
		case ']':
			inrange = false
		case '*':
			if !inrange {
				break Scan
			}
		}
	}
	return star, pattern[0:i], pattern[i:]
}

// matchChunk checks whether chunk matches the beginning of s.
// If so, it returns the remainder of s (after the match).
// Chunk is all single-character operators: literals, char classes, and ?.
func matchChunk(chunk, s string) (rest string, ok bool, err os.Error) {
	for len(chunk) > 0 {
		if len(s) == 0 {
			return
		}
		switch chunk[0] {
		case '[':
			// character class
			r, n := utf8.DecodeRuneInString(s)
			s = s[n:]
			chunk = chunk[1:]
			// possibly negated
			notNegated := true
			if len(chunk) > 0 && chunk[0] == '^' {
				notNegated = false
				chunk = chunk[1:]
			}
			// parse all ranges
			match := false
			nrange := 0
			for {
				if len(chunk) > 0 && chunk[0] == ']' && nrange > 0 {
					chunk = chunk[1:]
					break
				}
				var lo, hi int
				if lo, chunk, err = getEsc(chunk); err != nil {
					return
				}
				hi = lo
				if chunk[0] == '-' {
					if hi, chunk, err = getEsc(chunk[1:]); err != nil {
						return
					}
				}
				if lo <= r && r <= hi {
					match = true
				}
				nrange++
			}
			if match != notNegated {
				return
			}

		case '?':
			if s[0] == Separator {
				return
			}
			_, n := utf8.DecodeRuneInString(s)
			s = s[n:]
			chunk = chunk[1:]

		case '\\':
			chunk = chunk[1:]
			if len(chunk) == 0 {
				err = ErrBadPattern
				return
			}
			fallthrough

		default:
			if chunk[0] != s[0] {
				return
			}
			s = s[1:]
			chunk = chunk[1:]
		}
	}
	return s, true, nil
}

// getEsc gets a possibly-escaped character from chunk, for a character class.
func getEsc(chunk string) (r int, nchunk string, err os.Error) {
	if len(chunk) == 0 || chunk[0] == '-' || chunk[0] == ']' {
		err = ErrBadPattern
		return
	}
	if chunk[0] == '\\' {
		chunk = chunk[1:]
		if len(chunk) == 0 {
			err = ErrBadPattern
			return
		}
	}
	r, n := utf8.DecodeRuneInString(chunk)
	if r == utf8.RuneError && n == 1 {
		err = ErrBadPattern
	}
	nchunk = chunk[n:]
	if len(nchunk) == 0 {
		err = ErrBadPattern
	}
	return
}

// Glob returns the names of all files matching pattern or nil
// if there is no matching file. The syntax of patterns is the same
// as in Match. The pattern may describe hierarchical names such as
// /usr/*/bin/ed (assuming the Separator is '/').
//
func Glob(pattern string) (matches []string) {
	if !hasMeta(pattern) {
		if _, err := os.Stat(pattern); err == nil {
			return []string{pattern}
		}
		return nil
	}

	dir, file := Split(pattern)
	switch dir {
	case "":
		dir = "."
	case string(Separator):
		// nothing
	default:
		dir = dir[0 : len(dir)-1] // chop off trailing separator
	}

	if hasMeta(dir) {
		for _, d := range Glob(dir) {
			matches = glob(d, file, matches)
		}
	} else {
		return glob(dir, file, nil)
	}
	return matches
}

// glob searches for files matching pattern in the directory dir
// and appends them to matches.
func glob(dir, pattern string, matches []string) []string {
	fi, err := os.Stat(dir)
	if err != nil {
		return nil
	}
	if !fi.IsDirectory() {
		return matches
	}
	d, err := os.Open(dir, os.O_RDONLY, 0666)
	if err != nil {
		return nil
	}
	defer d.Close()

	names, err := d.Readdirnames(-1)
	if err != nil {
		return nil
	}
	sort.SortStrings(names)

	for _, n := range names {
		matched, err := Match(pattern, n)
		if err != nil {
			return matches
		}
		if matched {
			matches = append(matches, Join(dir, n))
		}
	}
	return matches
}

// hasMeta returns true if path contains any of the magic characters
// recognized by Match.
func hasMeta(path string) bool {
	// TODO(niemeyer): Should other magic characters be added here?
	return strings.IndexAny(path, "*?[") >= 0
}
<commit_msg>path/filepath.Glob: don't drop known matches on error. Fixes issue 1610.<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package filepath

import (
	"os"
	"sort"
	"strings"
	"utf8"
)

var ErrBadPattern = os.NewError("syntax error in pattern")

// Match returns true if name matches the shell file name pattern.
// The pattern syntax is:
//
//	pattern:
//		{ term }
//	term:
//		'*'         matches any sequence of non-Separator characters
//		'?'         matches any single non-Separator character
//		'[' [ '^' ] { character-range } ']'
//		            character class (must be non-empty)
//		c           matches character c (c != '*', '?', '\\', '[')
//		'\\' c      matches character c
//
//	character-range:
//		c           matches character c (c != '\\', '-', ']')
//		'\\' c      matches character c
//		lo '-' hi   matches character c for lo <= c <= hi
//
// Match requires pattern to match all of name, not just a substring.
// The only possible error return is when pattern is malformed.
//
func Match(pattern, name string) (matched bool, err os.Error) {
Pattern:
	for len(pattern) > 0 {
		var star bool
		var chunk string
		star, chunk, pattern = scanChunk(pattern)
		if star && chunk == "" {
			// Trailing * matches rest of string unless it has a /.
			return strings.Index(name, string(Separator)) < 0, nil
		}
		// Look for match at current position.
		t, ok, err := matchChunk(chunk, name)
		// if we're the last chunk, make sure we've exhausted the name
		// otherwise we'll give a false result even if we could still match
		// using the star
		if ok && (len(t) == 0 || len(pattern) > 0) {
			name = t
			continue
		}
		if err != nil {
			return false, err
		}
		if star {
			// Look for match skipping i+1 bytes.
			// Cannot skip /.
			for i := 0; i < len(name) && name[i] != Separator; i++ {
				t, ok, err := matchChunk(chunk, name[i+1:])
				if ok {
					// if we're the last chunk, make sure we exhausted the name
					if len(pattern) == 0 && len(t) > 0 {
						continue
					}
					name = t
					continue Pattern
				}
				if err != nil {
					return false, err
				}
			}
		}
		return false, nil
	}
	return len(name) == 0, nil
}

// scanChunk gets the next segment of pattern, which is a non-star string
// possibly preceded by a star.
func scanChunk(pattern string) (star bool, chunk, rest string) {
	for len(pattern) > 0 && pattern[0] == '*' {
		pattern = pattern[1:]
		star = true
	}
	inrange := false
	var i int
Scan:
	for i = 0; i < len(pattern); i++ {
		switch pattern[i] {
		case '\\':
			// error check handled in matchChunk: bad pattern.
			if i+1 < len(pattern) {
				i++
			}
		case '[':
			inrange = true
		case ']':
			inrange = false
		case '*':
			if !inrange {
				break Scan
			}
		}
	}
	return star, pattern[0:i], pattern[i:]
}

// matchChunk checks whether chunk matches the beginning of s.
// If so, it returns the remainder of s (after the match).
// Chunk is all single-character operators: literals, char classes, and ?.
func matchChunk(chunk, s string) (rest string, ok bool, err os.Error) {
	for len(chunk) > 0 {
		if len(s) == 0 {
			return
		}
		switch chunk[0] {
		case '[':
			// character class
			r, n := utf8.DecodeRuneInString(s)
			s = s[n:]
			chunk = chunk[1:]
			// possibly negated
			notNegated := true
			if len(chunk) > 0 && chunk[0] == '^' {
				notNegated = false
				chunk = chunk[1:]
			}
			// parse all ranges
			match := false
			nrange := 0
			for {
				if len(chunk) > 0 && chunk[0] == ']' && nrange > 0 {
					chunk = chunk[1:]
					break
				}
				var lo, hi int
				if lo, chunk, err = getEsc(chunk); err != nil {
					return
				}
				hi = lo
				if chunk[0] == '-' {
					if hi, chunk, err = getEsc(chunk[1:]); err != nil {
						return
					}
				}
				if lo <= r && r <= hi {
					match = true
				}
				nrange++
			}
			if match != notNegated {
				return
			}

		case '?':
			if s[0] == Separator {
				return
			}
			_, n := utf8.DecodeRuneInString(s)
			s = s[n:]
			chunk = chunk[1:]

		case '\\':
			chunk = chunk[1:]
			if len(chunk) == 0 {
				err = ErrBadPattern
				return
			}
			fallthrough

		default:
			if chunk[0] != s[0] {
				return
			}
			s = s[1:]
			chunk = chunk[1:]
		}
	}
	return s, true, nil
}

// getEsc gets a possibly-escaped character from chunk, for a character class.
func getEsc(chunk string) (r int, nchunk string, err os.Error) {
	if len(chunk) == 0 || chunk[0] == '-' || chunk[0] == ']' {
		err = ErrBadPattern
		return
	}
	if chunk[0] == '\\' {
		chunk = chunk[1:]
		if len(chunk) == 0 {
			err = ErrBadPattern
			return
		}
	}
	r, n := utf8.DecodeRuneInString(chunk)
	if r == utf8.RuneError && n == 1 {
		err = ErrBadPattern
	}
	nchunk = chunk[n:]
	if len(nchunk) == 0 {
		err = ErrBadPattern
	}
	return
}

// Glob returns the names of all files matching pattern or nil
// if there is no matching file. The syntax of patterns is the same
// as in Match. The pattern may describe hierarchical names such as
// /usr/*/bin/ed (assuming the Separator is '/').
//
func Glob(pattern string) (matches []string) {
	if !hasMeta(pattern) {
		if _, err := os.Stat(pattern); err == nil {
			return []string{pattern}
		}
		return nil
	}

	dir, file := Split(pattern)
	switch dir {
	case "":
		dir = "."
	case string(Separator):
		// nothing
	default:
		dir = dir[0 : len(dir)-1] // chop off trailing separator
	}

	if hasMeta(dir) {
		for _, d := range Glob(dir) {
			matches = glob(d, file, matches)
		}
	} else {
		return glob(dir, file, nil)
	}
	return matches
}

// glob searches for files matching pattern in the directory dir
// and appends them to matches. If the directory cannot be
// opened, it returns the existing matches. New matches are
// added in lexicographical order.
func glob(dir, pattern string, matches []string) (m []string) {
	m = matches
	fi, err := os.Stat(dir)
	if err != nil {
		return
	}
	if !fi.IsDirectory() {
		return
	}
	d, err := os.Open(dir, os.O_RDONLY, 0666)
	if err != nil {
		return
	}
	defer d.Close()

	names, err := d.Readdirnames(-1)
	if err != nil {
		return
	}
	sort.SortStrings(names)

	for _, n := range names {
		matched, err := Match(pattern, n)
		if err != nil {
			break
		}
		if matched {
			m = append(m, Join(dir, n))
		}
	}
	return
}

// hasMeta returns true if path contains any of the magic characters
// recognized by Match.
func hasMeta(path string) bool {
	// TODO(niemeyer): Should other magic characters be added here?
	return strings.IndexAny(path, "*?[") >= 0
}
<|endoftext|>