<commit_before>package terraform

import (
	"fmt"
	"log"
	"strings"

	"github.com/hashicorp/terraform/config"
)

// EvalCompareDiff is an EvalNode implementation that compares two diffs
// and errors if the diffs are not equal.
type EvalCompareDiff struct {
	Info     *InstanceInfo
	One, Two **InstanceDiff
}

// TODO: test
func (n *EvalCompareDiff) Eval(ctx EvalContext) (interface{}, error) {
	one, two := *n.One, *n.Two

	// If either are nil, let them be empty
	if one == nil {
		one = new(InstanceDiff)
		one.init()
	}
	if two == nil {
		two = new(InstanceDiff)
		two.init()
	}
	oneId := one.Attributes["id"]
	twoId := two.Attributes["id"]
	delete(one.Attributes, "id")
	delete(two.Attributes, "id")
	defer func() {
		if oneId != nil {
			one.Attributes["id"] = oneId
		}
		if twoId != nil {
			two.Attributes["id"] = twoId
		}
	}()

	if same, reason := one.Same(two); !same {
		log.Printf("[ERROR] %s: diffs didn't match", n.Info.Id)
		log.Printf("[ERROR] %s: reason: %s", n.Info.Id, reason)
		log.Printf("[ERROR] %s: diff one: %#v", n.Info.Id, one)
		log.Printf("[ERROR] %s: diff two: %#v", n.Info.Id, two)
		return nil, fmt.Errorf(
			"%s: diffs didn't match during apply. This is a bug with "+
				"Terraform and should be reported as a GitHub Issue.\n"+
				"\n"+
				"Please include the following information in your report:\n"+
				"\n"+
				"    Terraform Version: %s\n"+
				"    Resource ID: %s\n"+
				"    Mismatch reason: %s\n"+
				"    Diff One (usually from plan): %#v\n"+
				"    Diff Two (usually from apply): %#v\n"+
				"\n"+
				"Also include as much context as you can about your config, state, "+
				"and the steps you performed to trigger this error.\n",
			n.Info.Id, Version, n.Info.Id, reason, one, two)
	}

	return nil, nil
}

// EvalDiff is an EvalNode implementation that does a refresh for
// a resource.
type EvalDiff struct {
	Info        *InstanceInfo
	Config      **ResourceConfig
	Provider    *ResourceProvider
	Diff        **InstanceDiff
	State       **InstanceState
	OutputDiff  **InstanceDiff
	OutputState **InstanceState

	// Resource is needed to fetch the ignore_changes list so we can
	// filter user-requested ignored attributes from the diff.
	Resource *config.Resource
}

// TODO: test
func (n *EvalDiff) Eval(ctx EvalContext) (interface{}, error) {
	state := *n.State
	config := *n.Config
	provider := *n.Provider

	// Call pre-diff hook
	err := ctx.Hook(func(h Hook) (HookAction, error) {
		return h.PreDiff(n.Info, state)
	})
	if err != nil {
		return nil, err
	}

	// The state for the diff must never be nil
	diffState := state
	if diffState == nil {
		diffState = new(InstanceState)
	}
	diffState.init()

	// Diff!
	diff, err := provider.Diff(n.Info, diffState, config)
	if err != nil {
		return nil, err
	}
	if diff == nil {
		diff = new(InstanceDiff)
	}

	// Preserve the DestroyTainted flag
	if n.Diff != nil {
		diff.DestroyTainted = (*n.Diff).DestroyTainted
	}

	// Require a destroy if there is an ID and it requires new.
	if diff.RequiresNew() && state != nil && state.ID != "" {
		diff.Destroy = true
	}

	// If we're creating a new resource, compute its ID
	if diff.RequiresNew() || state == nil || state.ID == "" {
		var oldID string
		if state != nil {
			oldID = state.Attributes["id"]
		}

		// Add diff to compute new ID
		diff.init()
		diff.Attributes["id"] = &ResourceAttrDiff{
			Old:         oldID,
			NewComputed: true,
			RequiresNew: true,
			Type:        DiffAttrOutput,
		}
	}

	if err := n.processIgnoreChanges(diff); err != nil {
		return nil, err
	}

	// Call post-refresh hook
	err = ctx.Hook(func(h Hook) (HookAction, error) {
		return h.PostDiff(n.Info, diff)
	})
	if err != nil {
		return nil, err
	}

	// Update our output
	*n.OutputDiff = diff

	// Update the state if we care
	if n.OutputState != nil {
		*n.OutputState = state

		// Merge our state so that the state is updated with our plan
		if !diff.Empty() && n.OutputState != nil {
			*n.OutputState = state.MergeDiff(diff)
		}
	}

	return nil, nil
}

func (n *EvalDiff) processIgnoreChanges(diff *InstanceDiff) error {
	if diff == nil || n.Resource == nil || n.Resource.Id() == "" {
		return nil
	}
	ignoreChanges := n.Resource.Lifecycle.IgnoreChanges

	if len(ignoreChanges) == 0 {
		return nil
	}

	changeType := diff.ChangeType()

	// If we're just creating the resource, we shouldn't alter the
	// Diff at all
	if changeType == DiffCreate {
		return nil
	}

	ignorableAttrKeys := make(map[string]bool)
	for _, ignoredKey := range ignoreChanges {
		for k := range diff.Attributes {
			if strings.HasPrefix(k, ignoredKey) {
				ignorableAttrKeys[k] = true
			}
		}
	}

	// If we are replacing the resource, then we expect there to be a bunch of
	// extraneous attribute diffs we need to filter out for the other
	// non-requires-new attributes going from "" -> "configval" or "" ->
	// "<computed>". Filtering these out allows us to see if we might be able to
	// skip this diff altogether.
	if changeType == DiffDestroyCreate {
		for k, v := range diff.Attributes {
			if v.Empty() || v.NewComputed {
				ignorableAttrKeys[k] = true
			}
		}

		// Here we emulate the implementation of diff.RequiresNew() with one small
		// tweak, we ignore the "id" attribute diff that gets added by EvalDiff,
		// since that was added in reaction to RequiresNew being true.
		requiresNewAfterIgnores := false
		for k, v := range diff.Attributes {
			if k == "id" {
				continue
			}
			if _, ok := ignorableAttrKeys[k]; ok {
				continue
			}
			if v.RequiresNew == true {
				requiresNewAfterIgnores = true
			}
		}

		// If we still require resource replacement after ignores, we
		// can't touch the diff, as all of the attributes will be
		// required to process the replacement.
		if requiresNewAfterIgnores {
			return nil
		}

		// Here we undo the two reactions to RequireNew in EvalDiff - the "id"
		// attribute diff and the Destroy boolean field
		log.Printf("[DEBUG] Removing 'id' diff and setting Destroy to false " +
			"because after ignore_changes, this diff no longer requires replacement")
		delete(diff.Attributes, "id")
		diff.Destroy = false
	}

	// If we didn't hit any of our early exit conditions, we can filter the diff.
	for k := range ignorableAttrKeys {
		log.Printf("[DEBUG] [EvalIgnoreChanges] %s - Ignoring diff attribute: %s",
			n.Resource.Id(), k)
		delete(diff.Attributes, k)
	}

	return nil
}

// EvalDiffDestroy is an EvalNode implementation that returns a plain
// destroy diff.
type EvalDiffDestroy struct {
	Info   *InstanceInfo
	State  **InstanceState
	Output **InstanceDiff
}

// TODO: test
func (n *EvalDiffDestroy) Eval(ctx EvalContext) (interface{}, error) {
	state := *n.State

	// If there is no state or we don't have an ID, we're already destroyed
	if state == nil || state.ID == "" {
		return nil, nil
	}

	// Call pre-diff hook
	err := ctx.Hook(func(h Hook) (HookAction, error) {
		return h.PreDiff(n.Info, state)
	})
	if err != nil {
		return nil, err
	}

	// The diff
	diff := &InstanceDiff{Destroy: true}

	// Call post-diff hook
	err = ctx.Hook(func(h Hook) (HookAction, error) {
		return h.PostDiff(n.Info, diff)
	})
	if err != nil {
		return nil, err
	}

	// Update our output
	*n.Output = diff

	return nil, nil
}

// EvalDiffDestroyModule is an EvalNode implementation that writes the diff to
// the full diff.
type EvalDiffDestroyModule struct {
	Path []string
}

// TODO: test
func (n *EvalDiffDestroyModule) Eval(ctx EvalContext) (interface{}, error) {
	diff, lock := ctx.Diff()

	// Acquire the lock so that we can do this safely concurrently
	lock.Lock()
	defer lock.Unlock()

	// Write the diff
	modDiff := diff.ModuleByPath(n.Path)
	if modDiff == nil {
		modDiff = diff.AddModule(n.Path)
	}
	modDiff.Destroy = true

	return nil, nil
}

// EvalFilterDiff is an EvalNode implementation that filters the diff
// according to some filter.
type EvalFilterDiff struct {
	// Input and output
	Diff   **InstanceDiff
	Output **InstanceDiff

	// Destroy, if true, will only include a destroy diff if it is set.
	Destroy bool
}

func (n *EvalFilterDiff) Eval(ctx EvalContext) (interface{}, error) {
	if *n.Diff == nil {
		return nil, nil
	}

	input := *n.Diff
	result := new(InstanceDiff)

	if n.Destroy {
		if input.Destroy || input.RequiresNew() {
			result.Destroy = true
		}
	}

	if n.Output != nil {
		*n.Output = result
	}

	return nil, nil
}

// EvalReadDiff is an EvalNode implementation that writes the diff to
// the full diff.
type EvalReadDiff struct {
	Name string
	Diff **InstanceDiff
}

func (n *EvalReadDiff) Eval(ctx EvalContext) (interface{}, error) {
	diff, lock := ctx.Diff()

	// Acquire the lock so that we can do this safely concurrently
	lock.Lock()
	defer lock.Unlock()

	// Write the diff
	modDiff := diff.ModuleByPath(ctx.Path())
	if modDiff == nil {
		return nil, nil
	}

	*n.Diff = modDiff.Resources[n.Name]

	return nil, nil
}

// EvalWriteDiff is an EvalNode implementation that writes the diff to
// the full diff.
type EvalWriteDiff struct {
	Name string
	Diff **InstanceDiff
}

// TODO: test
func (n *EvalWriteDiff) Eval(ctx EvalContext) (interface{}, error) {
	diff, lock := ctx.Diff()

	// The diff to write, if its empty it should write nil
	var diffVal *InstanceDiff
	if n.Diff != nil {
		diffVal = *n.Diff
	}
	if diffVal.Empty() {
		diffVal = nil
	}

	// Acquire the lock so that we can do this safely concurrently
	lock.Lock()
	defer lock.Unlock()

	// Write the diff
	modDiff := diff.ModuleByPath(ctx.Path())
	if modDiff == nil {
		modDiff = diff.AddModule(ctx.Path())
	}
	if diffVal != nil {
		modDiff.Resources[n.Name] = diffVal
	} else {
		delete(modDiff.Resources, n.Name)
	}

	return nil, nil
}
<commit_msg>Tainted resource not recreated if ignore_changes used on any attributes.<commit_after>package terraform

import (
	"fmt"
	"log"
	"strings"

	"github.com/hashicorp/terraform/config"
)

// EvalCompareDiff is an EvalNode implementation that compares two diffs
// and errors if the diffs are not equal.
type EvalCompareDiff struct {
	Info     *InstanceInfo
	One, Two **InstanceDiff
}

// TODO: test
func (n *EvalCompareDiff) Eval(ctx EvalContext) (interface{}, error) {
	one, two := *n.One, *n.Two

	// If either are nil, let them be empty
	if one == nil {
		one = new(InstanceDiff)
		one.init()
	}
	if two == nil {
		two = new(InstanceDiff)
		two.init()
	}
	oneId := one.Attributes["id"]
	twoId := two.Attributes["id"]
	delete(one.Attributes, "id")
	delete(two.Attributes, "id")
	defer func() {
		if oneId != nil {
			one.Attributes["id"] = oneId
		}
		if twoId != nil {
			two.Attributes["id"] = twoId
		}
	}()

	if same, reason := one.Same(two); !same {
		log.Printf("[ERROR] %s: diffs didn't match", n.Info.Id)
		log.Printf("[ERROR] %s: reason: %s", n.Info.Id, reason)
		log.Printf("[ERROR] %s: diff one: %#v", n.Info.Id, one)
		log.Printf("[ERROR] %s: diff two: %#v", n.Info.Id, two)
		return nil, fmt.Errorf(
			"%s: diffs didn't match during apply. This is a bug with "+
				"Terraform and should be reported as a GitHub Issue.\n"+
				"\n"+
				"Please include the following information in your report:\n"+
				"\n"+
				"    Terraform Version: %s\n"+
				"    Resource ID: %s\n"+
				"    Mismatch reason: %s\n"+
				"    Diff One (usually from plan): %#v\n"+
				"    Diff Two (usually from apply): %#v\n"+
				"\n"+
				"Also include as much context as you can about your config, state, "+
				"and the steps you performed to trigger this error.\n",
			n.Info.Id, Version, n.Info.Id, reason, one, two)
	}

	return nil, nil
}

// EvalDiff is an EvalNode implementation that does a refresh for
// a resource.
type EvalDiff struct {
	Info        *InstanceInfo
	Config      **ResourceConfig
	Provider    *ResourceProvider
	Diff        **InstanceDiff
	State       **InstanceState
	OutputDiff  **InstanceDiff
	OutputState **InstanceState

	// Resource is needed to fetch the ignore_changes list so we can
	// filter user-requested ignored attributes from the diff.
	Resource *config.Resource
}

// TODO: test
func (n *EvalDiff) Eval(ctx EvalContext) (interface{}, error) {
	state := *n.State
	config := *n.Config
	provider := *n.Provider

	// Call pre-diff hook
	err := ctx.Hook(func(h Hook) (HookAction, error) {
		return h.PreDiff(n.Info, state)
	})
	if err != nil {
		return nil, err
	}

	// The state for the diff must never be nil
	diffState := state
	if diffState == nil {
		diffState = new(InstanceState)
	}
	diffState.init()

	// Diff!
	diff, err := provider.Diff(n.Info, diffState, config)
	if err != nil {
		return nil, err
	}
	if diff == nil {
		diff = new(InstanceDiff)
	}

	// Preserve the DestroyTainted flag
	if n.Diff != nil {
		diff.DestroyTainted = (*n.Diff).DestroyTainted
	}

	// Require a destroy if there is an ID and it requires new.
	if diff.RequiresNew() && state != nil && state.ID != "" {
		diff.Destroy = true
	}

	// If we're creating a new resource, compute its ID
	if diff.RequiresNew() || state == nil || state.ID == "" {
		var oldID string
		if state != nil {
			oldID = state.Attributes["id"]
		}

		// Add diff to compute new ID
		diff.init()
		diff.Attributes["id"] = &ResourceAttrDiff{
			Old:         oldID,
			NewComputed: true,
			RequiresNew: true,
			Type:        DiffAttrOutput,
		}
	}

	if err := n.processIgnoreChanges(diff); err != nil {
		return nil, err
	}

	// Call post-refresh hook
	err = ctx.Hook(func(h Hook) (HookAction, error) {
		return h.PostDiff(n.Info, diff)
	})
	if err != nil {
		return nil, err
	}

	// Update our output
	*n.OutputDiff = diff

	// Update the state if we care
	if n.OutputState != nil {
		*n.OutputState = state

		// Merge our state so that the state is updated with our plan
		if !diff.Empty() && n.OutputState != nil {
			*n.OutputState = state.MergeDiff(diff)
		}
	}

	return nil, nil
}

func (n *EvalDiff) processIgnoreChanges(diff *InstanceDiff) error {
	if diff == nil || n.Resource == nil || n.Resource.Id() == "" {
		return nil
	}
	ignoreChanges := n.Resource.Lifecycle.IgnoreChanges

	if len(ignoreChanges) == 0 {
		return nil
	}

	changeType := diff.ChangeType()

	// If we're just creating the resource, we shouldn't alter the
	// Diff at all
	if changeType == DiffCreate {
		return nil
	}

	// If the resource has been tainted we shouldn't alter the Diff
	if diff.DestroyTainted {
		return nil
	}

	ignorableAttrKeys := make(map[string]bool)
	for _, ignoredKey := range ignoreChanges {
		for k := range diff.Attributes {
			if strings.HasPrefix(k, ignoredKey) {
				ignorableAttrKeys[k] = true
			}
		}
	}

	// If we are replacing the resource, then we expect there to be a bunch of
	// extraneous attribute diffs we need to filter out for the other
	// non-requires-new attributes going from "" -> "configval" or "" ->
	// "<computed>". Filtering these out allows us to see if we might be able to
	// skip this diff altogether.
	if changeType == DiffDestroyCreate {
		for k, v := range diff.Attributes {
			if v.Empty() || v.NewComputed {
				ignorableAttrKeys[k] = true
			}
		}

		// Here we emulate the implementation of diff.RequiresNew() with one small
		// tweak, we ignore the "id" attribute diff that gets added by EvalDiff,
		// since that was added in reaction to RequiresNew being true.
		requiresNewAfterIgnores := false
		for k, v := range diff.Attributes {
			if k == "id" {
				continue
			}
			if _, ok := ignorableAttrKeys[k]; ok {
				continue
			}
			if v.RequiresNew == true {
				requiresNewAfterIgnores = true
			}
		}

		// If we still require resource replacement after ignores, we
		// can't touch the diff, as all of the attributes will be
		// required to process the replacement.
		if requiresNewAfterIgnores {
			return nil
		}

		// Here we undo the two reactions to RequireNew in EvalDiff - the "id"
		// attribute diff and the Destroy boolean field
		log.Printf("[DEBUG] Removing 'id' diff and setting Destroy to false " +
			"because after ignore_changes, this diff no longer requires replacement")
		delete(diff.Attributes, "id")
		diff.Destroy = false
	}

	// If we didn't hit any of our early exit conditions, we can filter the diff.
	for k := range ignorableAttrKeys {
		log.Printf("[DEBUG] [EvalIgnoreChanges] %s - Ignoring diff attribute: %s",
			n.Resource.Id(), k)
		delete(diff.Attributes, k)
	}

	return nil
}

// EvalDiffDestroy is an EvalNode implementation that returns a plain
// destroy diff.
type EvalDiffDestroy struct {
	Info   *InstanceInfo
	State  **InstanceState
	Output **InstanceDiff
}

// TODO: test
func (n *EvalDiffDestroy) Eval(ctx EvalContext) (interface{}, error) {
	state := *n.State

	// If there is no state or we don't have an ID, we're already destroyed
	if state == nil || state.ID == "" {
		return nil, nil
	}

	// Call pre-diff hook
	err := ctx.Hook(func(h Hook) (HookAction, error) {
		return h.PreDiff(n.Info, state)
	})
	if err != nil {
		return nil, err
	}

	// The diff
	diff := &InstanceDiff{Destroy: true}

	// Call post-diff hook
	err = ctx.Hook(func(h Hook) (HookAction, error) {
		return h.PostDiff(n.Info, diff)
	})
	if err != nil {
		return nil, err
	}

	// Update our output
	*n.Output = diff

	return nil, nil
}

// EvalDiffDestroyModule is an EvalNode implementation that writes the diff to
// the full diff.
type EvalDiffDestroyModule struct {
	Path []string
}

// TODO: test
func (n *EvalDiffDestroyModule) Eval(ctx EvalContext) (interface{}, error) {
	diff, lock := ctx.Diff()

	// Acquire the lock so that we can do this safely concurrently
	lock.Lock()
	defer lock.Unlock()

	// Write the diff
	modDiff := diff.ModuleByPath(n.Path)
	if modDiff == nil {
		modDiff = diff.AddModule(n.Path)
	}
	modDiff.Destroy = true

	return nil, nil
}

// EvalFilterDiff is an EvalNode implementation that filters the diff
// according to some filter.
type EvalFilterDiff struct {
	// Input and output
	Diff   **InstanceDiff
	Output **InstanceDiff

	// Destroy, if true, will only include a destroy diff if it is set.
	Destroy bool
}

func (n *EvalFilterDiff) Eval(ctx EvalContext) (interface{}, error) {
	if *n.Diff == nil {
		return nil, nil
	}

	input := *n.Diff
	result := new(InstanceDiff)

	if n.Destroy {
		if input.Destroy || input.RequiresNew() {
			result.Destroy = true
		}
	}

	if n.Output != nil {
		*n.Output = result
	}

	return nil, nil
}

// EvalReadDiff is an EvalNode implementation that writes the diff to
// the full diff.
type EvalReadDiff struct {
	Name string
	Diff **InstanceDiff
}

func (n *EvalReadDiff) Eval(ctx EvalContext) (interface{}, error) {
	diff, lock := ctx.Diff()

	// Acquire the lock so that we can do this safely concurrently
	lock.Lock()
	defer lock.Unlock()

	// Write the diff
	modDiff := diff.ModuleByPath(ctx.Path())
	if modDiff == nil {
		return nil, nil
	}

	*n.Diff = modDiff.Resources[n.Name]

	return nil, nil
}

// EvalWriteDiff is an EvalNode implementation that writes the diff to
// the full diff.
type EvalWriteDiff struct {
	Name string
	Diff **InstanceDiff
}

// TODO: test
func (n *EvalWriteDiff) Eval(ctx EvalContext) (interface{}, error) {
	diff, lock := ctx.Diff()

	// The diff to write, if its empty it should write nil
	var diffVal *InstanceDiff
	if n.Diff != nil {
		diffVal = *n.Diff
	}
	if diffVal.Empty() {
		diffVal = nil
	}

	// Acquire the lock so that we can do this safely concurrently
	lock.Lock()
	defer lock.Unlock()

	// Write the diff
	modDiff := diff.ModuleByPath(ctx.Path())
	if modDiff == nil {
		modDiff = diff.AddModule(ctx.Path())
	}
	if diffVal != nil {
		modDiff.Resources[n.Name] = diffVal
	} else {
		delete(modDiff.Resources, n.Name)
	}

	return nil, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"strconv"
	"strings"
	"text/template"
)

type message struct {
	Name string
	Body string
}

const (
	filename = "persist.txt"
)

func main() {
	http.HandleFunc("/", handler) // each request calls handler function
	http.HandleFunc("/receiver", receiver)
	http.HandleFunc("/demo_badge", func(w http.ResponseWriter, r *http.Request) {
		coverBadge(w, 0)
	})
	log.Fatal(http.ListenAndServe(":"+os.Getenv("PORT"), nil))
}

func handler(w http.ResponseWriter, r *http.Request) {

	buffer, err := ioutil.ReadFile(filename)
	if err != nil {
		log.Fatalln(err)
	}

	percentString := string(buffer)

	fmt.Println(percentString)
	percent, _ := strconv.ParseFloat(strings.TrimSpace(percentString), 64)
	fmt.Println(percent)
	coverage := struct{ Percent float64 }{percent}
	err = pageTemplate.Execute(w, coverage)
	if err != nil {
		log.Print(err)
	}

}

var pageTemplate = template.Must(template.New("pageTemplate").Parse(`
<!DOCTYPE html>

Most recent reported code coverage is {{.Percent}}%

`))

func receiver(rw http.ResponseWriter, req *http.Request) {

	file, err := os.Create(filename)
	if err != nil {
		log.Fatalln(err)
	}

	body, err := ioutil.ReadAll(req.Body)
	if err != nil {
		log.Fatalln(err)
	}
	log.Println(string(body))

	var t message
	err = json.Unmarshal(body, &t)
	if err != nil {
		log.Fatalln(err)
	}

	re := regexp.MustCompile(`\d+.*\d*%`)

	numericalValue := re.FindString(string(t.Body))

	_, err = io.WriteString(file, numericalValue[:len(numericalValue)-1])
	if err != nil {
		log.Fatalln(err)
	}
	file.Close()

	log.Println(t.Body)
}

func dummyFunction(i int, j int) int {
	return i + j
}
<commit_msg>solved issue with serving page template from https while including js libraries over http CDN<commit_after>package main

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"strconv"
	"strings"
	"text/template"
)

type message struct {
	Name string
	Body string
}

const (
	filename = "persist.txt"
)

func main() {
	http.HandleFunc("/", handler) // each request calls handler function
	http.HandleFunc("/receiver", receiver)
	http.HandleFunc("/demo_badge", func(w http.ResponseWriter, r *http.Request) {
		coverBadge(w, 0)
	})
	log.Fatal(http.ListenAndServe(":"+os.Getenv("PORT"), nil))
}

func handler(w http.ResponseWriter, r *http.Request) {

	buffer, err := ioutil.ReadFile(filename)
	if err != nil {
		log.Fatalln(err)
	}

	percentString := string(buffer)
	percent, _ := strconv.ParseFloat(strings.TrimSpace(percentString), 64)
	coverage := struct{ Percent float64 }{percent}
	err = pageTemplate.Execute(w, coverage)
	if err != nil {
		log.Print(err)
	}

}

var pageTemplate = template.Must(template.New("pageTemplate").Parse(`
<!DOCTYPE html>

Most recent reported code coverage is {{.Percent}}%

`))

func receiver(rw http.ResponseWriter, req *http.Request) {

	file, err := os.Create(filename)
	if err != nil {
		log.Fatalln(err)
	}

	body, err := ioutil.ReadAll(req.Body)
	if err != nil {
		log.Fatalln(err)
	}
	log.Println(string(body))

	var t message
	err = json.Unmarshal(body, &t)
	if err != nil {
		log.Fatalln(err)
	}

	re := regexp.MustCompile(`\d+.*\d*%`)

	numericalValue := re.FindString(string(t.Body))

	_, err = io.WriteString(file, numericalValue[:len(numericalValue)-1])
	if err != nil {
		log.Fatalln(err)
	}
	file.Close()

	log.Println(t.Body)
}

func dummyFunction(i int, j int) int {
	return i + j
}
<|endoftext|>


<commit_before>package golnet

import (
	"fmt"
	"io/ioutil"
	"net"
	"os"
)

type RequestParamFunc func(request []byte) (reply []byte)

func TCPServer(connection_string string, request_handler RequestParamFunc) {
	server, err := net.Listen("tcp", connection_string)
	if err != nil {
		fmt.Println("Error listening:", err.Error())
		os.Exit(1)
	}

	defer server.Close()
	fmt.Println("Listening at ", connection_string)
	for {
		connection, err := server.Accept()
		if err != nil {
			fmt.Println("Error accepting: ", err.Error())
			os.Exit(1)
		}

		go handleRequest(connection, request_handler)
	}
}

func handleRequest(conn net.Conn, request_handler RequestParamFunc) {
	request, read_err := ioutil.ReadAll(conn)
	if read_err != nil {
		fmt.Println("ERROR: Golnet TCP server failed to read from client.", read_err)
	}

	reply := request_handler(request)
	_, write_err := conn.Write(reply)
	if write_err != nil {
		fmt.Println("ERROR: Golnet TCP server failed to write to client.", write_err)
	}
	conn.Close()
}
<commit_msg>[golnet] just muted any happy path verbosity from tcpserver<commit_after>package golnet

import (
	"fmt"
	"io/ioutil"
	"net"
	"os"
)

type RequestParamFunc func(request []byte) (reply []byte)

func TCPServer(connection_string string, request_handler RequestParamFunc) {
	server, err := net.Listen("tcp", connection_string)
	if err != nil {
		fmt.Println("Error listening:", err.Error())
		os.Exit(1)
	}

	defer server.Close()
	for {
		connection, err := server.Accept()
		if err != nil {
			fmt.Println("Error accepting: ", err.Error())
			os.Exit(1)
		}

		go handleRequest(connection, request_handler)
	}
}

func handleRequest(conn net.Conn, request_handler RequestParamFunc) {
	request, read_err := ioutil.ReadAll(conn)
	if read_err != nil {
		fmt.Println("ERROR: Golnet TCP server failed to read from client.", read_err)
	}

	reply := request_handler(request)
	_, write_err := conn.Write(reply)
	if write_err != nil {
		fmt.Println("ERROR: Golnet TCP server failed to write to client.", write_err)
	}
	conn.Close()
}
<|endoftext|>


<commit_before>package lldbpi

import "github.com/maxymania/go-nntp-backends/groupdb"
import "github.com/cznic/exp/lldb"
import "io"
import "bytes"
import "os"
import "errors"

func openWAL(fn string) (lldb.Filer,io.Closer,bool,error) {
	first := false
	f,e := os.OpenFile(fn,os.O_RDWR,0660)
	if e!=nil {
		first = true
		f,e = os.OpenFile(fn,os.O_RDWR|os.O_CREATE,0660)
		if e!=nil { return nil,nil,false,e }
	}
	fl,e := os.OpenFile(fn+".wal",os.O_RDWR|os.O_CREATE,0660)
	if e!=nil { f.Close(); return nil,nil,false,e }
	filer,e := lldb.NewACIDFiler(lldb.NewSimpleFileFiler(f),fl)
	if e!=nil { f.Close(); fl.Close(); return nil,nil,false,e }
	return filer,fl,first,nil
}

func Open(opts *groupdb.Options) (groupdb.GroupDB,error) {
	var h1,h2,h3 int64
	f,c,first,e := openWAL(opts.FileName)
	if e!=nil { return nil,e }
	g := new(groupDB)
	g.dprov = opts.DayProvider
	g.filer = f
	g.closr = c
	g.alloc,e = lldb.NewAllocator(g.filer,&lldb.Options{})
	if e!=nil { f.Close(); c.Close(); return nil,e }
	if first {
		g.filer.BeginUpdate()
		defer g.filer.EndUpdate()
		i,e := g.alloc.Alloc([]byte("......"))
		if e!=nil { f.Close(); c.Close(); return nil,e }
		if i!=1 { f.Close(); c.Close(); return nil,errors.New("corrupted") }
		g.group,h1,e = lldb.CreateBTree(g.alloc,bytes.Compare)
		if e!=nil { f.Close(); c.Close(); return nil,e }
		g.grass,h2,e = lldb.CreateBTree(g.alloc,bytes.Compare)
		if e!=nil { f.Close(); c.Close(); return nil,e }
		g.tmlog,h3,e = lldb.CreateBTree(g.alloc,bytes.Compare)
		if e!=nil { f.Close(); c.Close(); return nil,e }
		v,e := lldb.EncodeScalars(h1,h2,h3)
		if e!=nil { f.Close(); c.Close(); return nil,e }
		e = g.alloc.Realloc(1,v)
		if e!=nil { f.Close(); c.Close(); return nil,e }
	} else {
		v,e := g.alloc.Get(nil,1)
		if e!=nil { f.Close(); c.Close(); return nil,e }
		s,e := lldb.DecodeScalars(v)
		if e!=nil { f.Close(); c.Close(); return nil,e }
		g.group,e = lldb.OpenBTree(g.alloc,bytes.Compare,s[0].(int64))
		if e!=nil { f.Close(); c.Close(); return nil,e }
		g.grass,e = lldb.OpenBTree(g.alloc,bytes.Compare,s[1].(int64))
		if e!=nil { f.Close(); c.Close(); return nil,e }
		g.tmlog,e = lldb.OpenBTree(g.alloc,bytes.Compare,s[2].(int64))
		if e!=nil { f.Close(); c.Close(); return nil,e }
	}
	return g,nil
}

func init() {
	groupdb.RegisterPlugin("lldb",Open)
}

<commit_msg>time-log should be sorted descending<commit_after>package lldbpi

import "github.com/maxymania/go-nntp-backends/groupdb"
import "github.com/cznic/exp/lldb"
import "io"
import "bytes"
import "os"
import "errors"

func openWAL(fn string) (lldb.Filer,io.Closer,bool,error) {
	first := false
	f,e := os.OpenFile(fn,os.O_RDWR,0660)
	if e!=nil {
		first = true
		f,e = os.OpenFile(fn,os.O_RDWR|os.O_CREATE,0660)
		if e!=nil { return nil,nil,false,e }
	}
	fl,e := os.OpenFile(fn+".wal",os.O_RDWR|os.O_CREATE,0660)
	if e!=nil { f.Close(); return nil,nil,false,e }
	filer,e := lldb.NewACIDFiler(lldb.NewSimpleFileFiler(f),fl)
	if e!=nil { f.Close(); fl.Close(); return nil,nil,false,e }
	return filer,fl,first,nil
}

func Open(opts *groupdb.Options) (groupdb.GroupDB,error) {
	var h1,h2,h3 int64
	f,c,first,e := openWAL(opts.FileName)
	if e!=nil { return nil,e }
	g := new(groupDB)
	g.dprov = opts.DayProvider
	g.filer = f
	g.closr = c
	g.alloc,e = lldb.NewAllocator(g.filer,&lldb.Options{})
	if e!=nil { f.Close(); c.Close(); return nil,e }
	if first {
		g.filer.BeginUpdate()
		defer g.filer.EndUpdate()
		i,e := g.alloc.Alloc([]byte("......"))
		if e!=nil { f.Close(); c.Close(); return nil,e }
		if i!=1 { f.Close(); c.Close(); return nil,errors.New("corrupted") }
		g.group,h1,e = lldb.CreateBTree(g.alloc,bytes.Compare)
		if e!=nil { f.Close(); c.Close(); return nil,e }
		g.grass,h2,e = lldb.CreateBTree(g.alloc,bytes.Compare)
		if e!=nil { f.Close(); c.Close(); return nil,e }
		g.tmlog,h3,e = lldb.CreateBTree(g.alloc,reverseComp)
		if e!=nil { f.Close(); c.Close(); return nil,e }
		v,e := lldb.EncodeScalars(h1,h2,h3)
		if e!=nil { f.Close(); c.Close(); return nil,e }
		e = g.alloc.Realloc(1,v)
		if e!=nil { f.Close(); c.Close(); return nil,e }
	} else {
		v,e := g.alloc.Get(nil,1)
		if e!=nil { f.Close(); c.Close(); return nil,e }
		s,e := lldb.DecodeScalars(v)
		if e!=nil { f.Close(); c.Close(); return nil,e }
		g.group,e = lldb.OpenBTree(g.alloc,bytes.Compare,s[0].(int64))
		if e!=nil { f.Close(); c.Close(); return nil,e }
		g.grass,e = lldb.OpenBTree(g.alloc,bytes.Compare,s[1].(int64))
		if e!=nil { f.Close(); c.Close(); return nil,e }
		g.tmlog,e = lldb.OpenBTree(g.alloc,bytes.Compare,s[2].(int64))
		if e!=nil { f.Close(); c.Close(); return nil,e }
	}
	return g,nil
}

func init() {
	groupdb.RegisterPlugin("lldb",Open)
}

<|endoftext|>


<commit_before>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package elastigo

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"strconv"
	"sync"
	"time"
)

const (
	// Max buffer size in bytes before flushing to elasticsearch
	BulkMaxBuffer = 16384
	// Max number of Docs to hold in buffer before forcing flush
	BulkMaxDocs = 100
	// Max delay before forcing a flush to Elasticearch
	BulkDelaySeconds = 5
	// maximum wait shutdown seconds
	MAX_SHUTDOWN_SECS = 5
)

type ErrorBuffer struct {
	Err error
	Buf *bytes.Buffer
}

// A bulk indexer creates goroutines, and channels for connecting and sending data
// to elasticsearch in bulk, using buffers.
type BulkIndexer struct {
	conn *Conn

	// We are creating a variable defining the func responsible for sending
	// to allow a mock sendor for test purposes
	Sender func(*bytes.Buffer) error

	// The refresh parameter can be set to true in order to refresh the
	// relevant primary and replica shards immediately after the bulk
	// operation has occurred
	Refresh bool

	// If we encounter an error in sending, we are going to retry for this long
	// before returning an error
	// if 0 it will not retry
	RetryForSeconds int

	// channel for getting errors
	ErrorChannel chan *ErrorBuffer

	// channel for sending to background indexer
	bulkChannel chan []byte

	// numErrors is a running total of errors seen
	numErrors uint64

	// shutdown channel
	shutdownChan chan chan struct{}
	// channel to shutdown timer
	timerDoneChan chan struct{}

	// Channel to send a complete byte.Buffer to the http sendor
	sendBuf chan *bytes.Buffer
	// byte buffer for docs that have been converted to bytes, but not yet sent
	buf *bytes.Buffer
	// Buffer for Max number of time before forcing flush
	BufferDelayMax time.Duration
	// Max buffer size in bytes before flushing to elasticsearch
	BulkMaxBuffer int // 1048576
	// Max number of Docs to hold in buffer before forcing flush
	BulkMaxDocs int // 100

	// Number of documents we have send through so far on this session
	docCt int
	// Max number of http conns in flight at one time
	maxConns int
	// If we are indexing enough docs per bufferdelaymax, we won't need to do time
	// based eviction, else we do.
	needsTimeBasedFlush bool
	// Lock for document writes/operations
	mu sync.Mutex
	// Wait Group for the http sends
	sendWg *sync.WaitGroup
}

func (b *BulkIndexer) NumErrors() uint64 {
	return b.numErrors
}

func (c *Conn) NewBulkIndexer(maxConns int) *BulkIndexer {
	b := BulkIndexer{conn: c, sendBuf: make(chan *bytes.Buffer, maxConns)}
	b.needsTimeBasedFlush = true
	b.buf = new(bytes.Buffer)
	b.maxConns = maxConns
	b.BulkMaxBuffer = BulkMaxBuffer
	b.BulkMaxDocs = BulkMaxDocs
	b.BufferDelayMax = time.Duration(BulkDelaySeconds) * time.Second
	b.bulkChannel = make(chan []byte, 100)
	b.sendWg = new(sync.WaitGroup)
	b.timerDoneChan = make(chan struct{})
	return &b
}

// A bulk indexer with more control over error handling
//    @maxConns is the max number of in flight http requests
//    @retrySeconds is # of seconds to wait before retrying falied requests
//
//   done := make(chan bool)
//   BulkIndexerGlobalRun(100, done)
func (c *Conn) NewBulkIndexerErrors(maxConns, retrySeconds int) *BulkIndexer {
	b := c.NewBulkIndexer(maxConns)
	b.RetryForSeconds = retrySeconds
	b.ErrorChannel = make(chan *ErrorBuffer, 20)
	return b
}

// Starts this bulk Indexer running, this Run opens a go routine so is
// Non blocking
func (b *BulkIndexer) Start() {
	b.shutdownChan = make(chan chan struct{})

	go func() {
		// XXX(j): Refactor this stuff to use an interface.
		if b.Sender == nil {
			b.Sender = b.Send
		}
		// Backwards compatibility
		b.startHttpSender()
		b.startDocChannel()
		b.startTimer()
		ch := <-b.shutdownChan
		time.Sleep(2 * time.Millisecond)
		b.Flush()
		b.shutdown()
		ch <- struct{}{}
	}()
}

// Stop stops the bulk indexer, blocking the caller until it is complete.
func (b *BulkIndexer) Stop() {
	ch := make(chan struct{})
	b.shutdownChan <- ch
	select {
	case <-ch:
		// done
	case <-time.After(time.Second * time.Duration(MAX_SHUTDOWN_SECS)):
		// timeout!
	}
}

// Make a channel that will close when the given WaitGroup is done.
func wgChan(wg *sync.WaitGroup) <-chan interface{} {
	ch := make(chan interface{})
	go func() {
		wg.Wait()
		close(ch)
	}()
	return ch
}

func (b *BulkIndexer) PendingDocuments() int {
	return b.docCt
}

// Flush all current documents to ElasticSearch
func (b *BulkIndexer) Flush() {
	b.mu.Lock()
	if b.docCt > 0 {
		b.send(b.buf)
	}
	b.mu.Unlock()
}

func (b *BulkIndexer) startHttpSender() {

	// this sends http requests to elasticsearch it uses maxConns to open up that
	// many goroutines, each of which will synchronously call ElasticSearch
	// in theory, the whole set will cause a backup all the way to IndexBulk if
	// we have consumed all maxConns
	for i := 0; i < b.maxConns; i++ {
		go func() {
			for buf := range b.sendBuf {
				b.sendWg.Add(1)
				// Copy for the potential re-send.
				bufCopy := bytes.NewBuffer(buf.Bytes())
				err := b.Sender(buf)

				// Perhaps a b.FailureStrategy(err)  ??  with different types of strategies
				//  1.  Retry, then panic
				//  2.  Retry then return error and let runner decide
				//  3.  Retry, then log to disk?   retry later?
				if err != nil {
					if b.RetryForSeconds > 0 {
						time.Sleep(time.Second * time.Duration(b.RetryForSeconds))
						err = b.Sender(bufCopy)
						if err == nil {
							// Successfully re-sent with no error
							b.sendWg.Done()
							continue
						}
					}
					if b.ErrorChannel != nil {
						b.ErrorChannel <- &ErrorBuffer{err, buf}
					}
				}
				b.sendWg.Done()
			}
		}()
	}
}

// start a timer for checking back and forcing flush ever BulkDelaySeconds seconds
// even if we haven't hit max messages/size
func (b *BulkIndexer) startTimer() {
	ticker := time.NewTicker(b.BufferDelayMax)
	go func() {
		for {
			select {
			case <-ticker.C:
				b.mu.Lock()
				// don't send unless last sendor was the time,
				// otherwise an indication of other thresholds being hit
				// where time isn't needed
				if b.buf.Len() > 0 && b.needsTimeBasedFlush {
					b.needsTimeBasedFlush = true
					b.send(b.buf)
				} else if b.buf.Len() > 0 {
					b.needsTimeBasedFlush = true
				}
				b.mu.Unlock()
			case <-b.timerDoneChan:
				// shutdown this go routine
				return
			}

		}
	}()
}

func (b *BulkIndexer) startDocChannel() {
	// This goroutine accepts incoming byte arrays from the IndexBulk function and
	// writes to buffer
	go func() {
		for docBytes := range b.bulkChannel {
			b.mu.Lock()
			b.docCt += 1
			b.buf.Write(docBytes)
			if b.buf.Len() >= b.BulkMaxBuffer || b.docCt >= b.BulkMaxDocs {
				b.needsTimeBasedFlush = false
				//log.Printf("Send due to size:  docs=%d  bufsize=%d", b.docCt, b.buf.Len())
				b.send(b.buf)
			}
			b.mu.Unlock()
		}
	}()
}

func (b *BulkIndexer) send(buf *bytes.Buffer) {
	//b2 := *b.buf
	b.sendBuf <- buf
	b.buf = new(bytes.Buffer)
	//	b.buf.Reset()
	b.docCt = 0
}

func (b *BulkIndexer) shutdown() {
	// This must be called after Flush()
	close(b.timerDoneChan)
	close(b.sendBuf)
	close(b.bulkChannel)
	<-wgChan(b.sendWg)
}

// The index bulk API adds or updates a typed JSON document to a specific index, making it searchable.
// it operates by buffering requests, and ocassionally flushing to elasticsearch
// http://www.elasticsearch.org/guide/reference/api/bulk.html
func (b *BulkIndexer) Index(index string, _type string, id, parent, ttl string, date *time.Time, data interface{}) error {
	//{ "index" : { "_index" : "test", "_type" : "type1", "_id" : "1" } }
	by, err := WriteBulkBytes("index", index, _type, id, parent, ttl, date, data)
	if err != nil {
		return err
	}
	b.bulkChannel <- by
	return nil
}

func (b *BulkIndexer) Update(index string, _type string, id, parent, ttl string, date *time.Time, data interface{}) error {
	//{ "index" : { "_index" : "test", "_type" : "type1", "_id" : "1" } }
	by, err := WriteBulkBytes("update", index, _type, id, parent, ttl, date, data)
	if err != nil {
		return err
	}
	b.bulkChannel <- by
	return nil
}

func (b *BulkIndexer) Delete(index, _type, id string) {
	queryLine := fmt.Sprintf("{\"delete\":{\"_index\":%q,\"_type\":%q,\"_id\":%q}}\n", index, _type, id)
	b.bulkChannel <- []byte(queryLine)
	return
}

func (b *BulkIndexer) UpdateWithWithScript(index string, _type string, id, parent, ttl string, date *time.Time, script string) error {

	var data map[string]interface{} = make(map[string]interface{})
	data["script"] = script
	return b.Update(index, _type, id, parent, ttl, date, data)
}

func (b *BulkIndexer) UpdateWithPartialDoc(index string, _type string, id, parent, ttl string, date *time.Time, partialDoc interface{}, upsert bool) error {

	var data map[string]interface{} = make(map[string]interface{})

	data["doc"] = partialDoc
	if upsert {
		data["doc_as_upsert"] = true
	}
	return b.Update(index, _type, id, parent, ttl, date, data)
}

// This does the actual send of a buffer, which has already been formatted
// into bytes of ES formatted bulk data
func (b *BulkIndexer) Send(buf *bytes.Buffer) error {
	type responseStruct struct {
		Took   int64                    `json:"took"`
		Errors bool                     `json:"errors"`
		Items  []map[string]interface{} `json:"items"`
	}

	response := responseStruct{}

	body, err := b.conn.DoCommand("POST", fmt.Sprintf("/_bulk?refresh=%t", b.Refresh), nil, buf)

	if err != nil {
		b.numErrors += 1
		return err
	}
	// check for response errors, bulk insert will give 200 OK but then include errors in response
	jsonErr := json.Unmarshal(body, &response)
	if jsonErr == nil {
		if response.Errors {
			b.numErrors += uint64(len(response.Items))
			return fmt.Errorf("Bulk Insertion Error. Failed item count [%d]", len(response.Items))
		}
	}
	return nil
}

// Given a set of arguments for index, type, id, data create a set of bytes that is formatted for bulkd index
// http://www.elasticsearch.org/guide/reference/api/bulk.html
func WriteBulkBytes(op string, index string, _type string, id, parent, ttl string, date *time.Time, data interface{}) ([]byte, error) {
	// only index and update are currently supported
	if op != "index" && op != "update" {
		return nil, errors.New(fmt.Sprintf("Operation '%s' is not yet supported", op))
	}

	// First line
	buf := bytes.Buffer{}
	buf.WriteString(fmt.Sprintf(`{"%s":{"_index":"`, op))
	buf.WriteString(index)
	buf.WriteString(`","_type":"`)
	buf.WriteString(_type)
	buf.WriteString(`"`)
	if len(id) > 0 {
		buf.WriteString(`,"_id":"`)
		buf.WriteString(id)
		buf.WriteString(`"`)
	}

	if len(parent) > 0 {
		buf.WriteString(`,"_parent":"`)
		buf.WriteString(parent)
		buf.WriteString(`"`)
	}

	if op == "update" {
		buf.WriteString(`,"_retry_on_conflict":3`)
	}

	if len(ttl) > 0 {
		buf.WriteString(`,"ttl":"`)
		buf.WriteString(ttl)
		buf.WriteString(`"`)
	}
	if date != nil {
		buf.WriteString(`,"_timestamp":"`)
		buf.WriteString(strconv.FormatInt(date.UnixNano()/1e6, 10))
		buf.WriteString(`"`)
	}

	buf.WriteString(`}}`)
	buf.WriteRune('\n')
	//buf.WriteByte('\n')
	switch v := data.(type) {
	case *bytes.Buffer:
		io.Copy(&buf, v)
	case []byte:
		buf.Write(v)
	case string:
		buf.WriteString(v)
	default:
		body, jsonErr := json.Marshal(data)
		if jsonErr != nil {
			return nil, jsonErr
		}
		buf.Write(body)
	}
	buf.WriteRune('\n')
	return buf.Bytes(), nil
}
<commit_msg>refs elastigo #241 / PR #242 - additional fixes to WaitGroup usage and making numErrors tracking atomic to prevent races<commit_after>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package elastigo

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"strconv"
	"sync"
	"sync/atomic"
	"time"
)

const (
	// Max buffer size in bytes before flushing to elasticsearch
	BulkMaxBuffer = 16384
	// Max number of Docs to hold in buffer before forcing flush
	BulkMaxDocs = 100
	// Max delay before forcing a flush to Elasticearch
	BulkDelaySeconds = 5
	// maximum wait shutdown seconds
	MAX_SHUTDOWN_SECS = 5
)

type ErrorBuffer struct {
	Err error
	Buf *bytes.Buffer
}

// A bulk indexer creates goroutines, and channels for connecting and sending data
// to elasticsearch in bulk, using buffers.
type BulkIndexer struct {
	conn *Conn

	// We are creating a variable defining the func responsible for sending
	// to allow a mock sendor for test purposes
	Sender func(*bytes.Buffer) error

	// The refresh parameter can be set to true in order to refresh the
	// relevant primary and replica shards immediately after the bulk
	// operation has occurred
	Refresh bool

	// If we encounter an error in sending, we are going to retry for this long
	// before returning an error
	// if 0 it will not retry
	RetryForSeconds int

	// channel for getting errors
	ErrorChannel chan *ErrorBuffer

	// channel for sending to background indexer
	bulkChannel chan []byte

	// numErrors is a running total of errors seen
	numErrors uint64

	// shutdown channel
	shutdownChan chan chan struct{}
	// channel to shutdown timer
	timerDoneChan chan struct{}

	// Channel to send a complete byte.Buffer to the http sendor
	sendBuf chan *bytes.Buffer
	// byte buffer for docs that have been converted to bytes, but not yet sent
	buf *bytes.Buffer
	// Buffer for Max number of time before forcing flush
	BufferDelayMax time.Duration
	// Max buffer size in bytes before flushing to elasticsearch
	BulkMaxBuffer int // 1048576
	// Max number of Docs to hold in buffer before forcing flush
	BulkMaxDocs int // 100

	// Number of documents we have send through so far on this session
	docCt int
	// Max number of http conns in flight at one time
	maxConns int
	// If we are indexing enough docs per bufferdelaymax, we won't need to do time
	// based eviction, else we do.
	needsTimeBasedFlush bool
	// Lock for document writes/operations
	mu sync.Mutex
	// Wait Group for the http sends
	sendWg *sync.WaitGroup
}

func (b *BulkIndexer) NumErrors() uint64 {
	return atomic.LoadUint64(&b.numErrors)
}

func (c *Conn) NewBulkIndexer(maxConns int) *BulkIndexer {
	b := BulkIndexer{conn: c, sendBuf: make(chan *bytes.Buffer, maxConns)}
	b.needsTimeBasedFlush = true
	b.buf = new(bytes.Buffer)
	b.maxConns = maxConns
	b.BulkMaxBuffer = BulkMaxBuffer
	b.BulkMaxDocs = BulkMaxDocs
	b.BufferDelayMax = time.Duration(BulkDelaySeconds) * time.Second
	b.bulkChannel = make(chan []byte, 100)
	b.sendWg = new(sync.WaitGroup)
	b.timerDoneChan = make(chan struct{})
	return &b
}

// A bulk indexer with more control over error handling
//    @maxConns is the max number of in flight http requests
//    @retrySeconds is # of seconds to wait before retrying falied requests
//
//   done := make(chan bool)
//   BulkIndexerGlobalRun(100, done)
func (c *Conn) NewBulkIndexerErrors(maxConns, retrySeconds int) *BulkIndexer {
	b := c.NewBulkIndexer(maxConns)
	b.RetryForSeconds = retrySeconds
	b.ErrorChannel = make(chan *ErrorBuffer, 20)
	return b
}

// Starts this bulk Indexer running, this Run opens a go routine so is
// Non blocking
func (b *BulkIndexer) Start() {
	b.shutdownChan = make(chan chan struct{})

	go func() {
		// XXX(j): Refactor this stuff to use an interface.
		if b.Sender == nil {
			b.Sender = b.Send
		}
		// Backwards compatibility
		b.startHttpSender()
		b.startDocChannel()
		b.startTimer()
		ch := <-b.shutdownChan
		time.Sleep(2 * time.Millisecond)
		b.Flush()
		b.shutdown()
		ch <- struct{}{}
	}()
}

// Stop stops the bulk indexer, blocking the caller until it is complete.
func (b *BulkIndexer) Stop() {
	ch := make(chan struct{})
	b.shutdownChan <- ch
	select {
	case <-ch:
		// done
	case <-time.After(time.Second * time.Duration(MAX_SHUTDOWN_SECS)):
		// timeout!
	}
}

func (b *BulkIndexer) PendingDocuments() int {
	return b.docCt
}

// Flush all current documents to ElasticSearch
func (b *BulkIndexer) Flush() {
	b.mu.Lock()
	if b.docCt > 0 {
		b.send(b.buf)
	}
	b.mu.Unlock()
}

func (b *BulkIndexer) startHttpSender() {

	// this sends http requests to elasticsearch it uses maxConns to open up that
	// many goroutines, each of which will synchronously call ElasticSearch
	// in theory, the whole set will cause a backup all the way to IndexBulk if
	// we have consumed all maxConns
	for i := 0; i < b.maxConns; i++ {
		b.sendWg.Add(1)
		go func() {
			for buf := range b.sendBuf {
				// Copy for the potential re-send.
				bufCopy := bytes.NewBuffer(buf.Bytes())
				err := b.Sender(buf)

				// Perhaps a b.FailureStrategy(err)  ??  with different types of strategies
				//  1.  Retry, then panic
				//  2.  Retry then return error and let runner decide
				//  3.  Retry, then log to disk?   retry later?
				if err != nil {
					if b.RetryForSeconds > 0 {
						time.Sleep(time.Second * time.Duration(b.RetryForSeconds))
						err = b.Sender(bufCopy)
						if err == nil {
							// Successfully re-sent with no error
							continue
						}
					}
					if b.ErrorChannel != nil {
						b.ErrorChannel <- &ErrorBuffer{err, buf}
					}
				}
			}
			b.sendWg.Done()
		}()
	}
}

// start a timer for checking back and forcing flush ever BulkDelaySeconds seconds
// even if we haven't hit max messages/size
func (b *BulkIndexer) startTimer() {
	ticker := time.NewTicker(b.BufferDelayMax)
	go func() {
		for {
			select {
			case <-ticker.C:
				b.mu.Lock()
				// don't send unless last sendor was the time,
				// otherwise an indication of other thresholds being hit
				// where time isn't needed
				if b.buf.Len() > 0 && b.needsTimeBasedFlush {
					b.needsTimeBasedFlush = true
					b.send(b.buf)
				} else if b.buf.Len() > 0 {
					b.needsTimeBasedFlush = true
				}
				b.mu.Unlock()
			case <-b.timerDoneChan:
				// shutdown this go routine
				return
			}

		}
	}()
}

func (b *BulkIndexer) startDocChannel() {
	// This goroutine accepts incoming byte arrays from the IndexBulk function and
	// writes to buffer
	go func() {
		for docBytes := range b.bulkChannel {
			b.mu.Lock()
			b.docCt += 1
			b.buf.Write(docBytes)
			if b.buf.Len() >= b.BulkMaxBuffer || b.docCt >= b.BulkMaxDocs {
				b.needsTimeBasedFlush = false
				//log.Printf("Send due to size:  docs=%d  bufsize=%d", b.docCt, b.buf.Len())
				b.send(b.buf)
			}
			b.mu.Unlock()
		}
	}()
}

func (b *BulkIndexer) send(buf *bytes.Buffer) {
	//b2 := *b.buf
	b.sendBuf <- buf
	b.buf = new(bytes.Buffer)
	//	b.buf.Reset()
	b.docCt = 0
}

func (b *BulkIndexer) shutdown() {
	// This must be called after Flush()
	close(b.timerDoneChan)
	close(b.sendBuf)
	close(b.bulkChannel)
	b.sendWg.Wait()
}

// The index bulk API adds or updates a typed JSON document to a specific index, making it searchable.
// it operates by buffering requests, and ocassionally flushing to elasticsearch
// http://www.elasticsearch.org/guide/reference/api/bulk.html
func (b *BulkIndexer) Index(index string, _type string, id, parent, ttl string, date *time.Time, data interface{}) error {
	//{ "index" : { "_index" : "test", "_type" : "type1", "_id" : "1" } }
	by, err := WriteBulkBytes("index", index, _type, id, parent, ttl, date, data)
	if err != nil {
		return err
	}
	b.bulkChannel <- by
	return nil
}

func (b *BulkIndexer) Update(index string, _type string, id, parent, ttl string, date *time.Time, data interface{}) error {
	//{ "index" : { "_index" : "test", "_type" : "type1", "_id" : "1" } }
	by, err := WriteBulkBytes("update", index, _type, id, parent, ttl, date, data)
	if err != nil {
		return err
	}
	b.bulkChannel <- by
	return nil
}

func (b *BulkIndexer) Delete(index, _type, id string) {
	queryLine := fmt.Sprintf("{\"delete\":{\"_index\":%q,\"_type\":%q,\"_id\":%q}}\n", index, _type, id)
	b.bulkChannel <- []byte(queryLine)
	return
}

func (b *BulkIndexer) UpdateWithWithScript(index string, _type string, id, parent, ttl string, date *time.Time, script string) error {

	var data map[string]interface{} = make(map[string]interface{})
	data["script"] = script
	return b.Update(index, _type, id, parent, ttl, date, data)
}

func (b *BulkIndexer) UpdateWithPartialDoc(index string, _type string, id, parent, ttl string, date *time.Time, partialDoc interface{}, upsert bool) error {

	var data map[string]interface{} = make(map[string]interface{})

	data["doc"] = partialDoc
	if upsert {
		data["doc_as_upsert"] = true
	}
	return b.Update(index, _type, id, parent, ttl, date, data)
}

// This does the actual send of a buffer, which has already been formatted
// into bytes of ES formatted bulk data
func (b *BulkIndexer) Send(buf *bytes.Buffer) error {
	type responseStruct struct {
		Took   int64                    `json:"took"`
		Errors bool                     `json:"errors"`
		Items  []map[string]interface{} `json:"items"`
	}

	response := responseStruct{}

	body, err := b.conn.DoCommand("POST", fmt.Sprintf("/_bulk?refresh=%t", b.Refresh), nil, buf)

	if err != nil {
		atomic.AddUint64(&b.numErrors, 1)
		return err
	}
	// check for response errors, bulk insert will give 200 OK but then include errors in response
	jsonErr := json.Unmarshal(body, &response)
	if jsonErr == nil {
		if response.Errors {
			atomic.AddUint64(&b.numErrors, uint64(len(response.Items)))
			return fmt.Errorf("Bulk Insertion Error. Failed item count [%d]", len(response.Items))
		}
	}
	return nil
}

// Given a set of arguments for index, type, id, data create a set of bytes that is formatted for bulkd index
// http://www.elasticsearch.org/guide/reference/api/bulk.html
func WriteBulkBytes(op string, index string, _type string, id, parent, ttl string, date *time.Time, data interface{}) ([]byte, error) {
	// only index and update are currently supported
	if op != "index" && op != "update" {
		return nil, errors.New(fmt.Sprintf("Operation '%s' is not yet supported", op))
	}

	// First line
	buf := bytes.Buffer{}
	buf.WriteString(fmt.Sprintf(`{"%s":{"_index":"`, op))
	buf.WriteString(index)
	buf.WriteString(`","_type":"`)
	buf.WriteString(_type)
	buf.WriteString(`"`)
	if len(id) > 0 {
		buf.WriteString(`,"_id":"`)
		buf.WriteString(id)
		buf.WriteString(`"`)
	}

	if len(parent) > 0 {
		buf.WriteString(`,"_parent":"`)
		buf.WriteString(parent)
		buf.WriteString(`"`)
	}

	if op == "update" {
		buf.WriteString(`,"_retry_on_conflict":3`)
	}

	if len(ttl) > 0 {
		buf.WriteString(`,"ttl":"`)
		buf.WriteString(ttl)
		buf.WriteString(`"`)
	}
	if date != nil {
		buf.WriteString(`,"_timestamp":"`)
		buf.WriteString(strconv.FormatInt(date.UnixNano()/1e6, 10))
		buf.WriteString(`"`)
	}

	buf.WriteString(`}}`)
	buf.WriteRune('\n')
	//buf.WriteByte('\n')
	switch v := data.(type) {
	case *bytes.Buffer:
		io.Copy(&buf, v)
	case []byte:
		buf.Write(v)
	case string:
		buf.WriteString(v)
	default:
		body, jsonErr := json.Marshal(data)
		if jsonErr != nil {
			return nil, jsonErr
		}
		buf.Write(body)
	}
	buf.WriteRune('\n')
	return buf.Bytes(), nil
}
<|endoftext|>


<commit_before>/*
Package jsonptrerror extends encoding/json.Decoder to return unmarshal errors
located with JSON Pointer (RFC 6091).

The current implementation keeps a duplicate copy of the JSON document in memory.
*/
package jsonptrerror

import (
	"bytes"
	"encoding/json"
	"io"

	"github.com/dolmen-go/jsonptr"
)

// UnmarshalTypeError is an extension of encoding/json.UnmarshalTypeError
// that also includes the error location as a JSON Pointer (RFC 6901)
type UnmarshalTypeError struct {
	json.UnmarshalTypeError
	Pointer jsonptr.Pointer
}

type decoder interface {
	Decode(interface{}) error
	UseNumber()
}

// Decoder is the same as encoding/json.Decoder, except Decode returns
// our UnmarshalTypeError (providing a JSON Pointer) instead of encoding/json.UnmarshalTypeError.
type Decoder struct {
	decoder decoder
	input   bytes.Buffer
	err     error
}

func NewDecoder(r io.Reader) *Decoder {
	var d Decoder
	d.decoder = json.NewDecoder(io.TeeReader(r, &d.input))
	return &d
}

func (d *Decoder) UseNumber() {
	d.decoder.UseNumber()
}

const (
	tokenTopValue = iota
	tokenArrayValue
	tokenArrayComma
	tokenObjectKey
	tokenObjectColon
	tokenObjectValue
	tokenObjectComma
)

func (d *Decoder) Decode(v interface{}) error {
	if d.err != nil {
		return d.err
	}
	d.err = d.decoder.Decode(v)
	if err, ok := d.err.(*json.UnmarshalTypeError); ok {
		offset := int(err.Offset)
		input := d.input.Bytes()
		i := 0
		type elem struct {
			container byte
			property  []byte
			index     int
		}
		var elemStack []elem
		var expectKey bool
		for {
			if i == offset {
				ptr := jsonptr.Pointer{}
				for _, e := range elemStack {
					switch e.container {
					case '{':
						var name string
						json.Unmarshal(e.property, &name)
						ptr.Property(name)
					case '[':
						ptr.Index(e.index)
					}
				}
				d.err = &UnmarshalTypeError{*err, ptr}
				break
			}
			switch input[i] {
			case '{':
				elemStack = append(elemStack, elem{container: '{'})
				expectKey = true
			case '[':
				elemStack = append(elemStack, elem{container: '[', index: 0})
			case '}', ']':
				elemStack = elemStack[:len(elemStack)-1]
			case '"':
				j := i
			str:
				for {
					i++
					switch input[i] {
					case '\\':
						i++
					case '"':
						break str
					}
				}
				if expectKey {
					elemStack[len(elemStack)-1].property = input[j : i+1]
					expectKey = false
				}
			case ',':
				if elemStack[len(elemStack)-1].container == '{' {
					expectKey = true
				} else {
					elemStack[len(elemStack)-1].index++
				}
			}
			i++
		}
	}
	if d.err != nil {
		d.decoder = nil
		d.input = bytes.Buffer{}
	}
	return d.err
}
<commit_msg>UnmarshalTypeError: override Error() method<commit_after>/*
Package jsonptrerror extends encoding/json.Decoder to return unmarshal errors
located with JSON Pointer (RFC 6091).

The current implementation keeps a duplicate copy of the JSON document in memory.
*/
package jsonptrerror

import (
	"bytes"
	"encoding/json"
	"io"

	"github.com/dolmen-go/jsonptr"
)

// UnmarshalTypeError is an extension of encoding/json.UnmarshalTypeError
// that also includes the error location as a JSON Pointer (RFC 6901)
type UnmarshalTypeError struct {
	json.UnmarshalTypeError
	Pointer jsonptr.Pointer
}

func (e UnmarshalTypeError) Error() string {
	return e.Pointer.String() + ": cannot unmarshal " + e.Value + " into Go value of type " + e.Type.String()
}

type decoder interface {
	Decode(interface{}) error
	UseNumber()
}

// Decoder is the same as encoding/json.Decoder, except Decode returns
// our UnmarshalTypeError (providing a JSON Pointer) instead of encoding/json.UnmarshalTypeError.
type Decoder struct {
	decoder decoder
	input   bytes.Buffer
	err     error
}

func NewDecoder(r io.Reader) *Decoder {
	var d Decoder
	d.decoder = json.NewDecoder(io.TeeReader(r, &d.input))
	return &d
}

func (d *Decoder) UseNumber() {
	d.decoder.UseNumber()
}

const (
	tokenTopValue = iota
	tokenArrayValue
	tokenArrayComma
	tokenObjectKey
	tokenObjectColon
	tokenObjectValue
	tokenObjectComma
)

func (d *Decoder) Decode(v interface{}) error {
	if d.err != nil {
		return d.err
	}
	d.err = d.decoder.Decode(v)
	if err, ok := d.err.(*json.UnmarshalTypeError); ok {
		offset := int(err.Offset)
		input := d.input.Bytes()
		i := 0
		type elem struct {
			container byte
			property  []byte
			index     int
		}
		var elemStack []elem
		var expectKey bool
		for {
			if i == offset {
				ptr := jsonptr.Pointer{}
				for _, e := range elemStack {
					switch e.container {
					case '{':
						var name string
						json.Unmarshal(e.property, &name)
						ptr.Property(name)
					case '[':
						ptr.Index(e.index)
					}
				}
				d.err = &UnmarshalTypeError{*err, ptr}
				break
			}
			switch input[i] {
			case '{':
				elemStack = append(elemStack, elem{container: '{'})
				expectKey = true
			case '[':
				elemStack = append(elemStack, elem{container: '[', index: 0})
			case '}', ']':
				elemStack = elemStack[:len(elemStack)-1]
			case '"':
				j := i
			str:
				for {
					i++
					switch input[i] {
					case '\\':
						i++
					case '"':
						break str
					}
				}
				if expectKey {
					elemStack[len(elemStack)-1].property = input[j : i+1]
					expectKey = false
				}
			case ',':
				if elemStack[len(elemStack)-1].container == '{' {
					expectKey = true
				} else {
					elemStack[len(elemStack)-1].index++
				}
			}
			i++
		}
	}
	if d.err != nil {
		d.decoder = nil
		d.input = bytes.Buffer{}
	}
	return d.err
}
<|endoftext|>


<commit_before>package maxminddb

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"math/big"
	"reflect"
)

type decoder struct {
	buffer      []byte
	pointerBase uint
}

func (d *decoder) decodeSlice(size uint, offset uint, result reflect.Value) (uint, error) {
	result.Set(reflect.MakeSlice(result.Type(), int(size), int(size)))
	result.SetCap(int(size))
	for i := 0; i < int(size); i++ {
		var err error
		offset, err = d.decode(offset, result.Index(i))
		if err != nil {
			return 0, err
		}
	}
	return offset, nil
}

func (d *decoder) decodeBool(size uint, offset uint) (bool, uint, error) {
	return size != 0, offset, nil
}

func (d *decoder) decodeBytes(size uint, offset uint) ([]byte, uint, error) {
	newOffset := offset + size
	return d.buffer[offset:newOffset], newOffset, nil
}

func (d *decoder) decodeFloat64(size uint, offset uint) (float64, uint, error) {
	newOffset := offset + size
	var dbl float64
	binary.Read(bytes.NewBuffer(d.buffer[offset:newOffset]), binary.BigEndian, &dbl)
	return dbl, newOffset, nil
}

func (d *decoder) decodeFloat32(size uint, offset uint) (float32, uint, error) {
	newOffset := offset + size
	var flt float32
	binary.Read(bytes.NewBuffer(d.buffer[offset:newOffset]), binary.BigEndian, &flt)
	return flt, newOffset, nil
}

func (d *decoder) decodeInt(size uint, offset uint) (int, uint, error) {
	newOffset := offset + size
	intBytes := d.buffer[offset:newOffset]
	if size != 4 {
		pad := make([]byte, 4-size)
		intBytes = append(pad, intBytes...)
	}

	var val int32
	binary.Read(bytes.NewBuffer(intBytes), binary.BigEndian, &val)

	return int(val), newOffset, nil
}

func (d *decoder) decodeMap(size uint, offset uint, result reflect.Value) (uint, error) {
	if result.IsNil() {
		result.Set(reflect.MakeMap(result.Type()))
	}

	for i := uint(0); i < size; i++ {
		key := reflect.New(result.Type().Key())

		var err error
		offset, err = d.decode(offset, key)
		if err != nil {
			return 0, err
		}

		value := reflect.New(result.Type().Elem())
		offset, err = d.decode(offset, value)
		if err != nil {
			return 0, err
		}
		result.SetMapIndex(key.Elem(), value.Elem())
	}
	return offset, nil
}

func (d *decoder) decodeStruct(size uint, offset uint, result reflect.Value) (uint, error) {

	resultType := result.Type()
	numFields := resultType.NumField()

	fields := make(map[string]reflect.Value)
	for i := 0; i < numFields; i++ {
		fieldType := resultType.Field(i)

		fieldName := fieldType.Name
		tag := fieldType.Tag.Get("maxminddb")
		if tag != "" {
			fieldName = tag
		}
		fields[fieldName] = result.Field(i)
	}

	for i := uint(0); i < size; i++ {

		var key string
		keyValue := reflect.ValueOf(&key)

		var err error
		offset, err = d.decode(offset, keyValue)
		if err != nil {
			return 0, err
		}
		field, ok := fields[key]
		if !ok {
			// XXX - Ideally we should not bother decoding values we skip.
			// This doesn't matter for the geoip2 reader as we want to decode
			// everything, but it may matter for other use cases that just
			// want one or two values quickly.
			var skip interface{}
			field = reflect.ValueOf(&skip)
		}
		offset, err = d.decode(offset, field)
		if err != nil {
			return 0, err
		}
	}
	return offset, nil
}

var pointerValueOffset = map[uint]uint{
	1: 0,
	2: 2048,
	3: 526336,
	4: 0,
}

func (d *decoder) decodePointer(size uint, offset uint, result reflect.Value) (uint, error) {
	pointerSize := ((size >> 3) & 0x3) + 1
	newOffset := offset + pointerSize
	pointerBytes := d.buffer[offset:newOffset]
	var packed []byte
	if pointerSize == 4 {
		packed = pointerBytes
	} else {
		packed = append([]byte{byte(size & 0x7)}, pointerBytes...)
	}
	unpacked := uint(uintFromBytes(packed))

	pointer := unpacked + d.pointerBase + pointerValueOffset[pointerSize]
	_, err := d.decode(pointer, result)
	return newOffset, err
}

func (d *decoder) decodeUint(size uint, offset uint) (uint64, uint, error) {
	newOffset := offset + size
	val := uintFromBytes(d.buffer[offset:newOffset])

	return val, newOffset, nil
}

func (d *decoder) decodeUint128(size uint, offset uint) (*big.Int, uint, error) {
	newOffset := offset + size
	val := new(big.Int)
	val.SetBytes(d.buffer[offset:newOffset])

	return val, newOffset, nil
}

func uintFromBytes(uintBytes []byte) uint64 {
	var val uint64
	for _, b := range uintBytes {
		val = (val << 8) | uint64(b)
	}
	return val
}

func (d *decoder) decodeString(size uint, offset uint) (string, uint, error) {
	newOffset := offset + size
	return string(d.buffer[offset:newOffset]), newOffset, nil
}

func (d *decoder) decode(offset uint, result reflect.Value) (uint, error) {
	newOffset := offset + 1
	ctrlByte := d.buffer[offset]

	typeNum := dataType(ctrlByte >> 5)
	// Extended type
	if typeNum == 0 {
		typeNum = dataType(d.buffer[newOffset] + 7)
		newOffset++
	}

	var size uint
	size, newOffset = d.sizeFromCtrlByte(ctrlByte, newOffset, typeNum)
	return d.decodeFromType(typeNum, size, newOffset, result)
}

type dataType int

const (
	_Extended dataType = iota
	_Pointer
	_String
	_Float64
	_Bytes
	_Uint16
	_Uint32
	_Map
	_Int32
	_Uint64
	_Uint128
	_Slice
	_Container
	_Marker
	_Bool
	_Float32
)

func (d *decoder) decodeFromType(dtype dataType, size uint, offset uint, result reflect.Value) (uint, error) {

	if result.Kind() == reflect.Ptr {
		result = reflect.Indirect(result)
	}

	switch dtype {
	case _Pointer:
		return d.decodePointer(size, offset, result)
	case _Bool:
		value, newOffset, err := d.decodeBool(size, offset)
		if err != nil {
			return 0, err
		}
		switch result.Kind() {
		default:
			return newOffset, fmt.Errorf("trying to unmarshal %v into %v", value, result.Type())
		case reflect.Bool:
			result.SetBool(value)
			return newOffset, nil
		case reflect.Interface:
			result.Set(reflect.ValueOf(value))
			return newOffset, nil
		}
	case _Int32:
		value, newOffset, err := d.decodeInt(size, offset)
		if err != nil {
			return 0, err
		}

		switch result.Kind() {
		default:
			return newOffset, fmt.Errorf("trying to unmarshal %v into %v", value, result.Type())
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
			result.SetInt(int64(value))
			return newOffset, nil
		case reflect.Interface:
			result.Set(reflect.ValueOf(value))
			return newOffset, nil
		}
	case _Uint16, _Uint32, _Uint64:
		value, newOffset, err := d.decodeUint(size, offset)
		if err != nil {
			return 0, err
		}

		switch result.Kind() {
		default:
			return newOffset, fmt.Errorf("trying to unmarshal %v into %v", value, result.Type())
		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
			result.SetUint(value)
			return newOffset, nil
		case reflect.Interface:
			result.Set(reflect.ValueOf(value))
			return newOffset, nil
		}
	case _Uint128:
		value, newOffset, err := d.decodeUint128(size, offset)
		if err != nil {
			return 0, err
		}

		// XXX - this should allow *big.Int rather than just bigInt
		// Currently this is reported as invalid
		switch result.Kind() {
		default:
			return newOffset, fmt.Errorf("trying to unmarshal %v into %v", value, result.Type())
		case reflect.Struct:
			result.Set(reflect.ValueOf(*value))
			return newOffset, nil
		case reflect.Interface, reflect.Ptr:
			result.Set(reflect.ValueOf(value))
			return newOffset, nil
		}
	case _Float32:
		value, newOffset, err := d.decodeFloat32(size, offset)
		if err != nil {
			return 0, err
		}

		switch result.Kind() {
		default:
			return newOffset, fmt.Errorf("trying to unmarshal %v into %v", value, result.Type())
		case reflect.Float32, reflect.Float64:
			result.SetFloat(float64(value))
			return newOffset, nil
		case reflect.Interface:
			result.Set(reflect.ValueOf(value))
			return newOffset, nil
		}
	case _Float64:
		value, newOffset, err := d.decodeFloat64(size, offset)
		if err != nil {
			return 0, err
		}
		switch result.Kind() {
		default:
			return newOffset, fmt.Errorf("trying to unmarshal %v into %v", value, result.Type())
		case reflect.Float32, reflect.Float64:
			result.SetFloat(value)
			return newOffset, nil
		case reflect.Interface:
			result.Set(reflect.ValueOf(value))
			return newOffset, nil
		}
	case _String:
		value, newOffset, err := d.decodeString(size, offset)

		if err != nil {
			return 0, err
		}
		switch result.Kind() {
		default:
			return newOffset, fmt.Errorf("trying to unmarshal %v into %v", value, result.Type())
		case reflect.String:
			result.SetString(value)
			return newOffset, nil
		case reflect.Interface:
			result.Set(reflect.ValueOf(value))
			return newOffset, nil
		}
	case _Bytes:
		value, newOffset, err := d.decodeBytes(size, offset)
		if err != nil {
			return 0, err
		}
		switch result.Kind() {
		default:
			return newOffset, fmt.Errorf("trying to unmarshal %v into %v", value, result.Type())
		case reflect.Slice:
			result.SetBytes(value)
			return newOffset, nil
		case reflect.Interface:
			result.Set(reflect.ValueOf(value))
			return newOffset, nil
		}
	case _Slice:
		switch result.Kind() {
		default:
			return 0, fmt.Errorf("trying to unmarshal an array into %v", result.Type())
		case reflect.Slice:
			return d.decodeSlice(size, offset, result)
		case reflect.Interface:
			a := []interface{}{}
			rv := reflect.ValueOf(&a).Elem()
			newOffset, err := d.decodeSlice(size, offset, rv)
			result.Set(rv)
			return newOffset, err
		}
	case _Map:
		switch result.Kind() {
		default:
			return 0, fmt.Errorf("trying to unmarshal a map into %v", result.Type())
		case reflect.Struct:
			return d.decodeStruct(size, offset, result)
		case reflect.Map:
			return d.decodeMap(size, offset, result)
		case reflect.Interface:
			rv := reflect.ValueOf(make(map[string]interface{}))
			newOffset, err := d.decodeMap(size, offset, rv)
			result.Set(rv)
			return newOffset, err
		}
	default:
		return 0, fmt.Errorf("unknown type: %d", dtype)
	}
}

func (d *decoder) sizeFromCtrlByte(ctrlByte byte, offset uint, typeNum dataType) (uint, uint) {
	size := uint(ctrlByte & 0x1f)
	if typeNum == _Extended {
		return size, offset
	}

	var bytesToRead uint
	if size > 28 {
		bytesToRead = size - 28
	}

	newOffset := offset + bytesToRead
	sizeBytes := d.buffer[offset:newOffset]

	switch {
	case size == 29:
		size = 29 + uint(sizeBytes[0])
	case size == 30:
		size = 285 + uint(uintFromBytes(sizeBytes))
	case size > 30:
		size = uint(uintFromBytes(sizeBytes)) + 65821
	}
	return size, newOffset
}
<commit_msg>Remove unnecessary SetCap, which is not supported by Go 1.1<commit_after>package maxminddb

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"math/big"
	"reflect"
)

type decoder struct {
	buffer      []byte
	pointerBase uint
}

func (d *decoder) decodeSlice(size uint, offset uint, result reflect.Value) (uint, error) {
	result.Set(reflect.MakeSlice(result.Type(), int(size), int(size)))
	for i := 0; i < int(size); i++ {
		var err error
		offset, err = d.decode(offset, result.Index(i))
		if err != nil {
			return 0, err
		}
	}
	return offset, nil
}

func (d *decoder) decodeBool(size uint, offset uint) (bool, uint, error) {
	return size != 0, offset, nil
}

func (d *decoder) decodeBytes(size uint, offset uint) ([]byte, uint, error) {
	newOffset := offset + size
	return d.buffer[offset:newOffset], newOffset, nil
}

func (d *decoder) decodeFloat64(size uint, offset uint) (float64, uint, error) {
	newOffset := offset + size
	var dbl float64
	binary.Read(bytes.NewBuffer(d.buffer[offset:newOffset]), binary.BigEndian, &dbl)
	return dbl, newOffset, nil
}

func (d *decoder) decodeFloat32(size uint, offset uint) (float32, uint, error) {
	newOffset := offset + size
	var flt float32
	binary.Read(bytes.NewBuffer(d.buffer[offset:newOffset]), binary.BigEndian, &flt)
	return flt, newOffset, nil
}

func (d *decoder) decodeInt(size uint, offset uint) (int, uint, error) {
	newOffset := offset + size
	intBytes := d.buffer[offset:newOffset]
	if size != 4 {
		pad := make([]byte, 4-size)
		intBytes = append(pad, intBytes...)
	}

	var val int32
	binary.Read(bytes.NewBuffer(intBytes), binary.BigEndian, &val)

	return int(val), newOffset, nil
}

func (d *decoder) decodeMap(size uint, offset uint, result reflect.Value) (uint, error) {
	if result.IsNil() {
		result.Set(reflect.MakeMap(result.Type()))
	}

	for i := uint(0); i < size; i++ {
		key := reflect.New(result.Type().Key())

		var err error
		offset, err = d.decode(offset, key)
		if err != nil {
			return 0, err
		}

		value := reflect.New(result.Type().Elem())
		offset, err = d.decode(offset, value)
		if err != nil {
			return 0, err
		}
		result.SetMapIndex(key.Elem(), value.Elem())
	}
	return offset, nil
}

func (d *decoder) decodeStruct(size uint, offset uint, result reflect.Value) (uint, error) {

	resultType := result.Type()
	numFields := resultType.NumField()

	fields := make(map[string]reflect.Value)
	for i := 0; i < numFields; i++ {
		fieldType := resultType.Field(i)

		fieldName := fieldType.Name
		tag := fieldType.Tag.Get("maxminddb")
		if tag != "" {
			fieldName = tag
		}
		fields[fieldName] = result.Field(i)
	}

	for i := uint(0); i < size; i++ {

		var key string
		keyValue := reflect.ValueOf(&key)

		var err error
		offset, err = d.decode(offset, keyValue)
		if err != nil {
			return 0, err
		}
		field, ok := fields[key]
		if !ok {
			// XXX - Ideally we should not bother decoding values we skip.
			// This doesn't matter for the geoip2 reader as we want to decode
			// everything, but it may matter for other use cases that just
			// want one or two values quickly.
			var skip interface{}
			field = reflect.ValueOf(&skip)
		}
		offset, err = d.decode(offset, field)
		if err != nil {
			return 0, err
		}
	}
	return offset, nil
}

var pointerValueOffset = map[uint]uint{
	1: 0,
	2: 2048,
	3: 526336,
	4: 0,
}

func (d *decoder) decodePointer(size uint, offset uint, result reflect.Value) (uint, error) {
	pointerSize := ((size >> 3) & 0x3) + 1
	newOffset := offset + pointerSize
	pointerBytes := d.buffer[offset:newOffset]
	var packed []byte
	if pointerSize == 4 {
		packed = pointerBytes
	} else {
		packed = append([]byte{byte(size & 0x7)}, pointerBytes...)
	}
	unpacked := uint(uintFromBytes(packed))

	pointer := unpacked + d.pointerBase + pointerValueOffset[pointerSize]
	_, err := d.decode(pointer, result)
	return newOffset, err
}

func (d *decoder) decodeUint(size uint, offset uint) (uint64, uint, error) {
	newOffset := offset + size
	val := uintFromBytes(d.buffer[offset:newOffset])

	return val, newOffset, nil
}

func (d *decoder) decodeUint128(size uint, offset uint) (*big.Int, uint, error) {
	newOffset := offset + size
	val := new(big.Int)
	val.SetBytes(d.buffer[offset:newOffset])

	return val, newOffset, nil
}

func uintFromBytes(uintBytes []byte) uint64 {
	var val uint64
	for _, b := range uintBytes {
		val = (val << 8) | uint64(b)
	}
	return val
}

func (d *decoder) decodeString(size uint, offset uint) (string, uint, error) {
	newOffset := offset + size
	return string(d.buffer[offset:newOffset]), newOffset, nil
}

func (d *decoder) decode(offset uint, result reflect.Value) (uint, error) {
	newOffset := offset + 1
	ctrlByte := d.buffer[offset]

	typeNum := dataType(ctrlByte >> 5)
	// Extended type
	if typeNum == 0 {
		typeNum = dataType(d.buffer[newOffset] + 7)
		newOffset++
	}

	var size uint
	size, newOffset = d.sizeFromCtrlByte(ctrlByte, newOffset, typeNum)
	return d.decodeFromType(typeNum, size, newOffset, result)
}

type dataType int

const (
	_Extended dataType = iota
	_Pointer
	_String
	_Float64
	_Bytes
	_Uint16
	_Uint32
	_Map
	_Int32
	_Uint64
	_Uint128
	_Slice
	_Container
	_Marker
	_Bool
	_Float32
)

func (d *decoder) decodeFromType(dtype dataType, size uint, offset uint, result reflect.Value) (uint, error) {

	if result.Kind() == reflect.Ptr {
		result = reflect.Indirect(result)
	}

	switch dtype {
	case _Pointer:
		return d.decodePointer(size, offset, result)
	case _Bool:
		value, newOffset, err := d.decodeBool(size, offset)
		if err != nil {
			return 0, err
		}
		switch result.Kind() {
		default:
			return newOffset, fmt.Errorf("trying to unmarshal %v into %v", value, result.Type())
		case reflect.Bool:
			result.SetBool(value)
			return newOffset, nil
		case reflect.Interface:
			result.Set(reflect.ValueOf(value))
			return newOffset, nil
		}
	case _Int32:
		value, newOffset, err := d.decodeInt(size, offset)
		if err != nil {
			return 0, err
		}

		switch result.Kind() {
		default:
			return newOffset, fmt.Errorf("trying to unmarshal %v into %v", value, result.Type())
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
			result.SetInt(int64(value))
			return newOffset, nil
		case reflect.Interface:
			result.Set(reflect.ValueOf(value))
			return newOffset, nil
		}
	case _Uint16, _Uint32, _Uint64:
		value, newOffset, err := d.decodeUint(size, offset)
		if err != nil {
			return 0, err
		}

		switch result.Kind() {
		default:
			return newOffset, fmt.Errorf("trying to unmarshal %v into %v", value, result.Type())
		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
			result.SetUint(value)
			return newOffset, nil
		case reflect.Interface:
			result.Set(reflect.ValueOf(value))
			return newOffset, nil
		}
	case _Uint128:
		value, newOffset, err := d.decodeUint128(size, offset)
		if err != nil {
			return 0, err
		}

		// XXX - this should allow *big.Int rather than just bigInt
		// Currently this is reported as invalid
		switch result.Kind() {
		default:
			return newOffset, fmt.Errorf("trying to unmarshal %v into %v", value, result.Type())
		case reflect.Struct:
			result.Set(reflect.ValueOf(*value))
			return newOffset, nil
		case reflect.Interface, reflect.Ptr:
			result.Set(reflect.ValueOf(value))
			return newOffset, nil
		}
	case _Float32:
		value, newOffset, err := d.decodeFloat32(size, offset)
		if err != nil {
			return 0, err
		}

		switch result.Kind() {
		default:
			return newOffset, fmt.Errorf("trying to unmarshal %v into %v", value, result.Type())
		case reflect.Float32, reflect.Float64:
			result.SetFloat(float64(value))
			return newOffset, nil
		case reflect.Interface:
			result.Set(reflect.ValueOf(value))
			return newOffset, nil
		}
	case _Float64:
		value, newOffset, err := d.decodeFloat64(size, offset)
		if err != nil {
			return 0, err
		}
		switch result.Kind() {
		default:
			return newOffset, fmt.Errorf("trying to unmarshal %v into %v", value, result.Type())
		case reflect.Float32, reflect.Float64:
			result.SetFloat(value)
			return newOffset, nil
		case reflect.Interface:
			result.Set(reflect.ValueOf(value))
			return newOffset, nil
		}
	case _String:
		value, newOffset, err := d.decodeString(size, offset)

		if err != nil {
			return 0, err
		}
		switch result.Kind() {
		default:
			return newOffset, fmt.Errorf("trying to unmarshal %v into %v", value, result.Type())
		case reflect.String:
			result.SetString(value)
			return newOffset, nil
		case reflect.Interface:
			result.Set(reflect.ValueOf(value))
			return newOffset, nil
		}
	case _Bytes:
		value, newOffset, err := d.decodeBytes(size, offset)
		if err != nil {
			return 0, err
		}
		switch result.Kind() {
		default:
			return newOffset, fmt.Errorf("trying to unmarshal %v into %v", value, result.Type())
		case reflect.Slice:
			result.SetBytes(value)
			return newOffset, nil
		case reflect.Interface:
			result.Set(reflect.ValueOf(value))
			return newOffset, nil
		}
	case _Slice:
		switch result.Kind() {
		default:
			return 0, fmt.Errorf("trying to unmarshal an array into %v", result.Type())
		case reflect.Slice:
			return d.decodeSlice(size, offset, result)
		case reflect.Interface:
			a := []interface{}{}
			rv := reflect.ValueOf(&a).Elem()
			newOffset, err := d.decodeSlice(size, offset, rv)
			result.Set(rv)
			return newOffset, err
		}
	case _Map:
		switch result.Kind() {
		default:
			return 0, fmt.Errorf("trying to unmarshal a map into %v", result.Type())
		case reflect.Struct:
			return d.decodeStruct(size, offset, result)
		case reflect.Map:
			return d.decodeMap(size, offset, result)
		case reflect.Interface:
			rv := reflect.ValueOf(make(map[string]interface{}))
			newOffset, err := d.decodeMap(size, offset, rv)
			result.Set(rv)
			return newOffset, err
		}
	default:
		return 0, fmt.Errorf("unknown type: %d", dtype)
	}
}

func (d *decoder) sizeFromCtrlByte(ctrlByte byte, offset uint, typeNum dataType) (uint, uint) {
	size := uint(ctrlByte & 0x1f)
	if typeNum == _Extended {
		return size, offset
	}

	var bytesToRead uint
	if size > 28 {
		bytesToRead = size - 28
	}

	newOffset := offset + bytesToRead
	sizeBytes := d.buffer[offset:newOffset]

	switch {
	case size == 29:
		size = 29 + uint(sizeBytes[0])
	case size == 30:
		size = 285 + uint(uintFromBytes(sizeBytes))
	case size > 30:
		size = uint(uintFromBytes(sizeBytes)) + 65821
	}
	return size, newOffset
}
<|endoftext|>


<commit_before>package unhtml

import (
	"encoding"
	"fmt"
	"io"
	"reflect"
	"strconv"
	"strings"

	"gopkg.in/xmlpath.v1"
)

// Set to true to enable debugging prints sprinkled in the code
// ALWAYS set this to false before pushing
const debug = false

type NoNodesAvailable string

func (e NoNodesAvailable) Error() string {
	return "No nodes found for path: " + string(e)
}

// Error returned if there was an issue with type compatibility
type UnmarshalTypeError struct {
	Value string
	Type  reflect.Type
}

func (e *UnmarshalTypeError) Error() string {
	return "unhtml: cannot unmarshal " + e.Value + " into Go value of type " + e.Type.String()
}

// Error returned if invalid input was given
type InvalidUnmarshalError struct {
	Type reflect.Type
}

func (e *InvalidUnmarshalError) Error() string {
	if e.Type == nil {
		return "unhtml: Unmarshal(nil)"
	}

	if e.Type.Kind() != reflect.Ptr {
		return "unhtml: Unmarshal(non-pointer " + e.Type.String() + ")"
	}

	return "unhtml: Unmarshal(nil " + e.Type.String() + ")"
}

// Unmarshaler is an interface that can be implemented to
// receive the raw resulting node to unmarshal into the
// type.
//
// This receives a []byte of the contents of the current node
// (which might, or might not be HTML depending on node).
type Unmarshaler interface {
	UnmarshalHTML([]byte) error
}

// Decoder
type Decoder struct {
	root *xmlpath.Node
}

// NewDecoder returns a new Decoder by using the contents of the
// io.Reader as HTML input. The io.Reader is consumed whole and
// contents parsed before this function returns.
//
// An error return means something went wrong parsing the HTML.
func NewDecoder(r io.Reader) (*Decoder, error) {
	root, err := xmlpath.ParseHTML(r)

	if err != nil {
		return nil, err
	}

	return &Decoder{root: root}, nil
}

// Unmarshal tries to fill the value given with the input previously
// given to the Decoder. See `unhtml.Unmarshal` for full docs.
func (d *Decoder) Unmarshal(res interface{}) error {
	st := &state{}

	st.unmarshal(d.root, reflect.ValueOf(res))

	return st.firstError
}

// UnmarshalRelative unmarshals from the node depicted by the path
// given. This allows you to move the root node before unmarshalling.
//
// UnmarshalRelative can return errors from the following pieces:
// - unhtml errors
// - xmlpath path compiling
// - encoding.TextUnmarshaler
// - unhtml.Unmarshaler
func (d *Decoder) UnmarshalRelative(path string, res interface{}) error {
	// Compile the path before doing anything else
	xpath, err := xmlpath.Compile(path)
	if err != nil {
		return err
	}

	var nodes []*xmlpath.Node
	var st = &state{}

	um, utm, v := indirect(reflect.ValueOf(res))

	isSlice := v.Kind() == reflect.Slice || v.Kind() == reflect.Array

	for iter := xpath.Iter(d.root); iter.Next(); {
		nodes = append(nodes, iter.Node())

		// Only use the first node we find if `res` is not a slice or array
		if !isSlice {
			break
		}
	}

	// No results were found
	if len(nodes) == 0 {
		return NoNodesAvailable(path)
	}

	node := nodes[0]
	if um != nil {
		return um.UnmarshalHTML(node.Bytes())
	} else if utm != nil {
		return utm.UnmarshalText(node.Bytes())
	} else if !isSlice {
		// no-interface, no-slice value, unmarshal normally
		st.unmarshal(node, v)
		return st.firstError
	}

	// Special casing []byte and []rune to fill as-is
	sliceType := v.Type().Elem().Kind()
	if sliceType == reflect.Uint8 || sliceType == reflect.Int32 {
		st.unmarshal(node, v)
		return st.firstError
	}

	// Multi-node, with slice or array argument, fill it with multinode
	st.multinode(nodes, v)
	return st.firstError
}

// state is used to keep track of errors that occurred, we don't want
// to return early with an error and leave the unmarshalling in an incomplete
// state.
type state struct {
	firstError error
}

// saveError saves the first error given to saveError and discards all others
func (d *state) saveError(e error) {
	if e != nil && d.firstError == nil {
		d.firstError = e
	}
}

func (d *state) multinode(nodes []*xmlpath.Node, value reflect.Value) {
	switch value.Kind() {
	case reflect.Array, reflect.Slice:
	default:
		if debug {
			fmt.Println("unsupported multinode: ", nodes)
		}
		err := &UnmarshalTypeError{"Multinode result", value.Type()}

		d.saveError(err)
		return
	}

	isSlice := value.Kind() == reflect.Slice

	for i, node := range nodes {
		if isSlice {
			if i >= value.Cap() {
				newcap := value.Cap() + value.Cap()/2
				if newcap < 4 {
					newcap = 4
				}
				newv := reflect.MakeSlice(value.Type(), value.Len(), newcap)
				reflect.Copy(newv, value)
				value.Set(newv)
			}
			if i >= value.Len() {
				value.SetLen(i + 1)
			}
		}

		if i < value.Len() {
			d.unmarshal(node, value.Index(i))
		}
	}
}

func (d *state) unmarshal(root *xmlpath.Node, rv reflect.Value) {
	m, tm, value := indirect(rv)

	if m != nil {
		err := m.UnmarshalHTML(root.Bytes())
		d.saveError(err)
		return
	} else if tm != nil {
		err := tm.UnmarshalText(root.Bytes())
		d.saveError(err)
		return
	}

	if !rv.IsValid() {
		err := &InvalidUnmarshalError{nil}
		d.saveError(err)
		return
	}

	s := root.String()
	switch value.Kind() {
	case reflect.Struct:
		d.unmarshalStruct(root, value)
	case reflect.Array:
	case reflect.Slice:
		// Short-path for []byte and []rune
		t := value.Type().Elem().Kind()
		if t == reflect.Uint8 {
			value.Set(reflect.ValueOf(root.Bytes()))
		} else if t == reflect.Int32 {
			value.Set(reflect.ValueOf([]rune(s)))
		}
	default:
		if debug {
			fmt.Println("Unsupported type used: ", value.Kind())
		}
		// TODO: Change behavior on unsupported type?
	case reflect.Interface:
		if value.NumMethod() != 0 {
			err := &UnmarshalTypeError{s, value.Type()}

			d.saveError(err)
			break
		}

		b := root.Bytes()

		value.Set(reflect.ValueOf(b))
	case reflect.String:
		value.SetString(s)
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		// Trim off any whitespace, since they're common in formatted HTML
		s = strings.TrimSpace(s)
		n, err := strconv.ParseInt(s, 10, 64)
		if err != nil || value.OverflowInt(n) {
			d.saveError(err)
			break
		}
		value.SetInt(n)
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
		// Trim off any whitespace, since they're common in formatted HTML
		s = strings.TrimSpace(s)
		n, err := strconv.ParseUint(s, 10, 64)
		if err != nil || value.OverflowUint(n) {
			d.saveError(err)
			break
		}
		value.SetUint(n)
	case reflect.Float32, reflect.Float64:
		// Trim off any whitespace, since they're common in formatted HTML
		s = strings.TrimSpace(s)
		n, err := strconv.ParseFloat(s, value.Type().Bits())
		if err != nil || value.OverflowFloat(n) {
			d.saveError(err)
			break
		}
		value.SetFloat(n)

	}
}

func (d *state) unmarshalStruct(root *xmlpath.Node, value reflect.Value) {
	valueType := value.Type()

	if value.Kind() != reflect.Struct {
		err := &InvalidUnmarshalError{
			Type: value.Type(),
		}

		d.saveError(err)
		return
	}

	for i := 0; i < value.NumField(); i++ {
		field := value.Field(i)
		structField := valueType.Field(i)

		// Find the struct tag if any
		path := structField.Tag.Get("unhtml")

		if path == "" {
			// Skip fields with no tag, since we require an xpath
			if debug {
				fmt.Println("Skipping field due to lack of xpath: ", field)
			}
			continue
		}

		if !field.CanSet() {
			if debug {
				fmt.Println("Skipping field due to unsettability: ", field)
			}
			// TODO: Some way to feedback to the user
			continue
		}

		var (
			nodes = make([]*xmlpath.Node, 0, 12)
			xpath = xmlpath.MustCompile(path)
		)

		for iter := xpath.Iter(root); iter.Next(); {
			node := iter.Node()

			nodes = append(nodes, node)
		}

		if debug {
			fmt.Printf("Executed %s with %d resulting nodes\n", path, len(nodes))
		}

		if len(nodes) > 1 {
			d.multinode(nodes, field)
			continue
		} else if len(nodes) < 1 {
			if debug {
				fmt.Println("Xpath did not match any nodes: ", path)
			}
			continue
		}

		d.unmarshal(nodes[0], field)
	}
}

// indirect walks down v allocating pointers as needed until it gets to a non-pointer
//
// indirect original can be found in the stdlib encoding/json, credit to Go authors
func indirect(v reflect.Value) (Unmarshaler, encoding.TextUnmarshaler, reflect.Value) {
	if v.Kind() != reflect.Ptr && v.Type().Name() != "" && v.CanAddr() {
		v = v.Addr()
	}

	for {
		if v.Kind() == reflect.Interface && !v.IsNil() {
			e := v.Elem()
			if e.Kind() == reflect.Ptr && !e.IsNil() {
				v = e
				continue
			}
		}

		if v.Kind() != reflect.Ptr {
			break
		}

		if v.IsNil() {
			v.Set(reflect.New(v.Type().Elem()))
		}
		if v.Type().NumMethod() > 0 {
			if u, ok := v.Interface().(Unmarshaler); ok {
				return u, nil, reflect.Value{}
			}

			if u, ok := v.Interface().(encoding.TextUnmarshaler); ok {
				return nil, u, reflect.Value{}
			}
		}
		v = v.Elem()
	}
	return nil, nil, v
}
<commit_msg>Updated documentation, added pkg-level Unmarshal<commit_after>package unhtml

import (
	"encoding"
	"fmt"
	"io"
	"reflect"
	"strconv"
	"strings"

	"gopkg.in/xmlpath.v1"
)

// Set to true to enable debugging prints sprinkled in the code
// ALWAYS set this to false before pushing
const debug = false

// NoNodesAvailable is returned when an xpath to *Relative functions
// are unable to find any matching nodes.
type NoNodesAvailable string

func (e NoNodesAvailable) Error() string {
	return "No nodes found for path: " + string(e)
}

// Error returned if there was an issue with type compatibility
type UnmarshalTypeError struct {
	Value string
	Type  reflect.Type
}

func (e *UnmarshalTypeError) Error() string {
	return "unhtml: cannot unmarshal " + e.Value + " into Go value of type " + e.Type.String()
}

// Error returned if invalid input was given
type InvalidUnmarshalError struct {
	Type reflect.Type
}

func (e *InvalidUnmarshalError) Error() string {
	if e.Type == nil {
		return "unhtml: Unmarshal(nil)"
	}

	if e.Type.Kind() != reflect.Ptr {
		return "unhtml: Unmarshal(non-pointer " + e.Type.String() + ")"
	}

	return "unhtml: Unmarshal(nil " + e.Type.String() + ")"
}

// Unmarshaler is an interface that can be implemented to
// receive the raw resulting node to unmarshal into the
// type.
//
// This receives a []byte of all text nodes found concatted
// together in the current node.
type Unmarshaler interface {
	UnmarshalHTML([]byte) error
}

// Unmarshal parses the HTML in reader and extracts data from it, results
// are stored in the value pointed to by result.
//
// rootpath can be given to move the root node before unmarshalling, pass
// an empty string to omit moving the root node.
//
// Unmarshal can store values into the following types:
//
// 	string, []byte, []rune
// 	Any size unsigned integer and signed integer
// 	float32, float64
// 	An Unmarshaller
// 	An encoding.TextUnmarshaller
// 	Struct fields with an `unhtml` tag
// 	Slices and arrays containing any of the above types
//
// Any xpaths intended for Unmarshalling need to be compatible with the supported
// features gopkg.in/xmlpath.v1, see http://godoc.org/gopkg.in/xmlpath.v1 for
// documentation on supported features.
func Unmarshal(r io.Reader, result interface{}, rootpath string) error {
	d, err := NewDecoder(r)

	if err != nil {
		return err
	}

	if rootpath == "" {
		return d.Unmarshal(result)
	} else {
		return d.UnmarshalRelative(rootpath, result)
	}
}

type Decoder struct {
	root *xmlpath.Node
}

// NewDecoder returns a new Decoder by using the contents of the
// io.Reader as HTML input. The io.Reader is consumed whole and
// contents parsed before this function returns.
//
// An error return means something went wrong parsing the HTML.
func NewDecoder(r io.Reader) (*Decoder, error) {
	root, err := xmlpath.ParseHTML(r)

	if err != nil {
		return nil, err
	}

	return &Decoder{root: root}, nil
}

// Unmarshal tries to fill the value given with the input previously
// given to the Decoder.
//
// Unmarshal only takes a struct as result type, use UnmarshalRelative
// for other types.
func (d *Decoder) Unmarshal(result interface{}) error {
	st := &state{}

	st.unmarshal(d.root, reflect.ValueOf(result))

	return st.firstError
}

// UnmarshalRelative unmarshals from the node depicted by the path
// given. This allows you to move the root node before unmarshalling.
//
// UnmarshalRelative can return the following errors:
// - any unhtml errors
// - xmlpath path compiling
// - encoding.TextUnmarshaler
// - unhtml.Unmarshaler
func (d *Decoder) UnmarshalRelative(path string, res interface{}) error {
	// Compile the path before doing anything else
	xpath, err := xmlpath.Compile(path)
	if err != nil {
		return err
	}

	var nodes []*xmlpath.Node
	var st = &state{}

	um, utm, v := indirect(reflect.ValueOf(res))

	isSlice := v.Kind() == reflect.Slice || v.Kind() == reflect.Array

	for iter := xpath.Iter(d.root); iter.Next(); {
		nodes = append(nodes, iter.Node())

		// Only use the first node we find if `res` is not a slice or array
		if !isSlice {
			break
		}
	}

	// No results were found
	if len(nodes) == 0 {
		return NoNodesAvailable(path)
	}

	node := nodes[0]
	if um != nil {
		return um.UnmarshalHTML(node.Bytes())
	} else if utm != nil {
		return utm.UnmarshalText(node.Bytes())
	} else if !isSlice {
		// no-interface, no-slice value, unmarshal normally
		st.unmarshal(node, v)
		return st.firstError
	}

	// Special casing []byte and []rune to fill as-is
	sliceType := v.Type().Elem().Kind()
	if sliceType == reflect.Uint8 || sliceType == reflect.Int32 {
		st.unmarshal(node, v)
		return st.firstError
	}

	// Multi-node, with slice or array argument, fill it with multinode
	st.multinode(nodes, v)
	return st.firstError
}

// state is used to keep track of errors that occurred, we don't want
// to return early with an error and leave the unmarshalling in an incomplete
// state.
type state struct {
	firstError error
}

// saveError saves the first error given to saveError and discards all others
func (d *state) saveError(e error) {
	if e != nil && d.firstError == nil {
		d.firstError = e
	}
}

// multinode handles slices and arrays
func (d *state) multinode(nodes []*xmlpath.Node, value reflect.Value) {
	switch value.Kind() {
	case reflect.Array, reflect.Slice:
	default:
		if debug {
			fmt.Println("unsupported multinode: ", nodes)
		}
		err := &UnmarshalTypeError{"multinode result", value.Type()}

		d.saveError(err)
		return
	}

	isSlice := value.Kind() == reflect.Slice

	for i, node := range nodes {
		if isSlice {
			if i >= value.Cap() {
				newcap := value.Cap() + value.Cap()/2
				if newcap < 4 {
					newcap = 4
				}
				newv := reflect.MakeSlice(value.Type(), value.Len(), newcap)
				reflect.Copy(newv, value)
				value.Set(newv)
			}
			if i >= value.Len() {
				value.SetLen(i + 1)
			}
		}

		if i < value.Len() {
			d.unmarshal(node, value.Index(i))
		}
	}
}

// unmarshal handles any non-slice, non-array and non-struct values
func (d *state) unmarshal(root *xmlpath.Node, rv reflect.Value) {
	m, tm, value := indirect(rv)

	if m != nil {
		err := m.UnmarshalHTML(root.Bytes())
		d.saveError(err)
		return
	} else if tm != nil {
		err := tm.UnmarshalText(root.Bytes())
		d.saveError(err)
		return
	}

	if !rv.IsValid() {
		err := &InvalidUnmarshalError{nil}
		d.saveError(err)
		return
	}

	s := root.String()
	switch value.Kind() {
	case reflect.Struct:
		d.unmarshalStruct(root, value)
	case reflect.Array:
	case reflect.Slice:
		// Short-path for []byte and []rune
		t := value.Type().Elem().Kind()
		if t == reflect.Uint8 {
			value.Set(reflect.ValueOf(root.Bytes()))
		} else if t == reflect.Int32 {
			value.Set(reflect.ValueOf([]rune(s)))
		}
	default:
		if debug {
			fmt.Println("Unsupported type used: ", value.Kind())
		}
		// TODO: Change behavior on unsupported type?
	case reflect.Interface:
		if value.NumMethod() != 0 {
			err := &UnmarshalTypeError{s, value.Type()}

			d.saveError(err)
			break
		}

		b := root.Bytes()

		value.Set(reflect.ValueOf(b))
	case reflect.String:
		value.SetString(s)
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		// Trim off any whitespace, since they're common in formatted HTML
		s = strings.TrimSpace(s)
		n, err := strconv.ParseInt(s, 10, 64)
		if err != nil || value.OverflowInt(n) {
			d.saveError(err)
			break
		}
		value.SetInt(n)
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
		// Trim off any whitespace, since they're common in formatted HTML
		s = strings.TrimSpace(s)
		n, err := strconv.ParseUint(s, 10, 64)
		if err != nil || value.OverflowUint(n) {
			d.saveError(err)
			break
		}
		value.SetUint(n)
	case reflect.Float32, reflect.Float64:
		// Trim off any whitespace, since they're common in formatted HTML
		s = strings.TrimSpace(s)
		n, err := strconv.ParseFloat(s, value.Type().Bits())
		if err != nil || value.OverflowFloat(n) {
			d.saveError(err)
			break
		}
		value.SetFloat(n)

	}
}

// unmarshalStruct handles struct values
func (d *state) unmarshalStruct(root *xmlpath.Node, value reflect.Value) {
	valueType := value.Type()

	if value.Kind() != reflect.Struct {
		err := &InvalidUnmarshalError{
			Type: value.Type(),
		}

		d.saveError(err)
		return
	}

	for i := 0; i < value.NumField(); i++ {
		field := value.Field(i)
		structField := valueType.Field(i)

		// Find the struct tag if any
		path := structField.Tag.Get("unhtml")

		if path == "" {
			// Skip fields with no tag, since we require an xpath
			if debug {
				fmt.Println("Skipping field due to lack of xpath: ", field)
			}
			continue
		}

		if !field.CanSet() {
			if debug {
				fmt.Println("Skipping field due to unsettability: ", field)
			}
			// TODO: Some way to feedback to the user
			continue
		}

		var (
			nodes = make([]*xmlpath.Node, 0, 12)
			xpath = xmlpath.MustCompile(path)
		)

		for iter := xpath.Iter(root); iter.Next(); {
			node := iter.Node()

			nodes = append(nodes, node)
		}

		if debug {
			fmt.Printf("Executed %s with %d resulting nodes\n", path, len(nodes))
		}

		if len(nodes) > 1 {
			d.multinode(nodes, field)
			continue
		} else if len(nodes) < 1 {
			if debug {
				fmt.Println("Xpath did not match any nodes: ", path)
			}
			continue
		}

		d.unmarshal(nodes[0], field)
	}
}

// indirect walks down v allocating pointers as needed until it gets to a non-pointer
//
// indirect original can be found in the stdlib encoding/json, credit to Go authors
func indirect(v reflect.Value) (Unmarshaler, encoding.TextUnmarshaler, reflect.Value) {
	if v.Kind() != reflect.Ptr && v.Type().Name() != "" && v.CanAddr() {
		v = v.Addr()
	}

	for {
		if v.Kind() == reflect.Interface && !v.IsNil() {
			e := v.Elem()
			if e.Kind() == reflect.Ptr && !e.IsNil() {
				v = e
				continue
			}
		}

		if v.Kind() != reflect.Ptr {
			break
		}

		if v.IsNil() {
			v.Set(reflect.New(v.Type().Elem()))
		}
		if v.Type().NumMethod() > 0 {
			if u, ok := v.Interface().(Unmarshaler); ok {
				return u, nil, reflect.Value{}
			}

			if u, ok := v.Interface().(encoding.TextUnmarshaler); ok {
				return nil, u, reflect.Value{}
			}
		}
		v = v.Elem()
	}
	return nil, nil, v
}
<|endoftext|>