<commit_before>package main

import (
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"regexp"

	"github.com/tmc/srcutils/pos"

	"golang.org/x/tools/go/loader"
	"golang.org/x/tools/go/pointer"
	"golang.org/x/tools/go/ssa"
)

type refactor struct {
	iprog         *loader.Program
	prog          *ssa.Program
	ptraCfg       *pointer.Config
	packageNameRe *regexp.Regexp
}

func newRefactor(args []string, packageNameRe *regexp.Regexp) (*refactor, error) {
	conf := loader.Config{
		Build:         &build.Default,
		SourceImports: true,
		ParserMode:    parser.ParseComments,
	}
	args, err := conf.FromArgs(args, true)
	if err != nil {
		return nil, err
	}
	if len(args) > 0 {
		return nil, fmt.Errorf("surplus arguments: %q", args)
	}

	iprog, err := conf.Load()
	if err != nil {
		return nil, err
	}

	var mode ssa.BuilderMode
	prog := ssa.Create(iprog, mode)
	prog.BuildAll()

	// For each initial package (specified on the command line),
	// if it has a main function, analyze that,
	// otherwise analyze its tests, if any.
	var testPkgs, mains []*ssa.Package
	for _, info := range iprog.InitialPackages() {
		initialPkg := prog.Package(info.Pkg)

		// Add package to the pointer analysis scope.
		if initialPkg.Func("main") != nil {
			mains = append(mains, initialPkg)
		} else {
			testPkgs = append(testPkgs, initialPkg)
		}
	}
	if testPkgs != nil {
		if p := prog.CreateTestMainPackage(testPkgs...); p != nil {
			mains = append(mains, p)
		}
	}
	if mains == nil {
		return nil, fmt.Errorf("analysis scope has no main and no tests")
	}

	return &refactor{
		iprog,
		prog,
		&pointer.Config{Mains: mains, BuildCallGraph: true},
		packageNameRe,
	}, nil
}

func (r *refactor) callers(qpos *pos.QueryPos) ([]*pos.QueryPos, error) {
	pkg := r.prog.Package(qpos.Info.Pkg)
	if pkg == nil {
		return nil, fmt.Errorf("no SSA package")
	}
	if !ssa.HasEnclosingFunction(pkg, qpos.Path) {
		return nil, fmt.Errorf("this position is not inside a function")
	}

	target := ssa.EnclosingFunction(pkg, qpos.Path)
	if target == nil {
		return nil, fmt.Errorf("no SSA function built for this location (dead code?)")
	}

	ptrAnalysis, err := pointer.Analyze(r.ptraCfg)
	if err != nil {
		return nil, err
	}

	cg := ptrAnalysis.CallGraph
	cg.DeleteSyntheticNodes()
	edges := cg.CreateNode(target).In

	callers := []*pos.QueryPos{}
	for _, edge := range edges {
		if edge.Caller.ID <= 1 {
			continue
		}
		caller, err := r.posToQueryPos(edge.Pos())
		if err != nil {
			return callers, err
		}
		callers = append(callers, caller)
	}

	return callers, nil
}

func (r *refactor) callersAndCallsites(qpos *pos.QueryPos, depth int) ([]*pos.QueryPos, []*pos.QueryPos, error) {
	allCallers, allCallsites := map[token.Pos]*pos.QueryPos{}, map[token.Pos]*pos.QueryPos{}

	err := r.addCallersAndCallsites(qpos, depth, allCallers, allCallsites)
	if err != nil {
		return nil, nil, err
	}

	resultCallers, resultCallsites := []*pos.QueryPos{}, []*pos.QueryPos{}
	for _, caller := range allCallers {
		resultCallers = append(resultCallers, caller)
	}
	for _, caller := range allCallsites {
		resultCallsites = append(resultCallsites, caller)
	}
	return resultCallers, resultCallsites, nil
}

func (r *refactor) addCallersAndCallsites(qpos *pos.QueryPos, depth int, allCallers, allCallsites map[token.Pos]*pos.QueryPos) error {
	if _, present := allCallers[qpos.Start]; present {
		return nil
	}
	if depth == 0 {
		return nil
	}
	allCallers[qpos.Start] = qpos
	callers, err := r.callers(qpos)
	if err != nil {
		return err
	}
	for _, caller := range callers {
		allCallsites[caller.Start] = caller

		parent, err := r.parentFunc(caller.Path)
		if err != nil {
			return err
		}
		if parent != nil {
			if err := r.addCallersAndCallsites(parent, depth-1, allCallers, allCallsites); err != nil {
				return err
			}
		}
	}
	return nil
}

func (r *refactor) queryPos(position string, reflection bool) (*pos.QueryPos, error) {
	return pos.ParseQueryPos(r.iprog, position, reflection)
}

func (r *refactor) posToQueryPos(pos token.Pos) (*pos.QueryPos, error) {
	p := r.prog.Fset.Position(pos)
	return r.queryPos(fmt.Sprintf("%s:#%d", p.Filename, p.Offset), false)
}

func (r *refactor) parentFunc(path []ast.Node) (*pos.QueryPos, error) {
	for _, node := range path {
		// TODO consider function literals
		if fn, ok := node.(*ast.FuncDecl); ok {
			return r.posToQueryPos(fn.Type.Params.Pos())
		}
	}
	return nil, fmt.Errorf("no parent found: %s", r.prog.Fset.Position(path[0].Pos()))
}
<commit_msg>reflect upstream api changes<commit_after>package main

import (
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"regexp"

	"github.com/tmc/srcutils/pos"

	"golang.org/x/tools/go/loader"
	"golang.org/x/tools/go/pointer"
	"golang.org/x/tools/go/ssa"
)

type refactor struct {
	iprog         *loader.Program
	prog          *ssa.Program
	ptraCfg       *pointer.Config
	packageNameRe *regexp.Regexp
}

func newRefactor(args []string, packageNameRe *regexp.Regexp) (*refactor, error) {
	conf := loader.Config{
		Build:      &build.Default,
		ParserMode: parser.ParseComments,
	}
	args, err := conf.FromArgs(args, true)
	if err != nil {
		return nil, err
	}
	if len(args) > 0 {
		return nil, fmt.Errorf("surplus arguments: %q", args)
	}

	iprog, err := conf.Load()
	if err != nil {
		return nil, err
	}

	var mode ssa.BuilderMode
	prog := ssa.NewProgram(iprog.Fset, mode)
	prog.Build()

	// For each initial package (specified on the command line),
	// if it has a main function, analyze that,
	// otherwise analyze its tests, if any.
	var testPkgs, mains []*ssa.Package
	for _, info := range iprog.InitialPackages() {
		initialPkg := prog.Package(info.Pkg)

		// Add package to the pointer analysis scope.
		if initialPkg.Func("main") != nil {
			mains = append(mains, initialPkg)
		} else {
			testPkgs = append(testPkgs, initialPkg)
		}
	}
	if testPkgs != nil {
		if p := prog.CreateTestMainPackage(testPkgs...); p != nil {
			mains = append(mains, p)
		}
	}
	if mains == nil {
		return nil, fmt.Errorf("analysis scope has no main and no tests")
	}

	return &refactor{
		iprog,
		prog,
		&pointer.Config{Mains: mains, BuildCallGraph: true},
		packageNameRe,
	}, nil
}

func (r *refactor) callers(qpos *pos.QueryPos) ([]*pos.QueryPos, error) {
	pkg := r.prog.Package(qpos.Info.Pkg)
	if pkg == nil {
		return nil, fmt.Errorf("no SSA package")
	}
	if !ssa.HasEnclosingFunction(pkg, qpos.Path) {
		return nil, fmt.Errorf("this position is not inside a function")
	}

	target := ssa.EnclosingFunction(pkg, qpos.Path)
	if target == nil {
		return nil, fmt.Errorf("no SSA function built for this location (dead code?)")
	}

	ptrAnalysis, err := pointer.Analyze(r.ptraCfg)
	if err != nil {
		return nil, err
	}

	cg := ptrAnalysis.CallGraph
	cg.DeleteSyntheticNodes()
	edges := cg.CreateNode(target).In

	callers := []*pos.QueryPos{}
	for _, edge := range edges {
		if edge.Caller.ID <= 1 {
			continue
		}
		caller, err := r.posToQueryPos(edge.Pos())
		if err != nil {
			return callers, err
		}
		callers = append(callers, caller)
	}

	return callers, nil
}

func (r *refactor) callersAndCallsites(qpos *pos.QueryPos, depth int) ([]*pos.QueryPos, []*pos.QueryPos, error) {
	allCallers, allCallsites := map[token.Pos]*pos.QueryPos{}, map[token.Pos]*pos.QueryPos{}

	err := r.addCallersAndCallsites(qpos, depth, allCallers, allCallsites)
	if err != nil {
		return nil, nil, err
	}

	resultCallers, resultCallsites := []*pos.QueryPos{}, []*pos.QueryPos{}
	for _, caller := range allCallers {
		resultCallers = append(resultCallers, caller)
	}
	for _, caller := range allCallsites {
		resultCallsites = append(resultCallsites, caller)
	}
	return resultCallers, resultCallsites, nil
}

func (r *refactor) addCallersAndCallsites(qpos *pos.QueryPos, depth int, allCallers, allCallsites map[token.Pos]*pos.QueryPos) error {
	if _, present := allCallers[qpos.Start]; present {
		return nil
	}
	if depth == 0 {
		return nil
	}
	allCallers[qpos.Start] = qpos
	callers, err := r.callers(qpos)
	if err != nil {
		return err
	}
	for _, caller := range callers {
		allCallsites[caller.Start] = caller

		parent, err := r.parentFunc(caller.Path)
		if err != nil {
			return err
		}
		if parent != nil {
			if err := r.addCallersAndCallsites(parent, depth-1, allCallers, allCallsites); err != nil {
				return err
			}
		}
	}
	return nil
}

func (r *refactor) queryPos(position string, reflection bool) (*pos.QueryPos, error) {
	return pos.ParseQueryPos(r.iprog, position, reflection)
}

func (r *refactor) posToQueryPos(pos token.Pos) (*pos.QueryPos, error) {
	p := r.prog.Fset.Position(pos)
	return r.queryPos(fmt.Sprintf("%s:#%d", p.Filename, p.Offset), false)
}

func (r *refactor) parentFunc(path []ast.Node) (*pos.QueryPos, error) {
	for _, node := range path {
		// TODO consider function literals
		if fn, ok := node.(*ast.FuncDecl); ok {
			return r.posToQueryPos(fn.Type.Params.Pos())
		}
	}
	return nil, fmt.Errorf("no parent found: %s", r.prog.Fset.Position(path[0].Pos()))
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/workfit/tester/assert"
	"testing"
)

func TestEnumParent(t *testing.T) {
	tests := []struct {
		description string
		strValues   map[string]string
		//If nil, expect no change from strValues
		expectedValues  map[string]string
		expectedParents map[string]string
	}{
		{
			"Single layer",
			map[string]string{
				"Phase":         "",
				"PhaseAnother":  "Another",
				"PhaseOverride": "Heyo",
			},
			nil,
			map[string]string{
				"Phase":         "Phase",
				"PhaseAnother":  "Phase",
				"PhaseOverride": "Phase",
			},
		},
		{
			"No Tree",
			map[string]string{
				"ColorBlue":  "Blue",
				"ColorGreen": "Green",
				"ColorRed":   "Red",
			},
			nil,
			nil,
		},
		{
			"Two layers",
			map[string]string{
				"Color":          "",
				"ColorBlue":      "Blue",
				"ColorBlue_One":  "Blue > One",
				"ColorBlue_Two":  "Blue > Two",
				"ColorGreen":     "Green",
				"ColorGreen_One": "Green > One",
			},
			map[string]string{
				"Color":          "",
				"ColorBlue":      "Blue",
				"ColorBlue_One":  "One",
				"ColorBlue_Two":  "Two",
				"ColorGreen":     "Green",
				"ColorGreen_One": "One",
			},
			map[string]string{
				"Color":          "Color",
				"ColorBlue":      "Color",
				"ColorBlue_One":  "ColorBlue",
				"ColorBlue_Two":  "ColorBlue",
				"ColorGreen":     "Color",
				"ColorGreen_One": "ColorGreen",
			},
		},
		{
			"Three layers",
			map[string]string{
				"Color":             "",
				"ColorBlue":         "Blue",
				"ColorBlue_One":     "Blue > One",
				"ColorBlue_Two":     "Blue > Two",
				"ColorBlue_One_One": "Blue > One > One",
				"ColorBlue_One_Two": "Blue > One > Two",
			},
			map[string]string{
				"Color":             "",
				"ColorBlue":         "Blue",
				"ColorBlue_One":     "One",
				"ColorBlue_Two":     "Two",
				"ColorBlue_One_One": "One",
				"ColorBlue_One_Two": "Two",
			},
			map[string]string{
				"Color":             "Color",
				"ColorBlue":         "Color",
				"ColorBlue_One":     "ColorBlue",
				"ColorBlue_Two":     "ColorBlue",
				"ColorBlue_One_One": "ColorBlue_One",
				"ColorBlue_One_Two": "ColorBlue_One",
			},
		},
		{
			"Single implied layer",
			map[string]string{
				"Color":         "",
				"ColorBlue_One": "Blue > One",
			},
			map[string]string{
				"Color":                "",
				"-9223372036854775808": "Blue",
				"ColorBlue_One":        "One",
			},
			map[string]string{
				"Color":                "Color",
				"-9223372036854775808": "Color",
				"ColorBlue_One":        "-9223372036854775808",
			},
		},
		{
			"Two implied layers",
			map[string]string{
				"Color":            "",
				"ColorGreen_One_A": "Green > One > A",
			},
			map[string]string{
				"Color":                "",
				"-9223372036854775808": "Green",
				"-9223372036854775807": "One",
				"ColorGreen_One_A":     "A",
			},
			map[string]string{
				"Color":                "Color",
				"-9223372036854775808": "Color",
				"-9223372036854775807": "-9223372036854775808",
				"ColorGreen_One_A":     "-9223372036854775807",
			},
		},
		{
			"Single word implied nesting",
			map[string]string{
				"Color":        "",
				"ColorBlue":    "Blue",
				"ColorBlueOne": "Blue One",
				"ColorBlueTwo": "Blue Two",
			},
			map[string]string{
				"Color":        "",
				"ColorBlue":    "Blue",
				"ColorBlueOne": "One",
				"ColorBlueTwo": "Two",
			},
			map[string]string{
				"Color":        "Color",
				"ColorBlue":    "Color",
				"ColorBlueOne": "ColorBlue",
				"ColorBlueTwo": "ColorBlue",
			},
		},
		{
			"Multi-Word implied nesting",
			map[string]string{
				"Color":             "",
				"ColorBlueGreen":    "Blue Green",
				"ColorBlueGreenOne": "Blue Green One",
				"ColorBlueGreenTwo": "Blue Green Two",
			},
			map[string]string{
				"Color":             "",
				"ColorBlueGreen":    "Blue Green",
				"ColorBlueGreenOne": "One",
				"ColorBlueGreenTwo": "Two",
			},
			map[string]string{
				"Color":             "Color",
				"ColorBlueGreen":    "Color",
				"ColorBlueGreenOne": "ColorBlueGreen",
				"ColorBlueGreenTwo": "ColorBlueGreen",
			},
		},
		{
			"Implied node with implied nesting",
			map[string]string{
				"Color":             "",
				"ColorBlueGreenOne": "Blue Green One",
				"ColorBlueGreenTwo": "Blue Green Two",
			},
			map[string]string{
				"Color":                "",
				"-9223372036854775808": "Blue Green",
				"ColorBlueGreenOne":    "One",
				"ColorBlueGreenTwo":    "Two",
			},
			map[string]string{
				"Color":                "Color",
				"-9223372036854775808": "Color",
				"ColorBlueGreenOne":    "-9223372036854775808",
				"ColorBlueGreenTwo":    "-9223372036854775808",
			},
		},
		/*
			{
				"Multiple implied layers",
				map[string]string{
					"Color":              "",
					"ColorBlueGreen":     "Blue Green",
					"ColorBlueGreenOne":  "Blue Green One",
					"ColorBlueGreenOneA": "Blue Green One A",
					"ColorBlueGreenOneB": "Blue Green One B",
					"ColorBlueGreenTwo":  "Blue Green Two",
				},
				map[string]string{
					"Color":              "",
					"ColorBlueGreen":     "Blue Green",
					"ColorBlueGreenOne":  "Blue Green > One",
					"ColorBlueGreenOneA": "Blue Green > One > A",
					"ColorBlueGreenOneB": "Blue Green > One > B",
					"ColorBlueGreenTwo":  "Blue Green > Two",
				},
				map[string]string{
					"Color":              "Color",
					"ColorBlueGreen":     "Color",
					"ColorBlueGreenOne":  "ColorBlueGreen",
					"ColorBlueGreenOneA": "ColorBlueGreenOne",
					"ColorBlueGreenOneB": "ColorBlueGreenOne",
					"ColorBlueGreenTwo":  "ColorBlueGreen",
				},
			},
			{
				"Multiple implied layers with implied node",
				map[string]string{
					"Color":              "",
					"ColorBlueGreen":     "Blue Green",
					"ColorBlueGreenOneA": "Blue Green One A",
					"ColorBlueGreenOneB": "Blue Green One B",
					"ColorBlueGreenTwo":  "Blue Green Two",
				},
				map[string]string{
					"Color":                "",
					"ColorBlueGreen":       "Blue Green",
					"-9223372036854775808": "Blue Green > One",
					"ColorBlueGreenOneA":   "Blue Green > One > A",
					"ColorBlueGreenOneB":   "Blue Green > One > B",
					"ColorBlueGreenTwo":    "Blue Green > Two",
				},
				map[string]string{
					"Color":                "Color",
					"ColorBlueGreen":       "Color",
					"-9223372036854775808": "ColorBlueGreen",
					"ColorBlueGreenOneA":   "-9223372036854775808",
					"ColorBlueGreenOneB":   "-9223372036854775808",
					"ColorBlueGreenTwo":    "ColorBlueGreen",
				},
			},
			{
				"Mix implicit and explicit layers",
				map[string]string{
					"Color":               "",
					"ColorBlueGreen":      "Blue Green",
					"ColorBlueGreenOne":   "Blue Green One",
					"ColorBlueGreenOne_A": "Blue Green One > A",
					"ColorBlueGreenOne_B": "Blue Green One > B",
					"ColorBlueGreenTwo":   "Blue Green Two",
				},
				map[string]string{
					"Color":              "",
					"ColorBlueGreen":     "Blue Green",
					"ColorBlueGreenOne":  "Blue Green > One",
					"ColorBlueGreenOneA": "Blue Green > One > A",
					"ColorBlueGreenOneB": "Blue Green > One > B",
					"ColorBlueGreenTwo":  "Blue Green > Two",
				},
				map[string]string{
					"Color":              "Color",
					"ColorBlueGreen":     "Color",
					"ColorBlueGreenOne":  "ColorBlueGreen",
					"ColorBlueGreenOneA": "ColorBlueGreenOne",
					"ColorBlueGreenOneB": "ColorBlueGreenOne",
					"ColorBlueGreenTwo":  "ColorBlueGreen",
				},
			},
		*/
	}

	for i, test := range tests {

		e := newEnum("test", transformNone)

		for key, val := range test.strValues {
			e.AddTransformKey(key, true, val, transformNone)
		}

		err := e.Process()
		assert.For(t, i).ThatActual(err).IsNil()

		actualValues := e.ValueMap()
		actualParents := e.Parents()
		if test.expectedValues == nil {
			//Expect no change from test.strValues
			assert.For(t, i).ThatActual(actualValues).Equals(test.strValues).ThenDiffOnFail()
		} else {
			assert.For(t, i).ThatActual(actualValues).Equals(test.expectedValues).ThenDiffOnFail()
		}
		assert.For(t, i).ThatActual(actualParents).Equals(test.expectedParents).ThenDiffOnFail()
	}
}
<commit_msg>Enable the rest of the tests, which (with tweaking to expect the right things) all work. Part of #628.<commit_after>package main

import (
	"github.com/workfit/tester/assert"
	"testing"
)

func TestEnumParent(t *testing.T) {
	tests := []struct {
		description string
		strValues   map[string]string
		//If nil, expect no change from strValues
		expectedValues  map[string]string
		expectedParents map[string]string
	}{
		{
			"Single layer",
			map[string]string{
				"Phase":         "",
				"PhaseAnother":  "Another",
				"PhaseOverride": "Heyo",
			},
			nil,
			map[string]string{
				"Phase":         "Phase",
				"PhaseAnother":  "Phase",
				"PhaseOverride": "Phase",
			},
		},
		{
			"No Tree",
			map[string]string{
				"ColorBlue":  "Blue",
				"ColorGreen": "Green",
				"ColorRed":   "Red",
			},
			nil,
			nil,
		},
		{
			"Two layers",
			map[string]string{
				"Color":          "",
				"ColorBlue":      "Blue",
				"ColorBlue_One":  "Blue > One",
				"ColorBlue_Two":  "Blue > Two",
				"ColorGreen":     "Green",
				"ColorGreen_One": "Green > One",
			},
			map[string]string{
				"Color":          "",
				"ColorBlue":      "Blue",
				"ColorBlue_One":  "One",
				"ColorBlue_Two":  "Two",
				"ColorGreen":     "Green",
				"ColorGreen_One": "One",
			},
			map[string]string{
				"Color":          "Color",
				"ColorBlue":      "Color",
				"ColorBlue_One":  "ColorBlue",
				"ColorBlue_Two":  "ColorBlue",
				"ColorGreen":     "Color",
				"ColorGreen_One": "ColorGreen",
			},
		},
		{
			"Three layers",
			map[string]string{
				"Color":             "",
				"ColorBlue":         "Blue",
				"ColorBlue_One":     "Blue > One",
				"ColorBlue_Two":     "Blue > Two",
				"ColorBlue_One_One": "Blue > One > One",
				"ColorBlue_One_Two": "Blue > One > Two",
			},
			map[string]string{
				"Color":             "",
				"ColorBlue":         "Blue",
				"ColorBlue_One":     "One",
				"ColorBlue_Two":     "Two",
				"ColorBlue_One_One": "One",
				"ColorBlue_One_Two": "Two",
			},
			map[string]string{
				"Color":             "Color",
				"ColorBlue":         "Color",
				"ColorBlue_One":     "ColorBlue",
				"ColorBlue_Two":     "ColorBlue",
				"ColorBlue_One_One": "ColorBlue_One",
				"ColorBlue_One_Two": "ColorBlue_One",
			},
		},
		{
			"Single implied layer",
			map[string]string{
				"Color":         "",
				"ColorBlue_One": "Blue > One",
			},
			map[string]string{
				"Color":                "",
				"-9223372036854775808": "Blue",
				"ColorBlue_One":        "One",
			},
			map[string]string{
				"Color":                "Color",
				"-9223372036854775808": "Color",
				"ColorBlue_One":        "-9223372036854775808",
			},
		},
		{
			"Two implied layers",
			map[string]string{
				"Color":            "",
				"ColorGreen_One_A": "Green > One > A",
			},
			map[string]string{
				"Color":                "",
				"-9223372036854775808": "Green",
				"-9223372036854775807": "One",
				"ColorGreen_One_A":     "A",
			},
			map[string]string{
				"Color":                "Color",
				"-9223372036854775808": "Color",
				"-9223372036854775807": "-9223372036854775808",
				"ColorGreen_One_A":     "-9223372036854775807",
			},
		},
		{
			"Single word implied nesting",
			map[string]string{
				"Color":        "",
				"ColorBlue":    "Blue",
				"ColorBlueOne": "Blue One",
				"ColorBlueTwo": "Blue Two",
			},
			map[string]string{
				"Color":        "",
				"ColorBlue":    "Blue",
				"ColorBlueOne": "One",
				"ColorBlueTwo": "Two",
			},
			map[string]string{
				"Color":        "Color",
				"ColorBlue":    "Color",
				"ColorBlueOne": "ColorBlue",
				"ColorBlueTwo": "ColorBlue",
			},
		},
		{
			"Multi-Word implied nesting",
			map[string]string{
				"Color":             "",
				"ColorBlueGreen":    "Blue Green",
				"ColorBlueGreenOne": "Blue Green One",
				"ColorBlueGreenTwo": "Blue Green Two",
			},
			map[string]string{
				"Color":             "",
				"ColorBlueGreen":    "Blue Green",
				"ColorBlueGreenOne": "One",
				"ColorBlueGreenTwo": "Two",
			},
			map[string]string{
				"Color":             "Color",
				"ColorBlueGreen":    "Color",
				"ColorBlueGreenOne": "ColorBlueGreen",
				"ColorBlueGreenTwo": "ColorBlueGreen",
			},
		},
		{
			"Implied node with implied nesting",
			map[string]string{
				"Color":             "",
				"ColorBlueGreenOne": "Blue Green One",
				"ColorBlueGreenTwo": "Blue Green Two",
			},
			map[string]string{
				"Color":                "",
				"-9223372036854775808": "Blue Green",
				"ColorBlueGreenOne":    "One",
				"ColorBlueGreenTwo":    "Two",
			},
			map[string]string{
				"Color":                "Color",
				"-9223372036854775808": "Color",
				"ColorBlueGreenOne":    "-9223372036854775808",
				"ColorBlueGreenTwo":    "-9223372036854775808",
			},
		},
		{
			"Multiple implied layers",
			map[string]string{
				"Color":              "",
				"ColorBlueGreen":     "Blue Green",
				"ColorBlueGreenOne":  "Blue Green One",
				"ColorBlueGreenOneA": "Blue Green One A",
				"ColorBlueGreenOneB": "Blue Green One B",
				"ColorBlueGreenTwo":  "Blue Green Two",
			},
			map[string]string{
				"Color":              "",
				"ColorBlueGreen":     "Blue Green",
				"ColorBlueGreenOne":  "One",
				"ColorBlueGreenOneA": "A",
				"ColorBlueGreenOneB": "B",
				"ColorBlueGreenTwo":  "Two",
			},
			map[string]string{
				"Color":              "Color",
				"ColorBlueGreen":     "Color",
				"ColorBlueGreenOne":  "ColorBlueGreen",
				"ColorBlueGreenOneA": "ColorBlueGreenOne",
				"ColorBlueGreenOneB": "ColorBlueGreenOne",
				"ColorBlueGreenTwo":  "ColorBlueGreen",
			},
		},

		{
			"Multiple implied layers with implied node",
			map[string]string{
				"Color":              "",
				"ColorBlueGreen":     "Blue Green",
				"ColorBlueGreenOneA": "Blue Green One A",
				"ColorBlueGreenOneB": "Blue Green One B",
				"ColorBlueGreenTwo":  "Blue Green Two",
			},
			map[string]string{
				"Color":                "",
				"ColorBlueGreen":       "Blue Green",
				"-9223372036854775808": "One",
				"ColorBlueGreenOneA":   "A",
				"ColorBlueGreenOneB":   "B",
				"ColorBlueGreenTwo":    "Two",
			},
			map[string]string{
				"Color":                "Color",
				"ColorBlueGreen":       "Color",
				"-9223372036854775808": "ColorBlueGreen",
				"ColorBlueGreenOneA":   "-9223372036854775808",
				"ColorBlueGreenOneB":   "-9223372036854775808",
				"ColorBlueGreenTwo":    "ColorBlueGreen",
			},
		},
		{
			"Mix implicit and explicit layers",
			map[string]string{
				"Color":               "",
				"ColorBlueGreen":      "Blue Green",
				"ColorBlueGreenOne":   "Blue Green One",
				"ColorBlueGreenOne_A": "Blue Green One > A",
				"ColorBlueGreenOne_B": "Blue Green One > B",
				"ColorBlueGreenTwo":   "Blue Green Two",
			},
			map[string]string{
				"Color":               "",
				"ColorBlueGreen":      "Blue Green",
				"ColorBlueGreenOne":   "One",
				"ColorBlueGreenOne_A": "A",
				"ColorBlueGreenOne_B": "B",
				"ColorBlueGreenTwo":   "Two",
			},
			map[string]string{
				"Color":               "Color",
				"ColorBlueGreen":      "Color",
				"ColorBlueGreenOne":   "ColorBlueGreen",
				"ColorBlueGreenOne_A": "ColorBlueGreenOne",
				"ColorBlueGreenOne_B": "ColorBlueGreenOne",
				"ColorBlueGreenTwo":   "ColorBlueGreen",
			},
		},
	}

	for i, test := range tests {

		e := newEnum("test", transformNone)

		for key, val := range test.strValues {
			e.AddTransformKey(key, true, val, transformNone)
		}

		err := e.Process()
		assert.For(t, i).ThatActual(err).IsNil()

		actualValues := e.ValueMap()
		actualParents := e.Parents()
		if test.expectedValues == nil {
			//Expect no change from test.strValues
			assert.For(t, i).ThatActual(actualValues).Equals(test.strValues).ThenDiffOnFail()
		} else {
			assert.For(t, i).ThatActual(actualValues).Equals(test.expectedValues).ThenDiffOnFail()
		}
		assert.For(t, i).ThatActual(actualParents).Equals(test.expectedParents).ThenDiffOnFail()
	}
}
<|endoftext|>


<commit_before>// Sample
package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"runtime"
	"strconv"
	"sync/atomic"
	"time"

	"github.com/m-lab/etl/bq"
	"github.com/m-lab/etl/etl"
	"github.com/m-lab/etl/metrics"
	"github.com/m-lab/etl/parser"
	"github.com/m-lab/etl/storage"
	"github.com/m-lab/etl/task"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	// Enable profiling. For more background and usage information, see:
	//   https://blog.golang.org/profiling-go-programs
	"net/http/pprof"
	// Enable exported debug vars.  See https://golang.org/pkg/expvar/
	_ "expvar"
)

func init() {
	// Always prepend the filename and line number.
	log.SetFlags(log.LstdFlags | log.Lshortfile)
}

// Task Queue can always submit to an admin restricted URL.
//   login: admin
// Return 200 status code.
// Track reqeusts that last longer than 24 hrs.
// Is task handling idempotent?

// Useful headers added by AppEngine when sending Tasks via Push.
//   X-AppEngine-QueueName
//   X-AppEngine-TaskETA
//   X-AppEngine-TaskName
//   X-AppEngine-TaskRetryCount
//   X-AppEngine-TaskExecutionCount

func handler(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path != "/" {
		http.NotFound(w, r)
		return
	}
	fmt.Fprint(w, "Hello world!")
}

// Basic throttling to restrict the number of tasks in flight.
const defaultMaxInFlight = 20

var maxInFlight int32 // Max number of concurrent workers (and tasks in flight).
var inFlight int32    // Current number of tasks in flight.

// Returns true if request should be rejected.
// If the max concurrency (MC) exceeds (or matches) the instances*workers, then
// most requests will be rejected, until the median number of workers is
// less than the throttle.
// ** So we should set max instances (MI) * max workers (MW) > max concurrency.
//
// We also want max_concurrency high enough that most instances have several
// jobs.  With MI=20, MW=25, MC=100, the average workers/instance is only 4, and
// we end up with many instances starved, so AppEngine was removing instances even
// though the queue throughput was poor.
// ** So we probably want MC/MI > MW/2, to prevent starvation.
//
// For now, assuming:
//    MC: 180,  MI: 20, MW: 10
//
// TODO - replace the atomic with a channel based semaphore and non-blocking
// select.
func shouldThrottle() bool {
	if atomic.AddInt32(&inFlight, 1) > maxInFlight {
		atomic.AddInt32(&inFlight, -1)
		return true
	}
	return false
}

func decrementInFlight() {
	atomic.AddInt32(&inFlight, -1)
}

func worker(w http.ResponseWriter, r *http.Request) {
	// TODO(dev) Check how many times a request has already been attempted.

	// These keep track of the (nested) state of the worker.
	metrics.WorkerState.WithLabelValues("worker").Inc()
	defer metrics.WorkerState.WithLabelValues("worker").Dec()

	// Throttle by grabbing a semaphore from channel.
	if shouldThrottle() {
		metrics.TaskCount.WithLabelValues("unknown", "TooManyRequests").Inc()
		w.WriteHeader(http.StatusTooManyRequests)
		fmt.Fprintf(w, `{"message": "Too many tasks."}`)
		return
	}
	// Decrement counter when worker finishes.
	defer decrementInFlight()

	metrics.WorkerCount.Inc()
	defer metrics.WorkerCount.Dec()

	var err error
	retryCountStr := r.Header.Get("X-AppEngine-TaskRetryCount")
	retryCount := 0
	if retryCountStr != "" {
		retryCount, err = strconv.Atoi(retryCountStr)
		if err != nil {
			log.Printf("Invalid retries string: %s\n", retryCountStr)
		}
	}
	executionCountStr := r.Header.Get("X-AppEngine-TaskExecutionCount")
	executionCount := 0
	if executionCountStr != "" {
		executionCount, err = strconv.Atoi(executionCountStr)
		if err != nil {
			log.Printf("Invalid execution count string: %s\n", executionCountStr)
		}
	}

	r.ParseForm()
	// Log request data.
	for key, value := range r.Form {
		log.Printf("Form:   %q == %q\n", key, value)
	}

	// This handles base64 encoding, and requires a gs:// prefix.
	fn, err := storage.GetFilename(r.FormValue("filename"))
	if err != nil {
		metrics.TaskCount.WithLabelValues("unknown", "BadRequest").Inc()
		log.Printf("Invalid filename: %s\n", fn)
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, `{"message": "Invalid filename."}`)
		return
	}

	// TODO(dev): log the originating task queue name from headers.
	log.Printf("Received filename: %q  Retries: %d, Executions: %d\n",
		fn, retryCount, executionCount)

	data, err := etl.ValidateTestPath(fn)
	if err != nil {
		log.Printf("Invalid filename: %v\n", err)
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, `{"message": "Invalid filename."}`)
		return
	}
	dataType := data.GetDataType()

	// Move this into Validate function
	if dataType == etl.INVALID {
		metrics.TaskCount.WithLabelValues("unknown", "BadRequest").Inc()
		log.Printf("Invalid filename: %s\n", fn)
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, `{"message": "Invalid filename."}`)
		return
	}

	client, err := storage.GetStorageClient(false)
	if err != nil {
		metrics.TaskCount.WithLabelValues("unknown", "ServiceUnavailable").Inc()
		log.Printf("Error getting storage client: %v\n", err)
		w.WriteHeader(http.StatusServiceUnavailable)
		fmt.Fprintf(w, `{"message": "Could not create client."}`)
		return
	}

	// TODO - add a timer for reading the file.
	tr, err := storage.NewETLSource(client, fn)
	if err != nil {
		metrics.TaskCount.WithLabelValues(string(dataType), "ETLSourceError").Inc()
		log.Printf("Error opening gcs file: %v", err)
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintf(w, `{"message": "Problem opening gcs file."}`)
		return
		// TODO - anything better we could do here?
	}
	defer tr.Close()

	dateFormat := "20060102"
	date, err := time.Parse(dateFormat, data.PackedDate)

	dataset, ok := os.LookupEnv("BIGQUERY_DATASET")
	if !ok {
		// TODO - make this fatal.
		dataset = "mlab_sandbox"
	}
	ins, err := bq.NewInserter(dataset, dataType, date)
	if err != nil {
		metrics.TaskCount.WithLabelValues(string(dataType), "NewInserterError").Inc()
		log.Printf("Error creating BQ Inserter:  %v", err)
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintf(w, `{"message": "Problem creating BQ inserter."}`)
		return
		// TODO - anything better we could do here?
	}

	// Wrap inserter to give insertion time metrics.
	ins = bq.DurationWrapper{ins}

	// Create parser, injecting Inserter
	p := parser.NewParser(dataType, ins)
	tsk := task.NewTask(fn, tr, p)

	files, err := tsk.ProcessAllTests()

	// Count the files processed per-host-module per-weekday.
	// TODO(soltesz): evaluate separating hosts and pods as separate metrics.
	metrics.FileCount.WithLabelValues(
		data.Host+"-"+data.Pod+"-"+data.Experiment,
		date.Weekday().String()).Add(float64(files))

	metrics.WorkerState.WithLabelValues("finish").Inc()
	defer metrics.WorkerState.WithLabelValues("finish").Dec()
	if err != nil {
		metrics.TaskCount.WithLabelValues(string(dataType), "TaskError").Inc()
		log.Printf("Error Processing Tests:  %v", err)
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintf(w, `{"message": "Error in ProcessAllTests"}`)
		return
		// TODO - anything better we could do here?
	}

	// TODO - if there are any errors, consider sending back a meaningful response
	// for web browser and queue-pusher debugging.
	fmt.Fprintf(w, `{"message": "Success"}`)

	metrics.TaskCount.WithLabelValues(string(dataType), "OK").Inc()
}

func healthCheckHandler(w http.ResponseWriter, r *http.Request) {
	// TODO(soltesz): provide a real health check.
	fmt.Fprint(w, "ok")
}

func setMaxInFlight() {
	maxInFlightString, ok := os.LookupEnv("MAX_WORKERS")
	if ok {
		maxInFlightInt, err := strconv.Atoi(maxInFlightString)
		if err == nil {
			maxInFlight = int32(maxInFlightInt)
		} else {
			log.Println("MAX_WORKERS not configured.  Using 20.")
			maxInFlight = defaultMaxInFlight
		}
	} else {
		log.Println("MAX_WORKERS not configured.  Using 20.")
		maxInFlight = defaultMaxInFlight
	}
}

func main() {
	// Define a custom serve mux for prometheus to listen on a separate port.
	// We listen on a separate port so we can forward this port on the host VM.
	// We cannot forward port 8080 because it is used by AppEngine.
	mux := http.NewServeMux()
	// Assign the default prometheus handler to the standard exporter path.
	mux.Handle("/metrics", promhttp.Handler())
	// Assign the pprof handling paths to the external port to access individual
	// instances.
	mux.HandleFunc("/debug/pprof/", pprof.Index)
	mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
	mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
	mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
	mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
	go http.ListenAndServe(":9090", mux)

	http.HandleFunc("/", handler)
	http.HandleFunc("/worker", metrics.DurationHandler("generic", worker))
	http.HandleFunc("/_ah/health", healthCheckHandler)

	// Enable block profiling
	runtime.SetBlockProfileRate(1000000) // One event per msec.

	setMaxInFlight()

	// We also setup another prometheus handler on a non-standard path. This
	// path name will be accessible through the AppEngine service address,
	// however it will be served by a random instance.
	http.Handle("/random-metrics", promhttp.Handler())
	http.ListenAndServe(":8080", nil)
}
<commit_msg>refactor worker for reuse<commit_after>// Sample
package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"runtime"
	"strconv"
	"sync/atomic"
	"time"

	"github.com/m-lab/etl/bq"
	"github.com/m-lab/etl/etl"
	"github.com/m-lab/etl/metrics"
	"github.com/m-lab/etl/parser"
	"github.com/m-lab/etl/storage"
	"github.com/m-lab/etl/task"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	// Enable profiling. For more background and usage information, see:
	//   https://blog.golang.org/profiling-go-programs
	"net/http/pprof"
	// Enable exported debug vars.  See https://golang.org/pkg/expvar/
	_ "expvar"
)

func init() {
	// Always prepend the filename and line number.
	log.SetFlags(log.LstdFlags | log.Lshortfile)
}

// Task Queue can always submit to an admin restricted URL.
//   login: admin
// Return 200 status code.
// Track reqeusts that last longer than 24 hrs.
// Is task handling idempotent?

// Useful headers added by AppEngine when sending Tasks via Push.
//   X-AppEngine-QueueName
//   X-AppEngine-TaskETA
//   X-AppEngine-TaskName
//   X-AppEngine-TaskRetryCount
//   X-AppEngine-TaskExecutionCount

func handler(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path != "/" {
		http.NotFound(w, r)
		return
	}
	fmt.Fprint(w, "Hello world!")
}

// Basic throttling to restrict the number of tasks in flight.
const defaultMaxInFlight = 20

var maxInFlight int32 // Max number of concurrent workers (and tasks in flight).
var inFlight int32    // Current number of tasks in flight.

// Returns true if request should be rejected.
// If the max concurrency (MC) exceeds (or matches) the instances*workers, then
// most requests will be rejected, until the median number of workers is
// less than the throttle.
// ** So we should set max instances (MI) * max workers (MW) > max concurrency.
//
// We also want max_concurrency high enough that most instances have several
// jobs.  With MI=20, MW=25, MC=100, the average workers/instance is only 4, and
// we end up with many instances starved, so AppEngine was removing instances even
// though the queue throughput was poor.
// ** So we probably want MC/MI > MW/2, to prevent starvation.
//
// For now, assuming:
//    MC: 180,  MI: 20, MW: 10
//
// TODO - replace the atomic with a channel based semaphore and non-blocking
// select.
func shouldThrottle() bool {
	if atomic.AddInt32(&inFlight, 1) > maxInFlight {
		atomic.AddInt32(&inFlight, -1)
		return true
	}
	return false
}

func decrementInFlight() {
	atomic.AddInt32(&inFlight, -1)
}

// TODO(gfr) unify counting for http and pubsub paths?
func worker(rwr http.ResponseWriter, rq *http.Request) {
	// Throttle by grabbing a semaphore from channel.
	if shouldThrottle() {
		metrics.TaskCount.WithLabelValues("unknown", "TooManyRequests").Inc()
		rwr.WriteHeader(http.StatusTooManyRequests)
		fmt.Fprintf(rwr, `{"message": "Too many tasks."}`)
		return
	}

	// Decrement counter when worker finishes.
	defer decrementInFlight()

	var err error
	retryCountStr := rq.Header.Get("X-AppEngine-TaskRetryCount")
	retryCount := 0
	if retryCountStr != "" {
		retryCount, err = strconv.Atoi(retryCountStr)
		if err != nil {
			log.Printf("Invalid retries string: %s\n", retryCountStr)
		}
	}
	executionCountStr := rq.Header.Get("X-AppEngine-TaskExecutionCount")
	executionCount := 0
	if executionCountStr != "" {
		executionCount, err = strconv.Atoi(executionCountStr)
		if err != nil {
			log.Printf("Invalid execution count string: %s\n", executionCountStr)
		}
	}

	rq.ParseForm()
	// Log request data.
	for key, value := range rq.Form {
		log.Printf("Form:   %q == %q\n", key, value)
	}

	rawFileName := rq.FormValue("filename")
	status, msg := subworker(rawFileName, executionCount, retryCount)
	rwr.WriteHeader(status)
	fmt.Fprintf(rwr, msg)
}

func subworker(rawFileName string, executionCount, retryCount int) (status int, msg string) {
	// TODO(dev) Check how many times a request has already been attempted.

	// These keep track of the (nested) state of the worker.
	metrics.WorkerState.WithLabelValues("worker").Inc()
	defer metrics.WorkerState.WithLabelValues("worker").Dec()

	metrics.WorkerCount.Inc()
	defer metrics.WorkerCount.Dec()

	var err error
	// This handles base64 encoding, and requires a gs:// prefix.
	fn, err := storage.GetFilename(rawFileName)
	if err != nil {
		metrics.TaskCount.WithLabelValues("unknown", "BadRequest").Inc()
		log.Printf("Invalid filename: %s\n", fn)
		return http.StatusBadRequest, `{"message": "Invalid filename."}`
	}

	// TODO(dev): log the originating task queue name from headers.
	log.Printf("Received filename: %q  Retries: %d, Executions: %d\n",
		fn, retryCount, executionCount)

	data, err := etl.ValidateTestPath(fn)
	if err != nil {
		log.Printf("Invalid filename: %v\n", err)
		return http.StatusBadRequest, `{"message": "Invalid filename."}`
	}
	dataType := data.GetDataType()

	// Move this into Validate function
	if dataType == etl.INVALID {
		metrics.TaskCount.WithLabelValues("unknown", "BadRequest").Inc()
		log.Printf("Invalid filename: %s\n", fn)
		return http.StatusBadRequest, `{"message": "Invalid filename."}`
	}

	client, err := storage.GetStorageClient(false)
	if err != nil {
		metrics.TaskCount.WithLabelValues("unknown", "ServiceUnavailable").Inc()
		log.Printf("Error getting storage client: %v\n", err)
		return http.StatusServiceUnavailable, `{"message": "Could not create client."}`
	}

	// TODO - add a timer for reading the file.
	tr, err := storage.NewETLSource(client, fn)
	if err != nil {
		metrics.TaskCount.WithLabelValues(string(dataType), "ETLSourceError").Inc()
		log.Printf("Error opening gcs file: %v", err)
		return http.StatusInternalServerError, `{"message": "Problem opening gcs file."}`
		// TODO - anything better we could do here?
	}
	defer tr.Close()

	dateFormat := "20060102"
	date, err := time.Parse(dateFormat, data.PackedDate)

	dataset, ok := os.LookupEnv("BIGQUERY_DATASET")
	if !ok {
		// TODO - make this fatal.
		dataset = "mlab_sandbox"
	}
	ins, err := bq.NewInserter(dataset, dataType, date)
	if err != nil {
		metrics.TaskCount.WithLabelValues(string(dataType), "NewInserterError").Inc()
		log.Printf("Error creating BQ Inserter:  %v", err)
		return http.StatusInternalServerError, `{"message": "Problem creating BQ inserter."}`
		// TODO - anything better we could do here?
	}

	// Wrap inserter to give insertion time metrics.
	ins = bq.DurationWrapper{ins}

	// Create parser, injecting Inserter
	p := parser.NewParser(dataType, ins)
	tsk := task.NewTask(fn, tr, p)

	files, err := tsk.ProcessAllTests()

	// Count the files processed per-host-module per-weekday.
	// TODO(soltesz): evaluate separating hosts and pods as separate metrics.
	metrics.FileCount.WithLabelValues(
		data.Host+"-"+data.Pod+"-"+data.Experiment,
		date.Weekday().String()).Add(float64(files))

	metrics.WorkerState.WithLabelValues("finish").Inc()
	defer metrics.WorkerState.WithLabelValues("finish").Dec()
	if err != nil {
		metrics.TaskCount.WithLabelValues(string(dataType), "TaskError").Inc()
		log.Printf("Error Processing Tests:  %v", err)
		return http.StatusInternalServerError, `{"message": "Error in ProcessAllTests"}`
		// TODO - anything better we could do here?
	}

	metrics.TaskCount.WithLabelValues(string(dataType), "OK").Inc()
	return http.StatusOK, `{"message": "Success"}`
}

func healthCheckHandler(w http.ResponseWriter, r *http.Request) {
	// TODO(soltesz): provide a real health check.
	fmt.Fprint(w, "ok")
}

func setMaxInFlight() {
	maxInFlightString, ok := os.LookupEnv("MAX_WORKERS")
	if ok {
		maxInFlightInt, err := strconv.Atoi(maxInFlightString)
		if err == nil {
			maxInFlight = int32(maxInFlightInt)
		} else {
			log.Println("MAX_WORKERS not configured.  Using 20.")
			maxInFlight = defaultMaxInFlight
		}
	} else {
		log.Println("MAX_WORKERS not configured.  Using 20.")
		maxInFlight = defaultMaxInFlight
	}
}

func main() {
	// Define a custom serve mux for prometheus to listen on a separate port.
	// We listen on a separate port so we can forward this port on the host VM.
	// We cannot forward port 8080 because it is used by AppEngine.
	mux := http.NewServeMux()
	// Assign the default prometheus handler to the standard exporter path.
	mux.Handle("/metrics", promhttp.Handler())
	// Assign the pprof handling paths to the external port to access individual
	// instances.
	mux.HandleFunc("/debug/pprof/", pprof.Index)
	mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
	mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
	mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
	mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
	go http.ListenAndServe(":9090", mux)

	http.HandleFunc("/", handler)
	http.HandleFunc("/worker", metrics.DurationHandler("generic", worker))
	http.HandleFunc("/_ah/health", healthCheckHandler)

	// Enable block profiling
	runtime.SetBlockProfileRate(1000000) // One event per msec.

	setMaxInFlight()

	// We also setup another prometheus handler on a non-standard path. This
	// path name will be accessible through the AppEngine service address,
	// however it will be served by a random instance.
	http.Handle("/random-metrics", promhttp.Handler())
	http.ListenAndServe(":8080", nil)
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/gin-gonic/gin"
	"net/http"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"github.com/gin-gonic/contrib/sessions"
	"encoding/json"
	"net/url"
	"crypto/rand"
	"encoding/hex"
	"golang.org/x/crypto/bcrypt"
	"regexp"
"strings"
)

func generateToken(n int) []byte {
	b := make([]byte, n)
	_, err := rand.Read(b)
	if err != nil {
		println(err.Error())
		return b
	}
	token := make([]byte, n * 2)
	hex.Encode(token, b)
	return token
}

func AccountRender(c *gin.Context) {
	render, _ := TemplateStorage[c.Request.URL.Path]
	render.Data = c.Keys
	c.Render(http.StatusOK, render)
}

func AccountVerificationRender(c *gin.Context) {
	account, _ := getAccount(c)
	user, _ := getUser(c)
	if account.IsVerified == "yes" {
		c.Redirect(http.StatusFound, user.DefaultReturnUrl())
		return
	}
	if len(account.VerificationToken) > 0 {

	} else {
		VerifyURL := generateToken(21)
		hash, err := bcrypt.GenerateFromPassword(VerifyURL, bcrypt.DefaultCost)
		if err != nil {
			println(err.Error())
			return
		}

		session, err := mgo.Dial("mongodb://localhost:27017")
		defer session.Close()
		if err != nil {
			println(err.Error())
		}
		d := session.DB("test")
		collection := d.C(ACCOUNTS)
		account.VerificationToken = string(hash)
		collection.UpdateId(account.ID, account)// todo how to update only part?
		verifyURL := "http" +"://"+ "localhost:3000" +"/account/verification/" + string(VerifyURL) + "/"
		c.Set("VerifyURL", verifyURL)

		mailConf := MailConfig{}
		mailConf.Data = c.Keys
		mailConf.From = config.SMTP.From.Name + " <" + config.SMTP.From.Address + ">"
		mailConf.To = user.Email
		mailConf.Subject = "Your " + config.ProjectName + " Account"
		mailConf.ReplyTo = user.Email
		mailConf.HtmlPath = "views/account/verification/email-html.html"

		if err := mailConf.SendMail(); err != nil {
			//todo it's not serious
		}

	}

	render, _ := TemplateStorage[c.Request.URL.Path]
	render.Data = c.Keys
	c.Render(http.StatusOK, render)
}

func Verify (c *gin.Context) {

	account, _ := getAccount(c)
	user, _ := getUser(c)
	err := bcrypt.CompareHashAndPassword([]byte(account.VerificationToken), []byte(c.Param("token")))
	if err == nil {
		session, err := mgo.Dial("mongodb://localhost:27017")
		defer session.Close()
		if err != nil {
			println(err.Error())
		}
		d := session.DB("test")
		collection := d.C(ACCOUNTS)
		account.VerificationToken = ""
		account.IsVerified = "yes"
		collection.UpdateId(account.ID, account)
	}
	c.Redirect(http.StatusFound, user.DefaultReturnUrl())
}

func ResendVerification (c *gin.Context) {
	account, _ := getAccount(c)
	user, _ := getUser(c)
	if account.IsVerified == "yes" {
		c.HTML(http.StatusOK, user.DefaultReturnUrl(), c.Keys)
		return
	}
	response := Response{} // todo sync.Pool
	response.Errors = []string{}
	response.ErrFor = make(map[string]string)
	defer response.Recover(c)

	var body struct {
		Username    string  `json:"username"`
		Email   string  `json:"email"`
		Password string  `json:"password"`
	}
	decoder := json.NewDecoder(c.Request.Body)
	err := decoder.Decode(&body)

	email := strings.ToLower(body.Email)
	if len(email) == 0 {
		response.ErrFor["email"] = "required"
	} else {
		r, err := regexp.MatchString(`^[a-zA-Z0-9\-\_\.\+]+@[a-zA-Z0-9\-\_\.]+\.[a-zA-Z0-9\-\_]+$`, email)
		if err != nil {
			println(err.Error())
		}
		if !r {
			response.ErrFor["email"] = `invalid email format`
		}
	}
	if response.HasErrors() {
		response.Fail(c)
		return
	}
	session, err := mgo.Dial("mongodb://localhost:27017")
	defer session.Close()
	if err != nil {
		println(err.Error())
	}
	d := session.DB("test")
	collection := d.C(USERS)
	{
		user_ := User{}
		collection.Find(bson.M{"email": email, "_id": bson.M{"$ne": user.ID}}).One(&user_)
		if len(user_.Username) > 0 {
			response.ErrFor["email"] = `email already taken`
		}
	}
	if response.HasErrors() {
		response.Fail(c)
		return
	}
	user.Email = email
	collection.UpdateId(user.ID, user)

	collection = d.C(ACCOUNTS)
	VerifyURL := generateToken(21)
	hash, err := bcrypt.GenerateFromPassword(VerifyURL, bcrypt.DefaultCost)
	if err != nil {
		println(err.Error())
		return
	}
	account.VerificationToken = string(hash)
	collection.UpdateId(account.ID, account)// todo how to update only part?
	verifyURL := "http" +"://"+ "localhost:3000" +"/account/verification/" + string(VerifyURL) + "/"
	c.Set("VerifyURL", verifyURL)
	mailConf := MailConfig{}
	mailConf.Data = c.Keys
	mailConf.From = config.SMTP.From.Name + " <" + config.SMTP.From.Address + ">"
	mailConf.To = user.Email
	mailConf.Subject = "Your " + config.ProjectName + " Account"
	mailConf.ReplyTo = user.Email
	mailConf.HtmlPath = "views/account/verification/email-html.html"

	if err := mailConf.SendMail(); err != nil {
		//todo it's not serious
	}

	response.Success = true
	c.JSON(http.StatusOK, response)
}

func AccountSettingsRender(c *gin.Context) {
	sess := sessions.Default(c)

	public := sess.Get("public")
	session, err := mgo.Dial("mongodb://localhost:27017")
	defer session.Close()
	if err != nil {
		println(err.Error())
	}
	d := session.DB("test")
	collection := d.C(USERS)
	us := User{} // todo pool
	err = collection.FindId(bson.ObjectIdHex(public.(string))).One(&us)
	if err != nil {
		println(err.Error())
	}
	if len(us.Username) != 0 {
		User, _ := json.Marshal(gin.H{
			"_id": us.ID.Hex(),
			"user": us.Username,
			"email": us.Email,
		})
		c.Set("User", url.QueryEscape(string(User)))
	}
	collection = d.C(ACCOUNTS)
	ac := Account{} // todo pool
	err = collection.FindId(bson.ObjectIdHex(us.Roles.Account.Hex())).One(&ac)
	if err != nil {
		println(err.Error())
	}
	if len(ac.ID) != 0 {
		Account, _ := json.Marshal(gin.H{
			"_id": ac.ID.Hex(),
			"name": gin.H{
				"first": ac.Name.First,
				"middle": ac.Name.Middle,
				"last": ac.Name.Last,
			},
			"company": ac.Company,
			"phone": ac.Phone,
			"zip": ac.Zip,
		})
		c.Set("Account", url.QueryEscape(string(Account)))
	}
	render, _ := TemplateStorage[c.Request.URL.Path]
	render.Data = c.Keys
	c.Render(http.StatusOK, render)
}<commit_msg>[login] settings<commit_after>package main

import (
	"github.com/gin-gonic/gin"
	"net/http"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"github.com/gin-gonic/contrib/sessions"
	"encoding/json"
	"net/url"
	"crypto/rand"
	"encoding/hex"
	"golang.org/x/crypto/bcrypt"
	"regexp"
"strings"
	"html/template"
)

func generateToken(n int) []byte {
	b := make([]byte, n)
	_, err := rand.Read(b)
	if err != nil {
		println(err.Error())
		return b
	}
	token := make([]byte, n * 2)
	hex.Encode(token, b)
	return token
}

func AccountRender(c *gin.Context) {
	render, _ := TemplateStorage[c.Request.URL.Path]
	render.Data = c.Keys
	c.Render(http.StatusOK, render)
}

func AccountVerificationRender(c *gin.Context) {
	account, _ := getAccount(c)
	user, _ := getUser(c)
	if account.IsVerified == "yes" {
		c.Redirect(http.StatusFound, user.DefaultReturnUrl())
		return
	}
	if len(account.VerificationToken) > 0 {

	} else {
		VerifyURL := generateToken(21)
		hash, err := bcrypt.GenerateFromPassword(VerifyURL, bcrypt.DefaultCost)
		if err != nil {
			println(err.Error())
			return
		}

		session, err := mgo.Dial("mongodb://localhost:27017")
		defer session.Close()
		if err != nil {
			println(err.Error())
		}
		d := session.DB("test")
		collection := d.C(ACCOUNTS)
		account.VerificationToken = string(hash)
		collection.UpdateId(account.ID, account)// todo how to update only part?
		verifyURL := "http" +"://"+ "localhost:3000" +"/account/verification/" + string(VerifyURL) + "/"
		c.Set("VerifyURL", verifyURL)

		mailConf := MailConfig{}
		mailConf.Data = c.Keys
		mailConf.From = config.SMTP.From.Name + " <" + config.SMTP.From.Address + ">"
		mailConf.To = user.Email
		mailConf.Subject = "Your " + config.ProjectName + " Account"
		mailConf.ReplyTo = user.Email
		mailConf.HtmlPath = "views/account/verification/email-html.html"

		if err := mailConf.SendMail(); err != nil {
			//todo it's not serious
		}

	}

	render, _ := TemplateStorage[c.Request.URL.Path]
	render.Data = c.Keys
	c.Render(http.StatusOK, render)
}

func Verify (c *gin.Context) {

	account, _ := getAccount(c)
	user, _ := getUser(c)
	err := bcrypt.CompareHashAndPassword([]byte(account.VerificationToken), []byte(c.Param("token")))
	if err == nil {
		session, err := mgo.Dial("mongodb://localhost:27017")
		defer session.Close()
		if err != nil {
			println(err.Error())
		}
		d := session.DB("test")
		collection := d.C(ACCOUNTS)
		account.VerificationToken = ""
		account.IsVerified = "yes"
		collection.UpdateId(account.ID, account)
	}
	c.Redirect(http.StatusFound, user.DefaultReturnUrl())
}

func ResendVerification (c *gin.Context) {
	account, _ := getAccount(c)
	user, _ := getUser(c)
	if account.IsVerified == "yes" {
		c.HTML(http.StatusOK, user.DefaultReturnUrl(), c.Keys)
		return
	}
	response := Response{} // todo sync.Pool
	response.Errors = []string{}
	response.ErrFor = make(map[string]string)
	defer response.Recover(c)

	var body struct {
		Username    string  `json:"username"`
		Email   string  `json:"email"`
		Password string  `json:"password"`
	}
	decoder := json.NewDecoder(c.Request.Body)
	err := decoder.Decode(&body)

	email := strings.ToLower(body.Email)
	if len(email) == 0 {
		response.ErrFor["email"] = "required"
	} else {
		r, err := regexp.MatchString(`^[a-zA-Z0-9\-\_\.\+]+@[a-zA-Z0-9\-\_\.]+\.[a-zA-Z0-9\-\_]+$`, email)
		if err != nil {
			println(err.Error())
		}
		if !r {
			response.ErrFor["email"] = `invalid email format`
		}
	}
	if response.HasErrors() {
		response.Fail(c)
		return
	}
	session, err := mgo.Dial("mongodb://localhost:27017")
	defer session.Close()
	if err != nil {
		println(err.Error())
	}
	d := session.DB("test")
	collection := d.C(USERS)
	{
		user_ := User{}
		collection.Find(bson.M{"email": email, "_id": bson.M{"$ne": user.ID}}).One(&user_)
		if len(user_.Username) > 0 {
			response.ErrFor["email"] = `email already taken`
		}
	}
	if response.HasErrors() {
		response.Fail(c)
		return
	}
	user.Email = email
	collection.UpdateId(user.ID, user)

	collection = d.C(ACCOUNTS)
	VerifyURL := generateToken(21)
	hash, err := bcrypt.GenerateFromPassword(VerifyURL, bcrypt.DefaultCost)
	if err != nil {
		println(err.Error())
		return
	}
	account.VerificationToken = string(hash)
	collection.UpdateId(account.ID, account)// todo how to update only part?
	verifyURL := "http" +"://"+ "localhost:3000" +"/account/verification/" + string(VerifyURL) + "/"
	c.Set("VerifyURL", verifyURL)
	mailConf := MailConfig{}
	mailConf.Data = c.Keys
	mailConf.From = config.SMTP.From.Name + " <" + config.SMTP.From.Address + ">"
	mailConf.To = user.Email
	mailConf.Subject = "Your " + config.ProjectName + " Account"
	mailConf.ReplyTo = user.Email
	mailConf.HtmlPath = "views/account/verification/email-html.html"

	if err := mailConf.SendMail(); err != nil {
		//todo it's not serious
	}

	response.Success = true
	c.JSON(http.StatusOK, response)
}

func AccountSettingsRender(c *gin.Context) {
	sess := sessions.Default(c)

	public := sess.Get("public")
	session, err := mgo.Dial("mongodb://localhost:27017")
	defer session.Close()
	if err != nil {
		println(err.Error())
	}
	d := session.DB("test")
	collection := d.C(USERS)
	us := User{} // todo pool
	err = collection.FindId(bson.ObjectIdHex(public.(string))).One(&us)
	if err != nil {
		println(err.Error())
	}
	if len(us.Username) != 0 {
		User, _ := json.Marshal(gin.H{
			"_id": us.ID.Hex(),
			"user": us.Username,
			"email": us.Email,
		})
		c.Set("User", template.JS(url.QueryEscape(string(User))))
	}
	collection = d.C(ACCOUNTS)
	ac := Account{} // todo pool
	err = collection.FindId(bson.ObjectIdHex(us.Roles.Account.Hex())).One(&ac)
	if err != nil {
		println(err.Error())
	}
	if len(ac.ID) != 0 {
		Account, _ := json.Marshal(gin.H{
			"_id": ac.ID.Hex(),
			"name": gin.H{
				"first": ac.Name.First,
				"middle": ac.Name.Middle,
				"last": ac.Name.Last,
			},
			"company": ac.Company,
			"phone": ac.Phone,
			"zip": ac.Zip,
		})
		c.Set("Account", template.JS(url.QueryEscape(string(Account))))
	}
	render, _ := TemplateStorage[c.Request.URL.Path]
	render.Data = c.Keys
	c.Render(http.StatusOK, render)
}<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strconv"
	"strings"
	"text/template"

	"github.com/coreos/coreos-kubernetes/multi-node/aws/pkg/config"
	"github.com/spf13/cobra"
)

var (
	cmdInit = &cobra.Command{
		Use:          "init",
		Short:        "Initialize default kube-aws cluster configuration",
		Long:         ``,
		RunE:         runCmdInit,
		SilenceUsage: true,
	}

	initOpts = config.Config{}
)

func init() {
	cmdRoot.AddCommand(cmdInit)
	cmdInit.Flags().StringVar(&initOpts.ClusterName, "cluster-name", "", "The name of this cluster. This will be the name of the cloudformation stack")
	cmdInit.Flags().StringVar(&initOpts.ExternalDNSName, "external-dns-name", "", "The hostname that will route to the api server")
	cmdInit.Flags().StringVar(&initOpts.Region, "region", "", "The AWS region to deploy to")
	cmdInit.Flags().StringVar(&initOpts.AvailabilityZone, "availability-zone", "", "The AWS availability-zone to deploy to")
	cmdInit.Flags().StringVar(&initOpts.KeyName, "key-name", "", "The AWS key-pair for ssh access to nodes")
	cmdInit.Flags().StringVar(&initOpts.KMSKeyARN, "kms-key-arn", "", "The ARN of the AWS KMS key for encrypting TLS assets")
}

func runCmdInit(cmd *cobra.Command, args []string) error {
	// Validate flags.
	required := []struct {
		name, val string
	}{
		{"--cluster-name", initOpts.ClusterName},
		{"--external-dns-name", initOpts.ExternalDNSName},
		{"--region", initOpts.Region},
		{"--availability-zone", initOpts.AvailabilityZone},
		{"--key-name", initOpts.KeyName},
		{"--kms-key-arn", initOpts.KMSKeyARN},
	}
	var missing []string
	for _, req := range required {
		if req.val == "" {
			missing = append(missing, strconv.Quote(req.name))
		}
	}
	if len(missing) != 0 {
		return fmt.Errorf("Missing required flag(s): %s", strings.Join(missing, ", "))
	}

	// Render the default cluster config.
	cfgTemplate, err := template.New("cluster.yaml").Parse(string(config.DefaultClusterConfig))
	if err != nil {
		return fmt.Errorf("Error parsing default config template: %v", err)
	}

	out, err := os.OpenFile(configPath, os.O_CREATE|os.O_WRONLY, 0600)
	if err != nil {
		return fmt.Errorf("Error opening %s : %v", configPath, err)
	}
	defer out.Close()
	if err := cfgTemplate.Execute(out, initOpts); err != nil {
		return fmt.Errorf("Error exec-ing default config template: %v", err)
	}

	successMsg :=
		`Success! Created %s

Next steps:
1. (Optional) Edit %s to parameterize the cluster.
2. Use the "kube-aws render" command to render the stack template.
`

	fmt.Printf(successMsg, configPath, configPath)
	return nil
}
<commit_msg>kube-aws: fail if configPath (cluster.yaml) already exists<commit_after>package main

import (
	"fmt"
	"os"
	"strconv"
	"strings"
	"text/template"

	"github.com/coreos/coreos-kubernetes/multi-node/aws/pkg/config"
	"github.com/spf13/cobra"
)

var (
	cmdInit = &cobra.Command{
		Use:          "init",
		Short:        "Initialize default kube-aws cluster configuration",
		Long:         ``,
		RunE:         runCmdInit,
		SilenceUsage: true,
	}

	initOpts = config.Config{}
)

func init() {
	cmdRoot.AddCommand(cmdInit)
	cmdInit.Flags().StringVar(&initOpts.ClusterName, "cluster-name", "", "The name of this cluster. This will be the name of the cloudformation stack")
	cmdInit.Flags().StringVar(&initOpts.ExternalDNSName, "external-dns-name", "", "The hostname that will route to the api server")
	cmdInit.Flags().StringVar(&initOpts.Region, "region", "", "The AWS region to deploy to")
	cmdInit.Flags().StringVar(&initOpts.AvailabilityZone, "availability-zone", "", "The AWS availability-zone to deploy to")
	cmdInit.Flags().StringVar(&initOpts.KeyName, "key-name", "", "The AWS key-pair for ssh access to nodes")
	cmdInit.Flags().StringVar(&initOpts.KMSKeyARN, "kms-key-arn", "", "The ARN of the AWS KMS key for encrypting TLS assets")
}

func runCmdInit(cmd *cobra.Command, args []string) error {
	// Validate flags.
	required := []struct {
		name, val string
	}{
		{"--cluster-name", initOpts.ClusterName},
		{"--external-dns-name", initOpts.ExternalDNSName},
		{"--region", initOpts.Region},
		{"--availability-zone", initOpts.AvailabilityZone},
		{"--key-name", initOpts.KeyName},
		{"--kms-key-arn", initOpts.KMSKeyARN},
	}
	var missing []string
	for _, req := range required {
		if req.val == "" {
			missing = append(missing, strconv.Quote(req.name))
		}
	}
	if len(missing) != 0 {
		return fmt.Errorf("Missing required flag(s): %s", strings.Join(missing, ", "))
	}

	// Render the default cluster config.
	cfgTemplate, err := template.New("cluster.yaml").Parse(string(config.DefaultClusterConfig))
	if err != nil {
		return fmt.Errorf("Error parsing default config template: %v", err)
	}

	out, err := os.OpenFile(configPath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0600)
	if err != nil {
		return fmt.Errorf("Error opening %s : %v", configPath, err)
	}
	defer out.Close()
	if err := cfgTemplate.Execute(out, initOpts); err != nil {
		return fmt.Errorf("Error exec-ing default config template: %v", err)
	}

	successMsg :=
		`Success! Created %s

Next steps:
1. (Optional) Edit %s to parameterize the cluster.
2. Use the "kube-aws render" command to render the stack template.
`

	fmt.Printf(successMsg, configPath, configPath)
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"time"
	"html/template"
	"net/http"
	"os"
	"strings"

	"github.com/howeyc/ledger"

	"github.com/BurntSushi/toml"
	"github.com/go-martini/martini"
)

type quickviewAccountConfig struct {
	Name      string
	ShortName string `toml:"short_name"`
}

type quickviewConfigStruct struct {
	Accounts []quickviewAccountConfig `toml:"account"`
}

func quickviewHandler(w http.ResponseWriter, r *http.Request) {
	if len(quickviewConfigFileName) == 0 {
		http.Redirect(w, r, "/accounts", http.StatusFound)
	}
	var quickviewConfigData quickviewConfigStruct
	if _, lerr := toml.DecodeFile(quickviewConfigFileName, &quickviewConfigData); lerr != nil || len(quickviewConfigData.Accounts) < 1 {
		http.Redirect(w, r, "/accounts", http.StatusFound)
	}

	shorten := func(accname string) string {
		for _, qvc := range quickviewConfigData.Accounts {
			if qvc.Name == accname {
				return qvc.ShortName
			}
		}
		return abbrev(accname)
	}

	funcMap := template.FuncMap{
		"abbrev": shorten,
	}

	t, err := parseAssetsWithFunc(funcMap, "templates/template.accounts.html", "templates/template.nav.html")
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	trans, terr := getTransactions()
	if terr != nil {
		http.Error(w, terr.Error(), 500)
		return
	}

	var pData pageData
	pData.Reports = reportConfigData.Reports
	pData.Portfolios = portfolioConfigData.Portfolios
	pData.Transactions = trans

	includeNames := make(map[string]bool)
	for _, qvc := range quickviewConfigData.Accounts {
		includeNames[qvc.Name] = true
	}

	balances := ledger.GetBalances(trans, []string{})
	for _, bal := range balances {
		if includeNames[bal.Name] {
			pData.Accounts = append(pData.Accounts, bal)
		}
	}

	err = t.Execute(w, pData)
	if err != nil {
		http.Error(w, err.Error(), 500)
	}
}

func addTransactionPostHandler(w http.ResponseWriter, r *http.Request) {
	strDate := r.FormValue("transactionDate")
	strPayee := r.FormValue("transactionPayee")

	var accountLines []string
	for i := 1; i < 20; i++ {
		strAcc := r.FormValue(fmt.Sprintf("transactionAccount%d", i))
		strAmt := r.FormValue(fmt.Sprintf("transactionAmount%d", i))
		accountLines = append(accountLines, strings.Trim(fmt.Sprintf("%s          %s",strAcc, strAmt), " \t"))
	}

	date, _ := time.Parse("2006-01-02", strDate)

	f, err := os.OpenFile(ledgerFileName, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
    if err != nil {
		http.Error(w, err.Error(), 500)
	}

	fmt.Fprintln(f, date.Format("2006/01/02"), strPayee)
	for _, accLine := range accountLines {
		if len(accLine) > 0 {
			fmt.Fprintf(f, "    %s", accLine)
			fmt.Fprintln(f, "")
		}
	}
	fmt.Fprintln(f, "")

	f.Close()
	getTransactions()

	http.Redirect(w, r, "/addtrans", http.StatusFound)
}

func addTransactionHandler(w http.ResponseWriter, r *http.Request) {
	funcMap := template.FuncMap{
		"abbrev": abbrev,
	}

	t, err := parseAssetsWithFunc(funcMap, "templates/template.addtransaction.html", "templates/template.nav.html")
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	trans, terr := getTransactions()
	if terr != nil {
		http.Error(w, terr.Error(), 500)
		return
	}

	balances := ledger.GetBalances(trans, []string{})

	var pData pageData
	pData.Reports = reportConfigData.Reports
	pData.Portfolios = portfolioConfigData.Portfolios
	pData.Accounts = balances
	pData.Transactions = trans

	err = t.Execute(w, pData)
	if err != nil {
		http.Error(w, err.Error(), 500)
	}
}

func accountsHandler(w http.ResponseWriter, r *http.Request) {
	funcMap := template.FuncMap{
		"abbrev": abbrev,
	}

	t, err := parseAssetsWithFunc(funcMap, "templates/template.accounts.html", "templates/template.nav.html")
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	trans, terr := getTransactions()
	if terr != nil {
		http.Error(w, terr.Error(), 500)
		return
	}

	balances := ledger.GetBalances(trans, []string{})

	var pData pageData
	pData.Reports = reportConfigData.Reports
	pData.Portfolios = portfolioConfigData.Portfolios
	pData.Accounts = balances
	pData.Transactions = trans

	err = t.Execute(w, pData)
	if err != nil {
		http.Error(w, err.Error(), 500)
	}
}

func accountHandler(w http.ResponseWriter, r *http.Request, params martini.Params) {
	accountName := params["accountName"]

	t, err := parseAssets("templates/template.account.html", "templates/template.nav.html")
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	trans, terr := getTransactions()
	if terr != nil {
		http.Error(w, terr.Error(), 500)
		return
	}

	var pageTrans []*ledger.Transaction
	for _, tran := range trans {
		for _, accChange := range tran.AccountChanges {
			if strings.Contains(accChange.Name, accountName) {
				pageTrans = append(pageTrans, &ledger.Transaction{
					Payee:          tran.Payee,
					Date:           tran.Date,
					AccountChanges: []ledger.Account{accChange},
				})
			}
		}
	}

	var pData pageData
	pData.Reports = reportConfigData.Reports
	pData.Transactions = pageTrans

	err = t.Execute(w, pData)
	if err != nil {
		http.Error(w, err.Error(), 500)
	}
}
<commit_msg>add missing portfolios to account view<commit_after>package main

import (
	"fmt"
	"html/template"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/howeyc/ledger"

	"github.com/BurntSushi/toml"
	"github.com/go-martini/martini"
)

type quickviewAccountConfig struct {
	Name      string
	ShortName string `toml:"short_name"`
}

type quickviewConfigStruct struct {
	Accounts []quickviewAccountConfig `toml:"account"`
}

func quickviewHandler(w http.ResponseWriter, r *http.Request) {
	if len(quickviewConfigFileName) == 0 {
		http.Redirect(w, r, "/accounts", http.StatusFound)
	}
	var quickviewConfigData quickviewConfigStruct
	if _, lerr := toml.DecodeFile(quickviewConfigFileName, &quickviewConfigData); lerr != nil || len(quickviewConfigData.Accounts) < 1 {
		http.Redirect(w, r, "/accounts", http.StatusFound)
	}

	shorten := func(accname string) string {
		for _, qvc := range quickviewConfigData.Accounts {
			if qvc.Name == accname {
				return qvc.ShortName
			}
		}
		return abbrev(accname)
	}

	funcMap := template.FuncMap{
		"abbrev": shorten,
	}

	t, err := parseAssetsWithFunc(funcMap, "templates/template.accounts.html", "templates/template.nav.html")
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	trans, terr := getTransactions()
	if terr != nil {
		http.Error(w, terr.Error(), 500)
		return
	}

	var pData pageData
	pData.Reports = reportConfigData.Reports
	pData.Portfolios = portfolioConfigData.Portfolios
	pData.Transactions = trans

	includeNames := make(map[string]bool)
	for _, qvc := range quickviewConfigData.Accounts {
		includeNames[qvc.Name] = true
	}

	balances := ledger.GetBalances(trans, []string{})
	for _, bal := range balances {
		if includeNames[bal.Name] {
			pData.Accounts = append(pData.Accounts, bal)
		}
	}

	err = t.Execute(w, pData)
	if err != nil {
		http.Error(w, err.Error(), 500)
	}
}

func addTransactionPostHandler(w http.ResponseWriter, r *http.Request) {
	strDate := r.FormValue("transactionDate")
	strPayee := r.FormValue("transactionPayee")

	var accountLines []string
	for i := 1; i < 20; i++ {
		strAcc := r.FormValue(fmt.Sprintf("transactionAccount%d", i))
		strAmt := r.FormValue(fmt.Sprintf("transactionAmount%d", i))
		accountLines = append(accountLines, strings.Trim(fmt.Sprintf("%s          %s", strAcc, strAmt), " \t"))
	}

	date, _ := time.Parse("2006-01-02", strDate)

	f, err := os.OpenFile(ledgerFileName, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
	if err != nil {
		http.Error(w, err.Error(), 500)
	}

	fmt.Fprintln(f, date.Format("2006/01/02"), strPayee)
	for _, accLine := range accountLines {
		if len(accLine) > 0 {
			fmt.Fprintf(f, "    %s", accLine)
			fmt.Fprintln(f, "")
		}
	}
	fmt.Fprintln(f, "")

	f.Close()
	getTransactions()

	http.Redirect(w, r, "/addtrans", http.StatusFound)
}

func addTransactionHandler(w http.ResponseWriter, r *http.Request) {
	funcMap := template.FuncMap{
		"abbrev": abbrev,
	}

	t, err := parseAssetsWithFunc(funcMap, "templates/template.addtransaction.html", "templates/template.nav.html")
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	trans, terr := getTransactions()
	if terr != nil {
		http.Error(w, terr.Error(), 500)
		return
	}

	balances := ledger.GetBalances(trans, []string{})

	var pData pageData
	pData.Reports = reportConfigData.Reports
	pData.Portfolios = portfolioConfigData.Portfolios
	pData.Accounts = balances
	pData.Transactions = trans

	err = t.Execute(w, pData)
	if err != nil {
		http.Error(w, err.Error(), 500)
	}
}

func accountsHandler(w http.ResponseWriter, r *http.Request) {
	funcMap := template.FuncMap{
		"abbrev": abbrev,
	}

	t, err := parseAssetsWithFunc(funcMap, "templates/template.accounts.html", "templates/template.nav.html")
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	trans, terr := getTransactions()
	if terr != nil {
		http.Error(w, terr.Error(), 500)
		return
	}

	balances := ledger.GetBalances(trans, []string{})

	var pData pageData
	pData.Reports = reportConfigData.Reports
	pData.Portfolios = portfolioConfigData.Portfolios
	pData.Accounts = balances
	pData.Transactions = trans

	err = t.Execute(w, pData)
	if err != nil {
		http.Error(w, err.Error(), 500)
	}
}

func accountHandler(w http.ResponseWriter, r *http.Request, params martini.Params) {
	accountName := params["accountName"]

	t, err := parseAssets("templates/template.account.html", "templates/template.nav.html")
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	trans, terr := getTransactions()
	if terr != nil {
		http.Error(w, terr.Error(), 500)
		return
	}

	var pageTrans []*ledger.Transaction
	for _, tran := range trans {
		for _, accChange := range tran.AccountChanges {
			if strings.Contains(accChange.Name, accountName) {
				pageTrans = append(pageTrans, &ledger.Transaction{
					Payee:          tran.Payee,
					Date:           tran.Date,
					AccountChanges: []ledger.Account{accChange},
				})
			}
		}
	}

	var pData pageData
	pData.Reports = reportConfigData.Reports
	pData.Portfolios = portfolioConfigData.Portfolios
	pData.Transactions = pageTrans

	err = t.Execute(w, pData)
	if err != nil {
		http.Error(w, err.Error(), 500)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"os"
	"sync"

	"github.com/thomersch/grandine/lib/cugdf"
	"github.com/thomersch/grandine/lib/spatial"

	"github.com/thomersch/gosmparse"
)

type nd struct {
	Lat, Lon float64
	Tags     map[string]interface{}
}
type wy struct {
	ID      int64
	NodeIDs []int64
	Tags    map[string]interface{}
}
type rl struct {
	Members []gosmparse.RelationMember
	Tags    map[string]interface{}
}

type dataHandler struct {
	conds []condition

	ec *elemCache

	nodes    []nd
	nodesMtx sync.Mutex
	ways     []wy
	waysMtx  sync.Mutex
	rels     []rl
	relsMtx  sync.Mutex
}

func (d *dataHandler) ReadNode(n gosmparse.Node) {
	for _, cond := range d.conds {
		if cond.Matches(n.Tags) {
			d.nodesMtx.Lock()
			d.nodes = append(d.nodes, nd{
				Lat:  n.Lat,
				Lon:  n.Lon,
				Tags: cond.Map(n.Tags),
			})
			d.nodesMtx.Unlock()
		}
	}
}

func (d *dataHandler) ReadWay(w gosmparse.Way) {
	for _, cond := range d.conds {
		if cond.Matches(w.Tags) {
			d.ec.AddNodes(w.NodeIDs...)
			d.ec.setMembers(w.ID, w.NodeIDs)

			d.waysMtx.Lock()
			d.ways = append(d.ways, wy{
				ID:      w.ID,
				NodeIDs: w.NodeIDs,
				Tags:    cond.Map(w.Tags),
			})
			d.waysMtx.Unlock()
		}
	}
}

func (d *dataHandler) ReadRelation(r gosmparse.Relation) {
	for _, cond := range d.conds {
		if cond.Matches(r.Tags) {
			d.relsMtx.Lock()
			d.rels = append(d.rels, rl{
				Members: r.Members,
				Tags:    cond.Map(r.Tags),
			})
			d.relsMtx.Unlock()

			for _, memb := range r.Members {
				switch memb.Type {
				case gosmparse.WayType:
					d.ec.AddWay(memb.ID)
				} // TODO: check if relations of nodes/relations are necessary
			}
		}
	}
}

type elemCache struct {
	nodes    map[int64]spatial.Point
	nodesMtx sync.Mutex
	ways     map[int64][]int64
	waysMtx  sync.Mutex
}

func NewElemCache() *elemCache {
	return &elemCache{
		nodes: map[int64]spatial.Point{},
		ways:  map[int64][]int64{},
	}
}

func (d *elemCache) AddNodes(nIDs ...int64) {
	d.nodesMtx.Lock()
	for _, nID := range nIDs {
		d.nodes[nID] = spatial.Point{}
	}
	d.nodesMtx.Unlock()
}

func (d *elemCache) AddWay(wID int64) {
	d.waysMtx.Lock()
	d.ways[wID] = []int64{}
	d.waysMtx.Unlock()
}

func (d *elemCache) SetCoord(nID int64, coord spatial.Point) {
	d.nodesMtx.Lock()
	d.nodes[nID] = coord
	d.nodesMtx.Unlock()
}

func (d *elemCache) setMembers(wID int64, members []int64) {
	d.waysMtx.Lock()
	d.ways[wID] = members
	d.waysMtx.Unlock()
}

func (d *elemCache) ReadWay(w gosmparse.Way) {
	d.waysMtx.Lock()
	_, ok := d.ways[w.ID]
	d.waysMtx.Unlock()
	if ok {
		d.setMembers(w.ID, w.NodeIDs)
		d.AddNodes(w.NodeIDs...)
	}
}

func (d *elemCache) Line(wID int64) spatial.Line {
	// check if mutex is needed
	membs, ok := d.ways[wID]
	if !ok {
		log.Fatalf("missing referenced way: %v", wID)
	}

	var l spatial.Line
	for _, memb := range membs {
		l = append(l, d.nodes[memb])
	}
	return l
}

// Interface enforces this. Probably I should change the behavior.
func (d *elemCache) ReadNode(n gosmparse.Node)         {}
func (d *elemCache) ReadRelation(r gosmparse.Relation) {}

type nodeCollector struct {
	ec *elemCache
}

func (d *nodeCollector) ReadNode(n gosmparse.Node) {
	d.ec.SetCoord(n.ID, spatial.Point{float64(n.Lon), float64(n.Lat)})
}
func (d *nodeCollector) ReadWay(w gosmparse.Way)           {}
func (d *nodeCollector) ReadRelation(r gosmparse.Relation) {}

type tagMapFn func(map[string]string) map[string]interface{}

type mapper struct {
	cond     condition
	mapper   tagMapFn
	elemType spatial.GeomType
}

type condition struct {
	// TODO: make it possible to specify condition type (node/way/rel)
	key    string
	value  string
	mapper tagMapFn
}

func (c *condition) Matches(kv map[string]string) bool {
	if v, ok := kv[c.key]; ok {
		if len(c.value) == 0 || c.value == v {
			return true
		}
	}
	return false
}

func (c *condition) Map(kv map[string]string) map[string]interface{} {
	return c.mapper(kv)
}

func main() {
	transportationMapFn := func(kv map[string]string) map[string]interface{} {
		var cl string
		if class, ok := kv["highway"]; ok {
			cl = class
		}
		return map[string]interface{}{
			"@layer": "transportation",
			"class":  cl,
		}
	}

	landuseMapFn := func(kv map[string]string) map[string]interface{} {
		return map[string]interface{}{
			"__type": "area",
			"@layer": "landcover",
			"class":  "wood",
		}
	}

	aerowayMapFn := func(kv map[string]string) map[string]interface{} {
		var cl string
		if class, ok := kv["aeroway"]; ok {
			cl = class
		}
		return map[string]interface{}{
			"@layer": "aeroway",
			"class":  cl,
		}
	}

	buildingMapFn := func(kv map[string]string) map[string]interface{} {
		return map[string]interface{}{
			"@layer":    "building",
			"@zoom:min": 14,
		}
	}

	waterwayMapFn := func(kv map[string]string) map[string]interface{} {
		var cl string
		if class, ok := kv["waterway"]; ok {
			cl = class
		}
		return map[string]interface{}{
			"@layer": "waterway",
			"class":  cl,
		}
	}

	conds := []condition{
		condition{"aeroway", "aerodrome", aerowayMapFn},
		condition{"aeroway", "apron", aerowayMapFn},
		condition{"aeroway", "heliport", aerowayMapFn},
		condition{"aeroway", "runway", aerowayMapFn},
		condition{"aeroway", "helipad", aerowayMapFn},
		condition{"aeroway", "taxiway", aerowayMapFn},
		condition{"highway", "primary", transportationMapFn},
		condition{"highway", "secondary", transportationMapFn},
		condition{"highway", "tertiary", transportationMapFn},
		condition{"building", "", buildingMapFn},
		condition{"landuse", "forest", landuseMapFn},
		condition{"railway", "rail", transportationMapFn},
		condition{"waterway", "river", waterwayMapFn},
	}

	source := flag.String("src", "osm.pbf", "")
	outfile := flag.String("out", "osm.cugdf", "")
	flag.Parse()

	f, err := os.Open(*source)
	if err != nil {
		log.Fatal(err)
	}
	dec := gosmparse.NewDecoder(f)

	// First pass
	ec := NewElemCache()
	dh := dataHandler{
		conds: conds,
		ec:    ec,
	}
	log.Println("Starting 3 step parsing")
	log.Println("Reading data (1/3)...")
	err = dec.Parse(&dh)
	if err != nil {
		log.Fatal(err)
	}

	_, err = f.Seek(0, 0) // jumps to beginning of file
	if err != nil {
		log.Fatal(err)
	}

	// Second pass
	log.Println("Collecting nodes (2/3)...")
	err = dec.Parse(ec)
	if err != nil {
		log.Fatal(err)
	}

	_, err = f.Seek(0, 0)
	if err != nil {
		log.Fatal(err)
	}

	// Third pass
	log.Println("Resolving dependent objects (3/3)...")
	rc := nodeCollector{
		ec: ec,
	}
	err = dec.Parse(&rc)
	if err != nil {
		log.Fatal(err)
	}

	var fc []spatial.Feature

	log.Println("Parsing completed.")

	log.Println("Collecting points...")
	for _, pt := range dh.nodes {
		props := map[string]interface{}{}
		for k, v := range pt.Tags {
			props[k] = v
		}
		fc = append(fc, spatial.Feature{
			Props:    props,
			Geometry: spatial.MustNewGeom(spatial.Point{float64(pt.Lon), float64(pt.Lat)}),
		})
	}

	log.Println("Assembling ways...")
	// TODO: auto-detect if linestring or polygon, based on tags
	for _, wy := range dh.ways {
		var (
			area  bool
			props = map[string]interface{}{}
			geom  interface{}
		)
		for k, v := range wy.Tags {
			if k == "__type" && v == "area" {
				area = true
				continue
			}
			props[k] = v
		}
		ln := ec.Line(wy.ID)
		if !ln.Clockwise() {
			ln.Reverse()
		}
		if area {
			geom = spatial.Polygon{ln}
		} else {
			geom = ln
		}

		fc = append(fc, spatial.Feature{
			Props:    props,
			Geometry: spatial.MustNewGeom(geom),
		})
	}

	log.Println("Assembling relations...")
	for _, rl := range dh.rels {
		if v, ok := rl.Tags["type"]; !ok || v != "multipolygon" {
			continue
		}
		var poly spatial.Polygon

		for _, memb := range rl.Members {
			if memb.Role == "outer" || memb.Role == "inner" {
				ring := ec.Line(memb.ID)
				if (memb.Role == "outer" && !ring.Clockwise()) || (memb.Role == "inner" && ring.Clockwise()) {
					ring.Reverse()
				}
				poly = append(poly, ring)
			}
		}
	}

	log.Println("Writing out")
	of, err := os.Create(*outfile)
	if err != nil {
		log.Fatal(err)
	}
	err = cugdf.Marshal(fc, of)
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>cmd/spatialize: ogc winding order for osm data<commit_after>package main

import (
	"flag"
	"log"
	"os"
	"sync"

	"github.com/thomersch/grandine/lib/cugdf"
	"github.com/thomersch/grandine/lib/spatial"

	"github.com/thomersch/gosmparse"
)

type nd struct {
	Lat, Lon float64
	Tags     map[string]interface{}
}
type wy struct {
	ID      int64
	NodeIDs []int64
	Tags    map[string]interface{}
}
type rl struct {
	Members []gosmparse.RelationMember
	Tags    map[string]interface{}
}

type dataHandler struct {
	conds []condition

	ec *elemCache

	nodes    []nd
	nodesMtx sync.Mutex
	ways     []wy
	waysMtx  sync.Mutex
	rels     []rl
	relsMtx  sync.Mutex
}

func (d *dataHandler) ReadNode(n gosmparse.Node) {
	for _, cond := range d.conds {
		if cond.Matches(n.Tags) {
			d.nodesMtx.Lock()
			d.nodes = append(d.nodes, nd{
				Lat:  n.Lat,
				Lon:  n.Lon,
				Tags: cond.Map(n.Tags),
			})
			d.nodesMtx.Unlock()
		}
	}
}

func (d *dataHandler) ReadWay(w gosmparse.Way) {
	for _, cond := range d.conds {
		if cond.Matches(w.Tags) {
			d.ec.AddNodes(w.NodeIDs...)
			d.ec.setMembers(w.ID, w.NodeIDs)

			d.waysMtx.Lock()
			d.ways = append(d.ways, wy{
				ID:      w.ID,
				NodeIDs: w.NodeIDs,
				Tags:    cond.Map(w.Tags),
			})
			d.waysMtx.Unlock()
		}
	}
}

func (d *dataHandler) ReadRelation(r gosmparse.Relation) {
	for _, cond := range d.conds {
		if cond.Matches(r.Tags) {
			d.relsMtx.Lock()
			d.rels = append(d.rels, rl{
				Members: r.Members,
				Tags:    cond.Map(r.Tags),
			})
			d.relsMtx.Unlock()

			for _, memb := range r.Members {
				switch memb.Type {
				case gosmparse.WayType:
					d.ec.AddWay(memb.ID)
				} // TODO: check if relations of nodes/relations are necessary
			}
		}
	}
}

type elemCache struct {
	nodes    map[int64]spatial.Point
	nodesMtx sync.Mutex
	ways     map[int64][]int64
	waysMtx  sync.Mutex
}

func NewElemCache() *elemCache {
	return &elemCache{
		nodes: map[int64]spatial.Point{},
		ways:  map[int64][]int64{},
	}
}

func (d *elemCache) AddNodes(nIDs ...int64) {
	d.nodesMtx.Lock()
	for _, nID := range nIDs {
		d.nodes[nID] = spatial.Point{}
	}
	d.nodesMtx.Unlock()
}

func (d *elemCache) AddWay(wID int64) {
	d.waysMtx.Lock()
	d.ways[wID] = []int64{}
	d.waysMtx.Unlock()
}

func (d *elemCache) SetCoord(nID int64, coord spatial.Point) {
	d.nodesMtx.Lock()
	d.nodes[nID] = coord
	d.nodesMtx.Unlock()
}

func (d *elemCache) setMembers(wID int64, members []int64) {
	d.waysMtx.Lock()
	d.ways[wID] = members
	d.waysMtx.Unlock()
}

func (d *elemCache) ReadWay(w gosmparse.Way) {
	d.waysMtx.Lock()
	_, ok := d.ways[w.ID]
	d.waysMtx.Unlock()
	if ok {
		d.setMembers(w.ID, w.NodeIDs)
		d.AddNodes(w.NodeIDs...)
	}
}

func (d *elemCache) Line(wID int64) spatial.Line {
	// check if mutex is needed
	membs, ok := d.ways[wID]
	if !ok {
		log.Fatalf("missing referenced way: %v", wID)
	}

	var l spatial.Line
	for _, memb := range membs {
		l = append(l, d.nodes[memb])
	}
	return l
}

// Interface enforces this. Probably I should change the behavior.
func (d *elemCache) ReadNode(n gosmparse.Node)         {}
func (d *elemCache) ReadRelation(r gosmparse.Relation) {}

type nodeCollector struct {
	ec *elemCache
}

func (d *nodeCollector) ReadNode(n gosmparse.Node) {
	d.ec.SetCoord(n.ID, spatial.Point{float64(n.Lon), float64(n.Lat)})
}
func (d *nodeCollector) ReadWay(w gosmparse.Way)           {}
func (d *nodeCollector) ReadRelation(r gosmparse.Relation) {}

type tagMapFn func(map[string]string) map[string]interface{}

type mapper struct {
	cond     condition
	mapper   tagMapFn
	elemType spatial.GeomType
}

type condition struct {
	// TODO: make it possible to specify condition type (node/way/rel)
	key    string
	value  string
	mapper tagMapFn
}

func (c *condition) Matches(kv map[string]string) bool {
	if v, ok := kv[c.key]; ok {
		if len(c.value) == 0 || c.value == v {
			return true
		}
	}
	return false
}

func (c *condition) Map(kv map[string]string) map[string]interface{} {
	return c.mapper(kv)
}

func main() {
	transportationMapFn := func(kv map[string]string) map[string]interface{} {
		var cl string
		if class, ok := kv["highway"]; ok {
			cl = class
		}
		return map[string]interface{}{
			"@layer": "transportation",
			"class":  cl,
		}
	}

	landuseMapFn := func(kv map[string]string) map[string]interface{} {
		return map[string]interface{}{
			"__type": "area",
			"@layer": "landcover",
			"class":  "wood",
		}
	}

	aerowayMapFn := func(kv map[string]string) map[string]interface{} {
		var cl string
		if class, ok := kv["aeroway"]; ok {
			cl = class
		}
		return map[string]interface{}{
			"@layer": "aeroway",
			"class":  cl,
		}
	}

	buildingMapFn := func(kv map[string]string) map[string]interface{} {
		return map[string]interface{}{
			"@layer":    "building",
			"@zoom:min": 14,
		}
	}

	waterwayMapFn := func(kv map[string]string) map[string]interface{} {
		var cl string
		if class, ok := kv["waterway"]; ok {
			cl = class
		}
		return map[string]interface{}{
			"@layer": "waterway",
			"class":  cl,
		}
	}

	conds := []condition{
		condition{"aeroway", "aerodrome", aerowayMapFn},
		condition{"aeroway", "apron", aerowayMapFn},
		condition{"aeroway", "heliport", aerowayMapFn},
		condition{"aeroway", "runway", aerowayMapFn},
		condition{"aeroway", "helipad", aerowayMapFn},
		condition{"aeroway", "taxiway", aerowayMapFn},
		condition{"highway", "primary", transportationMapFn},
		condition{"highway", "secondary", transportationMapFn},
		condition{"highway", "tertiary", transportationMapFn},
		condition{"building", "", buildingMapFn},
		condition{"landuse", "forest", landuseMapFn},
		condition{"railway", "rail", transportationMapFn},
		condition{"waterway", "river", waterwayMapFn},
	}

	source := flag.String("src", "osm.pbf", "")
	outfile := flag.String("out", "osm.cugdf", "")
	flag.Parse()

	f, err := os.Open(*source)
	if err != nil {
		log.Fatal(err)
	}
	dec := gosmparse.NewDecoder(f)

	// First pass
	ec := NewElemCache()
	dh := dataHandler{
		conds: conds,
		ec:    ec,
	}
	log.Println("Starting 3 step parsing")
	log.Println("Reading data (1/3)...")
	err = dec.Parse(&dh)
	if err != nil {
		log.Fatal(err)
	}

	_, err = f.Seek(0, 0) // jumps to beginning of file
	if err != nil {
		log.Fatal(err)
	}

	// Second pass
	log.Println("Collecting nodes (2/3)...")
	err = dec.Parse(ec)
	if err != nil {
		log.Fatal(err)
	}

	_, err = f.Seek(0, 0)
	if err != nil {
		log.Fatal(err)
	}

	// Third pass
	log.Println("Resolving dependent objects (3/3)...")
	rc := nodeCollector{
		ec: ec,
	}
	err = dec.Parse(&rc)
	if err != nil {
		log.Fatal(err)
	}

	var fc []spatial.Feature

	log.Println("Parsing completed.")

	log.Println("Collecting points...")
	for _, pt := range dh.nodes {
		props := map[string]interface{}{}
		for k, v := range pt.Tags {
			props[k] = v
		}
		fc = append(fc, spatial.Feature{
			Props:    props,
			Geometry: spatial.MustNewGeom(spatial.Point{float64(pt.Lon), float64(pt.Lat)}),
		})
	}

	log.Println("Assembling ways...")
	// TODO: auto-detect if linestring or polygon, based on tags
	for _, wy := range dh.ways {
		var (
			area  bool
			props = map[string]interface{}{}
			geom  interface{}
		)
		for k, v := range wy.Tags {
			if k == "__type" && v == "area" {
				area = true
				continue
			}
			props[k] = v
		}
		ln := ec.Line(wy.ID)
		if !ln.Clockwise() {
			ln.Reverse()
		}
		if area {
			geom = spatial.Polygon{ln}
		} else {
			geom = ln
		}

		fc = append(fc, spatial.Feature{
			Props:    props,
			Geometry: spatial.MustNewGeom(geom),
		})
	}

	log.Println("Assembling relations...")
	for _, rl := range dh.rels {
		if v, ok := rl.Tags["type"]; !ok || v != "multipolygon" {
			continue
		}
		var poly spatial.Polygon

		for _, memb := range rl.Members {
			if memb.Role == "outer" || memb.Role == "inner" {
				ring := ec.Line(memb.ID)
				if (memb.Role == "outer" && ring.Clockwise()) || (memb.Role == "inner" && !ring.Clockwise()) {
					ring.Reverse()
				}
				poly = append(poly, ring)
			}
		}
	}

	log.Println("Writing out")
	of, err := os.Create(*outfile)
	if err != nil {
		log.Fatal(err)
	}
	err = cugdf.Marshal(fc, of)
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/toomore/gogrs/twse"
	"github.com/toomore/gogrs/utils"
)

type checkGroup interface {
	String() string
	CheckFunc(...*twse.Data) bool
	Mindata() int
}

type check01 struct{}

func (check01) String() string {
	return "MA 3 > 6 > 18"
}

func (check01) Mindata() int {
	return 18
}

func (check01) CheckFunc(b ...*twse.Data) bool {
	defer wg.Done()
	if prepareData(b...)[0] != true {
		return false
	}
	var ma3 = b[0].MA(3)
	if days, ok := utils.CountCountineFloat64(utils.DeltaFloat64(ma3)); !ok || days == 0 {
		return false
	}
	var ma6 = b[0].MA(6)
	if days, ok := utils.CountCountineFloat64(utils.DeltaFloat64(ma6)); !ok || days == 0 {
		return false
	}
	var ma18 = b[0].MA(18)
	if days, ok := utils.CountCountineFloat64(utils.DeltaFloat64(ma18)); !ok || days == 0 {
		return false
	}
	//log.Println(ma3[len(ma3)-1], ma6[len(ma6)-1], ma18[len(ma18)-1])
	if ma3[len(ma3)-1] > ma6[len(ma6)-1] && ma6[len(ma6)-1] > ma18[len(ma18)-1] {
		return true
	}
	return false
}

type check02 struct{}

func (check02) String() string {
	return "量大於前三天 K 線收紅"
}

func (check02) Mindata() int {
	return 4
}

func (check02) CheckFunc(b ...*twse.Data) bool {
	defer wg.Done()
	return utils.ThanSumPastUint64((*b[0]).GetVolumeList(), 3, true) && (*b[0]).IsRed()
}

type check03 struct{}

func (check03) String() string {
	return "量或價走平 45 天"
}

func (check03) Mindata() int {
	return 45
}

func (check03) CheckFunc(b ...*twse.Data) bool {
	defer wg.Done()
	if !prepareData(b...)[0] {
		return false
	}
	var price = b[0].GetPriceList()
	var volume = b[0].GetVolumeList()
	return price[len(price)-1] > 10 &&
		(utils.SD(price[len(price)-46:]) < 0.25 ||
			utils.SDUint64(volume[len(volume)-46:]) < 0.25)
}

func prepareData(b ...*twse.Data) []bool {
	var result []bool
	var mindata int
	for i := range ckList {
		if ckList[i].Mindata() > mindata {
			mindata = ckList[i].Mindata()
		}
	}

	for i, _ := range b {
		result = make([]bool, len(b))
		b[i].Get()
		if b[i].Len() < mindata {
			start := b[i].Len()
			for {
				b[i].PlusData()
				if b[i].Len() > mindata {
					result[i] = true
					break
				}
				if b[i].Len() == start {
					result[i] = false
					break
				}
				start = b[i].Len()
			}
			if b[i].Len() < mindata {
				result[i] = false
			}
		} else {
			result[i] = true
		}
	}
	return result
}

func init() {
	ckList.Add(checkGroup(check01{}))
	ckList.Add(checkGroup(check02{}))
	ckList.Add(checkGroup(check03{}))
}
<commit_msg>Fixed for loop only i.<commit_after>package main

import (
	"github.com/toomore/gogrs/twse"
	"github.com/toomore/gogrs/utils"
)

type checkGroup interface {
	String() string
	CheckFunc(...*twse.Data) bool
	Mindata() int
}

type check01 struct{}

func (check01) String() string {
	return "MA 3 > 6 > 18"
}

func (check01) Mindata() int {
	return 18
}

func (check01) CheckFunc(b ...*twse.Data) bool {
	defer wg.Done()
	if prepareData(b...)[0] != true {
		return false
	}
	var ma3 = b[0].MA(3)
	if days, ok := utils.CountCountineFloat64(utils.DeltaFloat64(ma3)); !ok || days == 0 {
		return false
	}
	var ma6 = b[0].MA(6)
	if days, ok := utils.CountCountineFloat64(utils.DeltaFloat64(ma6)); !ok || days == 0 {
		return false
	}
	var ma18 = b[0].MA(18)
	if days, ok := utils.CountCountineFloat64(utils.DeltaFloat64(ma18)); !ok || days == 0 {
		return false
	}
	//log.Println(ma3[len(ma3)-1], ma6[len(ma6)-1], ma18[len(ma18)-1])
	if ma3[len(ma3)-1] > ma6[len(ma6)-1] && ma6[len(ma6)-1] > ma18[len(ma18)-1] {
		return true
	}
	return false
}

type check02 struct{}

func (check02) String() string {
	return "量大於前三天 K 線收紅"
}

func (check02) Mindata() int {
	return 4
}

func (check02) CheckFunc(b ...*twse.Data) bool {
	defer wg.Done()
	return utils.ThanSumPastUint64((*b[0]).GetVolumeList(), 3, true) && (*b[0]).IsRed()
}

type check03 struct{}

func (check03) String() string {
	return "量或價走平 45 天"
}

func (check03) Mindata() int {
	return 45
}

func (check03) CheckFunc(b ...*twse.Data) bool {
	defer wg.Done()
	if !prepareData(b...)[0] {
		return false
	}
	var price = b[0].GetPriceList()
	var volume = b[0].GetVolumeList()
	return price[len(price)-1] > 10 &&
		(utils.SD(price[len(price)-46:]) < 0.25 ||
			utils.SDUint64(volume[len(volume)-46:]) < 0.25)
}

func prepareData(b ...*twse.Data) []bool {
	var result []bool
	var mindata int
	for i := range ckList {
		if ckList[i].Mindata() > mindata {
			mindata = ckList[i].Mindata()
		}
	}

	for i := range b {
		result = make([]bool, len(b))
		b[i].Get()
		if b[i].Len() < mindata {
			start := b[i].Len()
			for {
				b[i].PlusData()
				if b[i].Len() > mindata {
					result[i] = true
					break
				}
				if b[i].Len() == start {
					result[i] = false
					break
				}
				start = b[i].Len()
			}
			if b[i].Len() < mindata {
				result[i] = false
			}
		} else {
			result[i] = true
		}
	}
	return result
}

func init() {
	ckList.Add(checkGroup(check01{}))
	ckList.Add(checkGroup(check02{}))
	ckList.Add(checkGroup(check03{}))
}
<|endoftext|>