<commit_before>/*
Copyright (c) 2016, Simon J Mudd
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

* Redistributions of source code must retain the above copyright notice, this
  list of conditions and the following disclaimer.

* Redistributions in binary form must reproduce the above copyright notice,
  this list of conditions and the following disclaimer in the documentation
  and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

// Package stopwatch implements simple stopwatch functionality
package stopwatch

import (
	"time"
)

// NamedStopwatch holds a map of string named stopwatches. Intended to be used when several
// Stopwatches are being used at once, and easy to use as they are name based.
type NamedStopwatch struct {
	stopwatches map[string]*Stopwatch
}

//
func (ns *NamedStopwatch) AddStopwatch(name string, s *Stopwatch) {
	if ns.stopwatches == nil {
		ns.stopwatches = make(map[string](*Stopwatch))
	}
	ns.stopwatches[name] = s
}

func (ns *NamedStopwatch) Start(name string) {
	if s, ok := ns.stopwatches[name]; ok {
		s.Start()
	}
}
func (ns *NamedStopwatch) Stop(name string) {
	if s, ok := ns.stopwatches[name]; ok {
		s.Stop()
	}
}

func (ns *NamedStopwatch) Reset(name string) {
	if s, ok := ns.stopwatches[name]; ok {
		s.Reset()
	}
}

func (ns *NamedStopwatch) Keys() []string {
	keys := []string{}
	for k, _ := range ns.stopwatches {
		keys = append(keys, k)
	}
	return keys
}

func (ns *NamedStopwatch) Elapsed(name string) time.Duration {
	if s, ok := ns.stopwatches[name]; ok {
		return s.Elapsed()
	}
	return time.Duration(0)
}

func (ns *NamedStopwatch) ElapsedSeconds(name string) float64 {
	if s, ok := ns.stopwatches[name]; ok {
		return s.ElapsedSeconds()
	}
	return float64(0)
}

func (ns *NamedStopwatch) ElapsedMilliSeconds(name string) float64 {
	if s, ok := ns.stopwatches[name]; ok {
		return s.ElapsedMilliSeconds()
	}
	return float64(0)
}
<commit_msg>add some comments<commit_after>/*
Copyright (c) 2016, Simon J Mudd
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

* Redistributions of source code must retain the above copyright notice, this
  list of conditions and the following disclaimer.

* Redistributions in binary form must reproduce the above copyright notice,
  this list of conditions and the following disclaimer in the documentation
  and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

// Package stopwatch implements simple stopwatch functionality
package stopwatch

import (
	"errors"
	"fmt"
	"time"
)

// NamedStopwatch holds a map of string named stopwatches. Intended to be used when several
// Stopwatches are being used at once, and easy to use as they are name based.
type NamedStopwatch struct {
	stopwatches map[string]*Stopwatch
}

// AddStopwatch adds a Stopwatch with the given name.
func (ns *NamedStopwatch) AddStopwatch(name string, s *Stopwatch) error {
	if ns.stopwatches == nil {
		ns.stopwatches = make(map[string](*Stopwatch))
	}
	if _, found := ns.stopwatches[name]; !found {
		return errors.New(fmt.Sprintf("Stopwatch name %q already exists", name))
	}
	ns.stopwatches[name] = s
	return nil
}

// Start starts a NamedStopwatch if it exists
func (ns *NamedStopwatch) Start(name string) {
	if s, ok := ns.stopwatches[name]; ok {
		s.Start()
	}
}

// Stop stops a NamedStopwatch if it exists
func (ns *NamedStopwatch) Stop(name string) {
	if s, ok := ns.stopwatches[name]; ok {
		s.Stop()
	}
}

// Reset resets a NamedStopwatch if it exists
func (ns *NamedStopwatch) Reset(name string) {
	if s, ok := ns.stopwatches[name]; ok {
		s.Reset()
	}
}

// Keys returns the known names of Stopwatches
func (ns *NamedStopwatch) Keys() []string {
	keys := []string{}
	for k, _ := range ns.stopwatches {
		keys = append(keys, k)
	}
	return keys
}

// Elapsed returns the elapsed time.Duration of the named stopwatch if it exists or 0
func (ns *NamedStopwatch) Elapsed(name string) time.Duration {
	if s, ok := ns.stopwatches[name]; ok {
		return s.Elapsed()
	}
	return time.Duration(0)
}

// Elapsed returns the elapsed time in seconds of the named stopwatch if it exists or 0
func (ns *NamedStopwatch) ElapsedSeconds(name string) float64 {
	if s, ok := ns.stopwatches[name]; ok {
		return s.ElapsedSeconds()
	}
	return float64(0)
}

// Elapsed returns the elapsed time in milliseconds of the named stopwatch if it exists or 0
func (ns *NamedStopwatch) ElapsedMilliSeconds(name string) float64 {
	if s, ok := ns.stopwatches[name]; ok {
		return s.ElapsedMilliSeconds()
	}
	return float64(0)
}
<|endoftext|>


<commit_before>/*
 * Copyright 2019 The Kythe Authors. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"regexp"
	"strings"

	"github.com/kythe/llvmbzlgen/cmakelib/ast"
	"github.com/kythe/llvmbzlgen/cmakelib/bindings"
	bzlpath "github.com/kythe/llvmbzlgen/path"
	"github.com/kythe/llvmbzlgen/writer"
)

// blockCounter counts active blocks of the given name for matching
// paired CMake commands.
type blockCounter struct {
	begin string // The beginning command, e.g. "if"
	end   string // The ending command, e.g. "endif"
	count int
}

// newCounter returns a new blockCounter instance which counts
// blocks delimited by begin and "end" + begin.
func newCounter(begin string) *blockCounter {
	return &blockCounter{begin, "end" + begin, 0}
}

// Count increments the internal counter if text matches the begin delimiter,
// decrements it if it matches the end delimiter and returns true if text
// matched a delimiter or the current count is greater than zero.
func (bc *blockCounter) Count(text string) bool {
	matched := true
	if text == bc.begin {
		bc.count += 1
	} else if text == bc.end {
		bc.count -= 1
	} else {
		matched = false
	}
	return matched || bc.count > 0
}

type eval struct {
	p *ast.Parser
	o options

	w    *writer.StarlarkWriter
	v    *bindings.Mapping
	root bzlpath.Path
	path bzlpath.Path
}

type options struct {
	macroName   string
	shouldPrint func(string) bool
	shouldAdd   func(string) bool
	excludePath func(string) bool
}

// Option is a configuration option for the CMake evaluator.
type Option func(*eval)

// PrintCommands configures the evaluator to print commands on the StarlarkWriter for which the supplied predicate returns true.
func PrintCommands(p func(string) bool) Option {
	return func(e *eval) { e.o.shouldPrint = p }
}

// RecurseCommands configures the evaluator to recurse into the subdirectory
// specified by the first argument to the command when the provided predicate returns true.
// By default only "add_subdirectory" is handled this way.
func RecurseCommands(p func(string) bool) Option {
	return func(e *eval) { e.o.shouldAdd = p }
}

// ExcludePaths configures the evaluator to omit particular paths entirely during traversal.
func ExcludePaths(p func(string) bool) Option {
	return func(e *eval) { e.o.excludePath = p }
}

// DefineVars configures the evaluator to predefine the specified variables.
func DefineVars(vars map[string]string) Option {
	return func(e *eval) {
		for k, v := range vars {
			e.v.Set(k, v)
		}
	}
}

// Matching compiles the provided pattern and returns a predicate for matching strings.
func Matching(pat string) func(string) bool {
	return regexp.MustCompile(pat).MatchString
}

// NewEvaluator returns a new CMake evaluator instance.
func NewEvaluator(w io.Writer, opts ...Option) *eval {
	e := &eval{
		p: ast.NewParser(),
		w: writer.NewStarlarkWriter(w),
		v: bindings.New(),
		o: options{
			macroName: "generated_cmake_targets",
			shouldAdd: func(n string) bool { return n == "add_subdirectory" },
		},
	}
	for _, o := range opts {
		o(e)
	}
	return e
}

// parse parses the provided input into a CMakeFile AST.
func (e *eval) parse(input io.Reader) (*ast.CMakeFile, error) {
	return e.p.Parse(input)
}

// parse parses the provided path into a CMakeFile AST.
func (e *eval) parseFile(path string) (*ast.CMakeFile, error) {
	input, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer input.Close()
	return e.parse(input)
}

// walk evaluates all of the provided CMakeLists.txt files into the body of a single Starlark macro..
func (e *eval) walk(paths []bzlpath.Path) error {
	if err := e.w.BeginMacro(e.o.macroName); err != nil {
		return err
	}
	root, paths := bzlpath.SplitCommonRoot(paths)
	e.root = root
	for _, p := range paths {
		if err := e.AddSubdirectory(p.String()); err != nil {
			return err
		}
	}
	return e.w.EndMacro()
}

// dispatchFunc is a function which handles the current command, updates the
// remaining list of commands and returns a dispatchFunc suitable for processing that remainder.
type dispatchFunc func(*commandList) (dispatchFunc, error)

// commandList is a slice of CMake CommandInvocation elements used for dispatch.
type commandList []ast.CommandInvocation

// Advance removes the top command from the list and returns true if there are more to process.
func (l *commandList) Advance() bool {
	if len(*l) > 0 {
		*l = (*l)[1:]
		return len(*l) > 0
	}
	return false
}

// Head returns the first command in the list, if any.
func (l *commandList) Head() *ast.CommandInvocation {
	if len(*l) > 0 {
		return &(*l)[0]
	}
	return nil
}

// shouldPrint returns true if the command given by name should be included in the Starlark output.
func (e *eval) shouldPrint(name string) bool {
	return e.o.shouldPrint != nil && e.o.shouldPrint(name)
}

// shouldAdd retruns true if the command given by name should be recursed into.
func (e *eval) shouldAdd(name string) bool {
	return e.o.shouldAdd != nil && e.o.shouldAdd(name)
}

// excludePath returns true if the path given by dirpath should be skipped.
func (e *eval) excludePath(dirpath string) bool {
	return e.o.excludePath != nil && e.o.excludePath(dirpath)
}

// dispatch evaluates the next command from cmds and returns a new dispatchFunc for handling the remainder.
func (e *eval) dispatch(cmds *commandList) (dispatchFunc, error) {
	name := string(cmds.Head().Name)
	if e.shouldPrint(name) {
		e.PrintCommand(cmds.Head())
	}

	switch name {
	// TODO(shahms): Actually process these.
	case "if", "function", "foreach", "macro":
		counter := newCounter(name)
		for counter.Count(name) && cmds.Advance() {
			name = string(cmds.Head().Name)
		}
		return e.dispatch, nil
	case "set":
		e.setVariable(cmds.Head().Arguments.Eval(e.v))
	case "unset":
		e.unsetVariable(cmds.Head().Arguments.Eval(e.v))
	}

	if e.shouldAdd(name) {
		args := cmds.Head().Arguments.Eval(e.v)
		if len(args) != 1 {
			return nil, fmt.Errorf("invalid number of arguments to directory command %s", cmds.Head().Pos)
		}
		if !e.excludePath(args[0]) {
			if err := e.AddSubdirectory(cmds.Head().Arguments.Eval(e.v)[0]); err != nil {
				return nil, err
			}
		}
	}
	cmds.Advance()
	return e.dispatch, nil
}

// setVariable sets the value of the variable designated by the remained, following the rules of
// https://cmake.org/cmake/help/v3.3/command/set.html#command:set
func (e *eval) setVariable(args []string) {
	if len(args) == 0 {
		log.Println("Cannot set a variable without a name")
		return
	}
	key, args := args[0], args[1:len(args)]
	switch {
	case len(args) > 0 && args[len(args)-1] == "PARENT_SCOPE":
		e.v.SetParent(key, strings.Join(args[0:len(args)-1], ";"))
	case len(args) >= 3 && args[len(args)-3] == "CACHE":
		e.v.SetCache(key, strings.Join(args[:len(args)-3], ";"))
	case len(args) >= 4 && args[len(args)-4] == "CACHE": // FORCE
		e.v.SetCache(key, strings.Join(args[:len(args)-4], ";"))
	default:
		e.v.Set(key, strings.Join(args, ";"))
	}
}

// unsetVariable unsets the value of the variable designated by the remained, following the rules of
// https://cmake.org/cmake/help/v3.3/command/set.html#command:unset
func (e *eval) unsetVariable(args []string) {
	switch {
	case len(args) == 0:
		log.Println("Cannot unset a variable without a name")
	case len(args) == 1:
		e.v.Set(args[0], "")
	case len(args) == 2 && args[1] == "PARENT_SCOPE":
		e.v.SetParent(args[0], "")
	case len(args) == 2 && args[1] == "CACHE":
		e.v.SetCache(args[0], "")
	default:
		log.Println("Ignoring invalid unset command")
	}
}

// AddSubdirectory recurses into the directory specified by dirpath and evaluates the CMakeLists.txt contained therein.
func (e *eval) AddSubdirectory(dirpath string) error {
	if err := e.enterDirectory(dirpath); err != nil {
		return err
	}
	file, err := e.parseFile(path.Join(e.root.String(), e.path.String(), "CMakeLists.txt"))
	if err != nil {
		return err
	}

	cmds := commandList(file.Commands)
	dispatch := e.dispatch
	for {
		dispatch, err := dispatch(&cmds)
		if err != nil {
			return err
		}
		if len(cmds) == 0 || dispatch == nil {
			break
		}

	}
	return e.exitDirectory(dirpath)
}

// ProjectRoot returns the path prefix for forming project-rooted absolute paths.
func (e *eval) ProjectRoot() string {
	// Use a fixed prefix so that paths formed by simple string concatenation don't
	// start with '//' which is often treated specially.
	return "/root"
}

// CurrentDirectory returns the relative, project-rooted path currently being traversed.
func (e *eval) CurrentDirectory() string {
	return path.Join(e.path...)
}

// enterDirectory pushes a new directory onto the stack, setting up necessary state, etc.
func (e *eval) enterDirectory(dirpath string) error {
	if err := e.w.PushDirectory(dirpath); err != nil {
		return err
	}
	e.v.Push()
	e.path = append(e.path, dirpath)
	e.v.Set("CMAKE_CURRENT_SOURCE_DIR", path.Join(e.ProjectRoot(), e.CurrentDirectory()))
	e.v.Set("CMAKE_CURRENT_BINARY_DIR", path.Join(e.ProjectRoot(), e.CurrentDirectory()))
	return nil
}

// exitDirectory pops the most recently entered directory off the stack.
func (e *eval) exitDirectory(path string) error {
	e.v.Pop()
	e.path = e.path[:len(e.path)-1]
	tail, err := e.w.PopDirectory()
	if tail != path {
		return fmt.Errorf("unexpected directory state %v != %v", tail, path)
	}
	return err
}

// PrintCommand writes the given command to the configured StarlarkWriter.
func (e *eval) PrintCommand(command *ast.CommandInvocation) error {
	return e.w.WriteCommand(string(command.Name), writer.ArgumentLiterals(command.Arguments.Eval(e.v)))
}

func main() {
	flag.Parse()
	eval := NewEvaluator(os.Stdout,
		ExcludePaths(Matching(`(^|/)(unittests|examples|cmake)($|/)`)),
		RecurseCommands(Matching(`add(_\w+)?_subdirectory`)),
		PrintCommands(Matching("^("+strings.Join([]string{
			"configure_file", "set",
			"add_llvm_library", "add_clang_library", "add_llvm_target",
			"add_tablegen", "tablegen", "clang_diag_gen", "clang_tablegen", "add_public_tablegen_target",
		}, "|")+")$")))
	if err := eval.walk(bzlpath.ToPaths(flag.Args())); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>fix(cmaketobzl): keep llvm_component_library (#12)<commit_after>/*
 * Copyright 2019 The Kythe Authors. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"regexp"
	"strings"

	"github.com/kythe/llvmbzlgen/cmakelib/ast"
	"github.com/kythe/llvmbzlgen/cmakelib/bindings"
	bzlpath "github.com/kythe/llvmbzlgen/path"
	"github.com/kythe/llvmbzlgen/writer"
)

// blockCounter counts active blocks of the given name for matching
// paired CMake commands.
type blockCounter struct {
	begin string // The beginning command, e.g. "if"
	end   string // The ending command, e.g. "endif"
	count int
}

// newCounter returns a new blockCounter instance which counts
// blocks delimited by begin and "end" + begin.
func newCounter(begin string) *blockCounter {
	return &blockCounter{begin, "end" + begin, 0}
}

// Count increments the internal counter if text matches the begin delimiter,
// decrements it if it matches the end delimiter and returns true if text
// matched a delimiter or the current count is greater than zero.
func (bc *blockCounter) Count(text string) bool {
	matched := true
	if text == bc.begin {
		bc.count += 1
	} else if text == bc.end {
		bc.count -= 1
	} else {
		matched = false
	}
	return matched || bc.count > 0
}

type eval struct {
	p *ast.Parser
	o options

	w    *writer.StarlarkWriter
	v    *bindings.Mapping
	root bzlpath.Path
	path bzlpath.Path
}

type options struct {
	macroName   string
	shouldPrint func(string) bool
	shouldAdd   func(string) bool
	excludePath func(string) bool
}

// Option is a configuration option for the CMake evaluator.
type Option func(*eval)

// PrintCommands configures the evaluator to print commands on the StarlarkWriter for which the supplied predicate returns true.
func PrintCommands(p func(string) bool) Option {
	return func(e *eval) { e.o.shouldPrint = p }
}

// RecurseCommands configures the evaluator to recurse into the subdirectory
// specified by the first argument to the command when the provided predicate returns true.
// By default only "add_subdirectory" is handled this way.
func RecurseCommands(p func(string) bool) Option {
	return func(e *eval) { e.o.shouldAdd = p }
}

// ExcludePaths configures the evaluator to omit particular paths entirely during traversal.
func ExcludePaths(p func(string) bool) Option {
	return func(e *eval) { e.o.excludePath = p }
}

// DefineVars configures the evaluator to predefine the specified variables.
func DefineVars(vars map[string]string) Option {
	return func(e *eval) {
		for k, v := range vars {
			e.v.Set(k, v)
		}
	}
}

// Matching compiles the provided pattern and returns a predicate for matching strings.
func Matching(pat string) func(string) bool {
	return regexp.MustCompile(pat).MatchString
}

// NewEvaluator returns a new CMake evaluator instance.
func NewEvaluator(w io.Writer, opts ...Option) *eval {
	e := &eval{
		p: ast.NewParser(),
		w: writer.NewStarlarkWriter(w),
		v: bindings.New(),
		o: options{
			macroName: "generated_cmake_targets",
			shouldAdd: func(n string) bool { return n == "add_subdirectory" },
		},
	}
	for _, o := range opts {
		o(e)
	}
	return e
}

// parse parses the provided input into a CMakeFile AST.
func (e *eval) parse(input io.Reader) (*ast.CMakeFile, error) {
	return e.p.Parse(input)
}

// parse parses the provided path into a CMakeFile AST.
func (e *eval) parseFile(path string) (*ast.CMakeFile, error) {
	input, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer input.Close()
	return e.parse(input)
}

// walk evaluates all of the provided CMakeLists.txt files into the body of a single Starlark macro..
func (e *eval) walk(paths []bzlpath.Path) error {
	if err := e.w.BeginMacro(e.o.macroName); err != nil {
		return err
	}
	root, paths := bzlpath.SplitCommonRoot(paths)
	e.root = root
	for _, p := range paths {
		if err := e.AddSubdirectory(p.String()); err != nil {
			return err
		}
	}
	return e.w.EndMacro()
}

// dispatchFunc is a function which handles the current command, updates the
// remaining list of commands and returns a dispatchFunc suitable for processing that remainder.
type dispatchFunc func(*commandList) (dispatchFunc, error)

// commandList is a slice of CMake CommandInvocation elements used for dispatch.
type commandList []ast.CommandInvocation

// Advance removes the top command from the list and returns true if there are more to process.
func (l *commandList) Advance() bool {
	if len(*l) > 0 {
		*l = (*l)[1:]
		return len(*l) > 0
	}
	return false
}

// Head returns the first command in the list, if any.
func (l *commandList) Head() *ast.CommandInvocation {
	if len(*l) > 0 {
		return &(*l)[0]
	}
	return nil
}

// shouldPrint returns true if the command given by name should be included in the Starlark output.
func (e *eval) shouldPrint(name string) bool {
	return e.o.shouldPrint != nil && e.o.shouldPrint(name)
}

// shouldAdd retruns true if the command given by name should be recursed into.
func (e *eval) shouldAdd(name string) bool {
	return e.o.shouldAdd != nil && e.o.shouldAdd(name)
}

// excludePath returns true if the path given by dirpath should be skipped.
func (e *eval) excludePath(dirpath string) bool {
	return e.o.excludePath != nil && e.o.excludePath(dirpath)
}

// dispatch evaluates the next command from cmds and returns a new dispatchFunc for handling the remainder.
func (e *eval) dispatch(cmds *commandList) (dispatchFunc, error) {
	name := string(cmds.Head().Name)
	if e.shouldPrint(name) {
		e.PrintCommand(cmds.Head())
	}

	switch name {
	// TODO(shahms): Actually process these.
	case "if", "function", "foreach", "macro":
		counter := newCounter(name)
		for counter.Count(name) && cmds.Advance() {
			name = string(cmds.Head().Name)
		}
		return e.dispatch, nil
	case "set":
		e.setVariable(cmds.Head().Arguments.Eval(e.v))
	case "unset":
		e.unsetVariable(cmds.Head().Arguments.Eval(e.v))
	}

	if e.shouldAdd(name) {
		args := cmds.Head().Arguments.Eval(e.v)
		if len(args) != 1 {
			return nil, fmt.Errorf("invalid number of arguments to directory command %s", cmds.Head().Pos)
		}
		if !e.excludePath(args[0]) {
			if err := e.AddSubdirectory(cmds.Head().Arguments.Eval(e.v)[0]); err != nil {
				return nil, err
			}
		}
	}
	cmds.Advance()
	return e.dispatch, nil
}

// setVariable sets the value of the variable designated by the remained, following the rules of
// https://cmake.org/cmake/help/v3.3/command/set.html#command:set
func (e *eval) setVariable(args []string) {
	if len(args) == 0 {
		log.Println("Cannot set a variable without a name")
		return
	}
	key, args := args[0], args[1:len(args)]
	switch {
	case len(args) > 0 && args[len(args)-1] == "PARENT_SCOPE":
		e.v.SetParent(key, strings.Join(args[0:len(args)-1], ";"))
	case len(args) >= 3 && args[len(args)-3] == "CACHE":
		e.v.SetCache(key, strings.Join(args[:len(args)-3], ";"))
	case len(args) >= 4 && args[len(args)-4] == "CACHE": // FORCE
		e.v.SetCache(key, strings.Join(args[:len(args)-4], ";"))
	default:
		e.v.Set(key, strings.Join(args, ";"))
	}
}

// unsetVariable unsets the value of the variable designated by the remained, following the rules of
// https://cmake.org/cmake/help/v3.3/command/set.html#command:unset
func (e *eval) unsetVariable(args []string) {
	switch {
	case len(args) == 0:
		log.Println("Cannot unset a variable without a name")
	case len(args) == 1:
		e.v.Set(args[0], "")
	case len(args) == 2 && args[1] == "PARENT_SCOPE":
		e.v.SetParent(args[0], "")
	case len(args) == 2 && args[1] == "CACHE":
		e.v.SetCache(args[0], "")
	default:
		log.Println("Ignoring invalid unset command")
	}
}

// AddSubdirectory recurses into the directory specified by dirpath and evaluates the CMakeLists.txt contained therein.
func (e *eval) AddSubdirectory(dirpath string) error {
	if err := e.enterDirectory(dirpath); err != nil {
		return err
	}
	file, err := e.parseFile(path.Join(e.root.String(), e.path.String(), "CMakeLists.txt"))
	if err != nil {
		return err
	}

	cmds := commandList(file.Commands)
	dispatch := e.dispatch
	for {
		dispatch, err := dispatch(&cmds)
		if err != nil {
			return err
		}
		if len(cmds) == 0 || dispatch == nil {
			break
		}

	}
	return e.exitDirectory(dirpath)
}

// ProjectRoot returns the path prefix for forming project-rooted absolute paths.
func (e *eval) ProjectRoot() string {
	// Use a fixed prefix so that paths formed by simple string concatenation don't
	// start with '//' which is often treated specially.
	return "/root"
}

// CurrentDirectory returns the relative, project-rooted path currently being traversed.
func (e *eval) CurrentDirectory() string {
	return path.Join(e.path...)
}

// enterDirectory pushes a new directory onto the stack, setting up necessary state, etc.
func (e *eval) enterDirectory(dirpath string) error {
	if err := e.w.PushDirectory(dirpath); err != nil {
		return err
	}
	e.v.Push()
	e.path = append(e.path, dirpath)
	e.v.Set("CMAKE_CURRENT_SOURCE_DIR", path.Join(e.ProjectRoot(), e.CurrentDirectory()))
	e.v.Set("CMAKE_CURRENT_BINARY_DIR", path.Join(e.ProjectRoot(), e.CurrentDirectory()))
	return nil
}

// exitDirectory pops the most recently entered directory off the stack.
func (e *eval) exitDirectory(path string) error {
	e.v.Pop()
	e.path = e.path[:len(e.path)-1]
	tail, err := e.w.PopDirectory()
	if tail != path {
		return fmt.Errorf("unexpected directory state %v != %v", tail, path)
	}
	return err
}

// PrintCommand writes the given command to the configured StarlarkWriter.
func (e *eval) PrintCommand(command *ast.CommandInvocation) error {
	return e.w.WriteCommand(string(command.Name), writer.ArgumentLiterals(command.Arguments.Eval(e.v)))
}

func main() {
	flag.Parse()
	eval := NewEvaluator(os.Stdout,
		ExcludePaths(Matching(`(^|/)(unittests|examples|cmake)($|/)`)),
		RecurseCommands(Matching(`add(_\w+)?_subdirectory`)),
		PrintCommands(Matching("^("+strings.Join([]string{
			"configure_file", "set",
			"add_llvm_library", "add_llvm_component_library", "add_clang_library", "add_llvm_target",
			"add_tablegen", "tablegen", "clang_diag_gen", "clang_tablegen", "add_public_tablegen_target",
		}, "|")+")$")))
	if err := eval.walk(bzlpath.ToPaths(flag.Args())); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package net

var head = `
<!doctype html>

    <meta charset="utf-8">
    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
    <meta name="description" content="">
    <meta name="author" content="">

    <title>`

var middle = `</title>

    <link href="https://cdn.bootcdn.net/ajax/libs/twitter-bootstrap/4.5.3/css/bootstrap.min.css" rel="stylesheet">

                 <form action="`

var tail = `" method="post" enctype="multipart/form-data">
                         <input type="file" name="file">

                         <input type="submit">
                 </form>

`

func GetUploadPage(title, path string) string {
	return head + title + middle + path + tail
}
<commit_msg>add extra info<commit_after>package net

var head = `
<!doctype html>

    <meta charset="utf-8">
    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
    <meta name="description" content="">
    <meta name="author" content="">

    <title>`

var middle = `</title>

    <link href="https://dilfish.icu/302/bootstrap.css" rel="stylesheet">

                 <form action="`

var tail = `" method="post" enctype="multipart/form-data">
                         <input type="file" name="file">

                         <input type="submit">
                 </form>

累积上传最多1G，单次最大10M

`

func GetUploadPage(title, path string) string {
	return head + title + middle + path + tail
}
<|endoftext|>


<commit_before>package network

import (
	"sync"
	"time"

	"github.com/golang/protobuf/proto"
	"github.com/micro/go-micro/client"
	rtr "github.com/micro/go-micro/client/selector/router"
	"github.com/micro/go-micro/proxy"
	"github.com/micro/go-micro/router"
	pb "github.com/micro/go-micro/router/proto"
	"github.com/micro/go-micro/server"
	"github.com/micro/go-micro/transport"
	"github.com/micro/go-micro/tunnel"
	trn "github.com/micro/go-micro/tunnel/transport"
	"github.com/micro/go-micro/util/log"
)

var (
	// ControlChannel is the name of the tunnel channel for passing contron message
	ControlChannel = "control"
)

// network implements Network interface
type network struct {
	// options configure the network
	options Options
	// rtr is network router
	router.Router
	// prx is network proxy
	proxy.Proxy
	// tun is network tunnel
	tunnel.Tunnel
	// srv is network server
	srv server.Server
	// client is network client
	client client.Client

	sync.RWMutex
	// connected marks the network as connected
	connected bool
	// closed closes the network
	closed chan bool
}

// newNetwork returns a new network node
func newNetwork(opts ...Option) Network {
	options := DefaultOptions()

	for _, o := range opts {
		o(&options)
	}

	// init tunnel address to the network bind address
	options.Tunnel.Init(
		tunnel.Address(options.Address),
	)

	// create tunnel client with tunnel transport
	tunTransport := transport.NewTransport(
		trn.WithTunnel(options.Tunnel),
	)

	// srv is network server
	srv := server.NewServer(
		server.Name(options.Name),
		server.Transport(tunTransport),
	)

	// client is network client
	client := client.NewClient(
		client.Transport(tunTransport),
		client.Selector(
			rtr.NewSelector(
				rtr.WithRouter(options.Router),
			),
		),
	)

	return &network{
		options: options,
		Router:  options.Router,
		Proxy:   options.Proxy,
		Tunnel:  options.Tunnel,
		srv:     srv,
		client:  client,
	}
}

// Name returns network name
func (n *network) Name() string {
	return n.options.Name
}

// Address returns network bind address
func (n *network) Address() string {
	return n.Tunnel.Address()
}

func (n *network) resolveNodes() ([]string, error) {
	// resolve the network address to network nodes
	records, err := n.options.Resolver.Resolve(n.options.Name)
	if err != nil {
		return nil, err
	}

	// collect network node addresses
	nodes := make([]string, len(records))
	for i, record := range records {
		nodes[i] = record.Address
	}

	return nodes, nil
}

func (n *network) resolve() {
	resolve := time.NewTicker(ResolveTime)
	defer resolve.Stop()

	for {
		select {
		case <-n.closed:
			return
		case <-resolve.C:
			nodes, err := n.resolveNodes()
			if err != nil {
				log.Debugf("Network failed to resolve nodes: %v", err)
				continue
			}
			// initialize the tunnel
			n.Tunnel.Init(
				tunnel.Nodes(nodes...),
			)
		}
	}
}

func (n *network) handleConn(conn tunnel.Conn, msg chan *transport.Message) {
	for {
		m := new(transport.Message)
		if err := conn.Recv(m); err != nil {
			// TODO: should we bail here?
			log.Debugf("Network tunnel advert receive error: %v", err)
			return
		}

		select {
		case msg <- m:
		case <-n.closed:
			return
		}
	}
}

func (n *network) process(l tunnel.Listener) {
	// receive control message queue
	recv := make(chan *transport.Message, 128)

	// accept a connection
	conn, err := l.Accept()
	if err != nil {
		// TODO: handle this
		log.Debugf("Network tunnel accept error: %v", err)
		return
	}

	go n.handleConn(conn, recv)

	for {
		select {
		case m := <-recv:
			// switch on type of message and take action
			switch m.Header["Micro-Method"] {
			case "advert":
				pbAdvert := &pb.Advert{}
				if err := proto.Unmarshal(m.Body, pbAdvert); err != nil {
					continue
				}

				var events []*router.Event
				for _, event := range pbAdvert.Events {
					route := router.Route{
						Service: event.Route.Service,
						Address: event.Route.Address,
						Gateway: event.Route.Gateway,
						Network: event.Route.Network,
						Link:    event.Route.Link,
						Metric:  int(event.Route.Metric),
					}
					e := &router.Event{
						Type:      router.EventType(event.Type),
						Timestamp: time.Unix(0, pbAdvert.Timestamp),
						Route:     route,
					}
					events = append(events, e)
				}
				advert := &router.Advert{
					Id:        pbAdvert.Id,
					Type:      router.AdvertType(pbAdvert.Type),
					Timestamp: time.Unix(0, pbAdvert.Timestamp),
					TTL:       time.Duration(pbAdvert.Ttl),
					Events:    events,
				}

				if err := n.Router.Process(advert); err != nil {
					log.Debugf("Network failed to process advert %s: %v", advert.Id, err)
					continue
				}
			}
		case <-n.closed:
			return
		}
	}
}

// advertise advertises routes to the network
func (n *network) advertise(client transport.Client, advertChan <-chan *router.Advert) {
	for {
		select {
		// process local adverts and randomly fire them at other nodes
		case advert := <-advertChan:
			// create a proto advert
			var events []*pb.Event
			for _, event := range advert.Events {
				route := &pb.Route{
					Service: event.Route.Service,
					Address: event.Route.Address,
					Gateway: n.options.Address,
					Network: "network",
					Link:    event.Route.Link,
					Metric:  int64(event.Route.Metric),
				}
				e := &pb.Event{
					Type:      pb.EventType(event.Type),
					Timestamp: event.Timestamp.UnixNano(),
					Route:     route,
				}
				events = append(events, e)
			}
			pbAdvert := &pb.Advert{
				Id:        advert.Id,
				Type:      pb.AdvertType(advert.Type),
				Timestamp: advert.Timestamp.UnixNano(),
				Events:    events,
			}
			body, err := proto.Marshal(pbAdvert)
			if err != nil {
				// TODO: should we bail here?
				log.Debugf("Network failed to marshal message: %v", err)
				continue
			}
			// create transport message and chuck it down the pipe
			m := transport.Message{
				Header: map[string]string{
					"Micro-Method": "advert",
				},
				Body: body,
			}

			if err := client.Send(&m); err != nil {
				log.Debugf("Network failed to send advert %s: %v", pbAdvert.Id, err)
				continue
			}
		case <-n.closed:
			return
		}
	}
}

// Connect connects the network
func (n *network) Connect() error {
	n.Lock()
	defer n.Unlock()

	// return if already connected
	if n.connected {
		return nil
	}

	// try to resolve network nodes
	nodes, err := n.resolveNodes()
	if err != nil {
		log.Debugf("Network failed to resolve nodes: %v", err)
	}

	// connect network tunnel
	if err := n.Tunnel.Connect(); err != nil {
		return err
	}

	// initialize the tunnel to resolved nodes
	n.Tunnel.Init(
		tunnel.Nodes(nodes...),
	)

	// dial into ControlChannel to send route adverts
	client, err := n.Tunnel.Dial(ControlChannel)
	if err != nil {
		return err
	}

	// listen on ControlChannel
	listener, err := n.Tunnel.Listen(ControlChannel)
	if err != nil {
		return err
	}

	// create closed channel
	n.closed = make(chan bool)

	// keep resolving network nodes
	go n.resolve()

	// start the router
	if err := n.options.Router.Start(); err != nil {
		return err
	}

	// start advertising routes
	advertChan, err := n.options.Router.Advertise()
	if err != nil {
		return err
	}

	// advertise routes
	go n.advertise(client, advertChan)
	// accept and process routes
	go n.process(listener)

	// start the server
	if err := n.srv.Start(); err != nil {
		return err
	}

	// set connected to true
	n.connected = true

	return nil
}

func (n *network) close() error {
	// stop the server
	if err := n.srv.Stop(); err != nil {
		return err
	}

	// stop the router
	if err := n.Router.Stop(); err != nil {
		return err
	}

	// close the tunnel
	if err := n.Tunnel.Close(); err != nil {
		return err
	}

	return nil
}

// Close closes network connection
func (n *network) Close() error {
	n.Lock()
	defer n.Unlock()

	if !n.connected {
		return nil
	}

	select {
	case <-n.closed:
		return nil
	default:
		close(n.closed)
		// set connected to false
		n.connected = false
	}

	return n.close()
}

// Client returns network client
func (n *network) Client() client.Client {
	return n.client
}

// Server returns network server
func (n *network) Server() server.Server {
	return n.srv
}
<commit_msg>Set Route.Link to "network" not Route.Network. Oops!<commit_after>package network

import (
	"sync"
	"time"

	"github.com/golang/protobuf/proto"
	"github.com/micro/go-micro/client"
	rtr "github.com/micro/go-micro/client/selector/router"
	"github.com/micro/go-micro/proxy"
	"github.com/micro/go-micro/router"
	pb "github.com/micro/go-micro/router/proto"
	"github.com/micro/go-micro/server"
	"github.com/micro/go-micro/transport"
	"github.com/micro/go-micro/tunnel"
	trn "github.com/micro/go-micro/tunnel/transport"
	"github.com/micro/go-micro/util/log"
)

var (
	// ControlChannel is the name of the tunnel channel for passing contron message
	ControlChannel = "control"
)

// network implements Network interface
type network struct {
	// options configure the network
	options Options
	// rtr is network router
	router.Router
	// prx is network proxy
	proxy.Proxy
	// tun is network tunnel
	tunnel.Tunnel
	// srv is network server
	srv server.Server
	// client is network client
	client client.Client

	sync.RWMutex
	// connected marks the network as connected
	connected bool
	// closed closes the network
	closed chan bool
}

// newNetwork returns a new network node
func newNetwork(opts ...Option) Network {
	options := DefaultOptions()

	for _, o := range opts {
		o(&options)
	}

	// init tunnel address to the network bind address
	options.Tunnel.Init(
		tunnel.Address(options.Address),
	)

	// create tunnel client with tunnel transport
	tunTransport := transport.NewTransport(
		trn.WithTunnel(options.Tunnel),
	)

	// srv is network server
	srv := server.NewServer(
		server.Name(options.Name),
		server.Transport(tunTransport),
	)

	// client is network client
	client := client.NewClient(
		client.Transport(tunTransport),
		client.Selector(
			rtr.NewSelector(
				rtr.WithRouter(options.Router),
			),
		),
	)

	return &network{
		options: options,
		Router:  options.Router,
		Proxy:   options.Proxy,
		Tunnel:  options.Tunnel,
		srv:     srv,
		client:  client,
	}
}

// Name returns network name
func (n *network) Name() string {
	return n.options.Name
}

// Address returns network bind address
func (n *network) Address() string {
	return n.Tunnel.Address()
}

func (n *network) resolveNodes() ([]string, error) {
	// resolve the network address to network nodes
	records, err := n.options.Resolver.Resolve(n.options.Name)
	if err != nil {
		return nil, err
	}

	// collect network node addresses
	nodes := make([]string, len(records))
	for i, record := range records {
		nodes[i] = record.Address
	}

	return nodes, nil
}

func (n *network) resolve() {
	resolve := time.NewTicker(ResolveTime)
	defer resolve.Stop()

	for {
		select {
		case <-n.closed:
			return
		case <-resolve.C:
			nodes, err := n.resolveNodes()
			if err != nil {
				log.Debugf("Network failed to resolve nodes: %v", err)
				continue
			}
			// initialize the tunnel
			n.Tunnel.Init(
				tunnel.Nodes(nodes...),
			)
		}
	}
}

func (n *network) handleConn(conn tunnel.Conn, msg chan *transport.Message) {
	for {
		m := new(transport.Message)
		if err := conn.Recv(m); err != nil {
			// TODO: should we bail here?
			log.Debugf("Network tunnel advert receive error: %v", err)
			return
		}

		select {
		case msg <- m:
		case <-n.closed:
			return
		}
	}
}

func (n *network) process(l tunnel.Listener) {
	// receive control message queue
	recv := make(chan *transport.Message, 128)

	// accept a connection
	conn, err := l.Accept()
	if err != nil {
		// TODO: handle this
		log.Debugf("Network tunnel accept error: %v", err)
		return
	}

	go n.handleConn(conn, recv)

	for {
		select {
		case m := <-recv:
			// switch on type of message and take action
			switch m.Header["Micro-Method"] {
			case "advert":
				pbAdvert := &pb.Advert{}
				if err := proto.Unmarshal(m.Body, pbAdvert); err != nil {
					continue
				}

				var events []*router.Event
				for _, event := range pbAdvert.Events {
					route := router.Route{
						Service: event.Route.Service,
						Address: event.Route.Address,
						Gateway: event.Route.Gateway,
						Network: event.Route.Network,
						Link:    event.Route.Link,
						Metric:  int(event.Route.Metric),
					}
					e := &router.Event{
						Type:      router.EventType(event.Type),
						Timestamp: time.Unix(0, pbAdvert.Timestamp),
						Route:     route,
					}
					events = append(events, e)
				}
				advert := &router.Advert{
					Id:        pbAdvert.Id,
					Type:      router.AdvertType(pbAdvert.Type),
					Timestamp: time.Unix(0, pbAdvert.Timestamp),
					TTL:       time.Duration(pbAdvert.Ttl),
					Events:    events,
				}

				if err := n.Router.Process(advert); err != nil {
					log.Debugf("Network failed to process advert %s: %v", advert.Id, err)
					continue
				}
			}
		case <-n.closed:
			return
		}
	}
}

// advertise advertises routes to the network
func (n *network) advertise(client transport.Client, advertChan <-chan *router.Advert) {
	for {
		select {
		// process local adverts and randomly fire them at other nodes
		case advert := <-advertChan:
			// create a proto advert
			var events []*pb.Event
			for _, event := range advert.Events {
				// NOTE: we override the Gateway and Link fields here
				route := &pb.Route{
					Service: event.Route.Service,
					Address: event.Route.Address,
					Gateway: n.options.Address,
					Network: event.Route.Network,
					Link:    "network",
					Metric:  int64(event.Route.Metric),
				}
				e := &pb.Event{
					Type:      pb.EventType(event.Type),
					Timestamp: event.Timestamp.UnixNano(),
					Route:     route,
				}
				events = append(events, e)
			}
			pbAdvert := &pb.Advert{
				Id:        advert.Id,
				Type:      pb.AdvertType(advert.Type),
				Timestamp: advert.Timestamp.UnixNano(),
				Events:    events,
			}
			body, err := proto.Marshal(pbAdvert)
			if err != nil {
				// TODO: should we bail here?
				log.Debugf("Network failed to marshal message: %v", err)
				continue
			}
			// create transport message and chuck it down the pipe
			m := transport.Message{
				Header: map[string]string{
					"Micro-Method": "advert",
				},
				Body: body,
			}

			if err := client.Send(&m); err != nil {
				log.Debugf("Network failed to send advert %s: %v", pbAdvert.Id, err)
				continue
			}
		case <-n.closed:
			return
		}
	}
}

// Connect connects the network
func (n *network) Connect() error {
	n.Lock()
	defer n.Unlock()

	// return if already connected
	if n.connected {
		return nil
	}

	// try to resolve network nodes
	nodes, err := n.resolveNodes()
	if err != nil {
		log.Debugf("Network failed to resolve nodes: %v", err)
	}

	// connect network tunnel
	if err := n.Tunnel.Connect(); err != nil {
		return err
	}

	// initialize the tunnel to resolved nodes
	n.Tunnel.Init(
		tunnel.Nodes(nodes...),
	)

	// dial into ControlChannel to send route adverts
	client, err := n.Tunnel.Dial(ControlChannel)
	if err != nil {
		return err
	}

	// listen on ControlChannel
	listener, err := n.Tunnel.Listen(ControlChannel)
	if err != nil {
		return err
	}

	// create closed channel
	n.closed = make(chan bool)

	// keep resolving network nodes
	go n.resolve()

	// start the router
	if err := n.options.Router.Start(); err != nil {
		return err
	}

	// start advertising routes
	advertChan, err := n.options.Router.Advertise()
	if err != nil {
		return err
	}

	// advertise routes
	go n.advertise(client, advertChan)
	// accept and process routes
	go n.process(listener)

	// start the server
	if err := n.srv.Start(); err != nil {
		return err
	}

	// set connected to true
	n.connected = true

	return nil
}

func (n *network) close() error {
	// stop the server
	if err := n.srv.Stop(); err != nil {
		return err
	}

	// stop the router
	if err := n.Router.Stop(); err != nil {
		return err
	}

	// close the tunnel
	if err := n.Tunnel.Close(); err != nil {
		return err
	}

	return nil
}

// Close closes network connection
func (n *network) Close() error {
	n.Lock()
	defer n.Unlock()

	if !n.connected {
		return nil
	}

	select {
	case <-n.closed:
		return nil
	default:
		close(n.closed)
		// set connected to false
		n.connected = false
	}

	return n.close()
}

// Client returns network client
func (n *network) Client() client.Client {
	return n.client
}

// Server returns network server
func (n *network) Server() server.Server {
	return n.srv
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package main

import (
	"errors"
	"fmt"
	"io"
	"log"
	"math/rand"
	"sync"
	"time"

	rpc "github.com/btcsuite/btcrpcclient"
	"github.com/btcsuite/btcutil"
	"github.com/btcsuite/btcwire"
	"github.com/conformal/btcjson"
)

// minFee is the minimum tx fee that can be paid
const minFee btcutil.Amount = 1e4 // 0.0001 BTC

// utxoQueue is the queue of utxos belonging to a actor
// utxos are queued after a block is received and are dispatched
// to their respective owner from com.poolUtxos
// they are dequeued from simulateTx and splitUtxos
type utxoQueue struct {
	utxos   []*TxOut
	enqueue chan *TxOut
	dequeue chan *TxOut
}

// Actor describes an actor on the simulation network.  Each actor runs
// independantly without external input to decide it's behavior.
type Actor struct {
	*Node
	quit             chan struct{}
	wg               sync.WaitGroup
	ownedAddresses   []btcutil.Address
	utxoQueue        *utxoQueue
	miningAddr       chan btcutil.Address
	walletPassphrase string
}

// TxOut is a valid tx output that can be used to generate transactions
type TxOut struct {
	OutPoint *btcwire.OutPoint
	Amount   btcutil.Amount
}

// NewActor creates a new actor which runs its own wallet process connecting
// to the btcd node server specified by node, and listening for simulator
// websocket connections on the specified port.
func NewActor(node *Node, port uint16) (*Actor, error) {
	// Please don't run this as root.
	if port < 1024 {
		return nil, errors.New("invalid actor port")
	}

	// Set btcwallet node args
	args, err := newBtcwalletArgs(port, node.Args.(*btcdArgs))
	if err != nil {
		return nil, err
	}

	logFile, err := getLogFile(args.prefix)
	if err != nil {
		log.Printf("Cannot get log file, logging disabled: %v", err)
	}
	btcwallet, err := NewNodeFromArgs(args, nil, logFile)
	if err != nil {
		return nil, err
	}

	a := Actor{
		Node:             btcwallet,
		quit:             make(chan struct{}),
		ownedAddresses:   make([]btcutil.Address, *maxAddresses),
		miningAddr:       make(chan btcutil.Address),
		walletPassphrase: "walletpass",
		utxoQueue: &utxoQueue{
			enqueue: make(chan *TxOut),
			dequeue: make(chan *TxOut),
		},
	}
	return &a, nil
}

// Start creates the command to execute a wallet process and starts the
// command in the background, attaching the command's stderr and stdout
// to the passed writers. Nil writers may be used to discard output.
//
// In addition to starting the wallet process, this runs goroutines to
// handle wallet notifications and requests the wallet process to create
// an intial encrypted wallet, so that it can actually send and receive BTC.
//
// If the RPC client connection cannot be established or wallet cannot
// be created, the wallet process is killed and the actor directory
// removed.
func (a *Actor) Start(stderr, stdout io.Writer, com *Communication) error {
	connected := make(chan struct{})
	var firstConn bool
	const timeoutSecs int64 = 3600 * 24

	if err := a.Node.Start(); err != nil {
		a.Shutdown()
		com.errChan <- struct{}{}
		return err
	}

	ntfnHandlers := &rpc.NotificationHandlers{
		OnBtcdConnected: func(conn bool) {
			if conn && !firstConn {
				firstConn = true
				connected <- struct{}{}
			}
		},
	}
	a.handlers = ntfnHandlers

	if err := a.Connect(); err != nil {
		a.Shutdown()
		com.errChan <- struct{}{}
		return err
	}

	// Wait for btcd to connect
	<-connected

	// Create the wallet.
	if err := a.client.CreateEncryptedWallet(a.walletPassphrase); err != nil {
		com.errChan <- struct{}{}
		return err
	}

	// Wait for wallet sync
	for i := 0; i < *maxConnRetries; i++ {
		if _, err := a.client.GetBalance(""); err != nil {
			time.Sleep(time.Duration(i) * 50 * time.Millisecond)
			continue
		}
		break
	}

	// Create wallet addresses and unlock wallet.
	log.Printf("%s: Creating wallet addresses...", a)
	for i := range a.ownedAddresses {
		fmt.Printf("\r%d/%d", i+1, len(a.ownedAddresses))
		addr, err := a.client.GetNewAddress()
		if err != nil {
			log.Printf("%s: Cannot create address #%d", a, i+1)
			com.errChan <- struct{}{}
			return err
		}
		a.ownedAddresses[i] = addr
	}
	fmt.Printf("\n")

	if err := a.client.WalletPassphrase(a.walletPassphrase, timeoutSecs); err != nil {
		log.Printf("%s: Cannot unlock wallet: %v", a, err)
		com.errChan <- struct{}{}
		return err
	}

	// Send a random address that will be used by the cpu miner.
	a.miningAddr <- a.ownedAddresses[rand.Int()%len(a.ownedAddresses)]

	// Start a goroutine that queues up a set of utxos belonging to this
	// actor. The utxos are sent from com.poolUtxos which in turn receives
	// block notifications from sim.go
	a.wg.Add(1)
	go a.queueUtxos()

	// Start a goroutine to simulate transactions.
	a.wg.Add(1)
	go a.simulateTx(com.downstream, com.txpool)

	// Start a goroutine to split utxos
	a.wg.Add(1)
	go a.splitUtxos(com.split, com.txpool)

	return nil
}

// simulateTx runs as a goroutine and simulates transactions between actors
//
// It receives a random address downstream, dequeues a utxo, sends a raw
// transaction to the address using the utxo as input
func (a *Actor) simulateTx(downstream <-chan btcutil.Address, txpool chan<- struct{}) {
	defer a.wg.Done()

	for {
		select {
		case utxo := <-a.utxoQueue.dequeue:
			select {
			case addr := <-downstream:
				// Create a raw transaction
				inputs := []btcjson.TransactionInput{{
					Txid: utxo.OutPoint.Hash.String(),
					Vout: utxo.OutPoint.Index,
				}}

				// Provide a fees of minFee to ensure the tx gets mined
				// the utxo amount is guaranteed to be > maxSplit*minFee
				amt := utxo.Amount - minFee
				amounts := map[btcutil.Address]btcutil.Amount{
					addr: amt,
				}

				err := a.sendRawTransaction(inputs, amounts)
				if err != nil {
					log.Printf("%s: Error sending raw transaction: %v", a, err)
					select {
					case txpool <- struct{}{}:
					case <-a.quit:
						return
					}
					continue
				}

			case <-a.quit:
				return
			}
		case <-a.quit:
			return
		}
	}
}

// splitUtxos runs as a goroutine and builds up a large set of utxos that
// can be used to simulate large tx/block ratios
//
// It receives a 'split' which is int that indicates the number of resultant utxos
// the tx is sent to addresses from the same actor since we're only interested in
// building up the utxo set
func (a *Actor) splitUtxos(split <-chan int, txpool chan<- struct{}) {
	defer a.wg.Done()

	for {
		select {
		case utxo := <-a.utxoQueue.dequeue:
			select {
			case split := <-split:
				// Create a raw transaction
				inputs := []btcjson.TransactionInput{{
					Txid: utxo.OutPoint.Hash.String(),
					Vout: utxo.OutPoint.Index,
				}}

				// Provide a fees of minFee to ensure the tx gets mined
				// the utxo amount is guaranteed to be > maxSplit*minFee
				amt := utxo.Amount - minFee
				amounts := map[btcutil.Address]btcutil.Amount{}

				// Create a output of random amount and sent it
				// to a random address from the address space
				// total number of outputs is split+1, taking into
				// account this utxo which is consumed in the process

				// set a rand start index for getting different random addrs
				randomIndex := rand.Int() % len(a.ownedAddresses)
				for i := 0; i <= split; i++ {
					var to btcutil.Address
					var change btcutil.Amount
					// pick a random address
					if randomIndex+i < len(a.ownedAddresses) {
						to = a.ownedAddresses[randomIndex+i]
					} else {
						// wrap around incase of an overflow
						to = a.ownedAddresses[randomIndex+i-len(a.ownedAddresses)]
					}
					if i == split {
						// last split, so just set the change
						change = amt
					} else {
						// pick a random change amount which is less than amt
						// but have a lower bound at minFee
						change = btcutil.Amount(rand.Int63n(int64(amt) / 2))
						if change < minFee {
							change = minFee
						}
						amt -= change
					}
					amounts[to] = change
				}

				err := a.sendRawTransaction(inputs, amounts)
				if err != nil {
					log.Printf("%s: Error sending raw transaction: %v", a, err)
					select {
					case txpool <- struct{}{}:
					case <-a.quit:
						return
					}
					continue
				}

			case <-a.quit:
				return
			}
		case <-a.quit:
			return
		}
	}
}

// sendRawTransaction creates a raw transaction, signs it and sends it
func (a *Actor) sendRawTransaction(inputs []btcjson.TransactionInput, amounts map[btcutil.Address]btcutil.Amount) error {
	msgTx, err := a.client.CreateRawTransaction(inputs, amounts)
	if err != nil {
		return err
	}
	// sign it
	msgTx, ok, err := a.client.SignRawTransaction(msgTx)
	if err != nil {
		return err
	}
	if !ok {
		return err
	}
	// and finally send it.
	if _, err := a.client.SendRawTransaction(msgTx, false); err != nil {
		return err
	}
	return nil
}

// queueUtxos receives utxos belonging to this actor and queues them up
func (a *Actor) queueUtxos() {
	defer a.wg.Done()

	enqueue := a.utxoQueue.enqueue
	var dequeue chan *TxOut
	var next *TxOut
out:
	for {
		select {
		case n, ok := <-enqueue:
			if !ok {
				// If no utxos are queued for handling,
				// the queue is finished.
				if len(a.utxoQueue.utxos) == 0 {
					break out
				}
				// nil channel so no more reads can occur.
				enqueue = nil
				continue
			}
			if len(a.utxoQueue.utxos) == 0 {
				next = n
				dequeue = a.utxoQueue.dequeue
			}
			a.utxoQueue.utxos = append(a.utxoQueue.utxos, n)
		case dequeue <- next:
			a.utxoQueue.utxos[0] = nil
			a.utxoQueue.utxos = a.utxoQueue.utxos[1:]
			if len(a.utxoQueue.utxos) != 0 {
				next = a.utxoQueue.utxos[0]
			} else {
				// If no more utxos can be enqueued, the
				// queue is finished.
				if enqueue == nil {
					break out
				}
				dequeue = nil
			}
		case <-a.quit:
			break out
		}
	}
	close(a.utxoQueue.dequeue)
}

// Shutdown performs a shutdown down the actor by first signalling
// all goroutines to stop, waiting for them to stop and them cleaning up
func (a *Actor) Shutdown() {
	select {
	case <-a.quit:
	default:
		close(a.quit)
		a.WaitForShutdown()
		a.Node.Shutdown()
	}
}

// WaitForShutdown waits until every actor goroutine has returned
func (a *Actor) WaitForShutdown() {
	a.wg.Wait()
}
<commit_msg>Update btcjson import paths to new location.<commit_after>// Copyright (c) 2014 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package main

import (
	"errors"
	"fmt"
	"io"
	"log"
	"math/rand"
	"sync"
	"time"

	"github.com/btcsuite/btcjson"
	rpc "github.com/btcsuite/btcrpcclient"
	"github.com/btcsuite/btcutil"
	"github.com/btcsuite/btcwire"
)

// minFee is the minimum tx fee that can be paid
const minFee btcutil.Amount = 1e4 // 0.0001 BTC

// utxoQueue is the queue of utxos belonging to a actor
// utxos are queued after a block is received and are dispatched
// to their respective owner from com.poolUtxos
// they are dequeued from simulateTx and splitUtxos
type utxoQueue struct {
	utxos   []*TxOut
	enqueue chan *TxOut
	dequeue chan *TxOut
}

// Actor describes an actor on the simulation network.  Each actor runs
// independantly without external input to decide it's behavior.
type Actor struct {
	*Node
	quit             chan struct{}
	wg               sync.WaitGroup
	ownedAddresses   []btcutil.Address
	utxoQueue        *utxoQueue
	miningAddr       chan btcutil.Address
	walletPassphrase string
}

// TxOut is a valid tx output that can be used to generate transactions
type TxOut struct {
	OutPoint *btcwire.OutPoint
	Amount   btcutil.Amount
}

// NewActor creates a new actor which runs its own wallet process connecting
// to the btcd node server specified by node, and listening for simulator
// websocket connections on the specified port.
func NewActor(node *Node, port uint16) (*Actor, error) {
	// Please don't run this as root.
	if port < 1024 {
		return nil, errors.New("invalid actor port")
	}

	// Set btcwallet node args
	args, err := newBtcwalletArgs(port, node.Args.(*btcdArgs))
	if err != nil {
		return nil, err
	}

	logFile, err := getLogFile(args.prefix)
	if err != nil {
		log.Printf("Cannot get log file, logging disabled: %v", err)
	}
	btcwallet, err := NewNodeFromArgs(args, nil, logFile)
	if err != nil {
		return nil, err
	}

	a := Actor{
		Node:             btcwallet,
		quit:             make(chan struct{}),
		ownedAddresses:   make([]btcutil.Address, *maxAddresses),
		miningAddr:       make(chan btcutil.Address),
		walletPassphrase: "walletpass",
		utxoQueue: &utxoQueue{
			enqueue: make(chan *TxOut),
			dequeue: make(chan *TxOut),
		},
	}
	return &a, nil
}

// Start creates the command to execute a wallet process and starts the
// command in the background, attaching the command's stderr and stdout
// to the passed writers. Nil writers may be used to discard output.
//
// In addition to starting the wallet process, this runs goroutines to
// handle wallet notifications and requests the wallet process to create
// an intial encrypted wallet, so that it can actually send and receive BTC.
//
// If the RPC client connection cannot be established or wallet cannot
// be created, the wallet process is killed and the actor directory
// removed.
func (a *Actor) Start(stderr, stdout io.Writer, com *Communication) error {
	connected := make(chan struct{})
	var firstConn bool
	const timeoutSecs int64 = 3600 * 24

	if err := a.Node.Start(); err != nil {
		a.Shutdown()
		com.errChan <- struct{}{}
		return err
	}

	ntfnHandlers := &rpc.NotificationHandlers{
		OnBtcdConnected: func(conn bool) {
			if conn && !firstConn {
				firstConn = true
				connected <- struct{}{}
			}
		},
	}
	a.handlers = ntfnHandlers

	if err := a.Connect(); err != nil {
		a.Shutdown()
		com.errChan <- struct{}{}
		return err
	}

	// Wait for btcd to connect
	<-connected

	// Create the wallet.
	if err := a.client.CreateEncryptedWallet(a.walletPassphrase); err != nil {
		com.errChan <- struct{}{}
		return err
	}

	// Wait for wallet sync
	for i := 0; i < *maxConnRetries; i++ {
		if _, err := a.client.GetBalance(""); err != nil {
			time.Sleep(time.Duration(i) * 50 * time.Millisecond)
			continue
		}
		break
	}

	// Create wallet addresses and unlock wallet.
	log.Printf("%s: Creating wallet addresses...", a)
	for i := range a.ownedAddresses {
		fmt.Printf("\r%d/%d", i+1, len(a.ownedAddresses))
		addr, err := a.client.GetNewAddress()
		if err != nil {
			log.Printf("%s: Cannot create address #%d", a, i+1)
			com.errChan <- struct{}{}
			return err
		}
		a.ownedAddresses[i] = addr
	}
	fmt.Printf("\n")

	if err := a.client.WalletPassphrase(a.walletPassphrase, timeoutSecs); err != nil {
		log.Printf("%s: Cannot unlock wallet: %v", a, err)
		com.errChan <- struct{}{}
		return err
	}

	// Send a random address that will be used by the cpu miner.
	a.miningAddr <- a.ownedAddresses[rand.Int()%len(a.ownedAddresses)]

	// Start a goroutine that queues up a set of utxos belonging to this
	// actor. The utxos are sent from com.poolUtxos which in turn receives
	// block notifications from sim.go
	a.wg.Add(1)
	go a.queueUtxos()

	// Start a goroutine to simulate transactions.
	a.wg.Add(1)
	go a.simulateTx(com.downstream, com.txpool)

	// Start a goroutine to split utxos
	a.wg.Add(1)
	go a.splitUtxos(com.split, com.txpool)

	return nil
}

// simulateTx runs as a goroutine and simulates transactions between actors
//
// It receives a random address downstream, dequeues a utxo, sends a raw
// transaction to the address using the utxo as input
func (a *Actor) simulateTx(downstream <-chan btcutil.Address, txpool chan<- struct{}) {
	defer a.wg.Done()

	for {
		select {
		case utxo := <-a.utxoQueue.dequeue:
			select {
			case addr := <-downstream:
				// Create a raw transaction
				inputs := []btcjson.TransactionInput{{
					Txid: utxo.OutPoint.Hash.String(),
					Vout: utxo.OutPoint.Index,
				}}

				// Provide a fees of minFee to ensure the tx gets mined
				// the utxo amount is guaranteed to be > maxSplit*minFee
				amt := utxo.Amount - minFee
				amounts := map[btcutil.Address]btcutil.Amount{
					addr: amt,
				}

				err := a.sendRawTransaction(inputs, amounts)
				if err != nil {
					log.Printf("%s: Error sending raw transaction: %v", a, err)
					select {
					case txpool <- struct{}{}:
					case <-a.quit:
						return
					}
					continue
				}

			case <-a.quit:
				return
			}
		case <-a.quit:
			return
		}
	}
}

// splitUtxos runs as a goroutine and builds up a large set of utxos that
// can be used to simulate large tx/block ratios
//
// It receives a 'split' which is int that indicates the number of resultant utxos
// the tx is sent to addresses from the same actor since we're only interested in
// building up the utxo set
func (a *Actor) splitUtxos(split <-chan int, txpool chan<- struct{}) {
	defer a.wg.Done()

	for {
		select {
		case utxo := <-a.utxoQueue.dequeue:
			select {
			case split := <-split:
				// Create a raw transaction
				inputs := []btcjson.TransactionInput{{
					Txid: utxo.OutPoint.Hash.String(),
					Vout: utxo.OutPoint.Index,
				}}

				// Provide a fees of minFee to ensure the tx gets mined
				// the utxo amount is guaranteed to be > maxSplit*minFee
				amt := utxo.Amount - minFee
				amounts := map[btcutil.Address]btcutil.Amount{}

				// Create a output of random amount and sent it
				// to a random address from the address space
				// total number of outputs is split+1, taking into
				// account this utxo which is consumed in the process

				// set a rand start index for getting different random addrs
				randomIndex := rand.Int() % len(a.ownedAddresses)
				for i := 0; i <= split; i++ {
					var to btcutil.Address
					var change btcutil.Amount
					// pick a random address
					if randomIndex+i < len(a.ownedAddresses) {
						to = a.ownedAddresses[randomIndex+i]
					} else {
						// wrap around incase of an overflow
						to = a.ownedAddresses[randomIndex+i-len(a.ownedAddresses)]
					}
					if i == split {
						// last split, so just set the change
						change = amt
					} else {
						// pick a random change amount which is less than amt
						// but have a lower bound at minFee
						change = btcutil.Amount(rand.Int63n(int64(amt) / 2))
						if change < minFee {
							change = minFee
						}
						amt -= change
					}
					amounts[to] = change
				}

				err := a.sendRawTransaction(inputs, amounts)
				if err != nil {
					log.Printf("%s: Error sending raw transaction: %v", a, err)
					select {
					case txpool <- struct{}{}:
					case <-a.quit:
						return
					}
					continue
				}

			case <-a.quit:
				return
			}
		case <-a.quit:
			return
		}
	}
}

// sendRawTransaction creates a raw transaction, signs it and sends it
func (a *Actor) sendRawTransaction(inputs []btcjson.TransactionInput, amounts map[btcutil.Address]btcutil.Amount) error {
	msgTx, err := a.client.CreateRawTransaction(inputs, amounts)
	if err != nil {
		return err
	}
	// sign it
	msgTx, ok, err := a.client.SignRawTransaction(msgTx)
	if err != nil {
		return err
	}
	if !ok {
		return err
	}
	// and finally send it.
	if _, err := a.client.SendRawTransaction(msgTx, false); err != nil {
		return err
	}
	return nil
}

// queueUtxos receives utxos belonging to this actor and queues them up
func (a *Actor) queueUtxos() {
	defer a.wg.Done()

	enqueue := a.utxoQueue.enqueue
	var dequeue chan *TxOut
	var next *TxOut
out:
	for {
		select {
		case n, ok := <-enqueue:
			if !ok {
				// If no utxos are queued for handling,
				// the queue is finished.
				if len(a.utxoQueue.utxos) == 0 {
					break out
				}
				// nil channel so no more reads can occur.
				enqueue = nil
				continue
			}
			if len(a.utxoQueue.utxos) == 0 {
				next = n
				dequeue = a.utxoQueue.dequeue
			}
			a.utxoQueue.utxos = append(a.utxoQueue.utxos, n)
		case dequeue <- next:
			a.utxoQueue.utxos[0] = nil
			a.utxoQueue.utxos = a.utxoQueue.utxos[1:]
			if len(a.utxoQueue.utxos) != 0 {
				next = a.utxoQueue.utxos[0]
			} else {
				// If no more utxos can be enqueued, the
				// queue is finished.
				if enqueue == nil {
					break out
				}
				dequeue = nil
			}
		case <-a.quit:
			break out
		}
	}
	close(a.utxoQueue.dequeue)
}

// Shutdown performs a shutdown down the actor by first signalling
// all goroutines to stop, waiting for them to stop and them cleaning up
func (a *Actor) Shutdown() {
	select {
	case <-a.quit:
	default:
		close(a.quit)
		a.WaitForShutdown()
		a.Node.Shutdown()
	}
}

// WaitForShutdown waits until every actor goroutine has returned
func (a *Actor) WaitForShutdown() {
	a.wg.Wait()
}
<|endoftext|>


<commit_before>package operator

import (
	"os"
	"strconv"
	"sync"

	"github.com/bahadley/esp/log"
)

const (
	defaultBufSz = 4
	defaultAggSz = 2

	envWinSz = "ESP_WINDOW_SIZE"
	envAggSz = "ESP_AGGREGATE_SIZE"
)

var (
	// Invariant:  Tupls are in descending order by SensorTuple.Timestamp.
	window []*SensorTuple

	// Length of window.
	bufSz uint32
	// Number of tuples that will compose the aggregate calculation.
	aggSz uint32

	// Used for window modification critical section.
	mutex sync.Mutex
)

func WindowInsert(msg string) error {
	newTuple := new(SensorTuple)

	err := Unmarshal(msg, newTuple)
	if err != nil {
		log.Warning.Printf("Failed to unmarshal tuple: %s", msg)
	} else {
		mutex.Lock()
		{
			if insert(newTuple) && window[bufSz-1] != nil {
				// A tuple was inserted and the window is full.
				avg := aggregate()
				aggTuple, err := Marshal(newTuple.Sensor, avg)
				if err != nil {
					log.Warning.Printf("Failed to marshal aggregate tuple for sensor: %s",
						newTuple.Sensor)
				} else {
					EgressChan <- aggTuple
				}
			}
		}
		mutex.Unlock()
	}

	return err
}

func insert(tmp *SensorTuple) bool {
	inserted := false

	if window[0] == nil {
		// Can only occur for the very first tuple received by the operator.
		// Having this case simplifies the remaining logic.
		window[0] = tmp
	} else {
		for idx, st := range window {
			if inserted ||
				(!inserted && st != nil && tmp.Timestamp.After(st.Timestamp)) {
				// Insert the new tuple and shift the subsequent tuples towards
				// the back of the window.  The last tuple will fall off if the
				// window is full.
				window[idx] = tmp
				tmp = st
				inserted = true
			}
		}
	}

	return inserted
}

func aggregate() float64 {
	// Calculate the aggregation and flush the tuples.
	sum := 0.0
	for idx := bufSz - aggSz; idx < bufSz; idx++ {
		sum += window[idx].Data
		window[idx] = nil
	}
	return sum / float64(aggSz)
}

func init() {
	envVal := os.Getenv(envWinSz)
	if len(envVal) == 0 {
		bufSz = defaultBufSz
	} else {
		val, err := strconv.Atoi(envVal)
		if err != nil {
			log.Error.Fatalf("Invalid environment variable: %s",
				envWinSz)
		}
		bufSz = uint32(val)
	}

	if bufSz <= 0 {
		log.Error.Fatalf("Invalid environment variable: %s <= 0",
			envWinSz)
	}

	envVal = os.Getenv(envAggSz)
	if len(envVal) == 0 {
		aggSz = defaultAggSz
	} else {
		val, err := strconv.Atoi(envVal)
		if err != nil {
			log.Error.Fatalf("Invalid environment variable: %s",
				envAggSz)
		}
		aggSz = uint32(val)
	}

	if bufSz < aggSz {
		log.Error.Fatalf("Invalid environment variables: %s < %s",
			envWinSz, envAggSz)
	}

	window = make([]*SensorTuple, bufSz)
}
<commit_msg>Fixed comment<commit_after>package operator

import (
	"os"
	"strconv"
	"sync"

	"github.com/bahadley/esp/log"
)

const (
	defaultBufSz = 4
	defaultAggSz = 2

	envWinSz = "ESP_WINDOW_SIZE"
	envAggSz = "ESP_AGGREGATE_SIZE"
)

var (
	// Invariant:  Tupls are in descending order by SensorTuple.Timestamp.
	window []*SensorTuple

	// Length of window.
	bufSz uint32
	// Number of tuples that will compose the aggregate calculation.
	aggSz uint32

	// Used for window modification critical section.
	mutex sync.Mutex
)

func WindowInsert(msg string) error {
	newTuple := new(SensorTuple)

	err := Unmarshal(msg, newTuple)
	if err != nil {
		log.Warning.Printf("Failed to unmarshal tuple: %s", msg)
	} else {
		mutex.Lock()
		{
			if insert(newTuple) && window[bufSz-1] != nil {
				// A tuple was inserted and the window is full.
				avg := aggregate()
				aggTuple, err := Marshal(newTuple.Sensor, avg)
				if err != nil {
					log.Warning.Printf("Failed to marshal aggregate tuple for sensor: %s",
						newTuple.Sensor)
				} else {
					EgressChan <- aggTuple
				}
			}
		}
		mutex.Unlock()
	}

	return err
}

func insert(tmp *SensorTuple) bool {
	inserted := false

	if window[0] == nil {
		// Can only occur for the very first tuple received by the operator.
		// Having this case simplifies the remaining logic.
		window[0] = tmp
	} else {
		for idx, st := range window {
			if inserted ||
				(!inserted && st != nil && tmp.Timestamp.After(st.Timestamp)) {
				// Insert the new tuple and shift the subsequent tuples towards
				// the back of the window.  The last tuple will fall off if the
				// window is full.
				window[idx] = tmp
				tmp = st
				inserted = true
			}
		}
	}

	return inserted
}

func aggregate() float64 {
	// Calculate the aggregation and flush the tuples that were used.
	sum := 0.0
	for idx := bufSz - aggSz; idx < bufSz; idx++ {
		sum += window[idx].Data
		window[idx] = nil
	}
	return sum / float64(aggSz)
}

func init() {
	envVal := os.Getenv(envWinSz)
	if len(envVal) == 0 {
		bufSz = defaultBufSz
	} else {
		val, err := strconv.Atoi(envVal)
		if err != nil {
			log.Error.Fatalf("Invalid environment variable: %s",
				envWinSz)
		}
		bufSz = uint32(val)
	}

	if bufSz <= 0 {
		log.Error.Fatalf("Invalid environment variable: %s <= 0",
			envWinSz)
	}

	envVal = os.Getenv(envAggSz)
	if len(envVal) == 0 {
		aggSz = defaultAggSz
	} else {
		val, err := strconv.Atoi(envVal)
		if err != nil {
			log.Error.Fatalf("Invalid environment variable: %s",
				envAggSz)
		}
		aggSz = uint32(val)
	}

	if bufSz < aggSz {
		log.Error.Fatalf("Invalid environment variables: %s < %s",
			envWinSz, envAggSz)
	}

	window = make([]*SensorTuple, bufSz)
}
<|endoftext|>


<commit_before>package gearcmd

import (
	"bufio"
	"bytes"
	"container/ring"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"strings"
	"syscall"
	"time"

	"github.com/Clever/baseworker-go"
	"github.com/Clever/gearcmd/argsparser"
	"gopkg.in/Clever/kayvee-go.v2"
)

// TaskConfig defines the configuration for the task.
type TaskConfig struct {
	FunctionName, FunctionCmd string
	WarningLines              int
	ParseArgs                 bool
	CmdTimeout                time.Duration
	RetryCount                int
}

// Process runs the Gearman job by running the configured task.
// We need to implement the Task interface so we return (byte[], error)
// though the byte[] is always nil.
func (conf TaskConfig) Process(job baseworker.Job) ([]byte, error) {
	// This wraps the actual processing to do some logging
	log.Println(kayvee.FormatLog("gearcmd", kayvee.Info, "START",
		map[string]interface{}{"function": conf.FunctionName, "job_id": getJobId(job), "job_data": string(job.Data())}))

	for {
		start := time.Now()
		err := conf.doProcess(job)
		end := time.Now()
		data := map[string]interface{}{
			"function": conf.FunctionName,
			"job_id":   getJobId(job),
			"job_data": string(job.Data()),
			"type":     "gauge",
		}

		var status string
		if err == nil {
			status = "success"
		} else {
			status = "failure"
		}
		log.Println(kayvee.FormatLog("gearman", kayvee.Info, status, map[string]interface{}{
			"type":     "counter",
			"function": conf.FunctionName,
		}))

		// Return if the job was successful.
		if err == nil {
			data["value"] = 1
			data["success"] = true
			log.Println(kayvee.FormatLog("gearcmd", kayvee.Info, "END", data))
			// Hopefully none of our jobs last long enough for a uint64...
			log.Printf(kayvee.FormatLog("gearcmd", kayvee.Info, "duration",
				map[string]interface{}{
					"value":    uint64(end.Sub(start).Seconds() * 1000),
					"type":     "gauge",
					"function": conf.FunctionName,
				},
			))
			return nil, nil
		}
		data["error_message"] = err.Error()
		data["value"] = 0
		data["success"] = false
		// Return if the job has no more retries.
		if conf.RetryCount <= 0 {
			log.Println(kayvee.FormatLog("gearcmd", kayvee.Error, "END", data))
			return nil, err
		}
		conf.RetryCount--
		log.Println(kayvee.FormatLog("gearcmd", kayvee.Error, "RETRY", data))
	}
}

// getJobId returns the jobId from the job handle
func getJobId(job baseworker.Job) string {
	splits := strings.Split(job.Handle(), ":")
	return splits[len(splits)-1]
}

func (conf TaskConfig) doProcess(job baseworker.Job) error {

	defer func() {
		// If we panicked then set the panic message as a warning. Gearman-go will
		// handle marking this job as failed.
		if r := recover(); r != nil {
			err := r.(error)
			job.SendWarning([]byte(err.Error()))
		}
	}()
	var args []string
	var err error
	if conf.ParseArgs {
		args, err = argsparser.ParseArgs(string(job.Data()))
		if err != nil {
			return err
		}
	} else {
		args = []string{string(job.Data())}

	}
	cmd := exec.Command(conf.FunctionCmd, args...)
	// create new pgid for this process so we can later kill all subprocess launched by it
	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}

	// Write the stdout and stderr of the process to both this process' stdout and stderr
	// and also write it to a byte buffer so that we can return it with the Gearman job
	// data as necessary.
	var stderrbuf bytes.Buffer
	cmd.Stderr = io.MultiWriter(os.Stderr, &stderrbuf)
	defer sendStderrWarnings(&stderrbuf, job, conf.WarningLines)

	stdoutReader, stdoutWriter := io.Pipe()
	cmd.Stdout = io.MultiWriter(os.Stdout, stdoutWriter)

	done := make(chan error)
	go func() {
		defer close(done)

		finishedProcessingStdout := make(chan error)
		go func() {
			finishedProcessingStdout <- streamToGearman(stdoutReader, job)
		}()

		// Save the cmdErr. We want to process stdout and stderr before we return it
		cmdErr := cmd.Run()
		stdoutWriter.Close()

		stdoutErr := <-finishedProcessingStdout
		if cmdErr != nil {
			done <- cmdErr
		} else if stdoutErr != nil {
			done <- stdoutErr
		}
	}()
	// No timeout
	if conf.CmdTimeout == 0 {
		// Will be nil if the channel was closed without any errors
		return <-done
	}
	select {
	case err := <-done:
		// Will be nil if the channel was closed without any errors
		return err
	case <-time.After(conf.CmdTimeout):
		// kill entire group of process spawned by our cmd.Process
		pgid, err := syscall.Getpgid(cmd.Process.Pid)
		fmt.Println("Killing pgid:", pgid)
		if err != nil {
			return fmt.Errorf("process timeout after %s. Unable to kill process, error: %s", conf.CmdTimeout.String(), err.Error())
		}
		syscall.Kill(-pgid, 9) // minus sign required to kill PGIDs
		return fmt.Errorf("process timed out after %s", conf.CmdTimeout.String())
	}
}

// This function streams the reader to the Gearman job (through job.SendData())
func streamToGearman(reader io.Reader, job baseworker.Job) error {
	buffer := make([]byte, 1024)
	for {
		n, err := reader.Read(buffer)
		// Process the data before processing the error (as per the io.Reader docs)
		if n > 0 {
			job.SendData(buffer[:n])
		}
		if err == io.EOF {
			return nil
		} else if err != nil {
			return err
		}
	}
}

// sendStderrWarnings sends the last X lines in the stderr output and to the job's warnings
// field
func sendStderrWarnings(buffer io.Reader, job baseworker.Job, warningLines int) error {
	scanner := bufio.NewScanner(buffer)
	// Create a circular buffer for the last X lines
	lastStderrLines := ring.New(warningLines)
	for scanner.Scan() {
		lastStderrLines = lastStderrLines.Next()
		lastStderrLines.Value = scanner.Bytes()
	}
	// Walk forward through the buffer to get all the last X entries. Note that we call next first
	// so that we start at the oldest entry.
	for i := 0; i < lastStderrLines.Len(); i++ {
		if lastStderrLines = lastStderrLines.Next(); lastStderrLines.Value != nil {
			job.SendWarning(append(lastStderrLines.Value.([]byte), byte('\n')))
		}
	}
	return scanner.Err()
}
<commit_msg>Added extra error handling, and use SIGTERM<commit_after>package gearcmd

import (
	"bufio"
	"bytes"
	"container/ring"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"strings"
	"syscall"
	"time"

	"github.com/Clever/baseworker-go"
	"github.com/Clever/gearcmd/argsparser"
	"gopkg.in/Clever/kayvee-go.v2"
)

// TaskConfig defines the configuration for the task.
type TaskConfig struct {
	FunctionName, FunctionCmd string
	WarningLines              int
	ParseArgs                 bool
	CmdTimeout                time.Duration
	RetryCount                int
}

// Process runs the Gearman job by running the configured task.
// We need to implement the Task interface so we return (byte[], error)
// though the byte[] is always nil.
func (conf TaskConfig) Process(job baseworker.Job) ([]byte, error) {
	// This wraps the actual processing to do some logging
	log.Println(kayvee.FormatLog("gearcmd", kayvee.Info, "START",
		map[string]interface{}{"function": conf.FunctionName, "job_id": getJobId(job), "job_data": string(job.Data())}))

	for {
		start := time.Now()
		err := conf.doProcess(job)
		end := time.Now()
		data := map[string]interface{}{
			"function": conf.FunctionName,
			"job_id":   getJobId(job),
			"job_data": string(job.Data()),
			"type":     "gauge",
		}

		var status string
		if err == nil {
			status = "success"
		} else {
			status = "failure"
		}
		log.Println(kayvee.FormatLog("gearman", kayvee.Info, status, map[string]interface{}{
			"type":     "counter",
			"function": conf.FunctionName,
		}))

		// Return if the job was successful.
		if err == nil {
			data["value"] = 1
			data["success"] = true
			log.Println(kayvee.FormatLog("gearcmd", kayvee.Info, "END", data))
			// Hopefully none of our jobs last long enough for a uint64...
			log.Printf(kayvee.FormatLog("gearcmd", kayvee.Info, "duration",
				map[string]interface{}{
					"value":    uint64(end.Sub(start).Seconds() * 1000),
					"type":     "gauge",
					"function": conf.FunctionName,
				},
			))
			return nil, nil
		}
		data["error_message"] = err.Error()
		data["value"] = 0
		data["success"] = false
		// Return if the job has no more retries.
		if conf.RetryCount <= 0 {
			log.Println(kayvee.FormatLog("gearcmd", kayvee.Error, "END", data))
			return nil, err
		}
		conf.RetryCount--
		log.Println(kayvee.FormatLog("gearcmd", kayvee.Error, "RETRY", data))
	}
}

// getJobId returns the jobId from the job handle
func getJobId(job baseworker.Job) string {
	splits := strings.Split(job.Handle(), ":")
	return splits[len(splits)-1]
}

func (conf TaskConfig) doProcess(job baseworker.Job) error {

	defer func() {
		// If we panicked then set the panic message as a warning. Gearman-go will
		// handle marking this job as failed.
		if r := recover(); r != nil {
			err := r.(error)
			job.SendWarning([]byte(err.Error()))
		}
	}()
	var args []string
	var err error
	if conf.ParseArgs {
		args, err = argsparser.ParseArgs(string(job.Data()))
		if err != nil {
			return err
		}
	} else {
		args = []string{string(job.Data())}

	}
	cmd := exec.Command(conf.FunctionCmd, args...)
	// create new pgid for this process so we can later kill all subprocess launched by it
	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}

	// Write the stdout and stderr of the process to both this process' stdout and stderr
	// and also write it to a byte buffer so that we can return it with the Gearman job
	// data as necessary.
	var stderrbuf bytes.Buffer
	cmd.Stderr = io.MultiWriter(os.Stderr, &stderrbuf)
	defer sendStderrWarnings(&stderrbuf, job, conf.WarningLines)

	stdoutReader, stdoutWriter := io.Pipe()
	cmd.Stdout = io.MultiWriter(os.Stdout, stdoutWriter)

	done := make(chan error)
	go func() {
		defer close(done)

		finishedProcessingStdout := make(chan error)
		go func() {
			finishedProcessingStdout <- streamToGearman(stdoutReader, job)
		}()

		// Save the cmdErr. We want to process stdout and stderr before we return it
		cmdErr := cmd.Run()
		stdoutWriter.Close()

		stdoutErr := <-finishedProcessingStdout
		if cmdErr != nil {
			done <- cmdErr
		} else if stdoutErr != nil {
			done <- stdoutErr
		}
	}()
	// No timeout
	if conf.CmdTimeout == 0 {
		// Will be nil if the channel was closed without any errors
		return <-done
	}
	select {
	case err := <-done:
		// Will be nil if the channel was closed without any errors
		return err
	case <-time.After(conf.CmdTimeout):
		// kill entire group of process spawned by our cmd.Process
		pgid, err := syscall.Getpgid(cmd.Process.Pid)
		fmt.Println("Killing pgid:", pgid)
		if err != nil {
			return fmt.Errorf("process timeout after %s. Unable to get pgid, error: %s", conf.CmdTimeout.String(), err.Error())
		}
		// minus sign required to kill PGIDs
		err = syscall.Kill(-pgid, syscall.SIGTERM)
		if err != nil {
			return fmt.Errorf("process timeout after %s. Unable to kill process, error: %s", conf.CmdTimeout.String(), err.Error())
		}
		return fmt.Errorf("process timed out after %s", conf.CmdTimeout.String())
	}
}

// This function streams the reader to the Gearman job (through job.SendData())
func streamToGearman(reader io.Reader, job baseworker.Job) error {
	buffer := make([]byte, 1024)
	for {
		n, err := reader.Read(buffer)
		// Process the data before processing the error (as per the io.Reader docs)
		if n > 0 {
			job.SendData(buffer[:n])
		}
		if err == io.EOF {
			return nil
		} else if err != nil {
			return err
		}
	}
}

// sendStderrWarnings sends the last X lines in the stderr output and to the job's warnings
// field
func sendStderrWarnings(buffer io.Reader, job baseworker.Job, warningLines int) error {
	scanner := bufio.NewScanner(buffer)
	// Create a circular buffer for the last X lines
	lastStderrLines := ring.New(warningLines)
	for scanner.Scan() {
		lastStderrLines = lastStderrLines.Next()
		lastStderrLines.Value = scanner.Bytes()
	}
	// Walk forward through the buffer to get all the last X entries. Note that we call next first
	// so that we start at the oldest entry.
	for i := 0; i < lastStderrLines.Len(); i++ {
		if lastStderrLines = lastStderrLines.Next(); lastStderrLines.Value != nil {
			job.SendWarning(append(lastStderrLines.Value.([]byte), byte('\n')))
		}
	}
	return scanner.Err()
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"
)

func TestCopyStaticFiles(t *testing.T) {
	outDir, err := ioutil.TempDir("", "copyStaticFiles")
	if err != nil {
		t.Fatal("Failed to create tempdir")
	}
	defer os.RemoveAll(outDir)

	if err := copyStaticFiles(outDir); err != nil {
		t.Fatalf("Error should not be raised: %#v", err)
	}

	files := []string{
		".gitignore",
		"main.go",
		filepath.Join("db", "db.go"),
		filepath.Join("middleware", "set_db.go"),
		filepath.Join("server", "server.go"),
	}

	for _, file := range files {
		_, err := os.Stat(filepath.Join(outDir, file))
		if err != nil {
			t.Fatalf("Static file is not copied: %s", file)
		}
	}
}

func TestGenerateController(t *testing.T) {
	model := &Model{
		Name: "User",
		Fields: map[string]string{
			"ID": "hoge",
		},
	}

	outDir, err := ioutil.TempDir("", "generateController")
	if err != nil {
		t.Fatal("Failed to create tempdir")
	}
	defer os.RemoveAll(outDir)

	if err := generateController(model, outDir); err != nil {
		t.Fatalf("Error should not be raised: %#v", err)
	}

	path := filepath.Join(outDir, "controllers", "user.go")
	_, err = os.Stat(path)
	if err != nil {
		t.Fatalf("Controller file is not generated: %s", path)
	}
}

func TestGenerateREADME(t *testing.T) {
	model := &Model{
		Name: "User",
		Fields: map[string]string{
			"ID": "hoge",
		},
	}
	models := []*Model{model}

	outDir, err := ioutil.TempDir("", "generateREADME")
	if err != nil {
		t.Fatal("Failed to create tempdir")
	}
	defer os.RemoveAll(outDir)

	if err := generateREADME(models, outDir); err != nil {
		t.Fatalf("Error should not be raised: %#v", err)
	}

	path := filepath.Join(outDir, "README.md")
	_, err = os.Stat(path)
	if err != nil {
		t.Fatalf("README is not generated: %s", path)
	}
}
<commit_msg>Write test for generateRouter<commit_after>package main

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"
)

func TestCopyStaticFiles(t *testing.T) {
	outDir, err := ioutil.TempDir("", "copyStaticFiles")
	if err != nil {
		t.Fatal("Failed to create tempdir")
	}
	defer os.RemoveAll(outDir)

	if err := copyStaticFiles(outDir); err != nil {
		t.Fatalf("Error should not be raised: %#v", err)
	}

	files := []string{
		".gitignore",
		"main.go",
		filepath.Join("db", "db.go"),
		filepath.Join("middleware", "set_db.go"),
		filepath.Join("server", "server.go"),
	}

	for _, file := range files {
		_, err := os.Stat(filepath.Join(outDir, file))
		if err != nil {
			t.Fatalf("Static file is not copied: %s", file)
		}
	}
}

func TestGenerateController(t *testing.T) {
	model := &Model{
		Name: "User",
		Fields: map[string]string{
			"ID": "hoge",
		},
	}

	outDir, err := ioutil.TempDir("", "generateController")
	if err != nil {
		t.Fatal("Failed to create tempdir")
	}
	defer os.RemoveAll(outDir)

	if err := generateController(model, outDir); err != nil {
		t.Fatalf("Error should not be raised: %#v", err)
	}

	path := filepath.Join(outDir, "controllers", "user.go")
	_, err = os.Stat(path)
	if err != nil {
		t.Fatalf("Controller file is not generated: %s", path)
	}
}

func TestGenerateREADME(t *testing.T) {
	model := &Model{
		Name: "User",
		Fields: map[string]string{
			"ID": "hoge",
		},
	}
	models := []*Model{model}

	outDir, err := ioutil.TempDir("", "generateREADME")
	if err != nil {
		t.Fatal("Failed to create tempdir")
	}
	defer os.RemoveAll(outDir)

	if err := generateREADME(models, outDir); err != nil {
		t.Fatalf("Error should not be raised: %#v", err)
	}

	path := filepath.Join(outDir, "README.md")
	_, err = os.Stat(path)
	if err != nil {
		t.Fatalf("README is not generated: %s", path)
	}
}

func TestGenerateRouter(t *testing.T) {
	model := &Model{
		Name: "User",
		Fields: map[string]string{
			"ID": "hoge",
		},
	}
	models := []*Model{model}

	outDir, err := ioutil.TempDir("", "generateRouter")
	if err != nil {
		t.Fatal("Failed to create tempdir")
	}
	defer os.RemoveAll(outDir)

	if err := generateRouter(models, outDir); err != nil {
		t.Fatalf("Error should not be raised: %#v", err)
	}

	path := filepath.Join(outDir, "router", "router.go")
	_, err = os.Stat(path)
	if err != nil {
		t.Fatalf("Router file is not generated: %s", path)
	}
}
<|endoftext|>