<commit_before>package system

import (
	"flag"
	"fmt"
	"go/parser"
	"go/token"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

func ensureMainEntrypoint(logger *logrus.Logger) error {
	// Don't do this for "go test" runs
	if flag.Lookup("test.v") != nil {
		logger.Debug("Skipping main() check for test")
		return nil
	}

	fset := token.NewFileSet()
	packageMap, parseErr := parser.ParseDir(fset, ".", nil, parser.PackageClauseOnly)
	if parseErr != nil {
		return errors.Errorf("Failed to parse source input: %s", parseErr.Error())
	}
	logger.WithFields(logrus.Fields{
		"SourcePackages": packageMap,
	}).Debug("Checking working directory")

	// If there isn't a main defined, we're in the wrong directory..
	mainPackageCount := 0
	for eachPackage := range packageMap {
		if eachPackage == "main" {
			mainPackageCount++
		}
	}
	if mainPackageCount <= 0 {
		unlikelyBinaryErr := fmt.Errorf("error: It appears your application's `func main() {}` is not in the current working directory. Please run this command in the same directory as `func main() {}`")
		return unlikelyBinaryErr
	}
	return nil
}

// GoVersion returns the configured go version for this system
func GoVersion(logger *logrus.Logger) (string, error) {
	runtimeVersion := runtime.Version()
	// Get the golang version from the output:
	// Matts-MBP:Sparta mweagle$ go version
	// go version go1.8.1 darwin/amd64
	golangVersionRE := regexp.MustCompile(`go(\d+\.\d+(\.\d+)?)`)
	matches := golangVersionRE.FindStringSubmatch(runtimeVersion)
	if len(matches) > 2 {
		return matches[1], nil
	}
	logger.WithFields(logrus.Fields{
		"Output": runtimeVersion,
	}).Warn("Unable to find Golang version using RegExp - using current version")
	return runtimeVersion, nil
}

// GoPath returns either $GOPATH or the new $HOME/go path
// introduced with Go 1.8
func GoPath() string {
	gopath := os.Getenv("GOPATH")
	if gopath == "" {
		home := os.Getenv("HOME")
		gopath = filepath.Join(home, "go")
	}
	return gopath
}

// BuildGoBinary is a helper to build a go binary with the given options
func BuildGoBinary(serviceName string,
	executableOutput string,
	useCGO bool,
	buildID string,
	userSuppliedBuildTags string,
	linkFlags string,
	noop bool,
	logger *logrus.Logger) error {

	// Before we do anything, let's make sure there's a `main` package in this directory.
	ensureMainPackageErr := ensureMainEntrypoint(logger)
	if ensureMainPackageErr != nil {
		return ensureMainPackageErr
	}
	// Go generate
	cmd := exec.Command("go", "generate")
	if logger.Level == logrus.DebugLevel {
		cmd = exec.Command("go", "generate", "-v", "-x")
	}
	cmd.Env = os.Environ()
	commandString := fmt.Sprintf("%s", cmd.Args)
	logger.Info(fmt.Sprintf("Running `%s`", strings.Trim(commandString, "[]")))
	goGenerateErr := RunOSCommand(cmd, logger)
	if nil != goGenerateErr {
		return goGenerateErr
	}
	// TODO: Smaller binaries via linker flags
	// Ref: https://blog.filippo.io/shrink-your-go-binaries-with-this-one-weird-trick/
	noopTag := ""
	if noop {
		noopTag = "noop "
	}

	buildTags := []string{
		"lambdabinary",
		"linux",
	}
	if noopTag != "" {
		buildTags = append(buildTags, noopTag)
	}
	if userSuppliedBuildTags != "" {
		userBuildTagsParts := strings.Split(userSuppliedBuildTags, " ")
		buildTags = append(buildTags, userBuildTagsParts...)
	}
	userBuildFlags := []string{"-tags", strings.Join(buildTags, " ")}

	// Append all the linker flags
	// Stamp the service name into the binary
	// We need to stamp the servicename into the aws binary so that if the user
	// chose some type of dynamic stack name at provision time, the name
	// we use at execution time has that value. This is necessary because
	// the function dispatch logic uses the AWS_LAMBDA_FUNCTION_NAME environment
	// variable to do the lookup. And in effect, this value has to be unique
	// across an account, since functions cannot have the same name
	// Custom flags for the binary
	linkerFlags := map[string]string{
		"StampedServiceName": serviceName,
		"StampedBuildID":     buildID,
	}
	for eachFlag, eachValue := range linkerFlags {
		linkFlags = fmt.Sprintf("%s -s -w -X github.com/mweagle/Sparta.%s=%s",
			linkFlags,
			eachFlag,
			eachValue)
	}
	linkFlags = strings.TrimSpace(linkFlags)
	if len(linkFlags) != 0 {
		userBuildFlags = append(userBuildFlags, "-ldflags", linkFlags)
	}
	// If this is CGO, do the Docker build if we're doing an actual
	// provision. Otherwise use the "normal" build to keep things
	// a bit faster.
	var cmdError error
	if useCGO {
		currentDir, currentDirErr := os.Getwd()
		if nil != currentDirErr {
			return currentDirErr
		}
		gopathVersion, gopathVersionErr := GoVersion(logger)
		if nil != gopathVersionErr {
			return gopathVersionErr
		}

		gopath := GoPath()
		containerGoPath := "/usr/src/gopath"
		// Get the package path in the current directory
		// so that we can it to the container path
		packagePath := strings.TrimPrefix(currentDir, gopath)
		volumeMountMapping := fmt.Sprintf("%s:%s", gopath, containerGoPath)
		containerSourcePath := fmt.Sprintf("%s%s", containerGoPath, packagePath)

		// If there's one from the environment, use that...
		// TODO

		// Otherwise, make one...

		// Any CGO paths?
		cgoLibPath := fmt.Sprintf("%s/cgo/lib", containerSourcePath)
		cgoIncludePath := fmt.Sprintf("%s/cgo/include", containerSourcePath)

		// Pass any SPARTA_* prefixed environment variables to the docker build
		//
		goosTarget := os.Getenv("SPARTA_GOOS")
		if goosTarget == "" {
			goosTarget = "linux"
		}
		goArch := os.Getenv("SPARTA_GOARCH")
		if goArch == "" {
			goArch = "amd64"
		}
		spartaEnvVars := []string{
			// "-e",
			// fmt.Sprintf("GOPATH=%s", containerGoPath),
			"-e",
			fmt.Sprintf("GOOS=%s", goosTarget),
			"-e",
			fmt.Sprintf("GOARCH=%s", goArch),
			"-e",
			fmt.Sprintf("CGO_LDFLAGS=-L%s", cgoLibPath),
			"-e",
			fmt.Sprintf("CGO_CFLAGS=-I%s", cgoIncludePath),
		}
		// User vars
		for _, eachPair := range os.Environ() {
			if strings.HasPrefix(eachPair, "SPARTA_") {
				spartaEnvVars = append(spartaEnvVars, "-e", eachPair)
			}
		}
		dockerBuildArgs := []string{
			"run",
			"--rm",
			"-v",
			volumeMountMapping,
			"-w",
			containerSourcePath}
		dockerBuildArgs = append(dockerBuildArgs, spartaEnvVars...)
		dockerBuildArgs = append(dockerBuildArgs,
			fmt.Sprintf("golang:%s", gopathVersion),
			"go",
			"build",
			"-o",
			executableOutput,
			"-buildmode=default",
		)
		dockerBuildArgs = append(dockerBuildArgs, userBuildFlags...)
		cmd = exec.Command("docker", dockerBuildArgs...)
		cmd.Env = os.Environ()
		logger.WithFields(logrus.Fields{
			"Name": executableOutput,
			"Args": dockerBuildArgs,
		}).Info("Building `cgo` library in Docker")
		cmdError = RunOSCommand(cmd, logger)

		// If this succeeded, let's find the .h file and move it into the scratch
		// Try to keep things tidy...
		if nil == cmdError {
			soExtension := filepath.Ext(executableOutput)
			headerFilepath := fmt.Sprintf("%s.h", strings.TrimSuffix(executableOutput, soExtension))
			_, headerFileErr := os.Stat(headerFilepath)
			if nil == headerFileErr {
				targetPath, targetPathErr := TemporaryFile(".sparta", filepath.Base(headerFilepath))
				if nil != targetPathErr {
					headerFileErr = targetPathErr
				} else {
					headerFileErr = os.Rename(headerFilepath, targetPath.Name())
				}
			}
			if nil != headerFileErr {
				logger.WithFields(logrus.Fields{
					"Path": headerFilepath,
				}).Warn("Failed to move .h file to scratch directory")
			}
		}
	} else {
		// Build the regular version
		buildArgs := []string{
			"build",
			"-o",
			executableOutput,
		}
		// Debug flags?
		if logger.Level == logrus.DebugLevel {
			buildArgs = append(buildArgs, "-v")
		}
		buildArgs = append(buildArgs, userBuildFlags...)
		buildArgs = append(buildArgs, ".")
		cmd = exec.Command("go", buildArgs...)
		cmd.Env = os.Environ()
		cmd.Env = append(cmd.Env, "GOOS=linux", "GOARCH=amd64")
		logger.WithFields(logrus.Fields{
			"Name": executableOutput,
		}).Info("Compiling binary")
		cmdError = RunOSCommand(cmd, logger)
	}
	return cmdError
}

// TemporaryFile creates a stable temporary filename in the current working
// directory
func TemporaryFile(scratchDir string, name string) (*os.File, error) {
	workingDir, err := os.Getwd()
	if nil != err {
		return nil, err
	}

	// Use a stable temporary name
	temporaryPath := filepath.Join(workingDir, scratchDir, name)
	buildDir := filepath.Dir(temporaryPath)
	mkdirErr := os.MkdirAll(buildDir, os.ModePerm)
	if nil != mkdirErr {
		return nil, mkdirErr
	}

	tmpFile, err := os.Create(temporaryPath)
	if err != nil {
		return nil, errors.New("Failed to create temporary file: " + err.Error())
	}

	return tmpFile, nil
}
<commit_msg>Stylistic updates<commit_after>package system

import (
	"flag"
	"fmt"
	"go/parser"
	"go/token"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

func ensureMainEntrypoint(logger *logrus.Logger) error {
	// Don't do this for "go test" runs
	if flag.Lookup("test.v") != nil {
		logger.Debug("Skipping main() check for test")
		return nil
	}

	fset := token.NewFileSet()
	packageMap, parseErr := parser.ParseDir(fset, ".", nil, parser.PackageClauseOnly)
	if parseErr != nil {
		return errors.Errorf("Failed to parse source input: %s", parseErr.Error())
	}
	logger.WithFields(logrus.Fields{
		"SourcePackages": packageMap,
	}).Debug("Checking working directory")

	// If there isn't a main defined, we're in the wrong directory..
	mainPackageCount := 0
	for eachPackage := range packageMap {
		if eachPackage == "main" {
			mainPackageCount++
		}
	}
	if mainPackageCount <= 0 {
		unlikelyBinaryErr := fmt.Errorf("error: It appears your application's `func main() {}` is not in the current working directory. Please run this command in the same directory as `func main() {}`")
		return unlikelyBinaryErr
	}
	return nil
}

// GoVersion returns the configured go version for this system
func GoVersion(logger *logrus.Logger) (string, error) {
	runtimeVersion := runtime.Version()
	// Get the golang version from the output:
	// Matts-MBP:Sparta mweagle$ go version
	// go version go1.8.1 darwin/amd64
	golangVersionRE := regexp.MustCompile(`go(\d+\.\d+(\.\d+)?)`)
	matches := golangVersionRE.FindStringSubmatch(runtimeVersion)
	if len(matches) > 2 {
		return matches[1], nil
	}
	logger.WithFields(logrus.Fields{
		"Output": runtimeVersion,
	}).Warn("Unable to find Golang version using RegExp - using current version")
	return runtimeVersion, nil
}

// GoPath returns either $GOPATH or the new $HOME/go path
// introduced with Go 1.8
func GoPath() string {
	gopath := os.Getenv("GOPATH")
	if gopath == "" {
		home := os.Getenv("HOME")
		gopath = filepath.Join(home, "go")
	}
	return gopath
}

// BuildGoBinary is a helper to build a go binary with the given options
func BuildGoBinary(serviceName string,
	executableOutput string,
	useCGO bool,
	buildID string,
	userSuppliedBuildTags string,
	linkFlags string,
	noop bool,
	logger *logrus.Logger) error {

	// Before we do anything, let's make sure there's a `main` package in this directory.
	ensureMainPackageErr := ensureMainEntrypoint(logger)
	if ensureMainPackageErr != nil {
		return ensureMainPackageErr
	}
	// Go generate
	cmd := exec.Command("go", "generate")
	if logger.Level == logrus.DebugLevel {
		cmd = exec.Command("go", "generate", "-v", "-x")
	}
	cmd.Env = os.Environ()

	logger.WithFields(logrus.Fields{
		"Command": strings.Join(cmd.Args, " "),
	}).Info("Build command")

	//	commandString := fmt.Sprintf("%s", cmd.Args)
	//logger.Info(fmt.Sprintf("Running `%s`", strings.Trim(commandString, "[]")))
	goGenerateErr := RunOSCommand(cmd, logger)
	if nil != goGenerateErr {
		return goGenerateErr
	}
	// TODO: Smaller binaries via linker flags
	// Ref: https://blog.filippo.io/shrink-your-go-binaries-with-this-one-weird-trick/
	noopTag := ""
	if noop {
		noopTag = "noop "
	}

	buildTags := []string{
		"lambdabinary",
		"linux",
	}
	if noopTag != "" {
		buildTags = append(buildTags, noopTag)
	}
	if userSuppliedBuildTags != "" {
		userBuildTagsParts := strings.Split(userSuppliedBuildTags, " ")
		buildTags = append(buildTags, userBuildTagsParts...)
	}
	userBuildFlags := []string{"-tags", strings.Join(buildTags, " ")}

	// Append all the linker flags
	// Stamp the service name into the binary
	// We need to stamp the servicename into the aws binary so that if the user
	// chose some type of dynamic stack name at provision time, the name
	// we use at execution time has that value. This is necessary because
	// the function dispatch logic uses the AWS_LAMBDA_FUNCTION_NAME environment
	// variable to do the lookup. And in effect, this value has to be unique
	// across an account, since functions cannot have the same name
	// Custom flags for the binary
	linkerFlags := map[string]string{
		"StampedServiceName": serviceName,
		"StampedBuildID":     buildID,
	}
	for eachFlag, eachValue := range linkerFlags {
		linkFlags = fmt.Sprintf("%s -s -w -X github.com/mweagle/Sparta.%s=%s",
			linkFlags,
			eachFlag,
			eachValue)
	}
	linkFlags = strings.TrimSpace(linkFlags)
	if len(linkFlags) != 0 {
		userBuildFlags = append(userBuildFlags, "-ldflags", linkFlags)
	}
	// If this is CGO, do the Docker build if we're doing an actual
	// provision. Otherwise use the "normal" build to keep things
	// a bit faster.
	var cmdError error
	if useCGO {
		currentDir, currentDirErr := os.Getwd()
		if nil != currentDirErr {
			return currentDirErr
		}
		gopathVersion, gopathVersionErr := GoVersion(logger)
		if nil != gopathVersionErr {
			return gopathVersionErr
		}

		gopath := GoPath()
		containerGoPath := "/usr/src/gopath"
		// Get the package path in the current directory
		// so that we can it to the container path
		packagePath := strings.TrimPrefix(currentDir, gopath)
		volumeMountMapping := fmt.Sprintf("%s:%s", gopath, containerGoPath)
		containerSourcePath := fmt.Sprintf("%s%s", containerGoPath, packagePath)

		// If there's one from the environment, use that...
		// TODO

		// Otherwise, make one...

		// Any CGO paths?
		cgoLibPath := fmt.Sprintf("%s/cgo/lib", containerSourcePath)
		cgoIncludePath := fmt.Sprintf("%s/cgo/include", containerSourcePath)

		// Pass any SPARTA_* prefixed environment variables to the docker build
		//
		goosTarget := os.Getenv("SPARTA_GOOS")
		if goosTarget == "" {
			goosTarget = "linux"
		}
		goArch := os.Getenv("SPARTA_GOARCH")
		if goArch == "" {
			goArch = "amd64"
		}
		spartaEnvVars := []string{
			// "-e",
			// fmt.Sprintf("GOPATH=%s", containerGoPath),
			"-e",
			fmt.Sprintf("GOOS=%s", goosTarget),
			"-e",
			fmt.Sprintf("GOARCH=%s", goArch),
			"-e",
			fmt.Sprintf("CGO_LDFLAGS=-L%s", cgoLibPath),
			"-e",
			fmt.Sprintf("CGO_CFLAGS=-I%s", cgoIncludePath),
		}
		// User vars
		for _, eachPair := range os.Environ() {
			if strings.HasPrefix(eachPair, "SPARTA_") {
				spartaEnvVars = append(spartaEnvVars, "-e", eachPair)
			}
		}
		dockerBuildArgs := []string{
			"run",
			"--rm",
			"-v",
			volumeMountMapping,
			"-w",
			containerSourcePath}
		dockerBuildArgs = append(dockerBuildArgs, spartaEnvVars...)
		dockerBuildArgs = append(dockerBuildArgs,
			fmt.Sprintf("golang:%s", gopathVersion),
			"go",
			"build",
			"-o",
			executableOutput,
			"-buildmode=default",
		)
		dockerBuildArgs = append(dockerBuildArgs, userBuildFlags...)
		cmd = exec.Command("docker", dockerBuildArgs...)
		cmd.Env = os.Environ()
		logger.WithFields(logrus.Fields{
			"Name": executableOutput,
			"Args": dockerBuildArgs,
		}).Info("Building `cgo` library in Docker")
		cmdError = RunOSCommand(cmd, logger)

		// If this succeeded, let's find the .h file and move it into the scratch
		// Try to keep things tidy...
		if nil == cmdError {
			soExtension := filepath.Ext(executableOutput)
			headerFilepath := fmt.Sprintf("%s.h", strings.TrimSuffix(executableOutput, soExtension))
			_, headerFileErr := os.Stat(headerFilepath)
			if nil == headerFileErr {
				targetPath, targetPathErr := TemporaryFile(".sparta", filepath.Base(headerFilepath))
				if nil != targetPathErr {
					headerFileErr = targetPathErr
				} else {
					headerFileErr = os.Rename(headerFilepath, targetPath.Name())
				}
			}
			if nil != headerFileErr {
				logger.WithFields(logrus.Fields{
					"Path": headerFilepath,
				}).Warn("Failed to move .h file to scratch directory")
			}
		}
	} else {
		// Build the regular version
		buildArgs := []string{
			"build",
			"-o",
			executableOutput,
		}
		// Debug flags?
		if logger.Level == logrus.DebugLevel {
			buildArgs = append(buildArgs, "-v")
		}
		buildArgs = append(buildArgs, userBuildFlags...)
		buildArgs = append(buildArgs, ".")
		cmd = exec.Command("go", buildArgs...)
		cmd.Env = os.Environ()
		cmd.Env = append(cmd.Env, "GOOS=linux", "GOARCH=amd64")
		logger.WithFields(logrus.Fields{
			"Path": executableOutput,
		}).Info("Compiling binary")
		cmdError = RunOSCommand(cmd, logger)
	}
	return cmdError
}

// TemporaryFile creates a stable temporary filename in the provided working
// directory
func TemporaryFile(scratchDir string, name string) (*os.File, error) {
	workingDir, err := os.Getwd()
	if nil != err {
		return nil, err
	}

	// Use a stable temporary name
	temporaryPath := filepath.Join(workingDir, scratchDir, name)
	buildDir := filepath.Dir(temporaryPath)
	mkdirErr := os.MkdirAll(buildDir, os.ModePerm)
	if nil != mkdirErr {
		return nil, mkdirErr
	}

	tmpFile, err := os.Create(temporaryPath)
	if err != nil {
		return nil, errors.New("Failed to create temporary file: " + err.Error())
	}
	return tmpFile, nil
}
<|endoftext|>


<commit_before>// Copyright 2020 The Xorm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tags

import (
	"reflect"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/xormplus/dialects"
	"github.com/xormplus/names"
	"github.com/xormplus/xorm/caches"
)

type ParseTableName1 struct{}

type ParseTableName2 struct{}

func (p ParseTableName2) TableName() string {
	return "p_parseTableName"
}

func TestParseTableName(t *testing.T) {
	parser := NewParser(
		"xorm",
		dialects.QueryDialect("mysql"),
		names.SnakeMapper{},
		names.SnakeMapper{},
		caches.NewManager(),
	)
	table, err := parser.Parse(reflect.ValueOf(new(ParseTableName1)))
	assert.NoError(t, err)
	assert.EqualValues(t, "parse_table_name1", table.Name)

	table, err = parser.Parse(reflect.ValueOf(new(ParseTableName2)))
	assert.NoError(t, err)
	assert.EqualValues(t, "p_parseTableName", table.Name)

	table, err = parser.Parse(reflect.ValueOf(ParseTableName2{}))
	assert.NoError(t, err)
	assert.EqualValues(t, "p_parseTableName", table.Name)
}
<commit_msg>fix import error<commit_after>// Copyright 2020 The Xorm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tags

import (
	"reflect"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/xormplus/xorm/caches"
	"github.com/xormplus/xorm/dialects"
	"github.com/xormplus/xorm/names"
)

type ParseTableName1 struct{}

type ParseTableName2 struct{}

func (p ParseTableName2) TableName() string {
	return "p_parseTableName"
}

func TestParseTableName(t *testing.T) {
	parser := NewParser(
		"xorm",
		dialects.QueryDialect("mysql"),
		names.SnakeMapper{},
		names.SnakeMapper{},
		caches.NewManager(),
	)
	table, err := parser.Parse(reflect.ValueOf(new(ParseTableName1)))
	assert.NoError(t, err)
	assert.EqualValues(t, "parse_table_name1", table.Name)

	table, err = parser.Parse(reflect.ValueOf(new(ParseTableName2)))
	assert.NoError(t, err)
	assert.EqualValues(t, "p_parseTableName", table.Name)

	table, err = parser.Parse(reflect.ValueOf(ParseTableName2{}))
	assert.NoError(t, err)
	assert.EqualValues(t, "p_parseTableName", table.Name)
}
<|endoftext|>


<commit_before>// Package xsdvalidate is a go package for xsd validation that utilizes libxml2.
//
// The goal of this package is to preload xsd files and use their in-memory representation to validate xml documents in a concurrent environment, eg. the post bodys of xml service endpoints and hand through libxml2 error messages. Similar packages on github either didn't provide error details or got stuck under load.
//
// libxml2-dev is needed, below an example how to install the latest sources as at the time of writing (Ubuntu, change prefix according to where libs and include files are located):
//  curl -sL ftp://xmlsoft.org/libxml2/libxml2-2.9.7.tar.gz | tar -xzf -
//  cd ./libxml2-2.9.7/
//  ./configure --prefix=/usr  --enable-static --with-threads --with-history
//  make
//  sudo make install
package xsdvalidate

import "C"
import (
	"sync"
	"sync/atomic"
	"time"
)

type guard struct {
	sync.Mutex
	initialized uint32
}

func (guard *guard) isInitialized() bool {
	if atomic.LoadUint32(&guard.initialized) == 0 {
		return false
	}
	return true
}

func (guard *guard) setInitialized(b bool) {
	switch b {
	case true:
		atomic.StoreUint32(&guard.initialized, 1)
	case false:
		atomic.StoreUint32(&guard.initialized, 0)
	}
}

var g guard

// Options type for parser/validation options.
type Options int16

// The parser options, ParsErrVerbose will slow down parsing considerably!
const (
	ParsErrDefault Options = 1 << iota // Default parser error output
	ParsErrVerbose                     // Verbose parser error output, considerably slower!
)

// Validation options for possible future enhancements.
const (
	ValidErrDefault Options = 1 << iota // Default validation error output
)

var quit chan struct{}

// Init initializes libxml2, see http://xmlsoft.org/threads.html.
func Init() error {
	g.Lock()
	defer g.Unlock()
	if g.isInitialized() {
		return Libxml2Error{errorMessage{"Libxml2 already initialized"}}
	}

	libXml2Init()
	g.setInitialized(true)
	return nil
}

// InitWithGc initializes lbxml2 with a goroutine that trims memory and runs the go gc every d duration.
// Not required but can help to keep the memory footprint at bay when doing tons of validations.
func InitWithGc(d time.Duration) {
	Init()
	quit = make(chan struct{})
	go gcTicker(d, quit)
}

// Cleanup cleans up libxml2 memory and finishes gc goroutine when running.
func Cleanup() {
	g.Lock()
	defer g.Unlock()
	libXml2Cleanup()
	g.setInitialized(false)
	if quit != nil {
		quit <- struct{}{}
		quit = nil
	}
}

// NewXmlHandlerMem creates a xml handler struct.
// Always use the Free() method when done using this handler or memory will be leaking.
// The go garbage collector will not collect the allocated resources.
func NewXmlHandlerMem(inXml []byte, options Options) (*XmlHandler, error) {
	if !g.isInitialized() {
		return nil, Libxml2Error{errorMessage{"Libxml2 not initialized"}}
	}

	xPtr, err := parseXmlMem(inXml, options)
	return &XmlHandler{xPtr}, err
}

// NewXsdHandlerUrl creates a xsd handler struct.
// Always use Free() method when done using this handler or memory will be leaking.
// The go garbage collector will not collect the allocated resources.
func NewXsdHandlerUrl(url string, options Options) (*XsdHandler, error) {
	g.Lock()
	defer g.Unlock()
	if !g.isInitialized() {
		return nil, Libxml2Error{errorMessage{"Libxml2 not initialized"}}
	}
	sPtr, err := parseUrlSchema(url, options)
	return &XsdHandler{sPtr}, err
}

// Validate validates an xmlHandler against an xsdHandler and returns a typed error.
// Both xmlHandler and xsdHandler have to be created first.
func (xsdHandler *XsdHandler) Validate(xmlHandler *XmlHandler, options Options) error {
	if !g.isInitialized() {
		return Libxml2Error{errorMessage{"Libxml2 not initialized"}}
	}

	if xsdHandler == nil || xsdHandler.schemaPtr == nil {
		return XsdParserError{errorMessage{"Xsd handler not properly initialized"}}

	}
	if xmlHandler == nil || xmlHandler.docPtr == nil {
		return XmlParserError{errorMessage{"Xml handler not properly initialized"}}
	}
	return validateWithXsd(xmlHandler, xsdHandler)

}

// ValidateXmlBuf validates a xml byte slice against an xsdHandler and returns a typed error.
// Rhe xsdHandler have to be created first.
// This is a shortcut to creating and freeing an xmlHandler.
// Also this is a bit faster less cgo calls are needed.
func (xsdHandler *XsdHandler) ValidateXmlMem(inXml []byte, parserOptions Options, validatorOptions Options) error {
	if !g.isInitialized() {
		return Libxml2Error{errorMessage{"Libxml2 not initialized"}}
	}
	if xsdHandler == nil || xsdHandler.schemaPtr == nil {
		return XsdParserError{errorMessage{"Xsd handler not properly initialized"}}

	}
	return validateBufWithXsd(inXml, parserOptions, xsdHandler)

}

// Free frees the wrapped schemaPtr, call this when this handler is not needed anymore.
func (xsdHandler *XsdHandler) Free() {
	freeSchemaPtr(xsdHandler)
}

// Free frees the wrapped xml docPtr, call this when this handler is not needed anymore.
func (xmlHandler *XmlHandler) Free() {
	freeDocPtr(xmlHandler)
}
<commit_msg>Cgo fixes, added ValidateXmlMem api call with example<commit_after>// Package xsdvalidate is a go package for xsd validation that utilizes libxml2.
//
// The goal of this package is to preload xsd files and use their in-memory representation to validate xml documents in a concurrent environment, eg. the post bodys of xml service endpoints and hand through libxml2 error messages. Similar packages on github either didn't provide error details or got stuck under load.
//
// libxml2-dev is needed, below an example how to install the latest sources as at the time of writing (Ubuntu, change prefix according to where libs and include files are located):
//  curl -sL ftp://xmlsoft.org/libxml2/libxml2-2.9.7.tar.gz | tar -xzf -
//  cd ./libxml2-2.9.7/
//  ./configure --prefix=/usr  --enable-static --with-threads --with-history
//  make
//  sudo make install
package xsdvalidate

import "C"
import (
	"sync"
	"sync/atomic"
	"time"
)

type guard struct {
	sync.Mutex
	initialized uint32
}

func (guard *guard) isInitialized() bool {
	if atomic.LoadUint32(&guard.initialized) == 0 {
		return false
	}
	return true
}

func (guard *guard) setInitialized(b bool) {
	switch b {
	case true:
		atomic.StoreUint32(&guard.initialized, 1)
	case false:
		atomic.StoreUint32(&guard.initialized, 0)
	}
}

var g guard

// Options type for parser/validation options.
type Options int16

// The parser options, ParsErrVerbose will slow down parsing considerably!
const (
	ParsErrDefault Options = 1 << iota // Default parser error output
	ParsErrVerbose                     // Verbose parser error output, considerably slower!
)

// Validation options for possible future enhancements.
const (
	ValidErrDefault Options = 1 << iota // Default validation error output
)

var quit chan struct{}

// Init initializes libxml2, see http://xmlsoft.org/threads.html.
func Init() error {
	g.Lock()
	defer g.Unlock()
	if g.isInitialized() {
		return Libxml2Error{errorMessage{"Libxml2 already initialized"}}
	}

	libXml2Init()
	g.setInitialized(true)
	return nil
}

// InitWithGc initializes lbxml2 with a goroutine that trims memory and runs the go gc every d duration.
// Not required but can help to keep the memory footprint at bay when doing tons of validations.
func InitWithGc(d time.Duration) {
	Init()
	quit = make(chan struct{})
	go gcTicker(d, quit)
}

// Cleanup cleans up libxml2 memory and finishes gc goroutine when running.
func Cleanup() {
	g.Lock()
	defer g.Unlock()
	libXml2Cleanup()
	g.setInitialized(false)
	if quit != nil {
		quit <- struct{}{}
		quit = nil
	}
}

// NewXmlHandlerMem creates a xml handler struct.
// If an error is returned it can be of type Libxml2Error or XmlParserError.
// Always use the Free() method when done using this handler or memory will be leaking.
// The go garbage collector will not collect the allocated resources.
func NewXmlHandlerMem(inXml []byte, options Options) (*XmlHandler, error) {
	if !g.isInitialized() {
		return nil, Libxml2Error{errorMessage{"Libxml2 not initialized"}}
	}

	xPtr, err := parseXmlMem(inXml, options)
	return &XmlHandler{xPtr}, err
}

// NewXsdHandlerUrl creates a xsd handler struct.
// Always use Free() method when done using this handler or memory will be leaking.
// If an error is returned it can be of type Libxml2Error or XsdParserError.
// The go garbage collector will not collect the allocated resources.
func NewXsdHandlerUrl(url string, options Options) (*XsdHandler, error) {
	g.Lock()
	defer g.Unlock()
	if !g.isInitialized() {
		return nil, Libxml2Error{errorMessage{"Libxml2 not initialized"}}
	}
	sPtr, err := parseUrlSchema(url, options)
	return &XsdHandler{sPtr}, err
}

// Validate validates an xmlHandler against an xsdHandler and returns a ValidationError.
// If an error is returned it is of type ValidationError.
// Both xmlHandler and xsdHandler have to be created first.
func (xsdHandler *XsdHandler) Validate(xmlHandler *XmlHandler, options Options) error {
	if !g.isInitialized() {
		return Libxml2Error{errorMessage{"Libxml2 not initialized"}}
	}

	if xsdHandler == nil || xsdHandler.schemaPtr == nil {
		return XsdParserError{errorMessage{"Xsd handler not properly initialized"}}

	}
	if xmlHandler == nil || xmlHandler.docPtr == nil {
		return XmlParserError{errorMessage{"Xml handler not properly initialized"}}
	}
	return validateWithXsd(xmlHandler, xsdHandler)

}

// ValidateXmlMem validates an xml byte slice against an xsdHandler.
// If an error is returned it can be of type XsdParserError or ValidationError.
// The xsdHandler has to be created first.
func (xsdHandler *XsdHandler) ValidateXmlMem(inXml []byte, parserOptions Options, validatorOptions Options) error {
	if !g.isInitialized() {
		return Libxml2Error{errorMessage{"Libxml2 not initialized"}}
	}
	if xsdHandler == nil || xsdHandler.schemaPtr == nil {
		return XsdParserError{errorMessage{"Xsd handler not properly initialized"}}

	}
	return validateBufWithXsd(inXml, parserOptions, xsdHandler)

}

// Free frees the wrapped schemaPtr, call this when this handler is not needed anymore.
func (xsdHandler *XsdHandler) Free() {
	freeSchemaPtr(xsdHandler)
}

// Free frees the wrapped xml docPtr, call this when this handler is not needed anymore.
func (xmlHandler *XmlHandler) Free() {
	freeDocPtr(xmlHandler)
}
<|endoftext|>


<commit_before>package main

import (
	"compress/gzip"
	"io"
	"net/http"
	"net/http/httputil"
	"os"
	"time"
	"net/http/pprof"

	"github.com/cleafy/promqueen/model"
	"github.com/sirupsen/logrus"
	"gopkg.in/alecthomas/kingpin.v2"
)

var (
	debug      = kingpin.Flag("debug", "Enable debug mode.").Bool()
	enableGZIP = kingpin.Flag("gzip", "Enable gzip mode.").Bool()
	interval   = kingpin.Flag("interval", "Timeout waiting for ping.").Default("60s").OverrideDefaultFromEnvar("ACTION_INTERVAL").Short('i').Duration()
	umap       = kingpin.Flag("umap", "stringmap [eg. service.name=http://get.uri:port/uri].").Short('u').StringMap()
	output     = kingpin.Flag("output", "Output file.").Short('o').OverrideDefaultFromEnvar("OUTPUT_FILE").Default("metrics").String()
	Version    = "unversioned"
	filewriter io.WriteCloser
)

func closeIfNotNil(wc io.WriteCloser) {
	if wc != nil {
		if err := wc.Close(); err != nil {
			logrus.Info(err)
		}
	}
}

func writerFor() (io.Writer, error) {
	if _, err := os.Stat(*output); !os.IsNotExist(err) && filewriter != nil {
		return filewriter, nil
	}

	file, err := os.OpenFile(*output, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
	if err != nil {
		return nil, err
	}
	closeIfNotNil(filewriter)
	if *enableGZIP {
		filewriter = gzip.NewWriter(file)
	} else {
		filewriter = file
	}
	return filewriter, nil
}

func main() {
	kingpin.Version(Version)
	kingpin.Parse()

	if *debug {
		logrus.SetLevel(logrus.DebugLevel)
	}

	if len(*umap) <= 0 {
		kingpin.Usage()
		return
	}

	ticker := time.NewTicker(*interval)

	for range ticker.C {
		for sname, url := range *umap {
			writer, err := writerFor()
			if err != nil {
				logrus.Errorf("writeFor failed with %v", err)
				continue
			}

			resp, err := http.Get(url)
			if err != nil {
				logrus.Errorf("http.Get: %v", err)
				continue
			}
			defer resp.Body.Close()

			dump, err := httputil.DumpResponse(resp, true)
			if err != nil {
				logrus.Errorf("httputil.DumpResponse: %v", err)
				continue
			}

			frame := model.NewFrame(sname, url, dump)

			err = model.WriteFrame(writer, frame)
			if err != nil {
				logrus.Errorf("model.WriteFrame failed with %v", err)
				continue
			}
		}
	}

	http.ListenAndServe("0.0.0.0:8080", nil)
}
<commit_msg>Fixed import for pprof<commit_after>package main

import (
	"compress/gzip"
	"io"
	"net/http"
	"net/http/httputil"
	"os"
	"time"
	_ "net/http/pprof"

	"github.com/cleafy/promqueen/model"
	"github.com/sirupsen/logrus"
	"gopkg.in/alecthomas/kingpin.v2"
)

var (
	debug      = kingpin.Flag("debug", "Enable debug mode.").Bool()
	enableGZIP = kingpin.Flag("gzip", "Enable gzip mode.").Bool()
	interval   = kingpin.Flag("interval", "Timeout waiting for ping.").Default("60s").OverrideDefaultFromEnvar("ACTION_INTERVAL").Short('i').Duration()
	umap       = kingpin.Flag("umap", "stringmap [eg. service.name=http://get.uri:port/uri].").Short('u').StringMap()
	output     = kingpin.Flag("output", "Output file.").Short('o').OverrideDefaultFromEnvar("OUTPUT_FILE").Default("metrics").String()
	Version    = "unversioned"
	filewriter io.WriteCloser
)

func closeIfNotNil(wc io.WriteCloser) {
	if wc != nil {
		if err := wc.Close(); err != nil {
			logrus.Info(err)
		}
	}
}

func writerFor() (io.Writer, error) {
	if _, err := os.Stat(*output); !os.IsNotExist(err) && filewriter != nil {
		return filewriter, nil
	}

	file, err := os.OpenFile(*output, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
	if err != nil {
		return nil, err
	}
	closeIfNotNil(filewriter)
	if *enableGZIP {
		filewriter = gzip.NewWriter(file)
	} else {
		filewriter = file
	}
	return filewriter, nil
}

func main() {
	kingpin.Version(Version)
	kingpin.Parse()

	if *debug {
		logrus.SetLevel(logrus.DebugLevel)
	}

	if len(*umap) <= 0 {
		kingpin.Usage()
		return
	}

	http.ListenAndServe("0.0.0.0:8080", nil)

	ticker := time.NewTicker(*interval)

	for range ticker.C {
		for sname, url := range *umap {
			writer, err := writerFor()
			if err != nil {
				logrus.Errorf("writeFor failed with %v", err)
				continue
			}

			resp, err := http.Get(url)
			if err != nil {
				logrus.Errorf("http.Get: %v", err)
				continue
			}
			defer resp.Body.Close()

			dump, err := httputil.DumpResponse(resp, true)
			if err != nil {
				logrus.Errorf("httputil.DumpResponse: %v", err)
				continue
			}

			frame := model.NewFrame(sname, url, dump)

			err = model.WriteFrame(writer, frame)
			if err != nil {
				logrus.Errorf("model.WriteFrame failed with %v", err)
				continue
			}
		}
	}

}
<|endoftext|>


<commit_before>package termite

import (
	"fmt"
	"github.com/hanwen/go-fuse/fuse"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
)

var _ = fmt.Println

type FsServer struct {
	contentServer  *ContentServer
	contentCache   *ContentCache
	Root           string
	excluded       map[string]bool
	excludePrivate bool

	hashCacheMutex sync.RWMutex
	hashCacheCond  *sync.Cond
	hashCache      map[string]string
	hashBusyMap    map[string]bool

	attrCacheMutex sync.RWMutex
	attrCache      map[string]*FileAttr
	attrCacheCond  *sync.Cond
	attrCacheBusy  map[string]bool

	// TODO - add counters and check that the rpcFs.fetchCond is
	// working.
}

var paranoia = false

func (me *FsServer) verify() {
	if !paranoia {
		return
	}
	me.attrCacheMutex.RLock()
	defer me.attrCacheMutex.RUnlock()

	for k, v := range me.attrCache {
		if v.Path != k {
			log.Panicf("attrCache mismatch %q %#v", k, v)
		}
		if _, ok := me.attrCacheBusy[k]; ok {
			log.Panicf("attrCacheBusy and attrCache entry for %q", k)
		}
	}
}

func NewFsServer(root string, cache *ContentCache, excluded []string) *FsServer {
	fs := &FsServer{
		contentCache:  cache,
		contentServer: &ContentServer{Cache: cache},
		Root:          root,
		hashCache:     make(map[string]string),
		hashBusyMap:   map[string]bool{},

		attrCache:      make(map[string]*FileAttr),
		attrCacheBusy:  map[string]bool{},
		excludePrivate: true,
	}

	fs.hashCacheCond = sync.NewCond(&fs.hashCacheMutex)
	fs.attrCacheCond = sync.NewCond(&fs.attrCacheMutex)
	fs.excluded = make(map[string]bool)
	for _, e := range excluded {
		if e[0] == '/' {
			panic("leading slash")
		}
		fs.excluded[e] = true
	}
	return fs
}

func (me FileAttr) Deletion() bool {
	return me.Status == fuse.ENOENT
}

func (me *FsServer) path(n string) string {
	if me.Root == "" {
		return n
	}
	return filepath.Join(me.Root, strings.TrimLeft(n, "/"))
}

func (me *FsServer) FileContent(req *ContentRequest, rep *ContentResponse) os.Error {
	return me.contentServer.FileContent(req, rep)
}

func (me *FsServer) GetAttr(req *AttrRequest, rep *AttrResponse) os.Error {
	log.Println("GetAttr req", req.Name)
	if req.Name != "" && req.Name[0] == '/' {
		panic("leading /")
	}

	a := me.oneGetAttr(req.Name)
	if a.Hash != "" {
		log.Printf("GetAttr %v %x", a, a.Hash)
	}
	rep.Attrs = append(rep.Attrs, a)
	return nil
}

func (me *FsServer) uncachedGetAttr(name string) (rep *FileAttr) {
	if me.excluded[name] {
		log.Printf("Denied access to excluded file %q", name)
		return &FileAttr{
			Path:   name,
			Status: fuse.ENOENT,
		}
	}
	p := me.path(name)
	fi, err := os.Lstat(p)
	rep = &FileAttr{
		FileInfo: fi,
		Status:   fuse.OsErrorToErrno(err),
		Path:     name,
	}

	// We don't want to expose the master's private files to the
	// world.
	if me.excludePrivate && fi != nil && fi.Mode&0077 == 0 {
		log.Printf("Denied access to private file %q", name)
		rep.FileInfo = nil
		rep.Status = fuse.EPERM
		fi = nil
	}

	if fi != nil {
		me.fillContent(rep)
	}
	return rep
}

func (me *FsServer) oneGetAttr(name string) (rep *FileAttr) {
	defer me.verify()
	me.attrCacheMutex.RLock()
	rep, ok := me.attrCache[name]
	me.attrCacheMutex.RUnlock()
	if ok {
		return rep
	}

	me.attrCacheMutex.Lock()
	defer me.attrCacheMutex.Unlock()
	for me.attrCacheBusy[name] && me.attrCache[name] == nil {
		me.attrCacheCond.Wait()
	}
	rep, ok = me.attrCache[name]
	if ok {
		return rep
	}
	me.attrCacheBusy[name] = true
	me.attrCacheMutex.Unlock()

	rep = me.uncachedGetAttr(name)

	me.attrCacheMutex.Lock()
	me.attrCache[name] = rep
	me.attrCacheCond.Broadcast()
	me.attrCacheBusy[name] = false, false
	return rep
}

func (me *FsServer) fillContent(rep *FileAttr) {
	if rep.FileInfo.IsSymlink() {
		rep.Link, _ = os.Readlink(me.path(rep.Path))
	}
	if rep.FileInfo.IsRegular() {
		rep.Hash = me.getHash(rep.Path)
		if rep.Hash == "" {
			// Typically happens if we want to open /etc/shadow as normal user.
			log.Println("fillContent returning EPERM for", rep.Path)
			rep.Status = fuse.EPERM
		}
	}
	if rep.FileInfo.IsDirectory() {
		p := me.path(rep.Path)
		d, e := ioutil.ReadDir(p)
		rep.NameModeMap = make(map[string]uint32)
		for _, v := range d {
			rep.NameModeMap[v.Name] = v.Mode
		}
		rep.Status = fuse.OsErrorToErrno(e)
	}
}

func (me *FsServer) updateFiles(infos []*FileAttr) {
	me.updateHashes(infos)
	me.updateAttrs(infos)
}

func updateAttributeMap(attributes map[string]*FileAttr, files []*FileAttr) {
	for _, r := range files {
		if len(r.Path) > 0 && r.Path[0] == '/' {
			panic("Leading slash.")
		}

		dir, basename := filepath.Split(r.Path)
		dir = strings.TrimRight(dir, string(filepath.Separator))
		if dir, ok := attributes[dir]; ok {
			if r.Deletion() {
				dir.NameModeMap[basename] = 0, false
			} else {
				dir.NameModeMap[basename] = r.Mode &^ 0777
			}
		}

		if r.FileInfo != nil && r.FileInfo.IsDirectory() && r.NameModeMap == nil {
			dirResp := attributes[r.Path]
			if dirResp == nil {
				copy := *r
				dirResp = &copy
			}

			dirResp.FileInfo = r.FileInfo
			dirResp.Status = r.Status
			if dirResp.NameModeMap == nil {
				dirResp.NameModeMap = map[string]uint32{}
			}
			continue
		}

		copy := *r
		attributes[r.Path] = &copy
	}
}

func (me *FsServer) updateAttrs(infos []*FileAttr) {
	defer me.verify()

	me.attrCacheMutex.Lock()
	defer me.attrCacheMutex.Unlock()

	updateAttributeMap(me.attrCache, infos)
}

func (me *FsServer) updateHashes(infos []*FileAttr) {
	defer me.verify()

	me.hashCacheMutex.Lock()
	defer me.hashCacheMutex.Unlock()

	for _, r := range infos {
		name := r.Path
		if !r.Status.Ok() || r.Link != "" {
			me.hashCache[name] = "", false
			me.hashBusyMap[name] = false, false
		}
		if r.Hash != "" {
			me.hashCache[name] = r.Hash
			me.hashBusyMap[name] = false, false
		}
	}
}

func (me *FsServer) dropHash(name string) {
	me.hashCacheMutex.Lock()
	defer me.hashCacheMutex.Unlock()
	me.hashCache[name] = "", false
	me.hashCacheCond.Broadcast()
}

func (me *FsServer) getHash(name string) (hash string) {
	fullPath := me.path(name)

	me.hashCacheMutex.RLock()
	hash = me.hashCache[name]
	me.hashCacheMutex.RUnlock()

	if hash != "" {
		return hash
	}

	me.hashCacheMutex.Lock()
	defer me.hashCacheMutex.Unlock()
	for me.hashBusyMap[name] && me.hashCache[name] == "" {
		me.hashCacheCond.Wait()
	}

	hash = me.hashCache[name]
	if hash != "" {
		return hash
	}
	me.hashBusyMap[name] = true
	me.hashCacheMutex.Unlock()

	// TODO - /usr should be configurable.
	if HasDirPrefix(fullPath, "/usr") && !HasDirPrefix(fullPath, "/usr/local") {
		hash = me.contentCache.SaveImmutablePath(fullPath)
	} else {
		hash = me.contentCache.SavePath(fullPath)
	}

	me.hashCacheMutex.Lock()
	me.hashCache[name] = hash
	me.hashBusyMap[name] = false, false
	me.hashCacheCond.Broadcast()

	return hash
}

func (me *FsServer) refreshAttributeCache(prefix string) FileSet {
	me.attrCacheMutex.Lock()
	defer me.attrCacheMutex.Unlock()

	if prefix != "" &&  prefix[0] == '/' {
		panic("leading /")
	}

	updated := []*FileAttr{}
	for key, attr := range me.attrCache {
		// TODO -should just do everything?
		if !strings.HasPrefix(key, prefix) {
			continue
		}

		fi, _ := os.Lstat(me.path(key))
		if fi == nil && attr.Status.Ok() {
			del := FileAttr{
				Path:   key,
				Status: fuse.ENOENT,
			}
			updated = append(updated, &del)
		}
		// TODO - does this handle symlinks corrrectly?
		if fi != nil && attr.FileInfo != nil && EncodeFileInfo(*attr.FileInfo) != EncodeFileInfo(*fi) {
			newEnt := FileAttr{
				Path:     key,
				Status:   fuse.OK,
				FileInfo: fi,
			}
			me.dropHash(key)

			me.fillContent(&newEnt)
			updated = append(updated, &newEnt)
		}
	}

	fs := FileSet{updated}
	fs.Sort()
	for _, u := range fs.Files {
		copy := *u
		me.attrCache[u.Path] = &copy
	}
	return fs
}

func (me *FsServer) copyCache() FileSet {
	me.attrCacheMutex.RLock()
	defer me.attrCacheMutex.RUnlock()

	dump := []*FileAttr{}
	for _, attr := range me.attrCache {
		copy := *attr
		dump = append(dump, &copy)
	}

	fs := FileSet{dump}
	fs.Sort()
	return fs
}
<commit_msg>Panic for missing NameModeMap.<commit_after>package termite

import (
	"fmt"
	"github.com/hanwen/go-fuse/fuse"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
)

var _ = fmt.Println

type FsServer struct {
	contentServer  *ContentServer
	contentCache   *ContentCache
	Root           string
	excluded       map[string]bool
	excludePrivate bool

	hashCacheMutex sync.RWMutex
	hashCacheCond  *sync.Cond
	hashCache      map[string]string
	hashBusyMap    map[string]bool

	attrCacheMutex sync.RWMutex
	attrCache      map[string]*FileAttr
	attrCacheCond  *sync.Cond
	attrCacheBusy  map[string]bool

	// TODO - add counters and check that the rpcFs.fetchCond is
	// working.
}

var paranoia = false

func (me *FsServer) verify() {
	if !paranoia {
		return
	}
	me.attrCacheMutex.RLock()
	defer me.attrCacheMutex.RUnlock()

	for k, v := range me.attrCache {
		if v.Path != k {
			log.Panicf("attrCache mismatch %q %#v", k, v)
		}
		if _, ok := me.attrCacheBusy[k]; ok {
			log.Panicf("attrCacheBusy and attrCache entry for %q", k)
		}
	}
}

func NewFsServer(root string, cache *ContentCache, excluded []string) *FsServer {
	fs := &FsServer{
		contentCache:  cache,
		contentServer: &ContentServer{Cache: cache},
		Root:          root,
		hashCache:     make(map[string]string),
		hashBusyMap:   map[string]bool{},

		attrCache:      make(map[string]*FileAttr),
		attrCacheBusy:  map[string]bool{},
		excludePrivate: true,
	}

	fs.hashCacheCond = sync.NewCond(&fs.hashCacheMutex)
	fs.attrCacheCond = sync.NewCond(&fs.attrCacheMutex)
	fs.excluded = make(map[string]bool)
	for _, e := range excluded {
		if e[0] == '/' {
			panic("leading slash")
		}
		fs.excluded[e] = true
	}
	return fs
}

func (me FileAttr) Deletion() bool {
	return me.Status == fuse.ENOENT
}

func (me *FsServer) path(n string) string {
	if me.Root == "" {
		return n
	}
	return filepath.Join(me.Root, strings.TrimLeft(n, "/"))
}

func (me *FsServer) FileContent(req *ContentRequest, rep *ContentResponse) os.Error {
	return me.contentServer.FileContent(req, rep)
}

func (me *FsServer) GetAttr(req *AttrRequest, rep *AttrResponse) os.Error {
	log.Println("GetAttr req", req.Name)
	if req.Name != "" && req.Name[0] == '/' {
		panic("leading /")
	}

	a := me.oneGetAttr(req.Name)
	if a.Hash != "" {
		log.Printf("GetAttr %v %x", a, a.Hash)
	}
	rep.Attrs = append(rep.Attrs, a)
	return nil
}

func (me *FsServer) uncachedGetAttr(name string) (rep *FileAttr) {
	if me.excluded[name] {
		log.Printf("Denied access to excluded file %q", name)
		return &FileAttr{
			Path:   name,
			Status: fuse.ENOENT,
		}
	}
	p := me.path(name)
	fi, err := os.Lstat(p)
	rep = &FileAttr{
		FileInfo: fi,
		Status:   fuse.OsErrorToErrno(err),
		Path:     name,
	}

	// We don't want to expose the master's private files to the
	// world.
	if me.excludePrivate && fi != nil && fi.Mode&0077 == 0 {
		log.Printf("Denied access to private file %q", name)
		rep.FileInfo = nil
		rep.Status = fuse.EPERM
		fi = nil
	}

	if fi != nil {
		me.fillContent(rep)
	}
	return rep
}

func (me *FsServer) oneGetAttr(name string) (rep *FileAttr) {
	defer me.verify()
	me.attrCacheMutex.RLock()
	rep, ok := me.attrCache[name]
	me.attrCacheMutex.RUnlock()
	if ok {
		return rep
	}

	me.attrCacheMutex.Lock()
	defer me.attrCacheMutex.Unlock()
	for me.attrCacheBusy[name] && me.attrCache[name] == nil {
		me.attrCacheCond.Wait()
	}
	rep, ok = me.attrCache[name]
	if ok {
		return rep
	}
	me.attrCacheBusy[name] = true
	me.attrCacheMutex.Unlock()

	rep = me.uncachedGetAttr(name)

	me.attrCacheMutex.Lock()
	me.attrCache[name] = rep
	me.attrCacheCond.Broadcast()
	me.attrCacheBusy[name] = false, false
	return rep
}

func (me *FsServer) fillContent(rep *FileAttr) {
	if rep.FileInfo.IsSymlink() {
		rep.Link, _ = os.Readlink(me.path(rep.Path))
	}
	if rep.FileInfo.IsRegular() {
		rep.Hash = me.getHash(rep.Path)
		if rep.Hash == "" {
			// Typically happens if we want to open /etc/shadow as normal user.
			log.Println("fillContent returning EPERM for", rep.Path)
			rep.Status = fuse.EPERM
		}
	}
	if rep.FileInfo.IsDirectory() {
		p := me.path(rep.Path)
		d, e := ioutil.ReadDir(p)
		rep.NameModeMap = make(map[string]uint32)
		for _, v := range d {
			rep.NameModeMap[v.Name] = v.Mode
		}
		rep.Status = fuse.OsErrorToErrno(e)
	}
}

func (me *FsServer) updateFiles(infos []*FileAttr) {
	me.updateHashes(infos)
	me.updateAttrs(infos)
}

func updateAttributeMap(attributes map[string]*FileAttr, files []*FileAttr) {
	for _, r := range files {
		if len(r.Path) > 0 && r.Path[0] == '/' {
			panic("Leading slash.")
		}

		dir, basename := filepath.Split(r.Path)
		dir = strings.TrimRight(dir, string(filepath.Separator))
		if dirAttr, ok := attributes[dir]; ok {
			if dirAttr.NameModeMap == nil {
				log.Panicf("parent dir has no NameModeMap: %q", dir)
			}
			if r.Deletion() {
				dirAttr.NameModeMap[basename] = 0, false
			} else {
				dirAttr.NameModeMap[basename] = r.Mode &^ 0777
			}
		}

		if r.FileInfo != nil && r.FileInfo.IsDirectory() && r.NameModeMap == nil {
			dirResp := attributes[r.Path]
			if dirResp == nil {
				copy := *r
				dirResp = &copy
			}

			dirResp.FileInfo = r.FileInfo
			dirResp.Status = r.Status
			if dirResp.NameModeMap == nil {
				dirResp.NameModeMap = map[string]uint32{}
			}
			continue
		}

		copy := *r
		attributes[r.Path] = &copy
	}
}

func (me *FsServer) updateAttrs(infos []*FileAttr) {
	defer me.verify()

	me.attrCacheMutex.Lock()
	defer me.attrCacheMutex.Unlock()

	updateAttributeMap(me.attrCache, infos)
}

func (me *FsServer) updateHashes(infos []*FileAttr) {
	defer me.verify()

	me.hashCacheMutex.Lock()
	defer me.hashCacheMutex.Unlock()

	for _, r := range infos {
		name := r.Path
		if !r.Status.Ok() || r.Link != "" {
			me.hashCache[name] = "", false
			me.hashBusyMap[name] = false, false
		}
		if r.Hash != "" {
			me.hashCache[name] = r.Hash
			me.hashBusyMap[name] = false, false
		}
	}
}

func (me *FsServer) dropHash(name string) {
	me.hashCacheMutex.Lock()
	defer me.hashCacheMutex.Unlock()
	me.hashCache[name] = "", false
	me.hashCacheCond.Broadcast()
}

func (me *FsServer) getHash(name string) (hash string) {
	fullPath := me.path(name)

	me.hashCacheMutex.RLock()
	hash = me.hashCache[name]
	me.hashCacheMutex.RUnlock()

	if hash != "" {
		return hash
	}

	me.hashCacheMutex.Lock()
	defer me.hashCacheMutex.Unlock()
	for me.hashBusyMap[name] && me.hashCache[name] == "" {
		me.hashCacheCond.Wait()
	}

	hash = me.hashCache[name]
	if hash != "" {
		return hash
	}
	me.hashBusyMap[name] = true
	me.hashCacheMutex.Unlock()

	// TODO - /usr should be configurable.
	if HasDirPrefix(fullPath, "/usr") && !HasDirPrefix(fullPath, "/usr/local") {
		hash = me.contentCache.SaveImmutablePath(fullPath)
	} else {
		hash = me.contentCache.SavePath(fullPath)
	}

	me.hashCacheMutex.Lock()
	me.hashCache[name] = hash
	me.hashBusyMap[name] = false, false
	me.hashCacheCond.Broadcast()

	return hash
}

func (me *FsServer) refreshAttributeCache(prefix string) FileSet {
	me.attrCacheMutex.Lock()
	defer me.attrCacheMutex.Unlock()

	if prefix != "" &&  prefix[0] == '/' {
		panic("leading /")
	}

	updated := []*FileAttr{}
	for key, attr := range me.attrCache {
		// TODO -should just do everything?
		if !strings.HasPrefix(key, prefix) {
			continue
		}

		fi, _ := os.Lstat(me.path(key))
		if fi == nil && attr.Status.Ok() {
			del := FileAttr{
				Path:   key,
				Status: fuse.ENOENT,
			}
			updated = append(updated, &del)
		}
		// TODO - does this handle symlinks corrrectly?
		if fi != nil && attr.FileInfo != nil && EncodeFileInfo(*attr.FileInfo) != EncodeFileInfo(*fi) {
			newEnt := FileAttr{
				Path:     key,
				Status:   fuse.OK,
				FileInfo: fi,
			}
			me.dropHash(key)

			me.fillContent(&newEnt)
			updated = append(updated, &newEnt)
		}
	}

	fs := FileSet{updated}
	fs.Sort()
	for _, u := range fs.Files {
		copy := *u
		me.attrCache[u.Path] = &copy
	}
	return fs
}

func (me *FsServer) copyCache() FileSet {
	me.attrCacheMutex.RLock()
	defer me.attrCacheMutex.RUnlock()

	dump := []*FileAttr{}
	for _, attr := range me.attrCache {
		copy := *attr
		dump = append(dump, &copy)
	}

	fs := FileSet{dump}
	fs.Sort()
	return fs
}
<|endoftext|>


<commit_before>package termite

import (
	"bufio"
	"io"
	"json"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

type localDeciderRule struct {
	Regexp string
	Local bool
}

type localDecider struct {
	rules []localDeciderRule
}

func newLocalDecider(input io.Reader) *localDecider {
	reader := bufio.NewReader(input)
	out := []byte{}
	for {
		line, _, err := reader.ReadLine()
		if err != nil {
			break
		}
		if len(line) == 0 || line[0] == '#' || strings.HasPrefix(string(line), "//") {
			continue
		}
		out = append(out, line...)
		out = append(out, '\n')
	}

	decider := localDecider{}
	err := json.Unmarshal([]byte(out), &decider.rules)
	if err != nil {
		log.Println(err)
		return nil
	}
	return &decider
}

func (me *localDecider) ShouldRunLocally(command string) bool {
	for _, r := range me.rules {
		m, err := regexp.MatchString(r.Regexp, command)
		if err != nil {
			log.Println("regexp error:", err)
			continue
		}
		if m {
			return r.Local
		}
	}

	return false
}

func LocalDecider(dir string) *localDecider {
	localRc := filepath.Join(dir, ".termite-localrc")

	f, _ := os.Open(localRc)
	if f != nil {
		defer f.Close()
		return newLocalDecider(f)
	}

	rules := []localDeciderRule{
		localDeciderRule{Regexp: ".*termite-make", Local: true},
		localDeciderRule{Regexp: ".*", Local: false},
	}
	return &localDecider{rules}
}
<commit_msg>Commit changes to localrun.go.<commit_after>package termite

import (
	"bufio"
	"io"
	"json"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

type localDeciderRule struct {
	Regexp  string
	Local   bool
	LocalRecurse bool
}

type localDecider struct {
	rules []localDeciderRule
}

func newLocalDecider(input io.Reader) *localDecider {
	reader := bufio.NewReader(input)
	out := []byte{}
	for {
		line, _, err := reader.ReadLine()
		if err != nil {
			break
		}
		if len(line) == 0 || line[0] == '#' || strings.HasPrefix(string(line), "//") {
			continue
		}
		out = append(out, line...)
		out = append(out, '\n')
	}

	decider := localDecider{}
	err := json.Unmarshal([]byte(out), &decider.rules)
	if err != nil {
		log.Println(err)
		return nil
	}
	return &decider
}

func (me *localDecider) ShouldRunLocally(command string) (local bool, recurse bool) {
	for _, r := range me.rules {
		m, err := regexp.MatchString(r.Regexp, command)
		if err != nil {
			log.Println("regexp error:", err)
			continue
		}
		if m {
			return r.Local, r.LocalRecurse
		}
	}

	return false, false
}

func LocalDecider(dir string) *localDecider {
	localRc := filepath.Join(dir, ".termite-localrc")

	f, _ := os.Open(localRc)
	if f != nil {
		defer f.Close()
		return newLocalDecider(f)
	}

	rules := []localDeciderRule{
		localDeciderRule{Regexp: ".*termite-make", Local: true},
		localDeciderRule{Regexp: ".*", Local: false},
	}
	return &localDecider{rules}
}
<|endoftext|>


<commit_before>// Copyright 2015, Joe Tsai. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE.md file.

package bits

import "io"
import "runtime"
import "testing"
import "github.com/stretchr/testify/assert"

func TestBuffer(t *testing.T) {
	type X struct {
		ra, wa bool
		rn, wn int64
		buf    []byte
	}
	state := func(bb *Buffer) X {
		return X{
			bb.ReadAligned(), bb.WriteAligned(),
			bb.BitsRead(), bb.BitsWritten(),
			nb(bb.Bytes()),
		}
	}

	var bit bool
	var dat byte
	var val uint
	var buf [1024]byte
	var cnt int
	var err error

	bb := NewBuffer(nil)
	assert.Equal(t, X{true, true, 0, 0, nil}, state(bb))

	_, err = bb.ReadBit()
	assert.Equal(t, io.EOF, err)
	assert.Equal(t, X{true, true, 0, 0, nil}, state(bb))

	err = bb.WriteBit(true)
	assert.Equal(t, nil, err)
	assert.Equal(t, X{true, false, 0, 1, []byte{0x01}}, state(bb))

	bit, err = bb.ReadBit()
	assert.Equal(t, true, bit)
	assert.Equal(t, nil, err)
	assert.Equal(t, X{false, false, 1, 1, []byte{0x01}}, state(bb))

	_, err = bb.ReadBit()
	assert.Equal(t, err, io.EOF)
	assert.Equal(t, X{false, false, 1, 1, []byte{0x01}}, state(bb))

	err = bb.WriteBit(true)
	assert.Equal(t, nil, err)
	assert.Equal(t, X{false, false, 1, 2, []byte{0x03}}, state(bb))

	err = bb.WriteBit(false)
	assert.Equal(t, nil, err)
	assert.Equal(t, X{false, false, 1, 3, []byte{0x03}}, state(bb))

	bit, err = bb.ReadBit()
	assert.Equal(t, true, bit)
	assert.Equal(t, nil, err)
	assert.Equal(t, X{false, false, 2, 3, []byte{0x03}}, state(bb))

	cnt, err = bb.WriteBits(0x0b, 5)
	assert.Equal(t, 5, cnt)
	assert.Equal(t, nil, err)
	assert.Equal(t, X{false, true, 2, 8, []byte{0x5b}}, state(bb))

	val, cnt, err = bb.ReadBits(6)
	assert.Equal(t, uint(0x16), val)
	assert.Equal(t, 6, cnt)
	assert.Equal(t, nil, err)
	assert.Equal(t, X{true, true, 8, 8, nil}, state(bb))

	err = bb.WriteByte(0xd3)
	assert.Equal(t, nil, err)
	assert.Equal(t, X{true, true, 8, 16, []byte{0xd3}}, state(bb))

	cnt, err = bb.Write([]byte{0xc3, 0x1c, 0x3b})
	assert.Equal(t, 3, cnt)
	assert.Equal(t, nil, err)
	assert.Equal(t, X{true, true, 8, 40, []byte{0xd3, 0xc3, 0x1c, 0x3b}}, state(bb))

	cnt, err = bb.WriteBits(0x7bfe3, 19)
	assert.Equal(t, 19, cnt)
	assert.Equal(t, nil, err)
	assert.Equal(t, X{true, false, 8, 59, []byte{0xd3, 0xc3, 0x1c, 0x3b, 0xe3, 0xbf, 0x07}}, state(bb))

	err = bb.WriteByte(0xff)
	assert.Equal(t, ErrAlign, err)
	assert.Equal(t, X{true, false, 8, 59, []byte{0xd3, 0xc3, 0x1c, 0x3b, 0xe3, 0xbf, 0x07}}, state(bb))

	cnt, err = bb.Write([]byte{0xff, 0x00})
	assert.Equal(t, 0, cnt)
	assert.Equal(t, ErrAlign, err)
	assert.Equal(t, X{true, false, 8, 59, []byte{0xd3, 0xc3, 0x1c, 0x3b, 0xe3, 0xbf, 0x07}}, state(bb))

	cnt, err = bb.Read(buf[:2])
	assert.Equal(t, 2, cnt)
	assert.Equal(t, nil, err)
	assert.Equal(t, []byte{0xd3, 0xc3}, nb(buf[:cnt]))
	assert.Equal(t, X{true, false, 24, 59, []byte{0x1c, 0x3b, 0xe3, 0xbf, 0x07}}, state(bb))

	dat, err = bb.ReadByte()
	assert.Equal(t, byte(0x1c), dat)
	assert.Equal(t, nil, err)
	assert.Equal(t, X{true, false, 32, 59, []byte{0x3b, 0xe3, 0xbf, 0x07}}, state(bb))

	val, cnt, err = bb.ReadBits(13)
	assert.Equal(t, uint(0x33b), val)
	assert.Equal(t, 13, cnt)
	assert.Equal(t, nil, err)
	assert.Equal(t, X{false, false, 45, 59, []byte{0xe3, 0xbf, 0x07}}, state(bb))

	cnt, err = bb.Read(buf[:])
	assert.Equal(t, 0, cnt)
	assert.Equal(t, ErrAlign, err)
	assert.Equal(t, X{false, false, 45, 59, []byte{0xe3, 0xbf, 0x07}}, state(bb))

	_, err = bb.ReadByte()
	assert.Equal(t, ErrAlign, err)
	assert.Equal(t, X{false, false, 45, 59, []byte{0xe3, 0xbf, 0x07}}, state(bb))

	val, cnt, err = bb.ReadBits(21)
	assert.Equal(t, uint(0x3dff), val)
	assert.Equal(t, 14, cnt)
	assert.Equal(t, io.ErrUnexpectedEOF, err)
	assert.Equal(t, X{false, false, 59, 59, []byte{0x07}}, state(bb))

	_, cnt, err = bb.ReadBits(3)
	assert.Equal(t, 0, cnt)
	assert.Equal(t, io.EOF, err)
	assert.Equal(t, X{false, false, 59, 59, []byte{0x07}}, state(bb))

	cnt, err = bb.WriteBits(0xabcde, 20)
	assert.Equal(t, 20, cnt)
	assert.Equal(t, nil, err)
	assert.Equal(t, X{false, false, 59, 79, []byte{0xf7, 0xe6, 0x55}}, state(bb))

	val, cnt, err = bb.ReadBits(5)
	assert.Equal(t, uint(0x1e), val)
	assert.Equal(t, 5, cnt)
	assert.Equal(t, nil, err)
	assert.Equal(t, X{true, false, 64, 79, []byte{0xe6, 0x55}}, state(bb))

	cnt, err = bb.Read(buf[:])
	assert.Equal(t, 1, cnt)
	assert.Equal(t, ErrAlign, err)
	assert.Equal(t, []byte{0xe6}, nb(buf[:cnt]))
	assert.Equal(t, X{true, false, 72, 79, []byte{0x55}}, state(bb))

	_, err = bb.ReadByte()
	assert.Equal(t, ErrAlign, err)
	assert.Equal(t, X{true, false, 72, 79, []byte{0x55}}, state(bb))

	err = bb.WriteBit(true)
	assert.Equal(t, nil, err)
	assert.Equal(t, X{true, true, 72, 80, []byte{0xd5}}, state(bb))

	cnt, err = bb.Read(buf[:])
	assert.Equal(t, 1, cnt)
	assert.Equal(t, io.EOF, err)
	assert.Equal(t, []byte{0xd5}, nb(buf[:cnt]))
	assert.Equal(t, X{true, true, 80, 80, nil}, state(bb))

	_, err = bb.ReadByte()
	assert.Equal(t, io.EOF, err)
	assert.Equal(t, X{true, true, 80, 80, nil}, state(bb))

	// Reset
	bb.Reset()
	assert.Equal(t, X{true, true, 0, 0, nil}, state(bb))

	cnt, err = bb.Read(buf[:])
	assert.Equal(t, 0, cnt)
	assert.Equal(t, io.EOF, err)
	assert.Equal(t, []byte(nil), nb(buf[:cnt]))
	assert.Equal(t, X{true, true, 0, 0, nil}, state(bb))

	// Reset with data
	bb.ResetData([]byte{0xab, 0xcd, 0xef})
	assert.Equal(t, X{true, true, 0, 24, []byte{0xab, 0xcd, 0xef}}, state(bb))

	cnt, err = bb.Read(buf[:])
	assert.Equal(t, 3, cnt)
	assert.Equal(t, io.EOF, err)
	assert.Equal(t, []byte{0xab, 0xcd, 0xef}, nb(buf[:cnt]))
	assert.Equal(t, X{true, true, 24, 24, nil}, state(bb))
}

func BenchmarkBufferWriter(b *testing.B) {
	cnt := 1 << 20 // 1 MiB
	bb := NewBuffer(make([]byte, 0, cnt))

	runtime.GC()
	b.ReportAllocs()
	b.SetBytes(int64(cnt))
	b.ResetTimer()

	for ni := 0; ni < b.N; ni++ {
		bb.Reset()
		for bi := 0; bi < cnt; bi++ {
			bb.WriteBit(true)
			bb.WriteBit(false)
			bb.WriteBit(true)
			bb.WriteBit(false)
			bb.WriteBit(true)
			bb.WriteBit(false)
			bb.WriteBit(true)
			bb.WriteBit(false)
		}
	}
}

func BenchmarkBufferReader(b *testing.B) {
	cnt := 1 << 20 // 1 MiB
	data := make([]byte, cnt)
	for i := range data {
		data[i] = 0x55
	}
	bb := NewBuffer(data)

	b.ReportAllocs()
	b.SetBytes(int64(cnt))
	b.ResetTimer()

	for ni := 0; ni < b.N; ni++ {
		bb.ResetData(data)
		for bi := 0; bi < cnt; bi++ {
			bb.ReadBit()
			bb.ReadBit()
			bb.ReadBit()
			bb.ReadBit()
			bb.ReadBit()
			bb.ReadBit()
			bb.ReadBit()
			bb.ReadBit()
		}
	}
}
<commit_msg>Explicit GC before benchmark<commit_after>// Copyright 2015, Joe Tsai. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE.md file.

package bits

import "io"
import "runtime"
import "testing"
import "github.com/stretchr/testify/assert"

func TestBuffer(t *testing.T) {
	type X struct {
		ra, wa bool
		rn, wn int64
		buf    []byte
	}
	state := func(bb *Buffer) X {
		return X{
			bb.ReadAligned(), bb.WriteAligned(),
			bb.BitsRead(), bb.BitsWritten(),
			nb(bb.Bytes()),
		}
	}

	var bit bool
	var dat byte
	var val uint
	var buf [1024]byte
	var cnt int
	var err error

	bb := NewBuffer(nil)
	assert.Equal(t, X{true, true, 0, 0, nil}, state(bb))

	_, err = bb.ReadBit()
	assert.Equal(t, io.EOF, err)
	assert.Equal(t, X{true, true, 0, 0, nil}, state(bb))

	err = bb.WriteBit(true)
	assert.Equal(t, nil, err)
	assert.Equal(t, X{true, false, 0, 1, []byte{0x01}}, state(bb))

	bit, err = bb.ReadBit()
	assert.Equal(t, true, bit)
	assert.Equal(t, nil, err)
	assert.Equal(t, X{false, false, 1, 1, []byte{0x01}}, state(bb))

	_, err = bb.ReadBit()
	assert.Equal(t, err, io.EOF)
	assert.Equal(t, X{false, false, 1, 1, []byte{0x01}}, state(bb))

	err = bb.WriteBit(true)
	assert.Equal(t, nil, err)
	assert.Equal(t, X{false, false, 1, 2, []byte{0x03}}, state(bb))

	err = bb.WriteBit(false)
	assert.Equal(t, nil, err)
	assert.Equal(t, X{false, false, 1, 3, []byte{0x03}}, state(bb))

	bit, err = bb.ReadBit()
	assert.Equal(t, true, bit)
	assert.Equal(t, nil, err)
	assert.Equal(t, X{false, false, 2, 3, []byte{0x03}}, state(bb))

	cnt, err = bb.WriteBits(0x0b, 5)
	assert.Equal(t, 5, cnt)
	assert.Equal(t, nil, err)
	assert.Equal(t, X{false, true, 2, 8, []byte{0x5b}}, state(bb))

	val, cnt, err = bb.ReadBits(6)
	assert.Equal(t, uint(0x16), val)
	assert.Equal(t, 6, cnt)
	assert.Equal(t, nil, err)
	assert.Equal(t, X{true, true, 8, 8, nil}, state(bb))

	err = bb.WriteByte(0xd3)
	assert.Equal(t, nil, err)
	assert.Equal(t, X{true, true, 8, 16, []byte{0xd3}}, state(bb))

	cnt, err = bb.Write([]byte{0xc3, 0x1c, 0x3b})
	assert.Equal(t, 3, cnt)
	assert.Equal(t, nil, err)
	assert.Equal(t, X{true, true, 8, 40, []byte{0xd3, 0xc3, 0x1c, 0x3b}}, state(bb))

	cnt, err = bb.WriteBits(0x7bfe3, 19)
	assert.Equal(t, 19, cnt)
	assert.Equal(t, nil, err)
	assert.Equal(t, X{true, false, 8, 59, []byte{0xd3, 0xc3, 0x1c, 0x3b, 0xe3, 0xbf, 0x07}}, state(bb))

	err = bb.WriteByte(0xff)
	assert.Equal(t, ErrAlign, err)
	assert.Equal(t, X{true, false, 8, 59, []byte{0xd3, 0xc3, 0x1c, 0x3b, 0xe3, 0xbf, 0x07}}, state(bb))

	cnt, err = bb.Write([]byte{0xff, 0x00})
	assert.Equal(t, 0, cnt)
	assert.Equal(t, ErrAlign, err)
	assert.Equal(t, X{true, false, 8, 59, []byte{0xd3, 0xc3, 0x1c, 0x3b, 0xe3, 0xbf, 0x07}}, state(bb))

	cnt, err = bb.Read(buf[:2])
	assert.Equal(t, 2, cnt)
	assert.Equal(t, nil, err)
	assert.Equal(t, []byte{0xd3, 0xc3}, nb(buf[:cnt]))
	assert.Equal(t, X{true, false, 24, 59, []byte{0x1c, 0x3b, 0xe3, 0xbf, 0x07}}, state(bb))

	dat, err = bb.ReadByte()
	assert.Equal(t, byte(0x1c), dat)
	assert.Equal(t, nil, err)
	assert.Equal(t, X{true, false, 32, 59, []byte{0x3b, 0xe3, 0xbf, 0x07}}, state(bb))

	val, cnt, err = bb.ReadBits(13)
	assert.Equal(t, uint(0x33b), val)
	assert.Equal(t, 13, cnt)
	assert.Equal(t, nil, err)
	assert.Equal(t, X{false, false, 45, 59, []byte{0xe3, 0xbf, 0x07}}, state(bb))

	cnt, err = bb.Read(buf[:])
	assert.Equal(t, 0, cnt)
	assert.Equal(t, ErrAlign, err)
	assert.Equal(t, X{false, false, 45, 59, []byte{0xe3, 0xbf, 0x07}}, state(bb))

	_, err = bb.ReadByte()
	assert.Equal(t, ErrAlign, err)
	assert.Equal(t, X{false, false, 45, 59, []byte{0xe3, 0xbf, 0x07}}, state(bb))

	val, cnt, err = bb.ReadBits(21)
	assert.Equal(t, uint(0x3dff), val)
	assert.Equal(t, 14, cnt)
	assert.Equal(t, io.ErrUnexpectedEOF, err)
	assert.Equal(t, X{false, false, 59, 59, []byte{0x07}}, state(bb))

	_, cnt, err = bb.ReadBits(3)
	assert.Equal(t, 0, cnt)
	assert.Equal(t, io.EOF, err)
	assert.Equal(t, X{false, false, 59, 59, []byte{0x07}}, state(bb))

	cnt, err = bb.WriteBits(0xabcde, 20)
	assert.Equal(t, 20, cnt)
	assert.Equal(t, nil, err)
	assert.Equal(t, X{false, false, 59, 79, []byte{0xf7, 0xe6, 0x55}}, state(bb))

	val, cnt, err = bb.ReadBits(5)
	assert.Equal(t, uint(0x1e), val)
	assert.Equal(t, 5, cnt)
	assert.Equal(t, nil, err)
	assert.Equal(t, X{true, false, 64, 79, []byte{0xe6, 0x55}}, state(bb))

	cnt, err = bb.Read(buf[:])
	assert.Equal(t, 1, cnt)
	assert.Equal(t, ErrAlign, err)
	assert.Equal(t, []byte{0xe6}, nb(buf[:cnt]))
	assert.Equal(t, X{true, false, 72, 79, []byte{0x55}}, state(bb))

	_, err = bb.ReadByte()
	assert.Equal(t, ErrAlign, err)
	assert.Equal(t, X{true, false, 72, 79, []byte{0x55}}, state(bb))

	err = bb.WriteBit(true)
	assert.Equal(t, nil, err)
	assert.Equal(t, X{true, true, 72, 80, []byte{0xd5}}, state(bb))

	cnt, err = bb.Read(buf[:])
	assert.Equal(t, 1, cnt)
	assert.Equal(t, io.EOF, err)
	assert.Equal(t, []byte{0xd5}, nb(buf[:cnt]))
	assert.Equal(t, X{true, true, 80, 80, nil}, state(bb))

	_, err = bb.ReadByte()
	assert.Equal(t, io.EOF, err)
	assert.Equal(t, X{true, true, 80, 80, nil}, state(bb))

	// Reset
	bb.Reset()
	assert.Equal(t, X{true, true, 0, 0, nil}, state(bb))

	cnt, err = bb.Read(buf[:])
	assert.Equal(t, 0, cnt)
	assert.Equal(t, io.EOF, err)
	assert.Equal(t, []byte(nil), nb(buf[:cnt]))
	assert.Equal(t, X{true, true, 0, 0, nil}, state(bb))

	// Reset with data
	bb.ResetData([]byte{0xab, 0xcd, 0xef})
	assert.Equal(t, X{true, true, 0, 24, []byte{0xab, 0xcd, 0xef}}, state(bb))

	cnt, err = bb.Read(buf[:])
	assert.Equal(t, 3, cnt)
	assert.Equal(t, io.EOF, err)
	assert.Equal(t, []byte{0xab, 0xcd, 0xef}, nb(buf[:cnt]))
	assert.Equal(t, X{true, true, 24, 24, nil}, state(bb))
}

func BenchmarkBufferWriter(b *testing.B) {
	cnt := 1 << 20 // 1 MiB
	bb := NewBuffer(make([]byte, 0, cnt))

	runtime.GC()
	b.ReportAllocs()
	b.SetBytes(int64(cnt))
	b.ResetTimer()

	for ni := 0; ni < b.N; ni++ {
		bb.Reset()
		for bi := 0; bi < cnt; bi++ {
			bb.WriteBit(true)
			bb.WriteBit(false)
			bb.WriteBit(true)
			bb.WriteBit(false)
			bb.WriteBit(true)
			bb.WriteBit(false)
			bb.WriteBit(true)
			bb.WriteBit(false)
		}
	}
}

func BenchmarkBufferReader(b *testing.B) {
	cnt := 1 << 20 // 1 MiB
	data := make([]byte, cnt)
	for i := range data {
		data[i] = 0x55
	}
	bb := NewBuffer(data)

	runtime.GC()
	b.ReportAllocs()
	b.SetBytes(int64(cnt))
	b.ResetTimer()

	for ni := 0; ni < b.N; ni++ {
		bb.ResetData(data)
		for bi := 0; bi < cnt; bi++ {
			bb.ReadBit()
			bb.ReadBit()
			bb.ReadBit()
			bb.ReadBit()
			bb.ReadBit()
			bb.ReadBit()
			bb.ReadBit()
			bb.ReadBit()
		}
	}
}
<|endoftext|>


<commit_before>package command

/*
 Cribbed **heavily** from https://github.com/hashicorp/consul
*/

import (
	"fmt"
	"github.com/mitchellh/cli"
	"os"
	"log"
	"strings"
	"github.com/PolkaBand/polka/utils"
	"text/template"
)

type GenerateCommand struct {
	Name string
	Ui   cli.Ui
}

type Concept struct {
	Name string
}

func (c *GenerateCommand) Help() string {
	helpText := `
Usage: polka generate [subcommand] concept

Generate a lambda function of type [subcommand] to implement the concept.

Subcommands:

	endpoint	an endpoint that supports CRUD and a health check on (concept)\n
`
	return strings.TrimSpace(helpText)
}

func GetEndpoints() []string {
	endpoints := []string{
		"create.js",
		"read.js",
		"update.js",
		"delete.js",
		"health.js",
	}
	return endpoints
}

func GenerateEndpoint(name string) {
	concept := Concept{name}
	for _, templateName := range GetEndpoints() {
		FindAndExecuteTemplate(templateName, concept)
	}
}

func MakeEndpointDir() (string, error) {
	if appDir, err := GetAppDir(); err == nil {
		endpointDir := fmt.Sprintf("%v/app/endpoint", appDir )
		log.Printf("trying to make %v\n", endpointDir )
		if err := os.MkdirAll(endpointDir, 0777); err == nil {
			log.Printf("should have made %v\n", endpointDir )
			return endpointDir, nil
		} else {
			return "", err
		}
	} else {
		return "", err
	}
}

func FindAndExecuteTemplate(templateName string, concept Concept) {

	if templateDir, err := utils.FindTemplateDir(); err == nil {

		if endpointDir, err := MakeEndpointDir(); err == nil {
			//TODO assume that js is the only implementation language for now
			templatesGlob := fmt.Sprintf("%v/js/endpoint/*.js", templateDir )

			if t, err := template.ParseGlob(templatesGlob); err == nil {

				outputFileName := fmt.Sprintf("%v/%v", endpointDir, templateName )
				log.Println(outputFileName)

				outputFilename, err := os.Create(outputFileName)

				if err = t.ExecuteTemplate(outputFilename, templateName, concept); err != nil {
					log.Fatal(err)
				}
			}
		}
	}
}

func (c *GenerateCommand) Run(args []string) int {
	if len(args) < 2 {
		fmt.Printf("%v", c.Help())
		return 1
	}

	subcommand := args[0]

	switch subcommand {
	case "endpoint":
		GenerateEndpoint(args[1])
	case "integration":
		log.Println("generating integrations are not implemented yet")
	}

	return 0
}

func (c *GenerateCommand) Synopsis() string {
	return "generates lambda functions for endpoints"
}
<commit_msg>refactored generate command<commit_after>package command

/*
 Cribbed **heavily** from https://github.com/hashicorp/consul
*/

import (
	"fmt"
	"github.com/mitchellh/cli"
	"os"
	"log"
	"strings"
	"github.com/PolkaBand/polka/utils"
	"text/template"
)

type GenerateCommand struct {
	Name string
	Ui   cli.Ui
}

type Concept struct {
	Name string
}

func (c *GenerateCommand) Help() string {
	helpText := `
Usage: polka generate [subcommand] concept

Generate a lambda function of type [subcommand] to implement the concept.

Subcommands:

	endpoint	an endpoint that supports CRUD and a health check on (concept)\n
`
	return strings.TrimSpace(helpText)
}

func GetEndpoints() []string {
	endpoints := []string{
		"create.js",
		"read.js",
		"update.js",
		"delete.js",
		"health.js",
	}
	return endpoints
}

func MakeAllEndpoints(name string) {
	concept := Concept{name}
	for _, templateName := range GetEndpoints() {
		FindAndExecuteTemplate(templateName, concept)
	}
}

func MakeEndpointDir() (string, error) {
	if appDir, err := GetAppDir(); err == nil {
		endpointDir := fmt.Sprintf("%v/app/endpoint", appDir )
		log.Printf("trying to make %v\n", endpointDir )
		if err := os.MkdirAll(endpointDir, 0777); err == nil {
			log.Printf("should have made %v\n", endpointDir )
			return endpointDir, nil
		} else {
			return "", err
		}
	} else {
		return "", err
	}
}

func FindAndExecuteTemplate(templateName string, concept Concept) {

	if templateDir, err := utils.FindTemplateDir(); err == nil {

		if endpointDir, err := MakeEndpointDir(); err == nil {
			//TODO assume that js is the only implementation language for now
			templatesGlob := fmt.Sprintf("%v/js/endpoint/*.js", templateDir )

			if t, err := template.ParseGlob(templatesGlob); err == nil {

				outputFileName := fmt.Sprintf("%v/%v", endpointDir, templateName )
				log.Println(outputFileName)

				outputFilename, err := os.Create(outputFileName)

				if err = t.ExecuteTemplate(outputFilename, templateName, concept); err != nil {
					log.Fatal(err)
				}
			}
		}
	}
}

func (c *GenerateCommand) Run(args []string) int {
	if len(args) < 2 {
		fmt.Printf("%v", c.Help())
		return 1
	}

	subcommand := args[0]

	switch subcommand {
	case "endpoint":
		MakeAllEndpoints(args[1])
	case "integration":
		log.Println("generating integrations are not implemented yet")
	}

	return 0
}

func (c *GenerateCommand) Synopsis() string {
	return "generates lambda functions for endpoints"
}
<|endoftext|>


<commit_before>package commands

import (
	"encoding/gob"
	"fmt"
	"net/url"
	"os"
	"time"

	"flag"
	"path/filepath"
	"strings"

	"github.com/astaxie/beego"
	"github.com/astaxie/beego/logs"
	"github.com/astaxie/beego/orm"
	"github.com/lifei6671/gocaptcha"
	"github.com/lifei6671/godoc/commands/migrate"
	"github.com/lifei6671/godoc/conf"
	"github.com/lifei6671/godoc/models"
	"github.com/lifei6671/godoc/utils"
	"log"
)

var (
	ConfigurationFile = "./conf/app.conf"
	WorkingDirectory  = "./"
	LogFile           = "./logs"
)

// RegisterDataBase 注册数据库
func RegisterDataBase() {
	adapter := beego.AppConfig.String("db_adapter")

	if adapter == "mysql" {
		host := beego.AppConfig.String("db_host")
		database := beego.AppConfig.String("db_database")
		username := beego.AppConfig.String("db_username")
		password := beego.AppConfig.String("db_password")
		timezone := beego.AppConfig.String("timezone")

		port := beego.AppConfig.String("db_port")

		dataSource := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8mb4&parseTime=true&loc=%s", username, password, host, port, database, url.QueryEscape(timezone))

		orm.RegisterDataBase("default", "mysql", dataSource)

		location, err := time.LoadLocation(timezone)
		if err == nil {
			orm.DefaultTimeLoc = location
		} else {
			log.Fatalln(err)
		}
	} else if adapter == "sqlite3" {
		database := beego.AppConfig.String("db_database")
		dbPath := filepath.Dir(database)
		os.MkdirAll(dbPath, 0777)

		orm.RegisterDataBase("default", "sqlite3", database)
	}
}

// RegisterModel 注册Model
func RegisterModel() {
	orm.RegisterModelWithPrefix(conf.GetDatabasePrefix(),
		new(models.Member),
		new(models.Book),
		new(models.Relationship),
		new(models.Option),
		new(models.Document),
		new(models.Attachment),
		new(models.Logger),
		new(models.MemberToken),
		new(models.DocumentHistory),
		new(models.Migration),
	)
	migrate.RegisterMigration()
}

// RegisterLogger 注册日志
func RegisterLogger(log string) {

	logs.SetLogFuncCall(true)
	logs.SetLogger("console")
	logs.EnableFuncCallDepth(true)
	logs.Async()

	logPath := filepath.Join(log, "log.log")

	if _, err := os.Stat(logPath); os.IsNotExist(err) {

		os.MkdirAll(log, 0777)
		logPath = strings.Replace(logPath,"\\","/","")

		if f, err := os.Create(logPath); err == nil {
			f.Close()
			beego.SetLogger("file", fmt.Sprintf(`{"filename":"%s"}`, logPath))
		}
	}

	beego.SetLogFuncCall(true)
	beego.BeeLogger.Async()
}

// RunCommand 注册orm命令行工具
func RegisterCommand() {

	if len(os.Args) >= 2 && os.Args[1] == "install" {
		ResolveCommand(os.Args[2:])
		Install()
	} else if len(os.Args) >= 2 && os.Args[1] == "version" {
		ResolveCommand(os.Args[2:])
		CheckUpdate()
	} else if len(os.Args) >= 2 && os.Args[1] == "migrate" {
		ResolveCommand(os.Args[2:])
		migrate.RunMigration()
	}
}

func RegisterFunction() {
	beego.AddFuncMap("config", models.GetOptionValue)

	beego.AddFuncMap("cdn", func(p string) string {
		cdn := beego.AppConfig.DefaultString("cdn", "")
		if strings.HasPrefix(p, "/") && strings.HasSuffix(cdn, "/") {
			return cdn + string(p[1:])
		}
		if !strings.HasPrefix(p, "/") && !strings.HasSuffix(cdn, "/") {
			return cdn + "/" + p
		}
		return cdn + p
	})

	beego.AddFuncMap("cdnjs", func(p string) string {
		cdn := beego.AppConfig.DefaultString("cdnjs", "")
		if strings.HasPrefix(p, "/") && strings.HasSuffix(cdn, "/") {
			return cdn + string(p[1:])
		}
		if !strings.HasPrefix(p, "/") && !strings.HasSuffix(cdn, "/") {
			return cdn + "/" + p
		}
		return cdn + p
	})
	beego.AddFuncMap("cdncss", func(p string) string {
		cdn := beego.AppConfig.DefaultString("cdncss", "")
		if strings.HasPrefix(p, "/") && strings.HasSuffix(cdn, "/") {
			return cdn + string(p[1:])
		}
		if !strings.HasPrefix(p, "/") && !strings.HasSuffix(cdn, "/") {
			return cdn + "/" + p
		}
		return cdn + p
	})
	beego.AddFuncMap("cdnimg", func(p string) string {
		cdn := beego.AppConfig.DefaultString("cdnimg", "")
		if strings.HasPrefix(p, "/") && strings.HasSuffix(cdn, "/") {
			return cdn + string(p[1:])
		}
		if !strings.HasPrefix(p, "/") && !strings.HasSuffix(cdn, "/") {
			return cdn + "/" + p
		}
		return cdn + p
	})
}

func ResolveCommand(args []string) {
	flagSet := flag.NewFlagSet("MinDoc command: ", flag.ExitOnError)
	flagSet.StringVar(&ConfigurationFile, "config", "", "MinDoc configuration file.")
	flagSet.StringVar(&WorkingDirectory, "dir", "", "MinDoc working directory.")
	flagSet.StringVar(&LogFile, "log", "", "MinDoc log file path.")

	flagSet.Parse(args)

	if WorkingDirectory == "" {
		if p, err := filepath.Abs(os.Args[0]); err == nil {
			WorkingDirectory = filepath.Dir(p)
		}
	}
	if LogFile == "" {
		LogFile = filepath.Join(WorkingDirectory,"logs")
	}
	if ConfigurationFile == "" {
		ConfigurationFile = filepath.Join(WorkingDirectory,"conf","app.conf")
		config :=  filepath.Join(WorkingDirectory,"conf","app.conf.example")
		if !utils.FileExists(ConfigurationFile) && utils.FileExists(config){
			utils.CopyFile(ConfigurationFile,config)
		}
	}

	err := beego.LoadAppConfig("ini", ConfigurationFile)

	if err != nil {
		log.Println("An error occurred:", err)
		os.Exit(1)
	}
	uploads := filepath.Join(WorkingDirectory, "uploads")

	os.MkdirAll(uploads,0666)

	beego.BConfig.WebConfig.StaticDir["/static"] = filepath.Join(WorkingDirectory, "static")
	beego.BConfig.WebConfig.StaticDir["/uploads"] = uploads
	beego.BConfig.WebConfig.ViewsPath = filepath.Join(WorkingDirectory, "views")

	fonts := filepath.Join(WorkingDirectory, "static", "fonts")

	if !utils.FileExists(fonts) {
		log.Fatal("Font path not exist.")
	}
	gocaptcha.ReadFonts(filepath.Join(WorkingDirectory, "static", "fonts"), ".ttf")

	RegisterDataBase()
	RegisterModel()
	RegisterLogger(LogFile)
}

func init() {

	gocaptcha.ReadFonts("./static/fonts", ".ttf")
	gob.Register(models.Member{})

	if p,err := filepath.Abs(os.Args[0]);err == nil{
		WorkingDirectory = filepath.Dir(p)
	}
}
<commit_msg>修复BUG<commit_after>package commands

import (
	"encoding/gob"
	"fmt"
	"net/url"
	"os"
	"time"

	"flag"
	"path/filepath"
	"strings"

	"github.com/astaxie/beego"
	"github.com/astaxie/beego/logs"
	"github.com/astaxie/beego/orm"
	"github.com/lifei6671/gocaptcha"
	"github.com/lifei6671/godoc/commands/migrate"
	"github.com/lifei6671/godoc/conf"
	"github.com/lifei6671/godoc/models"
	"github.com/lifei6671/godoc/utils"
	"log"
	"encoding/json"
)

var (
	ConfigurationFile = "./conf/app.conf"
	WorkingDirectory  = "./"
	LogFile           = "./logs"
)

// RegisterDataBase 注册数据库
func RegisterDataBase() {
	adapter := beego.AppConfig.String("db_adapter")

	if adapter == "mysql" {
		host := beego.AppConfig.String("db_host")
		database := beego.AppConfig.String("db_database")
		username := beego.AppConfig.String("db_username")
		password := beego.AppConfig.String("db_password")
		timezone := beego.AppConfig.String("timezone")

		port := beego.AppConfig.String("db_port")

		dataSource := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8mb4&parseTime=true&loc=%s", username, password, host, port, database, url.QueryEscape(timezone))

		orm.RegisterDataBase("default", "mysql", dataSource)

		location, err := time.LoadLocation(timezone)
		if err == nil {
			orm.DefaultTimeLoc = location
		} else {
			log.Fatalln(err)
		}
	} else if adapter == "sqlite3" {
		database := beego.AppConfig.String("db_database")
		dbPath := filepath.Dir(database)
		os.MkdirAll(dbPath, 0777)

		orm.RegisterDataBase("default", "sqlite3", database)
	}
}

// RegisterModel 注册Model
func RegisterModel() {
	orm.RegisterModelWithPrefix(conf.GetDatabasePrefix(),
		new(models.Member),
		new(models.Book),
		new(models.Relationship),
		new(models.Option),
		new(models.Document),
		new(models.Attachment),
		new(models.Logger),
		new(models.MemberToken),
		new(models.DocumentHistory),
		new(models.Migration),
	)
	migrate.RegisterMigration()
}

// RegisterLogger 注册日志
func RegisterLogger(log string) {

	logs.SetLogFuncCall(true)
	logs.SetLogger("console")
	logs.EnableFuncCallDepth(true)
	logs.Async()

	logPath := filepath.Join(log, "log.log")

	if _, err := os.Stat(logPath); os.IsNotExist(err) {

		os.MkdirAll(log, 0777)

		if f, err := os.Create(logPath); err == nil {
			f.Close()
			config := make(map[string]interface{},1)

			config["filename"] = logPath

			b,_ := json.Marshal(config)

			beego.SetLogger("file", string(b))
		}
	}

	beego.SetLogFuncCall(true)
	beego.BeeLogger.Async()
}

// RunCommand 注册orm命令行工具
func RegisterCommand() {

	if len(os.Args) >= 2 && os.Args[1] == "install" {
		ResolveCommand(os.Args[2:])
		Install()
	} else if len(os.Args) >= 2 && os.Args[1] == "version" {
		ResolveCommand(os.Args[2:])
		CheckUpdate()
	} else if len(os.Args) >= 2 && os.Args[1] == "migrate" {
		ResolveCommand(os.Args[2:])
		migrate.RunMigration()
	}
}

func RegisterFunction() {
	beego.AddFuncMap("config", models.GetOptionValue)

	beego.AddFuncMap("cdn", func(p string) string {
		cdn := beego.AppConfig.DefaultString("cdn", "")
		if strings.HasPrefix(p, "/") && strings.HasSuffix(cdn, "/") {
			return cdn + string(p[1:])
		}
		if !strings.HasPrefix(p, "/") && !strings.HasSuffix(cdn, "/") {
			return cdn + "/" + p
		}
		return cdn + p
	})

	beego.AddFuncMap("cdnjs", func(p string) string {
		cdn := beego.AppConfig.DefaultString("cdnjs", "")
		if strings.HasPrefix(p, "/") && strings.HasSuffix(cdn, "/") {
			return cdn + string(p[1:])
		}
		if !strings.HasPrefix(p, "/") && !strings.HasSuffix(cdn, "/") {
			return cdn + "/" + p
		}
		return cdn + p
	})
	beego.AddFuncMap("cdncss", func(p string) string {
		cdn := beego.AppConfig.DefaultString("cdncss", "")
		if strings.HasPrefix(p, "/") && strings.HasSuffix(cdn, "/") {
			return cdn + string(p[1:])
		}
		if !strings.HasPrefix(p, "/") && !strings.HasSuffix(cdn, "/") {
			return cdn + "/" + p
		}
		return cdn + p
	})
	beego.AddFuncMap("cdnimg", func(p string) string {
		cdn := beego.AppConfig.DefaultString("cdnimg", "")
		if strings.HasPrefix(p, "/") && strings.HasSuffix(cdn, "/") {
			return cdn + string(p[1:])
		}
		if !strings.HasPrefix(p, "/") && !strings.HasSuffix(cdn, "/") {
			return cdn + "/" + p
		}
		return cdn + p
	})
}

func ResolveCommand(args []string) {
	flagSet := flag.NewFlagSet("MinDoc command: ", flag.ExitOnError)
	flagSet.StringVar(&ConfigurationFile, "config", "", "MinDoc configuration file.")
	flagSet.StringVar(&WorkingDirectory, "dir", "", "MinDoc working directory.")
	flagSet.StringVar(&LogFile, "log", "", "MinDoc log file path.")

	flagSet.Parse(args)

	if WorkingDirectory == "" {
		if p, err := filepath.Abs(os.Args[0]); err == nil {
			WorkingDirectory = filepath.Dir(p)
		}
	}
	if LogFile == "" {
		LogFile = filepath.Join(WorkingDirectory,"logs")
	}
	if ConfigurationFile == "" {
		ConfigurationFile = filepath.Join(WorkingDirectory,"conf","app.conf")
		config :=  filepath.Join(WorkingDirectory,"conf","app.conf.example")
		if !utils.FileExists(ConfigurationFile) && utils.FileExists(config){
			utils.CopyFile(ConfigurationFile,config)
		}
	}

	err := beego.LoadAppConfig("ini", ConfigurationFile)

	if err != nil {
		log.Println("An error occurred:", err)
		os.Exit(1)
	}
	uploads := filepath.Join(WorkingDirectory, "uploads")

	os.MkdirAll(uploads,0666)

	beego.BConfig.WebConfig.StaticDir["/static"] = filepath.Join(WorkingDirectory, "static")
	beego.BConfig.WebConfig.StaticDir["/uploads"] = uploads
	beego.BConfig.WebConfig.ViewsPath = filepath.Join(WorkingDirectory, "views")

	fonts := filepath.Join(WorkingDirectory, "static", "fonts")

	if !utils.FileExists(fonts) {
		log.Fatal("Font path not exist.")
	}
	gocaptcha.ReadFonts(filepath.Join(WorkingDirectory, "static", "fonts"), ".ttf")

	RegisterDataBase()
	RegisterModel()
	RegisterLogger(LogFile)
}

func init() {

	gocaptcha.ReadFonts("./static/fonts", ".ttf")
	gob.Register(models.Member{})

	if p,err := filepath.Abs(os.Args[0]);err == nil{
		WorkingDirectory = filepath.Dir(p)
	}
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"io"
	"reflect"
	"strconv"
)

type optMap map[string]interface{}

// Request represents a call to a command from a consumer
type Request interface {
	Path() []string
	Option(name string) (interface{}, bool)
	SetOption(name string, val interface{})
	Arguments() []string
	Stream() io.Reader

	ConvertOptions(options map[string]Option) error
}

type request struct {
	path      []string
	options   optMap
	arguments []string
	in        io.Reader
}

// Path returns the command path of this request
func (r *request) Path() []string {
	return r.path
}

// Option returns the value of the option for given name.
func (r *request) Option(name string) (interface{}, bool) {
	val, err := r.options[name]
	return val, err
}

// SetOption sets the value of the option for given name.
func (r *request) SetOption(name string, val interface{}) {
	r.options[name] = val
}

// Arguments returns the arguments slice
func (r *request) Arguments() []string {
	return r.arguments
}

// Stream returns the input stream Reader
func (r *request) Stream() io.Reader {
	return r.in
}

type converter func(string) (interface{}, error)

var converters = map[reflect.Kind]converter{
	Bool: func(v string) (interface{}, error) {
		if v == "" {
			return true, nil
		}
		return strconv.ParseBool(v)
	},
	Int: func(v string) (interface{}, error) {
		return strconv.ParseInt(v, 0, 32)
	},
	Uint: func(v string) (interface{}, error) {
		return strconv.ParseInt(v, 0, 32)
	},
	Float: func(v string) (interface{}, error) {
		return strconv.ParseFloat(v, 64)
	},
}

func (r *request) ConvertOptions(options map[string]Option) error {
	converted := make(map[string]interface{})

	for k, v := range r.options {
		opt, ok := options[k]
		if !ok {
			return fmt.Errorf("Unrecognized option: '%s'", k)
		}

		kind := reflect.TypeOf(v).Kind()
		var value interface{}

		if kind != opt.Type {
			if kind == String {
				convert := converters[opt.Type]
				val, err := convert(v.(string))
				if err != nil {
					return fmt.Errorf("Could not convert string value '%s' to type '%s'",
						v, opt.Type.String())
				}
				value = val

			} else {
				return fmt.Errorf("Option '%s' should be type '%s', but got type '%s'",
					k, opt.Type.String(), kind.String())
			}
		} else {
			value = v
		}

		for _, name := range opt.Names {
			if _, ok := r.options[name]; name != k && ok {
				return fmt.Errorf("Duplicate command options were provided ('%s' and '%s')",
					k, name)
			}

			converted[name] = value
		}
	}

	r.options = converted
	return nil
}

// NewEmptyRequest initializes an empty request
func NewEmptyRequest() Request {
	return NewRequest(nil, nil, nil, nil)
}

// NewRequest returns a request initialized with given arguments
func NewRequest(path []string, opts optMap, args []string, in io.Reader) Request {
	if path == nil {
		path = make([]string, 0)
	}
	if opts == nil {
		opts = make(map[string]interface{})
	}
	if args == nil {
		args = make([]string, 0)
	}
	return &request{path, opts, args, in}
}
<commit_msg>commands: Added Request#SetStream<commit_after>package commands

import (
	"fmt"
	"io"
	"reflect"
	"strconv"
)

type optMap map[string]interface{}

// Request represents a call to a command from a consumer
type Request interface {
	Path() []string
	Option(name string) (interface{}, bool)
	SetOption(name string, val interface{})
	Arguments() []string
	Stream() io.Reader
	SetStream(io.Reader)

	ConvertOptions(options map[string]Option) error
}

type request struct {
	path      []string
	options   optMap
	arguments []string
	in        io.Reader
}

// Path returns the command path of this request
func (r *request) Path() []string {
	return r.path
}

// Option returns the value of the option for given name.
func (r *request) Option(name string) (interface{}, bool) {
	val, err := r.options[name]
	return val, err
}

// SetOption sets the value of the option for given name.
func (r *request) SetOption(name string, val interface{}) {
	r.options[name] = val
}

// Arguments returns the arguments slice
func (r *request) Arguments() []string {
	return r.arguments
}

// Stream returns the input stream Reader
func (r *request) Stream() io.Reader {
	return r.in
}

// SetStream sets the value of the input stream Reader
func (r *request) SetStream(in io.Reader) {
	r.in = in
}

type converter func(string) (interface{}, error)

var converters = map[reflect.Kind]converter{
	Bool: func(v string) (interface{}, error) {
		if v == "" {
			return true, nil
		}
		return strconv.ParseBool(v)
	},
	Int: func(v string) (interface{}, error) {
		return strconv.ParseInt(v, 0, 32)
	},
	Uint: func(v string) (interface{}, error) {
		return strconv.ParseInt(v, 0, 32)
	},
	Float: func(v string) (interface{}, error) {
		return strconv.ParseFloat(v, 64)
	},
}

func (r *request) ConvertOptions(options map[string]Option) error {
	converted := make(map[string]interface{})

	for k, v := range r.options {
		opt, ok := options[k]
		if !ok {
			return fmt.Errorf("Unrecognized option: '%s'", k)
		}

		kind := reflect.TypeOf(v).Kind()
		var value interface{}

		if kind != opt.Type {
			if kind == String {
				convert := converters[opt.Type]
				val, err := convert(v.(string))
				if err != nil {
					return fmt.Errorf("Could not convert string value '%s' to type '%s'",
						v, opt.Type.String())
				}
				value = val

			} else {
				return fmt.Errorf("Option '%s' should be type '%s', but got type '%s'",
					k, opt.Type.String(), kind.String())
			}
		} else {
			value = v
		}

		for _, name := range opt.Names {
			if _, ok := r.options[name]; name != k && ok {
				return fmt.Errorf("Duplicate command options were provided ('%s' and '%s')",
					k, name)
			}

			converted[name] = value
		}
	}

	r.options = converted
	return nil
}

// NewEmptyRequest initializes an empty request
func NewEmptyRequest() Request {
	return NewRequest(nil, nil, nil, nil)
}

// NewRequest returns a request initialized with given arguments
func NewRequest(path []string, opts optMap, args []string, in io.Reader) Request {
	if path == nil {
		path = make([]string, 0)
	}
	if opts == nil {
		opts = make(map[string]interface{})
	}
	if args == nil {
		args = make([]string, 0)
	}
	return &request{path, opts, args, in}
}
<|endoftext|>


<commit_before>package types

import (
	"encoding/binary"
	"time"
)

func TimeToTIME_MILLIS(t time.Time, adjustedToUTC bool) int64 {
	return TimeToTIME_MICROS(t, adjustedToUTC) / time.Millisecond.Microseconds()
}

func TimeToTIME_MICROS(t time.Time, adjustedToUTC bool) int64 {
	if adjustedToUTC {
		tu := t.UTC()
		h, m, s, ns := int64(tu.Hour()), int64(tu.Minute()), int64(tu.Second()), int64(tu.Nanosecond())
		nanos := h * time.Hour.Nanoseconds() + m * time.Minute.Nanoseconds() + s * time.Second.Nanoseconds() + ns * time.Nanosecond.Nanoseconds()
		return nanos / time.Microsecond.Nanoseconds()

	} else {
		h, m, s, ns := int64(t.Hour()), int64(t.Minute()), int64(t.Second()), int64(t.Nanosecond())
		nanos := h * time.Hour.Nanoseconds() + m * time.Minute.Nanoseconds() + s * time.Second.Nanoseconds() + ns * time.Nanosecond.Nanoseconds()
		return nanos / time.Microsecond.Nanoseconds()
	}
}

func TimeToTIMESTAMP_MILLIS(t time.Time, adjustedToUTC bool) int64 {
	return TimeToTIMESTAMP_MICROS(t, adjustedToUTC) / time.Millisecond.Microseconds()
}

func TIMESTAMP_MILLISToTime(millis int64, adjustedToUTC bool) time.Time {
	return TIMESTAMP_MICROSToTime(millis * time.Millisecond.Microseconds(), adjustedToUTC)
}

func TimeToTIMESTAMP_MICROS(t time.Time, adjustedToUTC bool) int64 {
	return TimeToTIMESTAMP_NANOS(t, adjustedToUTC) / time.Microsecond.Nanoseconds()
}

func TIMESTAMP_MICROSToTime(micros int64, adjustedToUTC bool) time.Time {
	return TIMESTAMP_NANOSToTime(micros * time.Microsecond.Nanoseconds(), adjustedToUTC)
}

func TimeToTIMESTAMP_NANOS(t time.Time, adjustedToUTC bool) int64 {
	if adjustedToUTC {
		return t.UnixNano()
	} else {
		epoch := time.Date(1970, 1, 1, 0, 0, 0, 0, t.Location())
		return t.Sub(epoch).Nanoseconds()
	}
}

func TIMESTAMP_NANOSToTime(nanos int64, adjustedToUTC bool) time.Time {
	if adjustedToUTC {
		return time.Unix(0, nanos).UTC()

	}else{
		epoch := time.Date(1970, 1, 1, 0, 0, 0, 0, time.Local)
		t := epoch.Add(time.Nanosecond * time.Duration(nanos))
		return t
	}
}

//From Spark
//https://github.com/apache/spark/blob/b9f2f78de59758d1932c1573338539e485a01112/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala#L47
const (
	JULIAN_DAY_OF_EPOCH int64 = 2440588
	MICROS_PER_DAY int64 = 3600 * 24 * 1000 * 1000
)

//From Spark
//https://github.com/apache/spark/blob/b9f2f78de59758d1932c1573338539e485a01112/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala#L180
func toJulianDay(t time.Time) (int32, int64) {
	utc := t.UTC()
	nanos := utc.UnixNano()
	micros := nanos / time.Microsecond.Nanoseconds()

	julianUs := micros + JULIAN_DAY_OF_EPOCH * MICROS_PER_DAY
	days := int32(julianUs / MICROS_PER_DAY)
	us := (julianUs % MICROS_PER_DAY) * 1000
	return days, us
}

//From Spark
//https://github.com/apache/spark/blob/b9f2f78de59758d1932c1573338539e485a01112/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala#L170
func fromJulianDay(days int32, nanos int64) time.Time {
	nanos = ((int64(days) - JULIAN_DAY_OF_EPOCH) * MICROS_PER_DAY + nanos / 1000) * 1000
	sec, nsec := nanos / time.Second.Nanoseconds(), nanos % time.Second.Nanoseconds()
	t := time.Unix(sec, nsec)
	return t.UTC()
}

// Reports the INT96 Timestamp in string format as required by Spark for Parquet files.
// Reference: https://stackoverflow.com/questions/53103762/cast-int96-timestamp-from-parquet-to-golang/53104516#53104516
func TimeToINT96(t time.Time) string {
	days, nanos := toJulianDay(t)

	bs1 := make([]byte, 8)
	binary.LittleEndian.PutUint64(bs1, uint64(nanos))

	bs2 := make([]byte, 4)
	binary.LittleEndian.PutUint32(bs2, uint32(days))

	bs := append(bs1, bs2...)
	return string(bs)
}

func INT96ToTime(int96 string) time.Time {
	nanos := binary.LittleEndian.Uint64([]byte(int96[:8]))
	days := binary.LittleEndian.Uint32([]byte(int96[8:]))

	return fromJulianDay(int32(days), int64(nanos))
}<commit_msg>del comments<commit_after>package types

import (
	"encoding/binary"
	"time"
)

func TimeToTIME_MILLIS(t time.Time, adjustedToUTC bool) int64 {
	return TimeToTIME_MICROS(t, adjustedToUTC) / time.Millisecond.Microseconds()
}

func TimeToTIME_MICROS(t time.Time, adjustedToUTC bool) int64 {
	if adjustedToUTC {
		tu := t.UTC()
		h, m, s, ns := int64(tu.Hour()), int64(tu.Minute()), int64(tu.Second()), int64(tu.Nanosecond())
		nanos := h * time.Hour.Nanoseconds() + m * time.Minute.Nanoseconds() + s * time.Second.Nanoseconds() + ns * time.Nanosecond.Nanoseconds()
		return nanos / time.Microsecond.Nanoseconds()

	} else {
		h, m, s, ns := int64(t.Hour()), int64(t.Minute()), int64(t.Second()), int64(t.Nanosecond())
		nanos := h * time.Hour.Nanoseconds() + m * time.Minute.Nanoseconds() + s * time.Second.Nanoseconds() + ns * time.Nanosecond.Nanoseconds()
		return nanos / time.Microsecond.Nanoseconds()
	}
}

func TimeToTIMESTAMP_MILLIS(t time.Time, adjustedToUTC bool) int64 {
	return TimeToTIMESTAMP_MICROS(t, adjustedToUTC) / time.Millisecond.Microseconds()
}

func TIMESTAMP_MILLISToTime(millis int64, adjustedToUTC bool) time.Time {
	return TIMESTAMP_MICROSToTime(millis * time.Millisecond.Microseconds(), adjustedToUTC)
}

func TimeToTIMESTAMP_MICROS(t time.Time, adjustedToUTC bool) int64 {
	return TimeToTIMESTAMP_NANOS(t, adjustedToUTC) / time.Microsecond.Nanoseconds()
}

func TIMESTAMP_MICROSToTime(micros int64, adjustedToUTC bool) time.Time {
	return TIMESTAMP_NANOSToTime(micros * time.Microsecond.Nanoseconds(), adjustedToUTC)
}

func TimeToTIMESTAMP_NANOS(t time.Time, adjustedToUTC bool) int64 {
	if adjustedToUTC {
		return t.UnixNano()
	} else {
		epoch := time.Date(1970, 1, 1, 0, 0, 0, 0, t.Location())
		return t.Sub(epoch).Nanoseconds()
	}
}

func TIMESTAMP_NANOSToTime(nanos int64, adjustedToUTC bool) time.Time {
	if adjustedToUTC {
		return time.Unix(0, nanos).UTC()

	}else{
		epoch := time.Date(1970, 1, 1, 0, 0, 0, 0, time.Local)
		t := epoch.Add(time.Nanosecond * time.Duration(nanos))
		return t
	}
}

//From Spark
//https://github.com/apache/spark/blob/b9f2f78de59758d1932c1573338539e485a01112/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala#L47
const (
	JULIAN_DAY_OF_EPOCH int64 = 2440588
	MICROS_PER_DAY int64 = 3600 * 24 * 1000 * 1000
)

//From Spark
//https://github.com/apache/spark/blob/b9f2f78de59758d1932c1573338539e485a01112/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala#L180
func toJulianDay(t time.Time) (int32, int64) {
	utc := t.UTC()
	nanos := utc.UnixNano()
	micros := nanos / time.Microsecond.Nanoseconds()

	julianUs := micros + JULIAN_DAY_OF_EPOCH * MICROS_PER_DAY
	days := int32(julianUs / MICROS_PER_DAY)
	us := (julianUs % MICROS_PER_DAY) * 1000
	return days, us
}

//From Spark
//https://github.com/apache/spark/blob/b9f2f78de59758d1932c1573338539e485a01112/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala#L170
func fromJulianDay(days int32, nanos int64) time.Time {
	nanos = ((int64(days) - JULIAN_DAY_OF_EPOCH) * MICROS_PER_DAY + nanos / 1000) * 1000
	sec, nsec := nanos / time.Second.Nanoseconds(), nanos % time.Second.Nanoseconds()
	t := time.Unix(sec, nsec)
	return t.UTC()
}

func TimeToINT96(t time.Time) string {
	days, nanos := toJulianDay(t)

	bs1 := make([]byte, 8)
	binary.LittleEndian.PutUint64(bs1, uint64(nanos))

	bs2 := make([]byte, 4)
	binary.LittleEndian.PutUint32(bs2, uint32(days))

	bs := append(bs1, bs2...)
	return string(bs)
}

func INT96ToTime(int96 string) time.Time {
	nanos := binary.LittleEndian.Uint64([]byte(int96[:8]))
	days := binary.LittleEndian.Uint32([]byte(int96[8:]))

	return fromJulianDay(int32(days), int64(nanos))
}<|endoftext|>