<commit_before>package main

import (
	"flag"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"runtime"
	"strings"
	"sync"
	"syscall"

	"github.com/cubicdaiya/nginx-build/builder"
	"github.com/cubicdaiya/nginx-build/command"
	"github.com/cubicdaiya/nginx-build/configure"
	"github.com/cubicdaiya/nginx-build/module3rd"
	"github.com/cubicdaiya/nginx-build/openssl"
	"github.com/cubicdaiya/nginx-build/util"
)

const (
	NginxBuildVersion = "0.9.4"
)

var (
	nginxBuildOptions Options
)

func init() {
	nginxBuildOptions = makeNginxBuildOptions()
}

func main() {
	// Parse flags
	for k, v := range nginxBuildOptions.Bools {
		v.Enabled = flag.Bool(k, false, v.Desc)
		nginxBuildOptions.Bools[k] = v
	}
	for k, v := range nginxBuildOptions.Values {
		v.Value = flag.String(k, v.Default, v.Desc)
		nginxBuildOptions.Values[k] = v
	}
	for k, v := range nginxBuildOptions.Numbers {
		v.Value = flag.Int(k, v.Default, v.Desc)
		nginxBuildOptions.Numbers[k] = v
	}

	// fake flag for --with-xxx=dynamic
	for i, arg := range os.Args {
		if strings.Contains(arg, "with-http_xslt_module=dynamic") {
			os.Args[i] = "--with-http_xslt_module_dynamic"
		}
		if strings.Contains(arg, "with-http_image_filter_module=dynamic") {
			os.Args[i] = "--with-http_image_filter_module_dynamic"
		}
		if strings.Contains(arg, "with-http_geoip_module=dynamic") {
			os.Args[i] = "--with-http_geoip_module_dynamic"
		}
		if strings.Contains(arg, "with-http_perl_module=dynamic") {
			os.Args[i] = "--with-http_perl_module_dynamic"
		}
		if strings.Contains(arg, "with-mail=dynamic") {
			os.Args[i] = "--with-mail_dynamic"
		}
		if strings.Contains(arg, "with-stream=dynamic") {
			os.Args[i] = "--with-stream_dynamic"
		}
	}

	var (
		configureOptions configure.Options
		multiflag        StringFlag
		multiflagDynamic StringFlag
	)

	argsString := configure.MakeArgsString()
	for k, v := range argsString {
		if k == "add-module" {
			flag.Var(&multiflag, k, v.Desc)
		} else if k == "add-dynamic-module" {
			flag.Var(&multiflagDynamic, k, v.Desc)
		} else {
			v.Value = flag.String(k, "", v.Desc)
			argsString[k] = v
		}
	}

	argsBool := configure.MakeArgsBool()
	for k, v := range argsBool {
		v.Enabled = flag.Bool(k, false, v.Desc)
		argsBool[k] = v
	}

	flag.CommandLine.SetOutput(os.Stdout)
	// The output of original flag.Usage() is too long
	defaultUsage := flag.Usage
	flag.Usage = usage
	flag.Parse()

	jobs := nginxBuildOptions.Numbers["j"].Value

	verbose := nginxBuildOptions.Bools["verbose"].Enabled
	pcreStatic := nginxBuildOptions.Bools["pcre"].Enabled
	openSSLStatic := nginxBuildOptions.Bools["openssl"].Enabled
	zlibStatic := nginxBuildOptions.Bools["zlib"].Enabled
	clear := nginxBuildOptions.Bools["clear"].Enabled
	versionPrint := nginxBuildOptions.Bools["version"].Enabled
	versionsPrint := nginxBuildOptions.Bools["versions"].Enabled
	openResty := nginxBuildOptions.Bools["openresty"].Enabled
	tengine := nginxBuildOptions.Bools["tengine"].Enabled
	configureOnly := nginxBuildOptions.Bools["configureonly"].Enabled
	helpAll := nginxBuildOptions.Bools["help-all"].Enabled

	version := nginxBuildOptions.Values["v"].Value
	nginxConfigurePath := nginxBuildOptions.Values["c"].Value
	modulesConfPath := nginxBuildOptions.Values["m"].Value
	workParentDir := nginxBuildOptions.Values["d"].Value
	pcreVersion := nginxBuildOptions.Values["pcreversion"].Value
	openSSLVersion := nginxBuildOptions.Values["opensslversion"].Value
	zlibVersion := nginxBuildOptions.Values["zlibversion"].Value
	openRestyVersion := nginxBuildOptions.Values["openrestyversion"].Value
	tengineVersion := nginxBuildOptions.Values["tengineversion"].Value
	patchPath := nginxBuildOptions.Values["patch"].Value
	patchOption := nginxBuildOptions.Values["patch-opt"].Value

	// Allow multiple flags for `--add-module`
	{
		tmp := argsString["add-module"]
		tmp_ := multiflag.String()
		tmp.Value = &tmp_
		argsString["add-module"] = tmp
	}

	// Allow multiple flags for `--add-dynamic-module`
	{
		tmp := argsString["add-dynamic-module"]
		tmp_ := multiflagDynamic.String()
		tmp.Value = &tmp_
		argsString["add-dynamic-module"] = tmp
	}

	configureOptions.Values = argsString
	configureOptions.Bools = argsBool

	if *helpAll {
		defaultUsage()
		return
	}

	if *versionPrint {
		printNginxBuildVersion()
		return
	}

	if *versionsPrint {
		printNginxVersions()
		return
	}

	printFirstMsg()

	// set verbose mode
	command.VerboseEnabled = *verbose

	var nginxBuilder builder.Builder
	if *openResty && *tengine {
		log.Fatal("select one between '-openresty' and '-tengine'.")
	}
	if *openResty {
		nginxBuilder = builder.MakeBuilder(builder.COMPONENT_OPENRESTY, *openRestyVersion)
	} else if *tengine {
		nginxBuilder = builder.MakeBuilder(builder.COMPONENT_TENGINE, *tengineVersion)
	} else {
		nginxBuilder = builder.MakeBuilder(builder.COMPONENT_NGINX, *version)
	}
	pcreBuilder := builder.MakeBuilder(builder.COMPONENT_PCRE, *pcreVersion)
	openSSLBuilder := builder.MakeBuilder(builder.COMPONENT_OPENSSL, *openSSLVersion)
	zlibBuilder := builder.MakeBuilder(builder.COMPONENT_ZLIB, *zlibVersion)

	// change default umask
	_ = syscall.Umask(0)

	versionCheck(*version)

	nginxConfigure, err := util.FileGetContents(*nginxConfigurePath)
	if err != nil {
		log.Fatal(err)
	}
	nginxConfigure = configure.Normalize(nginxConfigure)

	modules3rd, err := module3rd.Load(*modulesConfPath)
	if err != nil {
		log.Fatal(err)
	}

	if len(*workParentDir) == 0 {
		log.Fatal("set working directory with -d")
	}

	if !util.FileExists(*workParentDir) {
		err := os.Mkdir(*workParentDir, 0755)
		if err != nil {
			log.Fatalf("Failed to create working directory(%s) does not exist.", *workParentDir)
		}
	}

	var workDir string
	if *openResty {
		workDir = *workParentDir + "/openresty/" + *openRestyVersion
	} else if *tengine {
		workDir = *workParentDir + "/tengine/" + *tengineVersion
	} else {
		workDir = *workParentDir + "/nginx/" + *version
	}

	if *clear {
		err := util.ClearWorkDir(workDir)
		if err != nil {
			log.Fatal(err)
		}
	}

	if !util.FileExists(workDir) {
		err := os.MkdirAll(workDir, 0755)
		if err != nil {
			log.Fatalf("Failed to create working directory(%s) does not exist.", workDir)
		}
	}

	rootDir := util.SaveCurrentDir()
	err = os.Chdir(workDir)
	if err != nil {
		log.Fatal(err)
	}

	// remove nginx source code applyed patch
	if *patchPath != "" && util.FileExists(nginxBuilder.SourcePath()) {
		err := os.RemoveAll(nginxBuilder.SourcePath())
		if err != nil {
			log.Fatal(err)
		}
	}

	wg := new(sync.WaitGroup)
	if *pcreStatic {
		wg.Add(1)
		go func() {
			downloadAndExtractParallel(&pcreBuilder)
			wg.Done()
		}()
	}

	if *openSSLStatic {
		wg.Add(1)
		go func () {
			downloadAndExtractParallel(&openSSLBuilder)
			wg.Done()
		}()
	}

	if *zlibStatic {
		wg.Add(1)
		go func () {
			downloadAndExtractParallel(&zlibBuilder)
			wg.Done()
		}()
	}

	wg.Add(1)
	go func() {
		downloadAndExtractParallel(&nginxBuilder)
		wg.Done()
	}()

	if len(modules3rd) > 0 {
		wg.Add(len(modules3rd))
		for _, m := range modules3rd {
			go func(m module3rd.Module3rd) {
				module3rd.DownloadAndExtractParallel(m)
				wg.Done()
			}(m)
		}

	}

	// wait until all downloading processes by goroutine finish
	wg.Wait()

	if len(modules3rd) > 0 {
		for _, m := range modules3rd {
			if err := module3rd.Provide(&m); err != nil {
				log.Fatal(err)
			}
		}
	}

	// cd workDir/nginx-${version}
	os.Chdir(nginxBuilder.SourcePath())

	var dependencies []builder.StaticLibrary
	if *pcreStatic {
		dependencies = append(dependencies, builder.MakeStaticLibrary(&pcreBuilder))
	}

	if *openSSLStatic {
		dependencies = append(dependencies, builder.MakeStaticLibrary(&openSSLBuilder))
	}

	if *zlibStatic {
		dependencies = append(dependencies, builder.MakeStaticLibrary(&zlibBuilder))
	}

	log.Printf("Generate configure script for %s.....", nginxBuilder.SourcePath())

	if *pcreStatic && pcreBuilder.IsIncludeWithOption(nginxConfigure) {
		log.Println(pcreBuilder.WarnMsgWithLibrary())
	}

	if *openSSLStatic && openSSLBuilder.IsIncludeWithOption(nginxConfigure) {
		log.Println(openSSLBuilder.WarnMsgWithLibrary())

	}

	if *zlibStatic && zlibBuilder.IsIncludeWithOption(nginxConfigure) {
		log.Println(zlibBuilder.WarnMsgWithLibrary())
	}

	configureScript := configure.Generate(nginxConfigure, modules3rd, dependencies, configureOptions, rootDir, *openResty, *jobs)

	err = ioutil.WriteFile("./nginx-configure", []byte(configureScript), 0655)
	if err != nil {
		log.Fatalf("Failed to generate configure script for %s", nginxBuilder.SourcePath())
	}

	util.Patch(*patchPath, *patchOption, rootDir, false)

	// reverts source code with patch -R when the build was interrupted.
	if *patchPath != "" {
		sigChannel := make(chan os.Signal, 1)
		signal.Notify(sigChannel, os.Interrupt)
		go func() {
			<-sigChannel
			util.Patch(*patchPath, *patchOption, rootDir, true)
		}()
	}

	log.Printf("Configure %s.....", nginxBuilder.SourcePath())

	err = configure.Run()
	if err != nil {
		log.Printf("Failed to configure %s\n", nginxBuilder.SourcePath())
		util.Patch(*patchPath, *patchOption, rootDir, true)
		util.PrintFatalMsg(err, "nginx-configure.log")
	}

	if *configureOnly {
		util.Patch(*patchPath, *patchOption, rootDir, true)
		printLastMsg(workDir, nginxBuilder.SourcePath(), *openResty, *configureOnly)
		return
	}

	log.Printf("Build %s.....", nginxBuilder.SourcePath())

	if *openSSLStatic {
		// Workarounds for protecting a failure of building nginx with static-linked OpenSSL.

		// The build of old OpenSSL fails when multi-CPUs are used.
		// It is fixed in the commit below.
		//
		// https://github.com/openssl/openssl/commit/8e6bb99979b95ee8b878e22e043ceb78d79c32a1
		//
		// And backported to 1.0.1p and 1.0.2d.
		if !openssl.ParallelBuildAvailable(*openSSLVersion) {
			*jobs = 1
		}

		// Sometimes machine hardware name('uname -m') is different
		// from machine processor architecture name('uname -p') on Mac.
		// Specifically, `uname -p` is 'i386' and `uname -m` is 'x86_64'.
		// In this case, a build of OpenSSL fails.
		// So it needs to convince OpenSSL with KERNEL_BITS.
		if runtime.GOOS == "darwin" && runtime.GOARCH == "amd64" {
			os.Setenv("KERNEL_BITS", "64")
		}
	}

	err = builder.BuildNginx(*jobs)
	if err != nil {
		log.Printf("Failed to build %s\n", nginxBuilder.SourcePath())
		util.Patch(*patchPath, *patchOption, rootDir, true)
		util.PrintFatalMsg(err, "nginx-build.log")
	}

	printLastMsg(workDir, nginxBuilder.SourcePath(), *openResty, *configureOnly)
}
<commit_msg>patch: allowed multiflag for '--patch'.<commit_after>package main

import (
	"flag"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"runtime"
	"strings"
	"sync"
	"syscall"

	"github.com/cubicdaiya/nginx-build/builder"
	"github.com/cubicdaiya/nginx-build/command"
	"github.com/cubicdaiya/nginx-build/configure"
	"github.com/cubicdaiya/nginx-build/module3rd"
	"github.com/cubicdaiya/nginx-build/openssl"
	"github.com/cubicdaiya/nginx-build/util"
)

const (
	NginxBuildVersion = "0.9.4"
)

var (
	nginxBuildOptions Options
)

func init() {
	nginxBuildOptions = makeNginxBuildOptions()
}

func main() {
	var (
		multiflagPatch StringFlag
	)

	// Parse flags
	for k, v := range nginxBuildOptions.Bools {
		v.Enabled = flag.Bool(k, false, v.Desc)
		nginxBuildOptions.Bools[k] = v
	}
	for k, v := range nginxBuildOptions.Values {
		if k == "patch" {
			flag.Var(&multiflagPatch, k, v.Desc)
		} else {
			v.Value = flag.String(k, v.Default, v.Desc)
			nginxBuildOptions.Values[k] = v
		}
	}
	for k, v := range nginxBuildOptions.Numbers {
		v.Value = flag.Int(k, v.Default, v.Desc)
		nginxBuildOptions.Numbers[k] = v
	}

	// fake flag for --with-xxx=dynamic
	for i, arg := range os.Args {
		if strings.Contains(arg, "with-http_xslt_module=dynamic") {
			os.Args[i] = "--with-http_xslt_module_dynamic"
		}
		if strings.Contains(arg, "with-http_image_filter_module=dynamic") {
			os.Args[i] = "--with-http_image_filter_module_dynamic"
		}
		if strings.Contains(arg, "with-http_geoip_module=dynamic") {
			os.Args[i] = "--with-http_geoip_module_dynamic"
		}
		if strings.Contains(arg, "with-http_perl_module=dynamic") {
			os.Args[i] = "--with-http_perl_module_dynamic"
		}
		if strings.Contains(arg, "with-mail=dynamic") {
			os.Args[i] = "--with-mail_dynamic"
		}
		if strings.Contains(arg, "with-stream=dynamic") {
			os.Args[i] = "--with-stream_dynamic"
		}
	}

	var (
		configureOptions configure.Options
		multiflag        StringFlag
		multiflagDynamic StringFlag
	)

	argsString := configure.MakeArgsString()
	for k, v := range argsString {
		if k == "add-module" {
			flag.Var(&multiflag, k, v.Desc)
		} else if k == "add-dynamic-module" {
			flag.Var(&multiflagDynamic, k, v.Desc)
		} else {
			v.Value = flag.String(k, "", v.Desc)
			argsString[k] = v
		}
	}

	argsBool := configure.MakeArgsBool()
	for k, v := range argsBool {
		v.Enabled = flag.Bool(k, false, v.Desc)
		argsBool[k] = v
	}

	flag.CommandLine.SetOutput(os.Stdout)
	// The output of original flag.Usage() is too long
	defaultUsage := flag.Usage
	flag.Usage = usage
	flag.Parse()

	jobs := nginxBuildOptions.Numbers["j"].Value

	verbose := nginxBuildOptions.Bools["verbose"].Enabled
	pcreStatic := nginxBuildOptions.Bools["pcre"].Enabled
	openSSLStatic := nginxBuildOptions.Bools["openssl"].Enabled
	zlibStatic := nginxBuildOptions.Bools["zlib"].Enabled
	clear := nginxBuildOptions.Bools["clear"].Enabled
	versionPrint := nginxBuildOptions.Bools["version"].Enabled
	versionsPrint := nginxBuildOptions.Bools["versions"].Enabled
	openResty := nginxBuildOptions.Bools["openresty"].Enabled
	tengine := nginxBuildOptions.Bools["tengine"].Enabled
	configureOnly := nginxBuildOptions.Bools["configureonly"].Enabled
	helpAll := nginxBuildOptions.Bools["help-all"].Enabled

	version := nginxBuildOptions.Values["v"].Value
	nginxConfigurePath := nginxBuildOptions.Values["c"].Value
	modulesConfPath := nginxBuildOptions.Values["m"].Value
	workParentDir := nginxBuildOptions.Values["d"].Value
	pcreVersion := nginxBuildOptions.Values["pcreversion"].Value
	openSSLVersion := nginxBuildOptions.Values["opensslversion"].Value
	zlibVersion := nginxBuildOptions.Values["zlibversion"].Value
	openRestyVersion := nginxBuildOptions.Values["openrestyversion"].Value
	tengineVersion := nginxBuildOptions.Values["tengineversion"].Value
	patchOption := nginxBuildOptions.Values["patch-opt"].Value

	// Allow multiple flags for `--patch`
	{
		tmp := nginxBuildOptions.Values["patch"]
		tmp_ := multiflagPatch.String()
		tmp.Value = &tmp_
		nginxBuildOptions.Values["patch"] = tmp
	}

	// Allow multiple flags for `--add-module`
	{
		tmp := argsString["add-module"]
		tmp_ := multiflag.String()
		tmp.Value = &tmp_
		argsString["add-module"] = tmp
	}

	// Allow multiple flags for `--add-dynamic-module`
	{
		tmp := argsString["add-dynamic-module"]
		tmp_ := multiflagDynamic.String()
		tmp.Value = &tmp_
		argsString["add-dynamic-module"] = tmp
	}

	patchPath := nginxBuildOptions.Values["patch"].Value
	configureOptions.Values = argsString
	configureOptions.Bools = argsBool

	if *helpAll {
		defaultUsage()
		return
	}

	if *versionPrint {
		printNginxBuildVersion()
		return
	}

	if *versionsPrint {
		printNginxVersions()
		return
	}

	printFirstMsg()

	// set verbose mode
	command.VerboseEnabled = *verbose

	var nginxBuilder builder.Builder
	if *openResty && *tengine {
		log.Fatal("select one between '-openresty' and '-tengine'.")
	}
	if *openResty {
		nginxBuilder = builder.MakeBuilder(builder.COMPONENT_OPENRESTY, *openRestyVersion)
	} else if *tengine {
		nginxBuilder = builder.MakeBuilder(builder.COMPONENT_TENGINE, *tengineVersion)
	} else {
		nginxBuilder = builder.MakeBuilder(builder.COMPONENT_NGINX, *version)
	}
	pcreBuilder := builder.MakeBuilder(builder.COMPONENT_PCRE, *pcreVersion)
	openSSLBuilder := builder.MakeBuilder(builder.COMPONENT_OPENSSL, *openSSLVersion)
	zlibBuilder := builder.MakeBuilder(builder.COMPONENT_ZLIB, *zlibVersion)

	// change default umask
	_ = syscall.Umask(0)

	versionCheck(*version)

	nginxConfigure, err := util.FileGetContents(*nginxConfigurePath)
	if err != nil {
		log.Fatal(err)
	}
	nginxConfigure = configure.Normalize(nginxConfigure)

	modules3rd, err := module3rd.Load(*modulesConfPath)
	if err != nil {
		log.Fatal(err)
	}

	if len(*workParentDir) == 0 {
		log.Fatal("set working directory with -d")
	}

	if !util.FileExists(*workParentDir) {
		err := os.Mkdir(*workParentDir, 0755)
		if err != nil {
			log.Fatalf("Failed to create working directory(%s) does not exist.", *workParentDir)
		}
	}

	var workDir string
	if *openResty {
		workDir = *workParentDir + "/openresty/" + *openRestyVersion
	} else if *tengine {
		workDir = *workParentDir + "/tengine/" + *tengineVersion
	} else {
		workDir = *workParentDir + "/nginx/" + *version
	}

	if *clear {
		err := util.ClearWorkDir(workDir)
		if err != nil {
			log.Fatal(err)
		}
	}

	if !util.FileExists(workDir) {
		err := os.MkdirAll(workDir, 0755)
		if err != nil {
			log.Fatalf("Failed to create working directory(%s) does not exist.", workDir)
		}
	}

	rootDir := util.SaveCurrentDir()
	err = os.Chdir(workDir)
	if err != nil {
		log.Fatal(err)
	}

	// remove nginx source code applyed patch
	if *patchPath != "" && util.FileExists(nginxBuilder.SourcePath()) {
		err := os.RemoveAll(nginxBuilder.SourcePath())
		if err != nil {
			log.Fatal(err)
		}
	}

	wg := new(sync.WaitGroup)
	if *pcreStatic {
		wg.Add(1)
		go func() {
			downloadAndExtractParallel(&pcreBuilder)
			wg.Done()
		}()
	}

	if *openSSLStatic {
		wg.Add(1)
		go func() {
			downloadAndExtractParallel(&openSSLBuilder)
			wg.Done()
		}()
	}

	if *zlibStatic {
		wg.Add(1)
		go func() {
			downloadAndExtractParallel(&zlibBuilder)
			wg.Done()
		}()
	}

	wg.Add(1)
	go func() {
		downloadAndExtractParallel(&nginxBuilder)
		wg.Done()
	}()

	if len(modules3rd) > 0 {
		wg.Add(len(modules3rd))
		for _, m := range modules3rd {
			go func(m module3rd.Module3rd) {
				module3rd.DownloadAndExtractParallel(m)
				wg.Done()
			}(m)
		}

	}

	// wait until all downloading processes by goroutine finish
	wg.Wait()

	if len(modules3rd) > 0 {
		for _, m := range modules3rd {
			if err := module3rd.Provide(&m); err != nil {
				log.Fatal(err)
			}
		}
	}

	// cd workDir/nginx-${version}
	os.Chdir(nginxBuilder.SourcePath())

	var dependencies []builder.StaticLibrary
	if *pcreStatic {
		dependencies = append(dependencies, builder.MakeStaticLibrary(&pcreBuilder))
	}

	if *openSSLStatic {
		dependencies = append(dependencies, builder.MakeStaticLibrary(&openSSLBuilder))
	}

	if *zlibStatic {
		dependencies = append(dependencies, builder.MakeStaticLibrary(&zlibBuilder))
	}

	log.Printf("Generate configure script for %s.....", nginxBuilder.SourcePath())

	if *pcreStatic && pcreBuilder.IsIncludeWithOption(nginxConfigure) {
		log.Println(pcreBuilder.WarnMsgWithLibrary())
	}

	if *openSSLStatic && openSSLBuilder.IsIncludeWithOption(nginxConfigure) {
		log.Println(openSSLBuilder.WarnMsgWithLibrary())

	}

	if *zlibStatic && zlibBuilder.IsIncludeWithOption(nginxConfigure) {
		log.Println(zlibBuilder.WarnMsgWithLibrary())
	}

	configureScript := configure.Generate(nginxConfigure, modules3rd, dependencies, configureOptions, rootDir, *openResty, *jobs)

	err = ioutil.WriteFile("./nginx-configure", []byte(configureScript), 0655)
	if err != nil {
		log.Fatalf("Failed to generate configure script for %s", nginxBuilder.SourcePath())
	}

	if strings.Contains(*patchPath, ",") {
		pathes := strings.Split(*patchPath, ",")
		for _, path := range pathes {
			util.Patch(path, *patchOption, rootDir, false)
		}
	} else {
		util.Patch(*patchPath, *patchOption, rootDir, false)
	}

	// reverts source code with patch -R when the build was interrupted.
	if *patchPath != "" {
		sigChannel := make(chan os.Signal, 1)
		signal.Notify(sigChannel, os.Interrupt)
		go func() {
			<-sigChannel
			util.Patch(*patchPath, *patchOption, rootDir, true)
		}()
	}

	log.Printf("Configure %s.....", nginxBuilder.SourcePath())

	err = configure.Run()
	if err != nil {
		log.Printf("Failed to configure %s\n", nginxBuilder.SourcePath())
		util.Patch(*patchPath, *patchOption, rootDir, true)
		util.PrintFatalMsg(err, "nginx-configure.log")
	}

	if *configureOnly {
		util.Patch(*patchPath, *patchOption, rootDir, true)
		printLastMsg(workDir, nginxBuilder.SourcePath(), *openResty, *configureOnly)
		return
	}

	log.Printf("Build %s.....", nginxBuilder.SourcePath())

	if *openSSLStatic {
		// Workarounds for protecting a failure of building nginx with static-linked OpenSSL.

		// The build of old OpenSSL fails when multi-CPUs are used.
		// It is fixed in the commit below.
		//
		// https://github.com/openssl/openssl/commit/8e6bb99979b95ee8b878e22e043ceb78d79c32a1
		//
		// And backported to 1.0.1p and 1.0.2d.
		if !openssl.ParallelBuildAvailable(*openSSLVersion) {
			*jobs = 1
		}

		// Sometimes machine hardware name('uname -m') is different
		// from machine processor architecture name('uname -p') on Mac.
		// Specifically, `uname -p` is 'i386' and `uname -m` is 'x86_64'.
		// In this case, a build of OpenSSL fails.
		// So it needs to convince OpenSSL with KERNEL_BITS.
		if runtime.GOOS == "darwin" && runtime.GOARCH == "amd64" {
			os.Setenv("KERNEL_BITS", "64")
		}
	}

	err = builder.BuildNginx(*jobs)
	if err != nil {
		log.Printf("Failed to build %s\n", nginxBuilder.SourcePath())
		util.Patch(*patchPath, *patchOption, rootDir, true)
		util.PrintFatalMsg(err, "nginx-build.log")
	}

	printLastMsg(workDir, nginxBuilder.SourcePath(), *openResty, *configureOnly)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net/http"

	"github.com/dolaterio/dolaterio/api"
)

var (
	bind    = flag.String("bind", "127.0.0.1", "Bind IP")
	port    = flag.String("port", "8080", "API port")
	rdbHost = flag.String("rhost", "localhost", "RethinkDB host or IP")
	rdbPort = flag.String("rport", "28015", "RethinkDB port")
)

func main() {
	flag.Parse()

	err := api.Initialize()
	if err != nil {
		panic(err)
	}

	err = api.ConnectDb(*rdbHost, *rdbPort)
	if err != nil {
		panic(err)
	}

	handler, err := api.Handler()
	if err != nil {
		panic(err)
	}

	http.Handle("/", handler)
	address := fmt.Sprintf("%v:%v", *bind, *port)
	http.ListenAndServe(address, nil)
}
<commit_msg>:boom:<commit_after>package main

import (
	"flag"
	"fmt"
	"net/http"

	"github.com/dolaterio/dolaterio/api"
)

var (
	bind = flag.String("bind", "127.0.0.1", "Bind IP")
	port = flag.String("port", "8080", "API port")
)

func main() {
	flag.Parse()

	err := api.Initialize()
	if err != nil {
		panic(err)
	}

	err = api.ConnectDb()
	if err != nil {
		panic(err)
	}

	handler, err := api.Handler()
	if err != nil {
		panic(err)
	}

	http.Handle("/", handler)
	address := fmt.Sprintf("%v:%v", *bind, *port)
	http.ListenAndServe(address, nil)
}
<|endoftext|>


<commit_before>// Copyright 2018 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package web

import (
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"github.com/issue9/config"
	"github.com/issue9/logs/v2"
	lconf "github.com/issue9/logs/v2/config"
	"github.com/issue9/middleware"
	"github.com/issue9/middleware/compress"
	"github.com/issue9/middleware/recovery/errorhandler"
	"github.com/issue9/mux/v2"
	"golang.org/x/text/message"
	"gopkg.in/yaml.v2"

	"github.com/issue9/web/app"
	"github.com/issue9/web/context"
	"github.com/issue9/web/internal/webconfig"
	"github.com/issue9/web/mimetype"
	"github.com/issue9/web/module"
)

var (
	defaultConfigs *config.Manager
	defaultApp     *app.App
)

// Classic 初始化一个可运行的框架环境
//
// path 配置文件地址，该文件所在的目录，会成为项目所有配置文件的根地址。
func Classic(path string, get app.GetResultFunc) error {
	dir := filepath.Dir(path)
	filename := filepath.Base(path)

	mgr, err := config.NewManager(dir)
	if err != nil {
		return err
	}

	if err = mgr.AddUnmarshal(yaml.Unmarshal, ".yaml", ".yml"); err != nil {
		return err
	}
	if err = mgr.AddUnmarshal(json.Unmarshal, ".json"); err != nil {
		return err
	}
	if err = mgr.AddUnmarshal(xml.Unmarshal, ".xml"); err != nil {
		return err
	}

	if err = Init(mgr, filename, get); err != nil {
		return err
	}

	err = AddCompresses(map[string]compress.WriterFunc{
		"gzip":    compress.NewGzip,
		"deflate": compress.NewDeflate,
	})
	if err != nil {
		return err
	}

	err = Mimetypes().AddUnmarshals(map[string]mimetype.UnmarshalFunc{
		"application/json":    json.Unmarshal,
		"multipart/form-data": nil,
	})
	if err != nil {
		return err
	}

	return Mimetypes().AddMarshals(map[string]mimetype.MarshalFunc{
		"application/json":    json.Marshal,
		"multipart/form-data": nil,
	})
}

// Init 初始化整个应用环境
//
// 重复调用会直接 panic
func Init(mgr *config.Manager, configFilename string, get app.GetResultFunc) (err error) {
	if defaultApp != nil {
		panic("不能重复调用 Init")
	}

	if configFilename == "" {
		panic("参数 configFilename 不能为空")
	}

	webconf := &webconfig.WebConfig{}
	if err = mgr.LoadFile(configFilename, webconf); err != nil {
		return err
	}

	l := logs.New()
	if webconf.Logs != "" {
		lc := &lconf.Config{}
		if err = mgr.LoadFile(webconf.Logs, lc); err != nil {
			return err
		}
		l.Init(lc)
	}

	defaultConfigs = mgr
	defaultApp, err = app.New(webconf, l, get)
	return
}

// App 返回 defaultApp 实例
func App() *app.App {
	return defaultApp
}

// Grace 指定触发 Shutdown() 的信号，若为空，则任意信号都触发。
//
// 多次调用，则每次指定的信号都会起作用，如果由传递了相同的值，
// 则有可能多次触发 Shutdown()。
//
// NOTE: 传递空值，与不调用，其结果是不同的。
// 若是不调用，则不会处理任何信号；若是传递空值调用，则是处理任何要信号。
func Grace(sig ...os.Signal) {
	app.Grace(defaultApp, sig...)
}

// AddCompresses 添加压缩处理函数
func AddCompresses(m map[string]compress.WriterFunc) error {
	return defaultApp.AddCompresses(m)
}

// AddMiddlewares 设置全局的中间件，可多次调用。
func AddMiddlewares(m middleware.Middleware) {
	defaultApp.After(m)
}

// IsDebug 是否处在调试模式
func IsDebug() bool {
	return defaultApp.IsDebug()
}

// Mux 返回 mux.Mux 实例。
func Mux() *mux.Mux {
	return defaultApp.Mux()
}

// Mimetypes 返回 mimetype.Mimetypes
func Mimetypes() *mimetype.Mimetypes {
	return defaultApp.Mimetypes()
}

// Serve 运行路由，执行监听程序。
func Serve() error {
	return defaultApp.Serve()
}

// InitModules 初始化指定标签的模块
func InitModules(tag string) error {
	return defaultApp.Init(tag, INFO())
}

// Close 关闭服务。
//
// 无论配置文件如果设置，此函数都是直接关闭服务，不会等待。
func Close() error {
	return defaultApp.Close()
}

// Shutdown 关闭所有服务。
//
// 根据配置文件中的配置项，决定当前是直接关闭还是延时之后关闭。
func Shutdown() error {
	return defaultApp.Shutdown()
}

// URL 构建一条完整 URL
func URL(path string) string {
	return defaultApp.URL(path)
}

// Path 构建 URL 的 Path 部分
func Path(path string) string {
	return defaultApp.Path(path)
}

// Modules 当前系统使用的所有模块信息
func Modules() []*module.Module {
	return defaultApp.Modules.Modules()
}

// Services 返回所有的服务列表
func Services() []*module.Service {
	return defaultApp.Services()
}

// Tags 获取所有的子模块名称
func Tags() []string {
	return defaultApp.Tags()
}

// Server 获取 http.Server 实例
func Server() *http.Server {
	return defaultApp.Server()
}

// NewModule 注册一个模块
func NewModule(name, desc string, deps ...string) *Module {
	return defaultApp.NewModule(name, desc, deps...)
}

// File 获取配置目录下的文件。
func File(path string) string {
	return defaultConfigs.File(path)
}

// LoadFile 加载指定的配置文件内容到 v 中
func LoadFile(path string, v interface{}) error {
	return defaultConfigs.LoadFile(path, v)
}

// Load 加载指定的配置文件内容到 v 中
func Load(r io.Reader, typ string, v interface{}) error {
	return defaultConfigs.Load(r, typ, v)
}

// AddMessages 添加新的错误消息代码
func AddMessages(status int, messages map[int]string) {
	defaultApp.AddMessages(status, messages)
}

// ErrorHandlers 错误处理功能
func ErrorHandlers() *errorhandler.ErrorHandler {
	return defaultApp.ErrorHandlers()
}

// Messages 获取所有的错误消息代码
//
// 如果指定 p 的值，则返回本地化的消息内容。
func Messages(p *message.Printer) map[int]string {
	return defaultApp.Messages(p)
}

// Now 返回当前时间。
//
// 与 time.Now() 的区别在于 Now() 基于当前时区
func Now() time.Time {
	return time.Now().In(defaultApp.Location())
}

// ParseTime 分析时间格式，基于当前时间
func ParseTime(layout, value string) (time.Time, error) {
	return time.ParseInLocation(layout, value, defaultApp.Location())
}

// NewContext 根据当前配置，生成 context.Context 对象，若是出错则 panic
func NewContext(w http.ResponseWriter, r *http.Request) *Context {
	return context.New(w, r, defaultApp)
}

// INFO 获取 INFO 级别的 log.Logger 实例，在未指定 info 级别的日志时，该实例返回一个 nil。
func INFO() *log.Logger {
	return defaultApp.Logs().INFO()
}

// Info 相当于 INFO().Println(v...) 的简写方式
// Info 函数默认是带换行符的，若需要不带换行符的，请使用 DEBUG().Print() 函数代替。
// 其它相似函数也有类型功能。
func Info(v ...interface{}) {
	INFO().Output(2, fmt.Sprintln(v...))
}

// Infof 相当于 INFO().Printf(format, v...) 的简写方式
func Infof(format string, v ...interface{}) {
	INFO().Output(2, fmt.Sprintf(format, v...))
}

// DEBUG 获取 DEBUG 级别的 log.Logger 实例，在未指定 debug 级别的日志时，该实例返回一个 nil。
func DEBUG() *log.Logger {
	return defaultApp.Logs().DEBUG()
}

// Debug 相当于 DEBUG().Println(v...) 的简写方式
func Debug(v ...interface{}) {
	DEBUG().Output(2, fmt.Sprintln(v...))
}

// Debugf 相当于 DEBUG().Printf(format, v...) 的简写方式
func Debugf(format string, v ...interface{}) {
	DEBUG().Output(2, fmt.Sprintf(format, v...))
}

// TRACE 获取 TRACE 级别的 log.Logger 实例，在未指定 trace 级别的日志时，该实例返回一个 nil。
func TRACE() *log.Logger {
	return defaultApp.Logs().TRACE()
}

// Trace 相当于 TRACE().Println(v...) 的简写方式
func Trace(v ...interface{}) {
	TRACE().Output(2, fmt.Sprintln(v...))
}

// Tracef 相当于 TRACE().Printf(format, v...) 的简写方式
func Tracef(format string, v ...interface{}) {
	TRACE().Output(2, fmt.Sprintf(format, v...))
}

// WARN 获取 WARN 级别的 log.Logger 实例，在未指定 warn 级别的日志时，该实例返回一个 nil。
func WARN() *log.Logger {
	return defaultApp.Logs().WARN()
}

// Warn 相当于 WARN().Println(v...) 的简写方式
func Warn(v ...interface{}) {
	WARN().Output(2, fmt.Sprintln(v...))
}

// Warnf 相当于 WARN().Printf(format, v...) 的简写方式
func Warnf(format string, v ...interface{}) {
	WARN().Output(2, fmt.Sprintf(format, v...))
}

// ERROR 获取 ERROR 级别的 log.Logger 实例，在未指定 error 级别的日志时，该实例返回一个 nil。
func ERROR() *log.Logger {
	return defaultApp.Logs().ERROR()
}

// Error 相当于 ERROR().Println(v...) 的简写方式
func Error(v ...interface{}) {
	ERROR().Output(2, fmt.Sprintln(v...))
}

// Errorf 相当于 ERROR().Printf(format, v...) 的简写方式
func Errorf(format string, v ...interface{}) {
	ERROR().Output(2, fmt.Sprintf(format, v...))
}

// CRITICAL 获取 CRITICAL 级别的 log.Logger 实例，在未指定 critical 级别的日志时，该实例返回一个 nil。
func CRITICAL() *log.Logger {
	return defaultApp.Logs().CRITICAL()
}

// Critical 相当于 CRITICAL().Println(v...)的简写方式
func Critical(v ...interface{}) {
	CRITICAL().Output(2, fmt.Sprintln(v...))
}

// Criticalf 相当于 CRITICAL().Printf(format, v...) 的简写方式
func Criticalf(format string, v ...interface{}) {
	CRITICAL().Output(2, fmt.Sprintf(format, v...))
}

// Fatal 输出错误信息，然后退出程序。
func Fatal(code int, v ...interface{}) {
	defaultApp.Logs().Fatal(code, v...)
}

// Fatalf 输出错误信息，然后退出程序。
func Fatalf(code int, format string, v ...interface{}) {
	defaultApp.Logs().Fatalf(code, format, v...)
}

// Panic 输出错误信息，然后触发 panic。
func Panic(v ...interface{}) {
	defaultApp.Logs().Panic(v...)
}

// Panicf 输出错误信息，然后触发 panic。
func Panicf(format string, v ...interface{}) {
	defaultApp.Logs().Panicf(format, v...)
}

// FlushLogs 立即输出所有的日志信息。
func FlushLogs() {
	defaultApp.Logs().Flush()
}
<commit_msg>添加 Location 函数<commit_after>// Copyright 2018 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package web

import (
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"github.com/issue9/config"
	"github.com/issue9/logs/v2"
	lconf "github.com/issue9/logs/v2/config"
	"github.com/issue9/middleware"
	"github.com/issue9/middleware/compress"
	"github.com/issue9/middleware/recovery/errorhandler"
	"github.com/issue9/mux/v2"
	"golang.org/x/text/message"
	"gopkg.in/yaml.v2"

	"github.com/issue9/web/app"
	"github.com/issue9/web/context"
	"github.com/issue9/web/internal/webconfig"
	"github.com/issue9/web/mimetype"
	"github.com/issue9/web/module"
)

var (
	defaultConfigs *config.Manager
	defaultApp     *app.App
)

// Classic 初始化一个可运行的框架环境
//
// path 配置文件地址，该文件所在的目录，会成为项目所有配置文件的根地址。
func Classic(path string, get app.GetResultFunc) error {
	dir := filepath.Dir(path)
	filename := filepath.Base(path)

	mgr, err := config.NewManager(dir)
	if err != nil {
		return err
	}

	if err = mgr.AddUnmarshal(yaml.Unmarshal, ".yaml", ".yml"); err != nil {
		return err
	}
	if err = mgr.AddUnmarshal(json.Unmarshal, ".json"); err != nil {
		return err
	}
	if err = mgr.AddUnmarshal(xml.Unmarshal, ".xml"); err != nil {
		return err
	}

	if err = Init(mgr, filename, get); err != nil {
		return err
	}

	err = AddCompresses(map[string]compress.WriterFunc{
		"gzip":    compress.NewGzip,
		"deflate": compress.NewDeflate,
	})
	if err != nil {
		return err
	}

	err = Mimetypes().AddUnmarshals(map[string]mimetype.UnmarshalFunc{
		"application/json":    json.Unmarshal,
		"multipart/form-data": nil,
	})
	if err != nil {
		return err
	}

	return Mimetypes().AddMarshals(map[string]mimetype.MarshalFunc{
		"application/json":    json.Marshal,
		"multipart/form-data": nil,
	})
}

// Init 初始化整个应用环境
//
// 重复调用会直接 panic
func Init(mgr *config.Manager, configFilename string, get app.GetResultFunc) (err error) {
	if defaultApp != nil {
		panic("不能重复调用 Init")
	}

	if configFilename == "" {
		panic("参数 configFilename 不能为空")
	}

	webconf := &webconfig.WebConfig{}
	if err = mgr.LoadFile(configFilename, webconf); err != nil {
		return err
	}

	l := logs.New()
	if webconf.Logs != "" {
		lc := &lconf.Config{}
		if err = mgr.LoadFile(webconf.Logs, lc); err != nil {
			return err
		}
		l.Init(lc)
	}

	defaultConfigs = mgr
	defaultApp, err = app.New(webconf, l, get)
	return
}

// App 返回 defaultApp 实例
func App() *app.App {
	return defaultApp
}

// Grace 指定触发 Shutdown() 的信号，若为空，则任意信号都触发。
//
// 多次调用，则每次指定的信号都会起作用，如果由传递了相同的值，
// 则有可能多次触发 Shutdown()。
//
// NOTE: 传递空值，与不调用，其结果是不同的。
// 若是不调用，则不会处理任何信号；若是传递空值调用，则是处理任何要信号。
func Grace(sig ...os.Signal) {
	app.Grace(defaultApp, sig...)
}

// AddCompresses 添加压缩处理函数
func AddCompresses(m map[string]compress.WriterFunc) error {
	return defaultApp.AddCompresses(m)
}

// AddMiddlewares 设置全局的中间件，可多次调用。
func AddMiddlewares(m middleware.Middleware) {
	defaultApp.After(m)
}

// IsDebug 是否处在调试模式
func IsDebug() bool {
	return defaultApp.IsDebug()
}

// Mux 返回 mux.Mux 实例。
func Mux() *mux.Mux {
	return defaultApp.Mux()
}

// Mimetypes 返回 mimetype.Mimetypes
func Mimetypes() *mimetype.Mimetypes {
	return defaultApp.Mimetypes()
}

// Serve 运行路由，执行监听程序。
func Serve() error {
	return defaultApp.Serve()
}

// InitModules 初始化指定标签的模块
func InitModules(tag string) error {
	return defaultApp.Init(tag, INFO())
}

// Close 关闭服务。
//
// 无论配置文件如果设置，此函数都是直接关闭服务，不会等待。
func Close() error {
	return defaultApp.Close()
}

// Shutdown 关闭所有服务。
//
// 根据配置文件中的配置项，决定当前是直接关闭还是延时之后关闭。
func Shutdown() error {
	return defaultApp.Shutdown()
}

// URL 构建一条完整 URL
func URL(path string) string {
	return defaultApp.URL(path)
}

// Path 构建 URL 的 Path 部分
func Path(path string) string {
	return defaultApp.Path(path)
}

// Modules 当前系统使用的所有模块信息
func Modules() []*module.Module {
	return defaultApp.Modules.Modules()
}

// Services 返回所有的服务列表
func Services() []*module.Service {
	return defaultApp.Services()
}

// Tags 获取所有的子模块名称
func Tags() []string {
	return defaultApp.Tags()
}

// Server 获取 http.Server 实例
func Server() *http.Server {
	return defaultApp.Server()
}

// NewModule 注册一个模块
func NewModule(name, desc string, deps ...string) *Module {
	return defaultApp.NewModule(name, desc, deps...)
}

// File 获取配置目录下的文件。
func File(path string) string {
	return defaultConfigs.File(path)
}

// LoadFile 加载指定的配置文件内容到 v 中
func LoadFile(path string, v interface{}) error {
	return defaultConfigs.LoadFile(path, v)
}

// Load 加载指定的配置文件内容到 v 中
func Load(r io.Reader, typ string, v interface{}) error {
	return defaultConfigs.Load(r, typ, v)
}

// AddMessages 添加新的错误消息代码
func AddMessages(status int, messages map[int]string) {
	defaultApp.AddMessages(status, messages)
}

// ErrorHandlers 错误处理功能
func ErrorHandlers() *errorhandler.ErrorHandler {
	return defaultApp.ErrorHandlers()
}

// Messages 获取所有的错误消息代码
//
// 如果指定 p 的值，则返回本地化的消息内容。
func Messages(p *message.Printer) map[int]string {
	return defaultApp.Messages(p)
}

// Location 返回当前配置文件中指定的时区信息
func Location() *time.Location {
	return defaultApp.Location()
}

// Now 返回当前时间。
//
// 与 time.Now() 的区别在于 Now() 基于当前时区
func Now() time.Time {
	return time.Now().In(Location())
}

// ParseTime 分析时间格式，基于当前时间
func ParseTime(layout, value string) (time.Time, error) {
	return time.ParseInLocation(layout, value, Location())
}

// NewContext 根据当前配置，生成 context.Context 对象，若是出错则 panic
func NewContext(w http.ResponseWriter, r *http.Request) *Context {
	return context.New(w, r, defaultApp)
}

// INFO 获取 INFO 级别的 log.Logger 实例，在未指定 info 级别的日志时，该实例返回一个 nil。
func INFO() *log.Logger {
	return defaultApp.Logs().INFO()
}

// Info 相当于 INFO().Println(v...) 的简写方式
// Info 函数默认是带换行符的，若需要不带换行符的，请使用 DEBUG().Print() 函数代替。
// 其它相似函数也有类型功能。
func Info(v ...interface{}) {
	INFO().Output(2, fmt.Sprintln(v...))
}

// Infof 相当于 INFO().Printf(format, v...) 的简写方式
func Infof(format string, v ...interface{}) {
	INFO().Output(2, fmt.Sprintf(format, v...))
}

// DEBUG 获取 DEBUG 级别的 log.Logger 实例，在未指定 debug 级别的日志时，该实例返回一个 nil。
func DEBUG() *log.Logger {
	return defaultApp.Logs().DEBUG()
}

// Debug 相当于 DEBUG().Println(v...) 的简写方式
func Debug(v ...interface{}) {
	DEBUG().Output(2, fmt.Sprintln(v...))
}

// Debugf 相当于 DEBUG().Printf(format, v...) 的简写方式
func Debugf(format string, v ...interface{}) {
	DEBUG().Output(2, fmt.Sprintf(format, v...))
}

// TRACE 获取 TRACE 级别的 log.Logger 实例，在未指定 trace 级别的日志时，该实例返回一个 nil。
func TRACE() *log.Logger {
	return defaultApp.Logs().TRACE()
}

// Trace 相当于 TRACE().Println(v...) 的简写方式
func Trace(v ...interface{}) {
	TRACE().Output(2, fmt.Sprintln(v...))
}

// Tracef 相当于 TRACE().Printf(format, v...) 的简写方式
func Tracef(format string, v ...interface{}) {
	TRACE().Output(2, fmt.Sprintf(format, v...))
}

// WARN 获取 WARN 级别的 log.Logger 实例，在未指定 warn 级别的日志时，该实例返回一个 nil。
func WARN() *log.Logger {
	return defaultApp.Logs().WARN()
}

// Warn 相当于 WARN().Println(v...) 的简写方式
func Warn(v ...interface{}) {
	WARN().Output(2, fmt.Sprintln(v...))
}

// Warnf 相当于 WARN().Printf(format, v...) 的简写方式
func Warnf(format string, v ...interface{}) {
	WARN().Output(2, fmt.Sprintf(format, v...))
}

// ERROR 获取 ERROR 级别的 log.Logger 实例，在未指定 error 级别的日志时，该实例返回一个 nil。
func ERROR() *log.Logger {
	return defaultApp.Logs().ERROR()
}

// Error 相当于 ERROR().Println(v...) 的简写方式
func Error(v ...interface{}) {
	ERROR().Output(2, fmt.Sprintln(v...))
}

// Errorf 相当于 ERROR().Printf(format, v...) 的简写方式
func Errorf(format string, v ...interface{}) {
	ERROR().Output(2, fmt.Sprintf(format, v...))
}

// CRITICAL 获取 CRITICAL 级别的 log.Logger 实例，在未指定 critical 级别的日志时，该实例返回一个 nil。
func CRITICAL() *log.Logger {
	return defaultApp.Logs().CRITICAL()
}

// Critical 相当于 CRITICAL().Println(v...)的简写方式
func Critical(v ...interface{}) {
	CRITICAL().Output(2, fmt.Sprintln(v...))
}

// Criticalf 相当于 CRITICAL().Printf(format, v...) 的简写方式
func Criticalf(format string, v ...interface{}) {
	CRITICAL().Output(2, fmt.Sprintf(format, v...))
}

// Fatal 输出错误信息，然后退出程序。
func Fatal(code int, v ...interface{}) {
	defaultApp.Logs().Fatal(code, v...)
}

// Fatalf 输出错误信息，然后退出程序。
func Fatalf(code int, format string, v ...interface{}) {
	defaultApp.Logs().Fatalf(code, format, v...)
}

// Panic 输出错误信息，然后触发 panic。
func Panic(v ...interface{}) {
	defaultApp.Logs().Panic(v...)
}

// Panicf 输出错误信息，然后触发 panic。
func Panicf(format string, v ...interface{}) {
	defaultApp.Logs().Panicf(format, v...)
}

// FlushLogs 立即输出所有的日志信息。
func FlushLogs() {
	defaultApp.Logs().Flush()
}
<|endoftext|>


<commit_before>package fwk

import (
	"fmt"
	"os"
	"reflect"
	"sort"
)

var g_mgr *appmgr = nil

type fsm int

const (
	fsm_UNDEFINED  fsm = 0
	fsm_CONFIGURED fsm = 1
	fsm_STARTED    fsm = 2
	fsm_RUNNING    fsm = 3
	fsm_STOPPED    fsm = 4
	fsm_OFFLINE    fsm = 5
)

type appmgr struct {
	state fsm
	name  string

	props map[Component]map[string]interface{}
	dflow *dflowsvc
	store *datastore

	evtmax int64

	comps map[string]Component
	tsks  []Task
	svcs  []Svc
}

func NewApp() App {
	if g_mgr != nil {
		return g_mgr
	}

	var err Error
	var app *appmgr

	app = &appmgr{
		state:  fsm_UNDEFINED,
		name:   "app",
		props:  make(map[Component]map[string]interface{}),
		dflow:  nil,
		store:  nil,
		evtmax: 4,
		comps:  make(map[string]Component),
		tsks:   make([]Task, 0),
		svcs:   make([]Svc, 0),
	}

	svc, err := New("github.com/go-hep/fwk.datastore", "evtstore")
	if err != nil {
		fmt.Printf("**error** fwk.NewApp: could not create evtstore: %v\n", err)
		return nil
	}
	app.store = svc.(*datastore)

	//app.store = newDataStore("evtstore")
	err = app.AddSvc(app.store)
	if err != nil {
		fmt.Printf("**error** fwk.NewApp: could not create evtstore: %v\n", err)
		return nil
	}

	svc, err = New("github.com/go-hep/fwk.dflowsvc", "dataflow")
	if err != nil {
		fmt.Printf("**error** fwk.NewApp: could not create dataflow svc: %v\n", err)
		return nil
	}
	app.dflow = svc.(*dflowsvc)

	//app.dflow = newDflowSvc("dataflow")
	err = app.AddSvc(app.dflow)
	if err != nil {
		fmt.Printf("**error** fwk.NewApp: could not create dataflow svc: %v\n", err)
		return nil
	}

	g_mgr = app
	return app
}

func (app *appmgr) Name() string {
	return app.name
}

func (app *appmgr) SetName(n string) {
	app.name = n
}

func (app *appmgr) Component(n string) Component {
	c, ok := app.comps[n]
	if !ok {
		return nil
	}
	return c
}

func (app *appmgr) addComponent(c Component) Error {
	app.comps[c.Name()] = c
	return nil
}

func (app *appmgr) HasComponent(n string) bool {
	_, ok := app.comps[n]
	return ok
}

func (app *appmgr) Components() []Component {
	comps := make([]Component, 0, len(app.comps))
	for _, c := range app.comps {
		comps = append(comps, c)
	}
	return comps
}

func (app *appmgr) AddTask(tsk Task) Error {
	var err Error
	app.tsks = append(app.tsks, tsk)
	app.comps[tsk.Name()] = tsk
	return err
}

func (app *appmgr) DelTask(tsk Task) Error {
	var err Error
	tsks := make([]Task, 0, len(app.tsks))
	for _, t := range app.tsks {
		if t.Name() != tsk.Name() {
			tsks = append(tsks, t)
		}
	}
	app.tsks = tsks
	return err
}

func (app *appmgr) HasTask(n string) bool {
	for _, t := range app.tsks {
		if t.Name() == n {
			return true
		}
	}
	return false
}

func (app *appmgr) GetTask(n string) Task {
	for _, t := range app.tsks {
		if t.Name() == n {
			return t
		}
	}
	return nil
}

func (app *appmgr) Tasks() []Task {
	return app.tsks
}

func (app *appmgr) AddSvc(svc Svc) Error {
	var err Error
	app.svcs = append(app.svcs, svc)
	app.comps[svc.Name()] = svc
	return err
}

func (app *appmgr) DelSvc(svc Svc) Error {
	var err Error
	svcs := make([]Svc, 0, len(app.svcs))
	for _, s := range app.svcs {
		if s.Name() != svc.Name() {
			svcs = append(svcs, s)
		}
	}
	app.svcs = svcs
	return err
}

func (app *appmgr) HasSvc(n string) bool {
	for _, s := range app.svcs {
		if s.Name() == n {
			return true
		}
	}
	return false
}

func (app *appmgr) GetSvc(n string) Svc {
	for _, s := range app.svcs {
		if s.Name() == n {
			return s
		}
	}
	return nil
}

func (app *appmgr) Svcs() []Svc {
	return app.svcs
}

func (app *appmgr) DeclProp(c Component, name string, ptr interface{}) Error {
	_, ok := app.props[c]
	if !ok {
		app.props[c] = make(map[string]interface{})
	}
	switch reflect.TypeOf(ptr).Kind() {
	case reflect.Ptr:
		// ok
	default:
		return Errorf(
			"fwk.DeclProp: component [%s] didn't pass a pointer for the property [%s] (type=%T)",
			c.Name(),
			name,
			ptr,
		)
	}
	app.props[c][name] = ptr
	return nil
}

func (app *appmgr) SetProp(c Component, name string, value interface{}) Error {
	m, ok := app.props[c]
	if !ok {
		return Errorf(
			"fwk.SetProp: component [%s] didn't declare any property",
			c.Name(),
		)
	}
	rv := reflect.ValueOf(value)
	rt := rv.Type()
	ptr := reflect.ValueOf(m[name])
	dst := ptr.Elem().Type()
	if !rt.AssignableTo(dst) {
		return Errorf(
			"fwk.SetProp: component [%s] has property [%s] with type [%s]. got value=%v (type=%s)",
			c.Name(),
			name,
			dst.Name(),
			value,
			rt.Name(),
		)
	}
	ptr.Elem().Set(rv)
	return nil
}

func (app *appmgr) GetProp(c Component, name string) (interface{}, Error) {
	m, ok := app.props[c]
	if !ok {
		return nil, Errorf(
			"fwk.GetProp: component [%s] didn't declare any property",
			c.Name(),
		)
	}

	ptr, ok := m[name]
	if !ok {
		return nil, Errorf(
			"fwk.GetProp: component [%s] didn't declare any property with name [%s]",
			c.Name(),
			name,
		)
	}

	v := reflect.Indirect(reflect.ValueOf(ptr)).Interface()
	return v, nil
}

func (app *appmgr) Run() Error {
	var err Error
	var ctx Context

	if app.state == fsm_UNDEFINED {
		err = app.configure(ctx)
		if err != nil {
			return err
		}
	}

	if app.state == fsm_CONFIGURED {
		err = app.start(ctx)
		if err != nil {
			return err
		}
	}

	if app.state == fsm_STARTED {
		err = app.run(ctx)
		if err != nil {
			return err
		}
	}

	if app.state == fsm_RUNNING {
		err = app.stop(ctx)
		if err != nil {
			return err
		}
	}

	if app.state == fsm_STOPPED {
		err = app.shutdown(ctx)
		if err != nil {
			return err
		}
	}

	return err
}

func (app *appmgr) configure(ctx Context) Error {
	var err Error
	fmt.Printf(">>> app.configure...\n")
	for _, svc := range app.svcs {
		fmt.Printf(">>> configuring [%s]...\n", svc.Name())
		cfg, ok := svc.(Configurer)
		if !ok {
			continue
		}
		err = cfg.Configure(ctx)
		if err != nil {
			return err
		}
	}

	for _, tsk := range app.tsks {
		fmt.Printf(">>> configuring [%s]...\n", tsk.Name())
		cfg, ok := tsk.(Configurer)
		if !ok {
			continue
		}
		err = cfg.Configure(ctx)
		if err != nil {
			return err
		}
	}

	fmt.Printf(">>> --- [data flow] --- nodes...\n")
	for tsk, node := range app.dflow.nodes {
		fmt.Printf(">>> ---[%s]---\n", tsk)
		fmt.Printf("    in:  %v\n", node.in)
		fmt.Printf("    out: %v\n", node.out)
	}

	fmt.Printf(">>> --- [data flow] --- edges...\n")
	edges := make([]string, 0, len(app.dflow.edges))
	for n := range app.dflow.edges {
		edges = append(edges, n)
	}
	sort.Strings(edges)
	fmt.Printf(" edges: %v\n", edges)

	app.state = fsm_CONFIGURED
	fmt.Printf(">>> app.configure... [done]\n")
	return err
}

func (app *appmgr) start(ctx Context) Error {
	var err Error
	for _, svc := range app.svcs {
		fmt.Printf(">>> starting [%s]...\n", svc.Name())
		err = svc.StartSvc(ctx)
		if err != nil {
			return err
		}
	}

	for _, tsk := range app.tsks {
		fmt.Printf(">>> starting [%s]...\n", tsk.Name())
		err = tsk.StartTask(ctx)
		if err != nil {
			return err
		}
	}

	app.state = fsm_STARTED
	return err
}

func (app *appmgr) run(ctx Context) Error {
	var err Error
	app.state = fsm_RUNNING
	for ievt := int64(0); ievt < app.evtmax; ievt++ {
		fmt.Printf(">>> app.running evt=%d...\n", ievt)
		for k := range app.dflow.edges {
			app.store.store[k] = make(achan, 1)
		}
		errch := make(chan Error, len(app.tsks))
		for _, tsk := range app.tsks {
			go func(tsk Task) {
				//fmt.Printf(">>> running [%s]...\n", tsk.Name())
				ctx := context{
					id:    ievt,
					slot:  0,
					store: app.store,
					msg: msgstream{
						lvl: LvlInfo,
						w:   os.Stdout,
						n:   tsk.Name(),
					},
				}
				errch <- tsk.Process(ctx)
			}(tsk)
		}
		for i := 0; i < len(app.tsks); i++ {
			err := <-errch
			if err != nil {
				close(errch)
				return err
			}

		}
	}
	return err
}

func (app *appmgr) stop(ctx Context) Error {
	var err Error
	for _, tsk := range app.tsks {
		err = tsk.StopTask(ctx)
		if err != nil {
			return err
		}
	}

	for _, svc := range app.svcs {
		err = svc.StopSvc(ctx)
		if err != nil {
			return err
		}
	}

	app.state = fsm_STOPPED
	return err
}

func (app *appmgr) shutdown(ctx Context) Error {
	var err Error
	app.comps = nil
	app.tsks = nil
	app.svcs = nil
	app.state = fsm_OFFLINE

	app.props = nil
	app.dflow = nil
	app.store = nil

	g_mgr = nil
	return err
}

func init() {
	Register(reflect.TypeOf(appmgr{}))
}

// EOF
<commit_msg>app: use Deleter interface to clean-up store before event processing<commit_after>package fwk

import (
	"fmt"
	"os"
	"reflect"
	"sort"
)

var g_mgr *appmgr = nil

type fsm int

const (
	fsm_UNDEFINED  fsm = 0
	fsm_CONFIGURED fsm = 1
	fsm_STARTED    fsm = 2
	fsm_RUNNING    fsm = 3
	fsm_STOPPED    fsm = 4
	fsm_OFFLINE    fsm = 5
)

type appmgr struct {
	state fsm
	name  string

	props map[Component]map[string]interface{}
	dflow *dflowsvc
	store *datastore

	evtmax int64

	comps map[string]Component
	tsks  []Task
	svcs  []Svc
}

func NewApp() App {
	if g_mgr != nil {
		return g_mgr
	}

	var err Error
	var app *appmgr

	app = &appmgr{
		state:  fsm_UNDEFINED,
		name:   "app",
		props:  make(map[Component]map[string]interface{}),
		dflow:  nil,
		store:  nil,
		evtmax: 4,
		comps:  make(map[string]Component),
		tsks:   make([]Task, 0),
		svcs:   make([]Svc, 0),
	}

	svc, err := New("github.com/go-hep/fwk.datastore", "evtstore")
	if err != nil {
		fmt.Printf("**error** fwk.NewApp: could not create evtstore: %v\n", err)
		return nil
	}
	app.store = svc.(*datastore)

	//app.store = newDataStore("evtstore")
	err = app.AddSvc(app.store)
	if err != nil {
		fmt.Printf("**error** fwk.NewApp: could not create evtstore: %v\n", err)
		return nil
	}

	svc, err = New("github.com/go-hep/fwk.dflowsvc", "dataflow")
	if err != nil {
		fmt.Printf("**error** fwk.NewApp: could not create dataflow svc: %v\n", err)
		return nil
	}
	app.dflow = svc.(*dflowsvc)

	//app.dflow = newDflowSvc("dataflow")
	err = app.AddSvc(app.dflow)
	if err != nil {
		fmt.Printf("**error** fwk.NewApp: could not create dataflow svc: %v\n", err)
		return nil
	}

	g_mgr = app
	return app
}

func (app *appmgr) Name() string {
	return app.name
}

func (app *appmgr) SetName(n string) {
	app.name = n
}

func (app *appmgr) Component(n string) Component {
	c, ok := app.comps[n]
	if !ok {
		return nil
	}
	return c
}

func (app *appmgr) addComponent(c Component) Error {
	app.comps[c.Name()] = c
	return nil
}

func (app *appmgr) HasComponent(n string) bool {
	_, ok := app.comps[n]
	return ok
}

func (app *appmgr) Components() []Component {
	comps := make([]Component, 0, len(app.comps))
	for _, c := range app.comps {
		comps = append(comps, c)
	}
	return comps
}

func (app *appmgr) AddTask(tsk Task) Error {
	var err Error
	app.tsks = append(app.tsks, tsk)
	app.comps[tsk.Name()] = tsk
	return err
}

func (app *appmgr) DelTask(tsk Task) Error {
	var err Error
	tsks := make([]Task, 0, len(app.tsks))
	for _, t := range app.tsks {
		if t.Name() != tsk.Name() {
			tsks = append(tsks, t)
		}
	}
	app.tsks = tsks
	return err
}

func (app *appmgr) HasTask(n string) bool {
	for _, t := range app.tsks {
		if t.Name() == n {
			return true
		}
	}
	return false
}

func (app *appmgr) GetTask(n string) Task {
	for _, t := range app.tsks {
		if t.Name() == n {
			return t
		}
	}
	return nil
}

func (app *appmgr) Tasks() []Task {
	return app.tsks
}

func (app *appmgr) AddSvc(svc Svc) Error {
	var err Error
	app.svcs = append(app.svcs, svc)
	app.comps[svc.Name()] = svc
	return err
}

func (app *appmgr) DelSvc(svc Svc) Error {
	var err Error
	svcs := make([]Svc, 0, len(app.svcs))
	for _, s := range app.svcs {
		if s.Name() != svc.Name() {
			svcs = append(svcs, s)
		}
	}
	app.svcs = svcs
	return err
}

func (app *appmgr) HasSvc(n string) bool {
	for _, s := range app.svcs {
		if s.Name() == n {
			return true
		}
	}
	return false
}

func (app *appmgr) GetSvc(n string) Svc {
	for _, s := range app.svcs {
		if s.Name() == n {
			return s
		}
	}
	return nil
}

func (app *appmgr) Svcs() []Svc {
	return app.svcs
}

func (app *appmgr) DeclProp(c Component, name string, ptr interface{}) Error {
	_, ok := app.props[c]
	if !ok {
		app.props[c] = make(map[string]interface{})
	}
	switch reflect.TypeOf(ptr).Kind() {
	case reflect.Ptr:
		// ok
	default:
		return Errorf(
			"fwk.DeclProp: component [%s] didn't pass a pointer for the property [%s] (type=%T)",
			c.Name(),
			name,
			ptr,
		)
	}
	app.props[c][name] = ptr
	return nil
}

func (app *appmgr) SetProp(c Component, name string, value interface{}) Error {
	m, ok := app.props[c]
	if !ok {
		return Errorf(
			"fwk.SetProp: component [%s] didn't declare any property",
			c.Name(),
		)
	}
	rv := reflect.ValueOf(value)
	rt := rv.Type()
	ptr := reflect.ValueOf(m[name])
	dst := ptr.Elem().Type()
	if !rt.AssignableTo(dst) {
		return Errorf(
			"fwk.SetProp: component [%s] has property [%s] with type [%s]. got value=%v (type=%s)",
			c.Name(),
			name,
			dst.Name(),
			value,
			rt.Name(),
		)
	}
	ptr.Elem().Set(rv)
	return nil
}

func (app *appmgr) GetProp(c Component, name string) (interface{}, Error) {
	m, ok := app.props[c]
	if !ok {
		return nil, Errorf(
			"fwk.GetProp: component [%s] didn't declare any property",
			c.Name(),
		)
	}

	ptr, ok := m[name]
	if !ok {
		return nil, Errorf(
			"fwk.GetProp: component [%s] didn't declare any property with name [%s]",
			c.Name(),
			name,
		)
	}

	v := reflect.Indirect(reflect.ValueOf(ptr)).Interface()
	return v, nil
}

func (app *appmgr) Run() Error {
	var err Error
	var ctx Context

	if app.state == fsm_UNDEFINED {
		err = app.configure(ctx)
		if err != nil {
			return err
		}
	}

	if app.state == fsm_CONFIGURED {
		err = app.start(ctx)
		if err != nil {
			return err
		}
	}

	if app.state == fsm_STARTED {
		err = app.run(ctx)
		if err != nil {
			return err
		}
	}

	if app.state == fsm_RUNNING {
		err = app.stop(ctx)
		if err != nil {
			return err
		}
	}

	if app.state == fsm_STOPPED {
		err = app.shutdown(ctx)
		if err != nil {
			return err
		}
	}

	return err
}

func (app *appmgr) configure(ctx Context) Error {
	var err Error
	fmt.Printf(">>> app.configure...\n")
	for _, svc := range app.svcs {
		fmt.Printf(">>> configuring [%s]...\n", svc.Name())
		cfg, ok := svc.(Configurer)
		if !ok {
			continue
		}
		err = cfg.Configure(ctx)
		if err != nil {
			return err
		}
	}

	for _, tsk := range app.tsks {
		fmt.Printf(">>> configuring [%s]...\n", tsk.Name())
		cfg, ok := tsk.(Configurer)
		if !ok {
			continue
		}
		err = cfg.Configure(ctx)
		if err != nil {
			return err
		}
	}

	fmt.Printf(">>> --- [data flow] --- nodes...\n")
	for tsk, node := range app.dflow.nodes {
		fmt.Printf(">>> ---[%s]---\n", tsk)
		fmt.Printf("    in:  %v\n", node.in)
		fmt.Printf("    out: %v\n", node.out)
	}

	fmt.Printf(">>> --- [data flow] --- edges...\n")
	edges := make([]string, 0, len(app.dflow.edges))
	for n := range app.dflow.edges {
		edges = append(edges, n)
	}
	sort.Strings(edges)
	fmt.Printf(" edges: %v\n", edges)

	app.state = fsm_CONFIGURED
	fmt.Printf(">>> app.configure... [done]\n")
	return err
}

func (app *appmgr) start(ctx Context) Error {
	var err Error
	for _, svc := range app.svcs {
		fmt.Printf(">>> starting [%s]...\n", svc.Name())
		err = svc.StartSvc(ctx)
		if err != nil {
			return err
		}
	}

	for _, tsk := range app.tsks {
		fmt.Printf(">>> starting [%s]...\n", tsk.Name())
		err = tsk.StartTask(ctx)
		if err != nil {
			return err
		}
	}

	app.state = fsm_STARTED
	return err
}

func (app *appmgr) run(ctx Context) Error {
	var err Error
	app.state = fsm_RUNNING
	for ievt := int64(0); ievt < app.evtmax; ievt++ {
		fmt.Printf(">>> app.running evt=%d...\n", ievt)
		for k := range app.dflow.edges {
			ch, ok := app.store.store[k]
			if ok {
				select {
				case vv := <-ch:
					if vv, ok := vv.(Deleter); ok {
						err = vv.Delete()
						if err != nil {
							return err
						}
					}
				default:
				}
			}
			app.store.store[k] = make(achan, 1)
		}
		errch := make(chan Error, len(app.tsks))
		for _, tsk := range app.tsks {
			go func(tsk Task) {
				//fmt.Printf(">>> running [%s]...\n", tsk.Name())
				ctx := context{
					id:    ievt,
					slot:  0,
					store: app.store,
					msg: msgstream{
						lvl: LvlInfo,
						w:   os.Stdout,
						n:   tsk.Name(),
					},
				}
				errch <- tsk.Process(ctx)
			}(tsk)
		}
		for i := 0; i < len(app.tsks); i++ {
			err := <-errch
			if err != nil {
				close(errch)
				return err
			}

		}
	}
	return err
}

func (app *appmgr) stop(ctx Context) Error {
	var err Error
	for _, tsk := range app.tsks {
		err = tsk.StopTask(ctx)
		if err != nil {
			return err
		}
	}

	for _, svc := range app.svcs {
		err = svc.StopSvc(ctx)
		if err != nil {
			return err
		}
	}

	app.state = fsm_STOPPED
	return err
}

func (app *appmgr) shutdown(ctx Context) Error {
	var err Error
	app.comps = nil
	app.tsks = nil
	app.svcs = nil
	app.state = fsm_OFFLINE

	app.props = nil
	app.dflow = nil
	app.store = nil

	g_mgr = nil
	return err
}

func init() {
	Register(reflect.TypeOf(appmgr{}))
}

// EOF
<|endoftext|>


<commit_before>package simplyput

// TODO: PropertyList can support nested objects with named properties like "A.B.C", support nested JSON objects.
// TODO: Add rudimentary single-property queries, pagination, sorting, etc.
// TODO: Add memcache
// TODO: Support ETags, If-Modified-Since, etc. (http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html)
// TODO: PUT requests
// TODO: HEAD requests
// TODO: PATCH requests/semantics
// TODO: Batch requests (via multipart?)
// TODO: User POSTs a JSON schema, future requests are validated against that schema. Would anybody use that?

import (
	"appengine"
	"appengine/datastore"
	"appengine/urlfetch"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"
)

const (
	idKey        = "_id"
	createdKey   = "_created"
	updatedKey   = "_updated"
	defaultLimit = 10
)

func init() {
	http.HandleFunc("/datastore/v1dev/objects/", handle)
}

type userQuery struct {
	Limit, Offset int
	FilterKey, FilterType, FilterValue,
	StartCursor, EndCursor string
}

// getUserID gets the Google User ID for an access token.
func getUserID(accessToken string, client http.Client) (string, error) {
	resp, err := client.Get("https://www.googleapis.com/oauth2/v1/userinfo?access_token=" + accessToken)
	if err != nil {
		return "", err
	}
	var info struct {
		ID string
	}
	if err = json.NewDecoder(resp.Body).Decode(&info); err != nil {
		return "", err
	}
	resp.Body.Close()
	id := info.ID
	if id == "" {
		return "", errors.New("invalid auth")
	}
	return id, nil
}

// getKindAndID parses the kind and ID from a request path.
func getKindAndID(path string) (string, int64, error) {
	if match, err := regexp.MatchString("/datastore/v1dev/objects/[a-zA-Z]+/[0-9]+", path); err != nil {
		return "", int64(0), err
	} else if match {
		kind := path[len("/datastore/v1dev/objects/"):strings.LastIndex(path, "/")]
		idStr := path[strings.LastIndex(path, "/")+1:]
		id, err := strconv.ParseInt(idStr, 10, 64)
		if err != nil {
			return "", int64(0), err
		}
		return kind, id, nil
	}
	if match, err := regexp.MatchString("/datastore/v1dev/objects/[a-zA-Z]+", path); err != nil {
		return "", int64(0), err
	} else if match {
		kind := path[len("/datastore/v1dev/objects/"):]
		return kind, int64(0), nil
	}
	return "", int64(0), errors.New("invalid path")
}

// handle dispatches requests to the relevant API method and arranges certain common state
func handle(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	w.Header().Add("Access-Control-Allow-Origin", "*")

	r.ParseForm()
	client := urlfetch.Client(c)

	var userID string
	if appengine.IsDevAppServer() {
		// For local development, don't require an access token or user ID
		// If the user_id param is set, that's the user ID.
		userID = r.Form.Get("user_id")
	} else {
		// Get the access_token from the request and turn it into a user ID with which we will namespace Kinds in the datastore.
		accessToken := r.Form.Get("access_token")
		if accessToken == "" {
			h := r.Header.Get("Authorization")
			if strings.HasPrefix(h, "Bearer ") {
				accessToken = h[len("Bearer "):]
			}
		}
		if accessToken == "" {
			http.Error(w, "Unauthorized", http.StatusUnauthorized)
			return
		}
		var err error // Needed because otherwise the next line shadows userID...
		userID, err = getUserID(accessToken, *client)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	}

	kind, id, err := getKindAndID(r.URL.Path)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	dsKind := fmt.Sprintf("%s--%s", userID, kind)

	resp := make(map[string]interface{}, 0)
	errCode := http.StatusOK
	if id == int64(0) {
		switch r.Method {
		case "POST":
			resp, errCode = insert(c, dsKind, r.Body)
			r.Body.Close()
		case "GET":
			resp, errCode = list(c, dsKind, newUserQuery(r))
		default:
			http.Error(w, "Unsupported Method", http.StatusMethodNotAllowed)
			return
		}
	} else {
		switch r.Method {
		case "GET":
			resp, errCode = get(c, dsKind, id)
		case "DELETE":
			errCode = delete(c, dsKind, id)
		case "POST":
			// This is strictly "replace all properties/values", not "add new properties, update existing"
			resp, errCode = update(c, dsKind, id, r.Body)
			r.Body.Close()
		default:
			http.Error(w, "Unsupported Method", http.StatusMethodNotAllowed)
			return
		}
	}
	if errCode != http.StatusOK {
		http.Error(w, "", errCode)
		return
	}
	if err := json.NewEncoder(w).Encode(&resp); err != nil {
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}
	w.Header().Add("Content-Type", "application/json")
}

func newUserQuery(r *http.Request) userQuery {
	uq := userQuery{
		StartCursor: r.FormValue("start"),
		EndCursor:   r.FormValue("end"),
	}
	// TODO: MustParse for limit/offset, else panic
	uq.Limit, _ = strconv.Atoi(r.FormValue("limit"))
	uq.Offset, _ = strconv.Atoi(r.FormValue("offset"))

	// TODO: Support ?where=foo<bar queries (which may or may not be annoying to scope for users...)
	_ = r.FormValue("where")
	return uq
}

func delete(c appengine.Context, kind string, id int64) int {
	k := datastore.NewKey(c, kind, "", id, nil)
	if err := datastore.Delete(c, k); err != nil {
		if err == datastore.ErrNoSuchEntity {
			return http.StatusNotFound
		} else {
			c.Errorf("%v", err)
			return http.StatusInternalServerError
		}
	}
	return http.StatusOK
}

func get(c appengine.Context, kind string, id int64) (map[string]interface{}, int) {
	k := datastore.NewKey(c, kind, "", id, nil)
	var pl plist
	if err := datastore.Get(c, k, &pl); err != nil {
		if err == datastore.ErrNoSuchEntity {
			return nil, http.StatusNotFound
		}
		c.Errorf("%v", err)
		return nil, http.StatusInternalServerError
	}
	m := plistToMap(pl, k.IntID())
	m[idKey] = k.IntID()
	return m, http.StatusOK
}

func insert(c appengine.Context, kind string, r io.Reader) (map[string]interface{}, int) {
	var m map[string]interface{}
	if err := json.NewDecoder(r).Decode(&m); err != nil {
		c.Errorf("%v", err)
		return nil, http.StatusInternalServerError
	}
	m[createdKey] = time.Now().Unix()

	pl := mapToPlist("", m)

	k := datastore.NewIncompleteKey(c, kind, nil)
	k, err := datastore.Put(c, k, &pl)
	if err != nil {
		c.Errorf("%v", err)
		return nil, http.StatusInternalServerError
	}
	m[idKey] = k.IntID()
	return m, http.StatusOK
}

type prop struct {
	Name     string
	Value    interface{}
	Multiple bool
	NoIndex  bool
}
type plist []prop

// plistToMap transforms a plist such as you would get from the datastore into a map[string]interface{} suitable for JSON-encoding.
func plistToMap(pl plist, id int64) map[string]interface{} {
	m := make(map[string]interface{})
	for _, p := range pl {
		if strings.Contains(p.Name, ".") {
			parts := strings.Split(p.Name, ".")
			sub := m
			for _, p := range parts[:len(parts)-1] {
				// Traverse the path up until the leaf
				if i, exists := sub[p]; exists {
					// Already seen this path, traverse it
					if ii, ok := i.(map[string]interface{}); ok {
						sub = ii
					} else {
						// Got a sub-property of a non-map property. Uh oh...
						// Not sure it's worth failing/logging though...
					}
				} else {
					// First time down this path, add a new empty map
					next := map[string]interface{}{}
					sub[p] = next
					sub = next
				}
			}
			leaf := parts[len(parts)-1]
			if _, exists := sub[leaf]; exists {
				if !p.Multiple {
					// We would expect p.Multiple to be true here.
					// Not sure it's worth failing/logging though...
				}
				if _, isArr := sub[leaf].([]interface{}); isArr {
					// Already an array here, append to it
					sub[leaf] = append(sub[leaf].([]interface{}), p.Value)
				} else {
					// Already a single value here, should be an array now.
					sub[leaf] = []interface{}{sub[leaf], p.Value}
				}
			} else {
				sub[leaf] = p.Value
			}
		} else {
			if _, exists := m[p.Name]; exists {
				if !p.Multiple {
					// We would expect p.Multiple to be true here.
					// Not sure it's worth failing/logging though...
				}
				if _, isArr := m[p.Name].([]interface{}); isArr {
					// Already an array here, append to it
					m[p.Name] = append(m[p.Name].([]interface{}), p.Value)
				} else {
					// Already a single value here, should be an array now.
					m[p.Name] = []interface{}{m[p.Name], p.Value}
				}
			} else {
				m[p.Name] = p.Value
			}
		}
	}
	m[idKey] = id
	return m
}

// mapToPlist transforms a map[string]interface{} such as you would get from decoding JSON into a plist to store in the datastore.
func mapToPlist(prefix string, m map[string]interface{}) plist {
	pl := make(plist, 0, len(m))
	for k, v := range m {
		if m, nest := v.(map[string]interface{}); nest {
			// Generate a plist for this sub-map, and append it
			pl = append(pl, mapToPlist(prefix+k+".", m)...)
		} else if _, mult := v.([]interface{}); mult {
			// Generate a prop for every item in the slice
			for _, mv := range v.([]interface{}) {
				pl = append(pl, prop{
					Name:     prefix + k,
					Value:    mv,
					Multiple: true,
				})
			}
			// TODO: Apparently no way to store an empty list? That seems odd...
		} else {
			pl = append(pl, prop{
				Name:  prefix + k,
				Value: v,
			})
		}
	}
	return pl
}

func list(c appengine.Context, kind string, uq userQuery) (map[string]interface{}, int) {
	q := datastore.NewQuery(kind)

	if uq.Limit != 0 {
		q = q.Limit(uq.Limit)
	}
	if c, err := datastore.DecodeCursor(uq.StartCursor); err == nil {
		q.Start(c)
	}
	if c, err := datastore.DecodeCursor(uq.EndCursor); err == nil {
		q.End(c)
	}

	items := make([]map[string]interface{}, 0)

	var crs datastore.Cursor
	for t := q.Run(c); ; {
		var pl plist
		k, err := t.Next(&pl)
		if err == datastore.Done {
			break
		}
		if err != nil {
			c.Errorf("%v", err)
			return nil, http.StatusInternalServerError
		}
		m := plistToMap(pl, k.IntID())
		items = append(items, m)
		if crs, err = t.Cursor(); err != nil {
			c.Errorf("%v", err)
			return nil, http.StatusInternalServerError
		}
	}
	r := map[string]interface{}{
		"items":          items,
		"nextStartToken": crs.String(),
	}
	return r, http.StatusOK
}

func update(c appengine.Context, kind string, id int64, r io.Reader) (map[string]interface{}, int) {
	var m map[string]interface{}
	if err := json.NewDecoder(r).Decode(&m); err != nil {
		c.Errorf("%v", err)
		return nil, http.StatusInternalServerError
	}
	m[updatedKey] = time.Now().Unix()

	pl := mapToPlist("", m)

	k := datastore.NewKey(c, kind, "", id, nil)
	if _, err := datastore.Put(c, k, &pl); err != nil {
		c.Errorf("%v", err)
		return nil, http.StatusInternalServerError
	}
	m[idKey] = id
	return m, http.StatusOK
}
<commit_msg>Remove TODO about nested maps<commit_after>package simplyput

// TODO: Add rudimentary single-property queries, pagination, sorting, etc.
// TODO: Add memcache
// TODO: Support ETags, If-Modified-Since, etc. (http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html)
// TODO: PUT requests
// TODO: HEAD requests
// TODO: PATCH requests/semantics
// TODO: Batch requests (via multipart?)
// TODO: User POSTs a JSON schema, future requests are validated against that schema. Would anybody use that?

import (
	"appengine"
	"appengine/datastore"
	"appengine/urlfetch"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"
)

const (
	idKey        = "_id"
	createdKey   = "_created"
	updatedKey   = "_updated"
	defaultLimit = 10
)

func init() {
	http.HandleFunc("/datastore/v1dev/objects/", handle)
}

type userQuery struct {
	Limit, Offset int
	FilterKey, FilterType, FilterValue,
	StartCursor, EndCursor string
}

// getUserID gets the Google User ID for an access token.
func getUserID(accessToken string, client http.Client) (string, error) {
	resp, err := client.Get("https://www.googleapis.com/oauth2/v1/userinfo?access_token=" + accessToken)
	if err != nil {
		return "", err
	}
	var info struct {
		ID string
	}
	if err = json.NewDecoder(resp.Body).Decode(&info); err != nil {
		return "", err
	}
	resp.Body.Close()
	id := info.ID
	if id == "" {
		return "", errors.New("invalid auth")
	}
	return id, nil
}

// getKindAndID parses the kind and ID from a request path.
func getKindAndID(path string) (string, int64, error) {
	if match, err := regexp.MatchString("/datastore/v1dev/objects/[a-zA-Z]+/[0-9]+", path); err != nil {
		return "", int64(0), err
	} else if match {
		kind := path[len("/datastore/v1dev/objects/"):strings.LastIndex(path, "/")]
		idStr := path[strings.LastIndex(path, "/")+1:]
		id, err := strconv.ParseInt(idStr, 10, 64)
		if err != nil {
			return "", int64(0), err
		}
		return kind, id, nil
	}
	if match, err := regexp.MatchString("/datastore/v1dev/objects/[a-zA-Z]+", path); err != nil {
		return "", int64(0), err
	} else if match {
		kind := path[len("/datastore/v1dev/objects/"):]
		return kind, int64(0), nil
	}
	return "", int64(0), errors.New("invalid path")
}

// handle dispatches requests to the relevant API method and arranges certain common state
func handle(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	w.Header().Add("Access-Control-Allow-Origin", "*")

	r.ParseForm()
	client := urlfetch.Client(c)

	var userID string
	if appengine.IsDevAppServer() {
		// For local development, don't require an access token or user ID
		// If the user_id param is set, that's the user ID.
		userID = r.Form.Get("user_id")
	} else {
		// Get the access_token from the request and turn it into a user ID with which we will namespace Kinds in the datastore.
		accessToken := r.Form.Get("access_token")
		if accessToken == "" {
			h := r.Header.Get("Authorization")
			if strings.HasPrefix(h, "Bearer ") {
				accessToken = h[len("Bearer "):]
			}
		}
		if accessToken == "" {
			http.Error(w, "Unauthorized", http.StatusUnauthorized)
			return
		}
		var err error // Needed because otherwise the next line shadows userID...
		userID, err = getUserID(accessToken, *client)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	}

	kind, id, err := getKindAndID(r.URL.Path)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	dsKind := fmt.Sprintf("%s--%s", userID, kind)

	resp := make(map[string]interface{}, 0)
	errCode := http.StatusOK
	if id == int64(0) {
		switch r.Method {
		case "POST":
			resp, errCode = insert(c, dsKind, r.Body)
			r.Body.Close()
		case "GET":
			resp, errCode = list(c, dsKind, newUserQuery(r))
		default:
			http.Error(w, "Unsupported Method", http.StatusMethodNotAllowed)
			return
		}
	} else {
		switch r.Method {
		case "GET":
			resp, errCode = get(c, dsKind, id)
		case "DELETE":
			errCode = delete(c, dsKind, id)
		case "POST":
			// This is strictly "replace all properties/values", not "add new properties, update existing"
			resp, errCode = update(c, dsKind, id, r.Body)
			r.Body.Close()
		default:
			http.Error(w, "Unsupported Method", http.StatusMethodNotAllowed)
			return
		}
	}
	if errCode != http.StatusOK {
		http.Error(w, "", errCode)
		return
	}
	if err := json.NewEncoder(w).Encode(&resp); err != nil {
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}
	w.Header().Add("Content-Type", "application/json")
}

func newUserQuery(r *http.Request) userQuery {
	uq := userQuery{
		StartCursor: r.FormValue("start"),
		EndCursor:   r.FormValue("end"),
	}
	// TODO: MustParse for limit/offset, else panic
	uq.Limit, _ = strconv.Atoi(r.FormValue("limit"))
	uq.Offset, _ = strconv.Atoi(r.FormValue("offset"))

	// TODO: Support ?where=foo<bar queries (which may or may not be annoying to scope for users...)
	_ = r.FormValue("where")
	return uq
}

func delete(c appengine.Context, kind string, id int64) int {
	k := datastore.NewKey(c, kind, "", id, nil)
	if err := datastore.Delete(c, k); err != nil {
		if err == datastore.ErrNoSuchEntity {
			return http.StatusNotFound
		} else {
			c.Errorf("%v", err)
			return http.StatusInternalServerError
		}
	}
	return http.StatusOK
}

func get(c appengine.Context, kind string, id int64) (map[string]interface{}, int) {
	k := datastore.NewKey(c, kind, "", id, nil)
	var pl plist
	if err := datastore.Get(c, k, &pl); err != nil {
		if err == datastore.ErrNoSuchEntity {
			return nil, http.StatusNotFound
		}
		c.Errorf("%v", err)
		return nil, http.StatusInternalServerError
	}
	m := plistToMap(pl, k.IntID())
	m[idKey] = k.IntID()
	return m, http.StatusOK
}

func insert(c appengine.Context, kind string, r io.Reader) (map[string]interface{}, int) {
	var m map[string]interface{}
	if err := json.NewDecoder(r).Decode(&m); err != nil {
		c.Errorf("%v", err)
		return nil, http.StatusInternalServerError
	}
	m[createdKey] = time.Now().Unix()

	pl := mapToPlist("", m)

	k := datastore.NewIncompleteKey(c, kind, nil)
	k, err := datastore.Put(c, k, &pl)
	if err != nil {
		c.Errorf("%v", err)
		return nil, http.StatusInternalServerError
	}
	m[idKey] = k.IntID()
	return m, http.StatusOK
}

type prop struct {
	Name     string
	Value    interface{}
	Multiple bool
	NoIndex  bool
}
type plist []prop

// plistToMap transforms a plist such as you would get from the datastore into a map[string]interface{} suitable for JSON-encoding.
func plistToMap(pl plist, id int64) map[string]interface{} {
	m := make(map[string]interface{})
	for _, p := range pl {
		if strings.Contains(p.Name, ".") {
			parts := strings.Split(p.Name, ".")
			sub := m
			for _, p := range parts[:len(parts)-1] {
				// Traverse the path up until the leaf
				if i, exists := sub[p]; exists {
					// Already seen this path, traverse it
					if ii, ok := i.(map[string]interface{}); ok {
						sub = ii
					} else {
						// Got a sub-property of a non-map property. Uh oh...
						// Not sure it's worth failing/logging though...
					}
				} else {
					// First time down this path, add a new empty map
					next := map[string]interface{}{}
					sub[p] = next
					sub = next
				}
			}
			leaf := parts[len(parts)-1]
			if _, exists := sub[leaf]; exists {
				if !p.Multiple {
					// We would expect p.Multiple to be true here.
					// Not sure it's worth failing/logging though...
				}
				if _, isArr := sub[leaf].([]interface{}); isArr {
					// Already an array here, append to it
					sub[leaf] = append(sub[leaf].([]interface{}), p.Value)
				} else {
					// Already a single value here, should be an array now.
					sub[leaf] = []interface{}{sub[leaf], p.Value}
				}
			} else {
				sub[leaf] = p.Value
			}
		} else {
			if _, exists := m[p.Name]; exists {
				if !p.Multiple {
					// We would expect p.Multiple to be true here.
					// Not sure it's worth failing/logging though...
				}
				if _, isArr := m[p.Name].([]interface{}); isArr {
					// Already an array here, append to it
					m[p.Name] = append(m[p.Name].([]interface{}), p.Value)
				} else {
					// Already a single value here, should be an array now.
					m[p.Name] = []interface{}{m[p.Name], p.Value}
				}
			} else {
				m[p.Name] = p.Value
			}
		}
	}
	m[idKey] = id
	return m
}

// mapToPlist transforms a map[string]interface{} such as you would get from decoding JSON into a plist to store in the datastore.
func mapToPlist(prefix string, m map[string]interface{}) plist {
	pl := make(plist, 0, len(m))
	for k, v := range m {
		if m, nest := v.(map[string]interface{}); nest {
			// Generate a plist for this sub-map, and append it
			pl = append(pl, mapToPlist(prefix+k+".", m)...)
		} else if _, mult := v.([]interface{}); mult {
			// Generate a prop for every item in the slice
			for _, mv := range v.([]interface{}) {
				pl = append(pl, prop{
					Name:     prefix + k,
					Value:    mv,
					Multiple: true,
				})
			}
			// TODO: Apparently no way to store an empty list? That seems odd...
		} else {
			pl = append(pl, prop{
				Name:  prefix + k,
				Value: v,
			})
		}
	}
	return pl
}

func list(c appengine.Context, kind string, uq userQuery) (map[string]interface{}, int) {
	q := datastore.NewQuery(kind)

	if uq.Limit != 0 {
		q = q.Limit(uq.Limit)
	}
	if c, err := datastore.DecodeCursor(uq.StartCursor); err == nil {
		q.Start(c)
	}
	if c, err := datastore.DecodeCursor(uq.EndCursor); err == nil {
		q.End(c)
	}

	items := make([]map[string]interface{}, 0)

	var crs datastore.Cursor
	for t := q.Run(c); ; {
		var pl plist
		k, err := t.Next(&pl)
		if err == datastore.Done {
			break
		}
		if err != nil {
			c.Errorf("%v", err)
			return nil, http.StatusInternalServerError
		}
		m := plistToMap(pl, k.IntID())
		items = append(items, m)
		if crs, err = t.Cursor(); err != nil {
			c.Errorf("%v", err)
			return nil, http.StatusInternalServerError
		}
	}
	r := map[string]interface{}{
		"items":          items,
		"nextStartToken": crs.String(),
	}
	return r, http.StatusOK
}

func update(c appengine.Context, kind string, id int64, r io.Reader) (map[string]interface{}, int) {
	var m map[string]interface{}
	if err := json.NewDecoder(r).Decode(&m); err != nil {
		c.Errorf("%v", err)
		return nil, http.StatusInternalServerError
	}
	m[updatedKey] = time.Now().Unix()

	pl := mapToPlist("", m)

	k := datastore.NewKey(c, kind, "", id, nil)
	if _, err := datastore.Put(c, k, &pl); err != nil {
		c.Errorf("%v", err)
		return nil, http.StatusInternalServerError
	}
	m[idKey] = id
	return m, http.StatusOK
}
<|endoftext|>


<commit_before>package libxml2

import (
	"fmt"
	"os"
	"testing"

	"github.com/lestrrat/go-libxml2/dom"
	"github.com/lestrrat/go-libxml2/parser"
	"github.com/lestrrat/go-libxml2/types"
	"github.com/stretchr/testify/assert"
)

func TestEncoding(t *testing.T) {
	for _, enc := range []string{`utf-8`, `sjis`, `euc-jp`} {
		fn := fmt.Sprintf(`test/%s.xml`, enc)
		f, err := os.Open(fn)
		if err != nil {
			t.Errorf("Failed to open %s: %s", fn, err)
			return
		}
		defer f.Close()

		p := parser.New()
		doc, err := p.ParseReader(f)
		if err != nil {
			t.Errorf("Failed to parse %s: %s", fn, err)
			return
		}

		if doc.Encoding() != enc {
			t.Errorf("Expected encoding %s, got %s", enc, doc.Encoding())
			return
		}
	}
}

func TestNamespacedReconciliation(t *testing.T) {
	d := dom.CreateDocument()
	root, err := d.CreateElement("foo")
	if !assert.NoError(t, err, "failed to create document") {
		return
	}
	d.SetDocumentElement(root)
	if !assert.NoError(t, root.SetNamespace("http://default", "root"), "SetNamespace should succeed") {
		return
	}

	if !assert.NoError(t, root.SetNamespace("http://children", "child", false), "SetNamespace (no-activate) should succeed") {
		return
	}

	n, err := d.CreateElementNS("http://default", "branch")
	if !assert.NoError(t, err, "CreateElementNS should succeed") {
		return
	}
	root.AddChild(n)

	_, err = n.GetAttribute("xmlns")
	if !assert.Error(t, err, "GetAttribute should fail with not found") ||
		!assert.Equal(t, "attribute not found", err.Error(), "error matches") {
		return
	}

	var c types.Element
	for _, name := range []string{"a", "b", "c"} {
		child, err := d.CreateElementNS("http://children", "child:"+name)
		if !assert.NoError(t, err, "CreateElementNS should succeed") {
			return
		}
		if name == "c" {
			c = child
		}
		n.AddChild(child)
		_, err = n.GetAttribute("xmlns:child")
		if !assert.Error(t, err, "GetAttribute should fail with not found") ||
			!assert.Equal(t, "attribute not found", err.Error(), "error matches") {
			return
		}
	}

	if !assert.NoError(t, c.SetAttribute("xmlns:foo", "http://children"), "SetAttribute should succeeed") {
		return
	}

	attr, err := c.GetAttribute("xmlns:foo")
	if !assert.NoError(t, err, "xmlns:foo should exist") {
		return
	}
	if !assert.Equal(t, "http://children", attr.Value(), "attribute matches") {
		return
	}

	child, err := d.CreateElementNS("http://other", "branch")
	if !assert.NoError(t, err, "creating element with default namespace") {
		return
	}
	n.AddChild(child)

	// XXX This still fails
	/*
		attr, err = child.GetAttribute("xmlns")
		if !assert.NoError(t, err, "GetAttribute should succeed") {
			return
		}
		if !assert.Equal(t, "http://other", attr.Value(), "attribute matches") {
			return
		}
	*/

	t.Logf("%s", d.String())
}
<commit_msg>Add test for #7<commit_after>package libxml2

import (
	"fmt"
	"os"
	"testing"

	"github.com/lestrrat/go-libxml2/dom"
	"github.com/lestrrat/go-libxml2/parser"
	"github.com/lestrrat/go-libxml2/types"
	"github.com/lestrrat/go-libxml2/xpath"
	"github.com/stretchr/testify/assert"
)

func TestEncoding(t *testing.T) {
	for _, enc := range []string{`utf-8`, `sjis`, `euc-jp`} {
		fn := fmt.Sprintf(`test/%s.xml`, enc)
		f, err := os.Open(fn)
		if err != nil {
			t.Errorf("Failed to open %s: %s", fn, err)
			return
		}
		defer f.Close()

		p := parser.New()
		doc, err := p.ParseReader(f)
		if err != nil {
			t.Errorf("Failed to parse %s: %s", fn, err)
			return
		}

		if doc.Encoding() != enc {
			t.Errorf("Expected encoding %s, got %s", enc, doc.Encoding())
			return
		}
	}
}

func TestNamespacedReconciliation(t *testing.T) {
	d := dom.CreateDocument()
	root, err := d.CreateElement("foo")
	if !assert.NoError(t, err, "failed to create document") {
		return
	}
	d.SetDocumentElement(root)
	if !assert.NoError(t, root.SetNamespace("http://default", "root"), "SetNamespace should succeed") {
		return
	}

	if !assert.NoError(t, root.SetNamespace("http://children", "child", false), "SetNamespace (no-activate) should succeed") {
		return
	}

	n, err := d.CreateElementNS("http://default", "branch")
	if !assert.NoError(t, err, "CreateElementNS should succeed") {
		return
	}
	root.AddChild(n)

	_, err = n.GetAttribute("xmlns")
	if !assert.Error(t, err, "GetAttribute should fail with not found") ||
		!assert.Equal(t, "attribute not found", err.Error(), "error matches") {
		return
	}

	var c types.Element
	for _, name := range []string{"a", "b", "c"} {
		child, err := d.CreateElementNS("http://children", "child:"+name)
		if !assert.NoError(t, err, "CreateElementNS should succeed") {
			return
		}
		if name == "c" {
			c = child
		}
		n.AddChild(child)
		_, err = n.GetAttribute("xmlns:child")
		if !assert.Error(t, err, "GetAttribute should fail with not found") ||
			!assert.Equal(t, "attribute not found", err.Error(), "error matches") {
			return
		}
	}

	if !assert.NoError(t, c.SetAttribute("xmlns:foo", "http://children"), "SetAttribute should succeeed") {
		return
	}

	attr, err := c.GetAttribute("xmlns:foo")
	if !assert.NoError(t, err, "xmlns:foo should exist") {
		return
	}
	if !assert.Equal(t, "http://children", attr.Value(), "attribute matches") {
		return
	}

	child, err := d.CreateElementNS("http://other", "branch")
	if !assert.NoError(t, err, "creating element with default namespace") {
		return
	}
	n.AddChild(child)

	// XXX This still fails
	/*
		attr, err = child.GetAttribute("xmlns")
		if !assert.NoError(t, err, "GetAttribute should succeed") {
			return
		}
		if !assert.Equal(t, "http://other", attr.Value(), "attribute matches") {
			return
		}
	*/

	t.Logf("%s", d.String())
}

func TestRegressionGH7(t *testing.T) {
	doc, err := ParseHTMLString(`

Hello, World!

`)

	if !assert.NoError(t, err, "ParseHTMLString should succeed") {
		return
	}

	nodes := xpath.NodeList(doc.Find(`./body/div/div`))
	if !assert.NotEmpty(t, nodes, "Find should succeed") {
		return
	}

	v, err := nodes.Literal()
	if !assert.NoError(t, err, "Literal() should succeed") {
		return
	}
	if !assert.NotEmpty(t, v, "Literal() should return some string") {
		return
	}
	t.Logf("v = '%s'", v)
}
<|endoftext|>


<commit_before>// Copyright 2016 The appc Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package docker2aci

import "github.com/appc/spec/schema"

var Version = "0.11.0"
var AppcVersion = schema.AppContainerVersion
<commit_msg>version: bump to v0.11.0+git<commit_after>// Copyright 2016 The appc Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package docker2aci

import "github.com/appc/spec/schema"

var Version = "0.11.0+git"
var AppcVersion = schema.AppContainerVersion
<|endoftext|>


<commit_before>package main

import (
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
)

type FStore struct {
	Path    string
	TmpPath string
}

func (store FStore) AddFile(r io.Reader) (string, error) {
	file, err := ioutil.TempFile(store.TmpPath, "rabbity-tmp")
	if err != nil {
		return "", err
	}
	written, hashsum, err := Sha3HashCopy(file, r)
	if err != nil {
		return "", err
	}
	if err := file.Close(); err != nil {
		return "", err
	}
	log.Printf("Wrote %v bytes to %v (hashsum is %v)", written, file.Name(), hashsum)
	log.Printf("Moving %v to %v", file.Name(), store.getHashPath(hashsum))
	err = os.Rename(file.Name(), store.getHashPath(hashsum))
	return hashsum, nil
}

func (store FStore) getHashPath(hashsum string) string {
	dirPath := filepath.Join(store.Path, string(hashsum[0]), string(hashsum[1]))
	err := os.MkdirAll(dirPath, 0700)
	if err != nil {
		panic(err)
	}
	return filepath.Join(dirPath, hashsum[2:])
}

func (store FStore) getFilePath(hashsum string) string {
	return filepath.Join(store.Path, string(hashsum[0]),
		string(hashsum[1]), hashsum[2:])
}

func (store FStore) HasFile(hashsum string) bool {
	filePath := store.getFilePath(hashsum)
	if _, err := os.Stat(filePath); err == nil {
		return true
	}
	return false
}

func (store FStore) GetFile(hashsum string) (io.Reader, error) {
	return os.Open(store.getFilePath(hashsum))
}
<commit_msg>refer to fstore in methods by reference<commit_after>package main

import (
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
)

type FStore struct {
	Path    string
	TmpPath string
}

func (store *FStore) AddFile(r io.Reader) (string, error) {
	file, err := ioutil.TempFile(store.TmpPath, "rabbity-tmp")
	if err != nil {
		return "", err
	}
	written, hashsum, err := Sha3HashCopy(file, r)
	if err != nil {
		return "", err
	}
	if err := file.Close(); err != nil {
		return "", err
	}
	log.Printf("Wrote %v bytes to %v (hashsum is %v)", written, file.Name(), hashsum)
	log.Printf("Moving %v to %v", file.Name(), store.getHashPath(hashsum))
	err = os.Rename(file.Name(), store.getHashPath(hashsum))
	return hashsum, nil
}

func (store *FStore) getHashPath(hashsum string) string {
	dirPath := filepath.Join(store.Path, string(hashsum[0]), string(hashsum[1]))
	err := os.MkdirAll(dirPath, 0700)
	if err != nil {
		panic(err)
	}
	return filepath.Join(dirPath, hashsum[2:])
}

func (store *FStore) getFilePath(hashsum string) string {
	return filepath.Join(store.Path, string(hashsum[0]),
		string(hashsum[1]), hashsum[2:])
}

func (store *FStore) HasFile(hashsum string) bool {
	filePath := store.getFilePath(hashsum)
	if _, err := os.Stat(filePath); err == nil {
		return true
	}
	return false
}

func (store *FStore) GetFile(hashsum string) (io.Reader, error) {
	return os.Open(store.getFilePath(hashsum))
}
<|endoftext|>


<commit_before>package app_files

import (
	"crypto/sha1"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/cloudfoundry/cli/cf/models"
	"github.com/cloudfoundry/gofileutils/fileutils"
)

type AppFiles interface {
	AppFilesInDir(dir string) (appFiles []models.AppFileFields, err error)
	CopyFiles(appFiles []models.AppFileFields, fromDir, toDir string) (err error)
	CountFiles(directory string) int64
	WalkAppFiles(dir string, onEachFile func(string, string) error) (err error)
}

type ApplicationFiles struct{}

func (appfiles ApplicationFiles) AppFilesInDir(dir string) (appFiles []models.AppFileFields, err error) {
	dir, err = filepath.Abs(dir)
	if err != nil {
		return
	}

	err = appfiles.WalkAppFiles(dir, func(fileName string, fullPath string) error {
		fileInfo, err := os.Lstat(fullPath)
		if err != nil {
			return err
		}

		appFile := models.AppFileFields{
			Path: filepath.ToSlash(fileName),
			Size: fileInfo.Size(),
		}

		if fileInfo.IsDir() {
			appFile.Sha1 = "0"
			appFile.Size = 0
		} else {
			hash := sha1.New()
			file, err := os.Open(fullPath)
			if err != nil {
				return err
			}
			defer file.Close()

			_, err = io.Copy(hash, file)
			if err != nil {
				return err
			}

			appFile.Sha1 = fmt.Sprintf("%x", hash.Sum(nil))
		}

		appFiles = append(appFiles, appFile)

		return nil
	})

	return
}

func (appfiles ApplicationFiles) CopyFiles(appFiles []models.AppFileFields, fromDir, toDir string) (err error) {
	if err != nil {
		return
	}

	for _, file := range appFiles {
		fromPath := filepath.Join(fromDir, file.Path)
		toPath := filepath.Join(toDir, file.Path)
		err = copyPathToPath(fromPath, toPath)
		if err != nil {
			return
		}
	}
	return
}

func (appfiles ApplicationFiles) CountFiles(directory string) int64 {
	var count int64
	appfiles.WalkAppFiles(directory, func(_, _ string) error {
		count++
		return nil
	})
	return count
}

func (appfiles ApplicationFiles) WalkAppFiles(dir string, onEachFile func(string, string) error) (err error) {
	cfIgnore := loadIgnoreFile(dir)
	walkFunc := func(fullPath string, f os.FileInfo, inErr error) (err error) {
		err = inErr
		if err != nil {
			return
		}

		if fullPath == dir {
			return
		}

		if !f.Mode().IsRegular() && !f.IsDir() {
			return
		}

		fileRelativePath, _ := filepath.Rel(dir, fullPath)
		fileRelativeUnixPath := filepath.ToSlash(fileRelativePath)

		if !cfIgnore.FileShouldBeIgnored(fileRelativeUnixPath) {
			err = onEachFile(fileRelativePath, fullPath)
		}

		return
	}

	err = filepath.Walk(dir, walkFunc)
	return
}

func copyPathToPath(fromPath, toPath string) error {
	srcFileInfo, err := os.Stat(fromPath)
	if err != nil {
		return err
	}

	if srcFileInfo.IsDir() {
		err = os.MkdirAll(toPath, srcFileInfo.Mode())
		if err != nil {
			return err
		}
	} else {
		var dst *os.File
		dst, err = fileutils.Create(toPath)
		if err != nil {
			return err
		}
		defer dst.Close()

		dst.Chmod(srcFileInfo.Mode())

		src, err := os.Open(fromPath)
		if err != nil {
			return err
		}
		defer src.Close()

		_, err = io.Copy(dst, src)
		if err != nil {
			return err
		}
	}

	return nil
}

func loadIgnoreFile(dir string) CfIgnore {
	fileContents, err := ioutil.ReadFile(filepath.Join(dir, ".cfignore"))
	if err != nil {
		return NewCfIgnore("")
	}

	return NewCfIgnore(string(fileContents))
}
<commit_msg>Remove copyPathToPath<commit_after>package app_files

import (
	"crypto/sha1"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/cloudfoundry/cli/cf/models"
	"github.com/cloudfoundry/gofileutils/fileutils"
)

type AppFiles interface {
	AppFilesInDir(dir string) (appFiles []models.AppFileFields, err error)
	CopyFiles(appFiles []models.AppFileFields, fromDir, toDir string) (err error)
	CountFiles(directory string) int64
	WalkAppFiles(dir string, onEachFile func(string, string) error) (err error)
}

type ApplicationFiles struct{}

func (appfiles ApplicationFiles) AppFilesInDir(dir string) (appFiles []models.AppFileFields, err error) {
	dir, err = filepath.Abs(dir)
	if err != nil {
		return
	}

	err = appfiles.WalkAppFiles(dir, func(fileName string, fullPath string) error {
		fileInfo, err := os.Lstat(fullPath)
		if err != nil {
			return err
		}

		appFile := models.AppFileFields{
			Path: filepath.ToSlash(fileName),
			Size: fileInfo.Size(),
		}

		if fileInfo.IsDir() {
			appFile.Sha1 = "0"
			appFile.Size = 0
		} else {
			hash := sha1.New()
			file, err := os.Open(fullPath)
			if err != nil {
				return err
			}
			defer file.Close()

			_, err = io.Copy(hash, file)
			if err != nil {
				return err
			}

			appFile.Sha1 = fmt.Sprintf("%x", hash.Sum(nil))
		}

		appFiles = append(appFiles, appFile)

		return nil
	})

	return
}

func (appfiles ApplicationFiles) CopyFiles(appFiles []models.AppFileFields, fromDir, toDir string) error {
	for _, file := range appFiles {
		fromPath := filepath.Join(fromDir, file.Path)
		srcFileInfo, err := os.Stat(fromPath)
		if err != nil {
			return err
		}

		toPath := filepath.Join(toDir, file.Path)

		if srcFileInfo.IsDir() {
			err = os.MkdirAll(toPath, srcFileInfo.Mode())
			if err != nil {
				return err
			}
			continue
		}

		var dst *os.File
		dst, err = fileutils.Create(toPath)
		if err != nil {
			return err
		}
		defer dst.Close()

		dst.Chmod(srcFileInfo.Mode())

		src, err := os.Open(fromPath)
		if err != nil {
			return err
		}
		defer src.Close()

		_, err = io.Copy(dst, src)
		if err != nil {
			return err
		}
	}

	return nil
}

func (appfiles ApplicationFiles) CountFiles(directory string) int64 {
	var count int64
	appfiles.WalkAppFiles(directory, func(_, _ string) error {
		count++
		return nil
	})
	return count
}

func (appfiles ApplicationFiles) WalkAppFiles(dir string, onEachFile func(string, string) error) (err error) {
	cfIgnore := loadIgnoreFile(dir)
	walkFunc := func(fullPath string, f os.FileInfo, inErr error) (err error) {
		err = inErr
		if err != nil {
			return
		}

		if fullPath == dir {
			return
		}

		if !f.Mode().IsRegular() && !f.IsDir() {
			return
		}

		fileRelativePath, _ := filepath.Rel(dir, fullPath)
		fileRelativeUnixPath := filepath.ToSlash(fileRelativePath)

		if !cfIgnore.FileShouldBeIgnored(fileRelativeUnixPath) {
			err = onEachFile(fileRelativePath, fullPath)
		}

		return
	}

	err = filepath.Walk(dir, walkFunc)
	return
}

func loadIgnoreFile(dir string) CfIgnore {
	fileContents, err := ioutil.ReadFile(filepath.Join(dir, ".cfignore"))
	if err != nil {
		return NewCfIgnore("")
	}

	return NewCfIgnore(string(fileContents))
}
<|endoftext|>