<commit_before>/*
Open Source Initiative OSI - The MIT License (MIT):Licensing
The MIT License (MIT)
Copyright (c) 2017 Ralph Caraveo (deckarep@gmail.com)
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

package cmd

import (
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/deckarep/blade/lib/recipe"
	bladessh "github.com/deckarep/blade/lib/ssh"
	"github.com/spf13/cobra"
)

// Flag variables
var (
	retries     int
	concurrency int
	hosts       string
	port        int
	user        string
	quiet       bool
	verbose     bool
)

// blade ssh deploy-cloud-server-a // matches a recipe and therefore will follow the recipe guidelines against servers defined in recipe
// blade ssh deploy 127.0.0.1,128.0.0.1,129.0.0.1 //matches a recipe but uses the servers noted instead
// A recipe can be defined as the following
// 1. a single and simple command, with no servers specified, it's on you to provide a list.
// 2. a complex command, your recipe defines the server list.
// 3. a timed, tuned sequence of commands...perhaps it denotes a deploy order of multiple clusters, some sleep time, and perhaps even prompting to continue.
// 4. also, blade can cache server fetches...and has a default timeout
// 5. recipes are meant to be shared in some repository, this means that you can share with the team
// 6. recipes can be overriden per user, usually not a good idea...but possible still.
// 7. a recipe can be either:
//    a. a single command no servers specified/servers specified
//    b. a series of steps no servers specified/servers specified
//    c. a series of steps with sleep intervals baked in/continue prompt baked in
//    d. an aggregate, intended to execute on all servers but returning aggregates
//    e. banner prompts baked in, when dealing with production services (think: are you sure you want to continue?)
//    f. validation commands, intended on revealing some truthyness about infastructure
//    g. all recipes should emit proper log reports, and can tie into checklist as a service
//    h. integration into hipchat/slack/etc. when key events occur.
//    i. ability to view a simplified output of progress, notify end-user with gosx-notifier
//    j. ability to tie into ANY query infrastructure by allowing you to either pass in comma delimited servers
//          - or add something like this to your recipe: `ips prod redis-hosts -c`
// 8. blade generate where generate a database of all your recipes as Cobra Commands.
// 9. A seperate bolt database stores hosts cache for speed.

func init() {
	generateCommandLine()
	runCmd.PersistentFlags().StringVarP(&hosts,
		"servers", "s", "", "servers flag is one or more comma-delimited servers.")
	runCmd.PersistentFlags().IntVarP(&concurrency,
		"concurrency", "c", 0, "Max concurrency when running ssh commands")
	runCmd.PersistentFlags().IntVarP(&retries,
		"retries", "r", 3, "Number of times to retry until a successful command returns")
	runCmd.PersistentFlags().IntVarP(&port,
		"port", "p", 22, "The ssh port to use")
	runCmd.PersistentFlags().StringVarP(&user,
		"user", "u", "root", "user for ssh host login.")
	runCmd.PersistentFlags().BoolVarP(&quiet,
		"quiet", "q", false, "quiet mode will keep Blade as silent as possible.")
	runCmd.PersistentFlags().BoolVarP(&verbose,
		"verbose", "v", false, "verbose mode will keep Blade as verbose as possible.")
	RootCmd.AddCommand(runCmd)
}

var runCmd = &cobra.Command{
	Use:   "run",
	Short: "run [command]",
}

func scanBladeFolder(rootFolder string) []string {
	fileList := []string{}
	err := filepath.Walk(rootFolder, func(path string, f os.FileInfo, err error) error {
		fileList = append(fileList, path)
		return nil
	})

	if err != nil {
		log.Fatal("Failed to generate recipe data")
	}

	return fileList
}

func validateFlags() {
	if concurrency < 0 {
		log.Fatal("The specified --concurrency flag must not be a negative number.")
	}

	if port < 22 {
		log.Fatal("The specified --port flag must be 22 or greater.")
	}

	if quiet && verbose {
		log.Fatal("You must specify either --quiet or --verbose but not both.")
	}

	if retries < 0 {
		log.Fatal("The specified --retries flag must not be a negative number.")
	}
}

func applyRecipeFlagOverrides(currentRecipe *recipe.BladeRecipeYaml, cobraCommand *cobra.Command) {
	if len(currentRecipe.Args) > 0 {
		for _, arg := range currentRecipe.Args {
			arg.AttachFlag(cobraCommand)
		}
	}
}

func applyFlagOverrides(recipe *recipe.BladeRecipeYaml, modifier *bladessh.SessionModifier) {
	if hosts != "" {
		modifier.FlagOverrides.Hosts = strings.Split(strings.TrimSpace(hosts), ",")
	}

	if concurrency > 0 {
		modifier.FlagOverrides.Concurrency = concurrency
	}

	if port > 0 {
		modifier.FlagOverrides.Port = port
	}
}

func generateCommandLine() {
	fileList := scanBladeFolder("recipes/")
	commands := make(map[string]*cobra.Command)

	// For now let's skip the global.blade.yaml file.
	for _, file := range fileList {
		if strings.HasSuffix(file, ".blade.yaml") &&
			!strings.Contains(file, "global.blade.yaml") {
			parts := strings.Split(file, "/")
			var lastCommand *cobra.Command
			lastCommand = nil

			currentRecipe, err := recipe.LoadRecipeYaml(file)
			if err != nil {
				log.Println("Found a broken recipe...skipping: ", err.Error())
				continue
			}

			// parts[1:] drop the /recipe part.
			remainingParts := parts[1:]
			currentRecipe.Name = strings.TrimSuffix(strings.Join(remainingParts, "."), ".blade.toml")
			currentRecipe.Filename = file

			for _, p := range remainingParts {
				var currentCommand *cobra.Command

				// Known bug, we need to dedup these, but add them to a map based on their full path.
				// Reason is: if you have the same folder name in different hiearchies you'll collide.
				// This way we can add docs to describe command hiearchies when user uses the --help system.
				recipeAlreadyFound := false
				if _, ok := commands[p]; !ok {
					// If not found create it.
					currentCommand = &cobra.Command{
						Use:   p,
						Short: currentRecipe.Help.Short,
						Long:  currentRecipe.Help.Long,
					}
					commands[p] = currentCommand
				} else {
					// If found use it.
					currentCommand = commands[p]
					recipeAlreadyFound = true
				}

				// If we're not a dir but a blade.toml...set it up to Run.
				if strings.HasSuffix(p, "blade.yaml") {
					// Set the Use to just {recipe-name} of {recipe-name}.blade.toml.
					currentCommand.Use = strings.TrimSuffix(p, ".blade.yaml")
					applyRecipeFlagOverrides(currentRecipe, currentCommand)
					currentCommand.Run = func(cmd *cobra.Command, args []string) {
						// Apply validation of flags if used.
						validateFlags()

						modifier := bladessh.NewSessionModifier()

						// Apply flag overrides to the recipe here.
						applyFlagOverrides(currentRecipe, modifier)

						// Finally kick off session of requests.
						bladessh.StartSession(currentRecipe, modifier)
					}
				}

				// Only add recipe nodes we haven't already found.
				if !recipeAlreadyFound {
					if lastCommand == nil {
						runCmd.AddCommand(currentCommand)
					} else {
						lastCommand.AddCommand(currentCommand)
					}
				}

				lastCommand = currentCommand
			}
		}
	}
}
<commit_msg>Include ~/.blade/recipes for searching paths<commit_after>/*
Open Source Initiative OSI - The MIT License (MIT):Licensing
The MIT License (MIT)
Copyright (c) 2017 Ralph Caraveo (deckarep@gmail.com)
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

package cmd

import (
	"log"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/deckarep/blade/lib/recipe"
	bladessh "github.com/deckarep/blade/lib/ssh"
	"github.com/spf13/cobra"
)

// Flag variables
var (
	retries     int
	concurrency int
	hosts       string
	port        int
	user        string
	quiet       bool
	verbose     bool
)

// blade ssh deploy-cloud-server-a // matches a recipe and therefore will follow the recipe guidelines against servers defined in recipe
// blade ssh deploy 127.0.0.1,128.0.0.1,129.0.0.1 //matches a recipe but uses the servers noted instead
// A recipe can be defined as the following
// 1. a single and simple command, with no servers specified, it's on you to provide a list.
// 2. a complex command, your recipe defines the server list.
// 3. a timed, tuned sequence of commands...perhaps it denotes a deploy order of multiple clusters, some sleep time, and perhaps even prompting to continue.
// 4. also, blade can cache server fetches...and has a default timeout
// 5. recipes are meant to be shared in some repository, this means that you can share with the team
// 6. recipes can be overriden per user, usually not a good idea...but possible still.
// 7. a recipe can be either:
//    a. a single command no servers specified/servers specified
//    b. a series of steps no servers specified/servers specified
//    c. a series of steps with sleep intervals baked in/continue prompt baked in
//    d. an aggregate, intended to execute on all servers but returning aggregates
//    e. banner prompts baked in, when dealing with production services (think: are you sure you want to continue?)
//    f. validation commands, intended on revealing some truthyness about infastructure
//    g. all recipes should emit proper log reports, and can tie into checklist as a service
//    h. integration into hipchat/slack/etc. when key events occur.
//    i. ability to view a simplified output of progress, notify end-user with gosx-notifier
//    j. ability to tie into ANY query infrastructure by allowing you to either pass in comma delimited servers
//          - or add something like this to your recipe: `ips prod redis-hosts -c`
// 8. blade generate where generate a database of all your recipes as Cobra Commands.
// 9. A seperate bolt database stores hosts cache for speed.

func init() {
	generateCommandLine()
	runCmd.PersistentFlags().StringVarP(&hosts,
		"servers", "s", "", "servers flag is one or more comma-delimited servers.")
	runCmd.PersistentFlags().IntVarP(&concurrency,
		"concurrency", "c", 0, "Max concurrency when running ssh commands")
	runCmd.PersistentFlags().IntVarP(&retries,
		"retries", "r", 3, "Number of times to retry until a successful command returns")
	runCmd.PersistentFlags().IntVarP(&port,
		"port", "p", 22, "The ssh port to use")
	runCmd.PersistentFlags().StringVarP(&user,
		"user", "u", "root", "user for ssh host login.")
	runCmd.PersistentFlags().BoolVarP(&quiet,
		"quiet", "q", false, "quiet mode will keep Blade as silent as possible.")
	runCmd.PersistentFlags().BoolVarP(&verbose,
		"verbose", "v", false, "verbose mode will keep Blade as verbose as possible.")
	RootCmd.AddCommand(runCmd)
}

var runCmd = &cobra.Command{
	Use:   "run",
	Short: "run [command]",
}

func validateFlags() {
	if concurrency < 0 {
		log.Fatal("The specified --concurrency flag must not be a negative number.")
	}

	if port < 22 {
		log.Fatal("The specified --port flag must be 22 or greater.")
	}

	if quiet && verbose {
		log.Fatal("You must specify either --quiet or --verbose but not both.")
	}

	if retries < 0 {
		log.Fatal("The specified --retries flag must not be a negative number.")
	}
}

func applyRecipeFlagOverrides(currentRecipe *recipe.BladeRecipeYaml, cobraCommand *cobra.Command) {
	if len(currentRecipe.Args) > 0 {
		for _, arg := range currentRecipe.Args {
			arg.AttachFlag(cobraCommand)
		}
	}
}

func applyFlagOverrides(recipe *recipe.BladeRecipeYaml, modifier *bladessh.SessionModifier) {
	if hosts != "" {
		modifier.FlagOverrides.Hosts = strings.Split(strings.TrimSpace(hosts), ",")
	}

	if concurrency > 0 {
		modifier.FlagOverrides.Concurrency = concurrency
	}

	if port > 0 {
		modifier.FlagOverrides.Port = port
	}
}

func searchFolders(folders ...string) []string {
	var results []string

	for _, folder := range folders {
		if _, err := os.Stat(folder); err == nil {
			results = append(results, walkFolder(folder)...)
		}
	}

	return results
}

func walkFolder(rootFolder string) []string {
	fileList := []string{}
	err := filepath.Walk(rootFolder, func(path string, f os.FileInfo, err error) error {
		fileList = append(fileList, path)
		return nil
	})

	if err != nil {
		log.Fatal("Failed to generate recipe data")
	}

	return fileList
}

func userHomeDir() string {
	const bladeFolder = ".blade"

	if runtime.GOOS == "windows" {
		home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
		if home == "" {
			home = os.Getenv("USERPROFILE")
		}
		return path.Join(home, bladeFolder)
	}
	return path.Join(os.Getenv("HOME"), bladeFolder)
}

func generateCommandLine() {
	fileList := searchFolders(userHomeDir(), "recipes")
	commands := make(map[string]*cobra.Command)

	// For now let's skip the global.blade.yaml file.
	for _, file := range fileList {
		if strings.HasSuffix(file, ".blade.yaml") &&
			!strings.Contains(file, "global.blade.yaml") {
			parts := strings.Split(file, "/")
			var lastCommand *cobra.Command
			lastCommand = nil

			currentRecipe, err := recipe.LoadRecipeYaml(file)
			if err != nil {
				log.Println("Found a broken recipe...skipping: ", err.Error())
				continue
			}

			// Find and drop all /recipes folders including all parent dirs.
			var recipeIndex = 0
			for index, part := range parts {
				if part == "recipes" {
					recipeIndex = index
					break
				}
			}

			remainingParts := parts[recipeIndex+1:]
			currentRecipe.Name = strings.TrimSuffix(strings.Join(remainingParts, "."), ".blade.toml")
			currentRecipe.Filename = file

			for _, p := range remainingParts {
				var currentCommand *cobra.Command

				// Known bug, we need to dedup these, but add them to a map based on their full path.
				// Reason is: if you have the same folder name in different hiearchies you'll collide.
				// This way we can add docs to describe command hiearchies when user uses the --help system.
				recipeAlreadyFound := false
				if _, ok := commands[p]; !ok {
					// If not found create it.
					currentCommand = &cobra.Command{
						Use:   p,
						Short: currentRecipe.Help.Short,
						Long:  currentRecipe.Help.Long,
					}
					commands[p] = currentCommand
				} else {
					// If found use it.
					currentCommand = commands[p]
					recipeAlreadyFound = true
				}

				// If we're not a dir but a blade.toml...set it up to Run.
				if strings.HasSuffix(p, "blade.yaml") {
					// Set the Use to just {recipe-name} of {recipe-name}.blade.toml.
					currentCommand.Use = strings.TrimSuffix(p, ".blade.yaml")
					applyRecipeFlagOverrides(currentRecipe, currentCommand)
					currentCommand.Run = func(cmd *cobra.Command, args []string) {
						// Apply validation of flags if used.
						validateFlags()

						modifier := bladessh.NewSessionModifier()

						// Apply flag overrides to the recipe here.
						applyFlagOverrides(currentRecipe, modifier)

						// Finally kick off session of requests.
						bladessh.StartSession(currentRecipe, modifier)
					}
				}

				// Only add recipe nodes we haven't already found.
				if !recipeAlreadyFound {
					if lastCommand == nil {
						runCmd.AddCommand(currentCommand)
					} else {
						lastCommand.AddCommand(currentCommand)
					}
				}

				lastCommand = currentCommand
			}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package cmd

import (
	"context"
	"fmt"
	"net"
	"net/http"
	_ "net/http/pprof" // Used for debugging if enabled and a web server is running
	"os"
	"strings"

	"code.gitea.io/gitea/modules/graceful"
	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/setting"
	"code.gitea.io/gitea/routers"
	"code.gitea.io/gitea/routers/install"

	context2 "github.com/gorilla/context"
	"github.com/urfave/cli"
	ini "gopkg.in/ini.v1"
)

// CmdWeb represents the available web sub-command.
var CmdWeb = cli.Command{
	Name:  "web",
	Usage: "Start Gitea web server",
	Description: `Gitea web server is the only thing you need to run,
and it takes care of all the other things for you`,
	Action: runWeb,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "port, p",
			Value: "3000",
			Usage: "Temporary port number to prevent conflict",
		},
		cli.StringFlag{
			Name:  "install-port",
			Value: "3000",
			Usage: "Temporary port number to run the install page on to prevent conflict",
		},
		cli.StringFlag{
			Name:  "pid, P",
			Value: setting.PIDFile,
			Usage: "Custom pid file path",
		},
		cli.BoolFlag{
			Name:  "quiet, q",
			Usage: "Only display Fatal logging errors until logging is set-up",
		},
		cli.BoolFlag{
			Name:  "verbose",
			Usage: "Set initial logging to TRACE level until logging is properly set-up",
		},
	},
}

func runHTTPRedirector() {
	source := fmt.Sprintf("%s:%s", setting.HTTPAddr, setting.PortToRedirect)
	dest := strings.TrimSuffix(setting.AppURL, "/")
	log.Info("Redirecting: %s to %s", source, dest)

	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		target := dest + r.URL.Path
		if len(r.URL.RawQuery) > 0 {
			target += "?" + r.URL.RawQuery
		}
		http.Redirect(w, r, target, http.StatusTemporaryRedirect)
	})

	var err = runHTTP("tcp", source, "HTTP Redirector", context2.ClearHandler(handler))

	if err != nil {
		log.Fatal("Failed to start port redirection: %v", err)
	}
}

func runWeb(ctx *cli.Context) error {
	if ctx.Bool("verbose") {
		_ = log.DelLogger("console")
		log.NewLogger(0, "console", "console", fmt.Sprintf(`{"level": "trace", "colorize": %t, "stacktraceLevel": "none"}`, log.CanColorStdout))
	} else if ctx.Bool("quiet") {
		_ = log.DelLogger("console")
		log.NewLogger(0, "console", "console", fmt.Sprintf(`{"level": "fatal", "colorize": %t, "stacktraceLevel": "none"}`, log.CanColorStdout))
	}

	managerCtx, cancel := context.WithCancel(context.Background())
	graceful.InitManager(managerCtx)
	defer cancel()

	if os.Getppid() > 1 && len(os.Getenv("LISTEN_FDS")) > 0 {
		log.Info("Restarting Gitea on PID: %d from parent PID: %d", os.Getpid(), os.Getppid())
	} else {
		log.Info("Starting Gitea on PID: %d", os.Getpid())
	}

	// Set pid file setting
	if ctx.IsSet("pid") {
		setting.PIDFile = ctx.String("pid")
		setting.WritePIDFile = true
	}

	// Perform pre-initialization
	needsInstall := install.PreloadSettings(graceful.GetManager().HammerContext())
	if needsInstall {
		// Flag for port number in case first time run conflict
		if ctx.IsSet("port") {
			if err := setPort(ctx.String("port")); err != nil {
				return err
			}
		}
		if ctx.IsSet("install-port") {
			if err := setPort(ctx.String("install-port")); err != nil {
				return err
			}
		}
		c := install.Routes()
		err := listen(c, false)
		select {
		case <-graceful.GetManager().IsShutdown():
			<-graceful.GetManager().Done()
			log.Info("PID: %d Gitea Web Finished", os.Getpid())
			log.Close()
			return err
		default:
		}
	} else {
		NoInstallListener()
	}

	if setting.EnablePprof {
		go func() {
			log.Info("Starting pprof server on localhost:6060")
			log.Info("%v", http.ListenAndServe("localhost:6060", nil))
		}()
	}

	log.Info("Global init")
	// Perform global initialization
	routers.GlobalInit(graceful.GetManager().HammerContext())

	// Override the provided port number within the configuration
	if ctx.IsSet("port") {
		if err := setPort(ctx.String("port")); err != nil {
			return err
		}
	}

	// Set up Chi routes
	c := routers.NormalRoutes()
	err := listen(c, true)
	<-graceful.GetManager().Done()
	log.Info("PID: %d Gitea Web Finished", os.Getpid())
	log.Close()
	return err
}

func setPort(port string) error {
	setting.AppURL = strings.Replace(setting.AppURL, setting.HTTPPort, port, 1)
	setting.HTTPPort = port

	switch setting.Protocol {
	case setting.UnixSocket:
	case setting.FCGI:
	case setting.FCGIUnix:
	default:
		defaultLocalURL := string(setting.Protocol) + "://"
		if setting.HTTPAddr == "0.0.0.0" {
			defaultLocalURL += "localhost"
		} else {
			defaultLocalURL += setting.HTTPAddr
		}
		defaultLocalURL += ":" + setting.HTTPPort + "/"

		// Save LOCAL_ROOT_URL if port changed
		setting.CreateOrAppendToCustomConf(func(cfg *ini.File) {
			cfg.Section("server").Key("LOCAL_ROOT_URL").SetValue(defaultLocalURL)
		})
	}
	return nil
}

func listen(m http.Handler, handleRedirector bool) error {
	listenAddr := setting.HTTPAddr
	if setting.Protocol != setting.UnixSocket && setting.Protocol != setting.FCGIUnix {
		listenAddr = net.JoinHostPort(listenAddr, setting.HTTPPort)
	}
	log.Info("Listen: %v://%s%s", setting.Protocol, listenAddr, setting.AppSubURL)

	if setting.LFS.StartServer {
		log.Info("LFS server enabled")
	}

	var err error
	switch setting.Protocol {
	case setting.HTTP:
		if handleRedirector {
			NoHTTPRedirector()
		}
		err = runHTTP("tcp", listenAddr, "Web", context2.ClearHandler(m))
	case setting.HTTPS:
		if setting.EnableLetsEncrypt {
			err = runLetsEncrypt(listenAddr, setting.Domain, setting.LetsEncryptDirectory, setting.LetsEncryptEmail, context2.ClearHandler(m))
			break
		}
		if handleRedirector {
			if setting.RedirectOtherPort {
				go runHTTPRedirector()
			} else {
				NoHTTPRedirector()
			}
		}
		err = runHTTPS("tcp", listenAddr, "Web", setting.CertFile, setting.KeyFile, context2.ClearHandler(m))
	case setting.FCGI:
		if handleRedirector {
			NoHTTPRedirector()
		}
		err = runFCGI("tcp", listenAddr, "FCGI Web", context2.ClearHandler(m))
	case setting.UnixSocket:
		if handleRedirector {
			NoHTTPRedirector()
		}
		err = runHTTP("unix", listenAddr, "Web", context2.ClearHandler(m))
	case setting.FCGIUnix:
		if handleRedirector {
			NoHTTPRedirector()
		}
		err = runFCGI("unix", listenAddr, "Web", context2.ClearHandler(m))
	default:
		log.Fatal("Invalid protocol: %s", setting.Protocol)
	}

	if err != nil {
		log.Critical("Failed to start server: %v", err)
	}
	log.Info("HTTP Listener: %s Closed", listenAddr)
	return err
}
<commit_msg>Ensure that template compilation panics are sent to the logs (#16788)<commit_after>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package cmd

import (
	"context"
	"fmt"
	"net"
	"net/http"
	_ "net/http/pprof" // Used for debugging if enabled and a web server is running
	"os"
	"strings"

	"code.gitea.io/gitea/modules/graceful"
	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/setting"
	"code.gitea.io/gitea/routers"
	"code.gitea.io/gitea/routers/install"

	context2 "github.com/gorilla/context"
	"github.com/urfave/cli"
	ini "gopkg.in/ini.v1"
)

// CmdWeb represents the available web sub-command.
var CmdWeb = cli.Command{
	Name:  "web",
	Usage: "Start Gitea web server",
	Description: `Gitea web server is the only thing you need to run,
and it takes care of all the other things for you`,
	Action: runWeb,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "port, p",
			Value: "3000",
			Usage: "Temporary port number to prevent conflict",
		},
		cli.StringFlag{
			Name:  "install-port",
			Value: "3000",
			Usage: "Temporary port number to run the install page on to prevent conflict",
		},
		cli.StringFlag{
			Name:  "pid, P",
			Value: setting.PIDFile,
			Usage: "Custom pid file path",
		},
		cli.BoolFlag{
			Name:  "quiet, q",
			Usage: "Only display Fatal logging errors until logging is set-up",
		},
		cli.BoolFlag{
			Name:  "verbose",
			Usage: "Set initial logging to TRACE level until logging is properly set-up",
		},
	},
}

func runHTTPRedirector() {
	source := fmt.Sprintf("%s:%s", setting.HTTPAddr, setting.PortToRedirect)
	dest := strings.TrimSuffix(setting.AppURL, "/")
	log.Info("Redirecting: %s to %s", source, dest)

	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		target := dest + r.URL.Path
		if len(r.URL.RawQuery) > 0 {
			target += "?" + r.URL.RawQuery
		}
		http.Redirect(w, r, target, http.StatusTemporaryRedirect)
	})

	var err = runHTTP("tcp", source, "HTTP Redirector", context2.ClearHandler(handler))

	if err != nil {
		log.Fatal("Failed to start port redirection: %v", err)
	}
}

func runWeb(ctx *cli.Context) error {
	if ctx.Bool("verbose") {
		_ = log.DelLogger("console")
		log.NewLogger(0, "console", "console", fmt.Sprintf(`{"level": "trace", "colorize": %t, "stacktraceLevel": "none"}`, log.CanColorStdout))
	} else if ctx.Bool("quiet") {
		_ = log.DelLogger("console")
		log.NewLogger(0, "console", "console", fmt.Sprintf(`{"level": "fatal", "colorize": %t, "stacktraceLevel": "none"}`, log.CanColorStdout))
	}
	defer func() {
		if panicked := recover(); panicked != nil {
			log.Fatal("PANIC: %v\n%s", panicked, string(log.Stack(2)))
		}
	}()

	managerCtx, cancel := context.WithCancel(context.Background())
	graceful.InitManager(managerCtx)
	defer cancel()

	if os.Getppid() > 1 && len(os.Getenv("LISTEN_FDS")) > 0 {
		log.Info("Restarting Gitea on PID: %d from parent PID: %d", os.Getpid(), os.Getppid())
	} else {
		log.Info("Starting Gitea on PID: %d", os.Getpid())
	}

	// Set pid file setting
	if ctx.IsSet("pid") {
		setting.PIDFile = ctx.String("pid")
		setting.WritePIDFile = true
	}

	// Perform pre-initialization
	needsInstall := install.PreloadSettings(graceful.GetManager().HammerContext())
	if needsInstall {
		// Flag for port number in case first time run conflict
		if ctx.IsSet("port") {
			if err := setPort(ctx.String("port")); err != nil {
				return err
			}
		}
		if ctx.IsSet("install-port") {
			if err := setPort(ctx.String("install-port")); err != nil {
				return err
			}
		}
		c := install.Routes()
		err := listen(c, false)
		select {
		case <-graceful.GetManager().IsShutdown():
			<-graceful.GetManager().Done()
			log.Info("PID: %d Gitea Web Finished", os.Getpid())
			log.Close()
			return err
		default:
		}
	} else {
		NoInstallListener()
	}

	if setting.EnablePprof {
		go func() {
			log.Info("Starting pprof server on localhost:6060")
			log.Info("%v", http.ListenAndServe("localhost:6060", nil))
		}()
	}

	log.Info("Global init")
	// Perform global initialization
	routers.GlobalInit(graceful.GetManager().HammerContext())

	// Override the provided port number within the configuration
	if ctx.IsSet("port") {
		if err := setPort(ctx.String("port")); err != nil {
			return err
		}
	}

	// Set up Chi routes
	c := routers.NormalRoutes()
	err := listen(c, true)
	<-graceful.GetManager().Done()
	log.Info("PID: %d Gitea Web Finished", os.Getpid())
	log.Close()
	return err
}

func setPort(port string) error {
	setting.AppURL = strings.Replace(setting.AppURL, setting.HTTPPort, port, 1)
	setting.HTTPPort = port

	switch setting.Protocol {
	case setting.UnixSocket:
	case setting.FCGI:
	case setting.FCGIUnix:
	default:
		defaultLocalURL := string(setting.Protocol) + "://"
		if setting.HTTPAddr == "0.0.0.0" {
			defaultLocalURL += "localhost"
		} else {
			defaultLocalURL += setting.HTTPAddr
		}
		defaultLocalURL += ":" + setting.HTTPPort + "/"

		// Save LOCAL_ROOT_URL if port changed
		setting.CreateOrAppendToCustomConf(func(cfg *ini.File) {
			cfg.Section("server").Key("LOCAL_ROOT_URL").SetValue(defaultLocalURL)
		})
	}
	return nil
}

func listen(m http.Handler, handleRedirector bool) error {
	listenAddr := setting.HTTPAddr
	if setting.Protocol != setting.UnixSocket && setting.Protocol != setting.FCGIUnix {
		listenAddr = net.JoinHostPort(listenAddr, setting.HTTPPort)
	}
	log.Info("Listen: %v://%s%s", setting.Protocol, listenAddr, setting.AppSubURL)

	if setting.LFS.StartServer {
		log.Info("LFS server enabled")
	}

	var err error
	switch setting.Protocol {
	case setting.HTTP:
		if handleRedirector {
			NoHTTPRedirector()
		}
		err = runHTTP("tcp", listenAddr, "Web", context2.ClearHandler(m))
	case setting.HTTPS:
		if setting.EnableLetsEncrypt {
			err = runLetsEncrypt(listenAddr, setting.Domain, setting.LetsEncryptDirectory, setting.LetsEncryptEmail, context2.ClearHandler(m))
			break
		}
		if handleRedirector {
			if setting.RedirectOtherPort {
				go runHTTPRedirector()
			} else {
				NoHTTPRedirector()
			}
		}
		err = runHTTPS("tcp", listenAddr, "Web", setting.CertFile, setting.KeyFile, context2.ClearHandler(m))
	case setting.FCGI:
		if handleRedirector {
			NoHTTPRedirector()
		}
		err = runFCGI("tcp", listenAddr, "FCGI Web", context2.ClearHandler(m))
	case setting.UnixSocket:
		if handleRedirector {
			NoHTTPRedirector()
		}
		err = runHTTP("unix", listenAddr, "Web", context2.ClearHandler(m))
	case setting.FCGIUnix:
		if handleRedirector {
			NoHTTPRedirector()
		}
		err = runFCGI("unix", listenAddr, "Web", context2.ClearHandler(m))
	default:
		log.Fatal("Invalid protocol: %s", setting.Protocol)
	}

	if err != nil {
		log.Critical("Failed to start server: %v", err)
	}
	log.Info("HTTP Listener: %s Closed", listenAddr)
	return err
}
<|endoftext|>


<commit_before>package collectd

import (
	"errors"
	"fmt"
	"log"
	"net"
	"sync"
	"time"

	"github.com/influxdb/influxdb"
	"github.com/kimor79/gollectd"
)

// DefaultPort for collectd is 25826
const DefaultPort = 25826

// SeriesWriter defines the interface for the destination of the data.
type SeriesWriter interface {
	WriteSeries(database, retentionPolicy string, points []influxdb.Point) (uint64, error)
}

type Server struct {
	mu sync.Mutex
	wg sync.WaitGroup

	conn *net.UDPConn

	writer      SeriesWriter
	Database    string
	typesdb     gollectd.Types
	typesdbpath string
}

func NewServer(w SeriesWriter, typesDBPath string) *Server {
	s := Server{
		writer:      w,
		typesdbpath: typesDBPath,
		typesdb:     make(gollectd.Types),
	}

	return &s
}

func ListenAndServe(s *Server, iface string) error {
	if iface == "" { // Make sure we have an address
		return errors.New("bind address required")
	} else if s.Database == "" { // Make sure they have a database
		return errors.New("database was not specified in config")
	}

	addr, err := net.ResolveUDPAddr("udp", iface)
	if err != nil {
		return fmt.Errorf("unable to resolve UDP address: %v", err)
	}

	s.typesdb, err = gollectd.TypesDBFile(s.typesdbpath)
	if err != nil {
		return fmt.Errorf("unable to parse typesDBFile: %v", err)
	}

	conn, err := net.ListenUDP("udp", addr)
	if err != nil {
		return fmt.Errorf("unable to listen on UDP: %v", err)
	}
	s.conn = conn

	s.wg.Add(1)
	go s.serve(conn)

	return nil
}

func (s *Server) serve(conn *net.UDPConn) {
	defer s.wg.Done()

	// From https://collectd.org/wiki/index.php/Binary_protocol
	//   1024 bytes (payload only, not including UDP / IP headers)
	//   In versions 4.0 through 4.7, the receive buffer has a fixed size
	//   of 1024 bytes. When longer packets are received, the trailing data
	//   is simply ignored. Since version 4.8, the buffer size can be
	//   configured. Version 5.0 will increase the default buffer size to
	//   1452 bytes (the maximum payload size when using UDP/IPv6 over
	//   Ethernet).
	buffer := make([]byte, 1452)

	for {
		n, _, err := conn.ReadFromUDP(buffer)
		if err != nil && s.conn != nil {
			log.Printf("Collectd ReadFromUDP error: %s", err)
			continue
		}
		log.Printf("received %d bytes", n)
		if n > 0 {
			s.handleMessage(buffer[:n])
		}
		if s.conn == nil {
			// we closed the connection, time to go
			return
		}
	}
}

func (s *Server) handleMessage(buffer []byte) {
	log.Printf("handling message")
	packets, err := gollectd.Packets(buffer, s.typesdb)
	if err != nil {
		log.Printf("Collectd parse error: %s", err)
		return
	}

	for _, packet := range *packets {
		points := Unmarshal(&packet)
		for _, p := range points {
			_, err := s.writer.WriteSeries(s.Database, "", []influxdb.Point{p})
			if err != nil {
				log.Printf("Collectd cannot write data: %s", err)
				continue
			}
		}
	}
}

// Close shuts down the server's listeners.
func (s *Server) Close() error {
	// Notify other goroutines of shutdown.
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.conn == nil {
		return errors.New("server already closed")
	}
	s.conn.Close()
	s.conn = nil

	// Wait for all goroutines to shutdown.
	s.wg.Wait()
	log.Printf("all waitgroups finished")

	return nil
}

func Unmarshal(data *gollectd.Packet) []influxdb.Point {
	// Prefer high resolution timestamp.
	var timestamp time.Time
	if data.TimeHR > 0 {
		// TimeHR is "near" nanosecond measurement, but not exactly nanasecond time
		// Since we store time in microseconds, we round here (mostly so tests will work easier)
		sec := data.TimeHR >> 30
		// Shifting, masking, and dividing by 1 billion to get nanoseconds.
		nsec := ((data.TimeHR & 0x3FFFFFFF) << 30) / 1000 / 1000 / 1000
		timestamp = time.Unix(int64(sec), int64(nsec)).UTC().Round(time.Microsecond)
	} else {
		// If we don't have high resolution time, fall back to basic unix time
		timestamp = time.Unix(int64(data.Time), 0).UTC()
	}

	var points []influxdb.Point
	for i := range data.Values {
		name := fmt.Sprintf("%s_%s", data.Plugin, data.Values[i].Name)
		tags := make(map[string]string)
		fields := make(map[string]interface{})

		fields[name] = data.Values[i].Value

		if data.Hostname != "" {
			tags["host"] = data.Hostname
		}
		if data.PluginInstance != "" {
			tags["instance"] = data.PluginInstance
		}
		if data.Type != "" {
			tags["type"] = data.Type
		}
		if data.TypeInstance != "" {
			tags["type_instance"] = data.TypeInstance
		}
		p := influxdb.Point{
			Name:      name,
			Tags:      tags,
			Timestamp: timestamp,
			Fields:    fields,
		}

		points = append(points, p)
	}
	return points
}
<commit_msg>remove boring log in collectd.go<commit_after>package collectd

import (
	"errors"
	"fmt"
	"log"
	"net"
	"sync"
	"time"

	"github.com/influxdb/influxdb"
	"github.com/kimor79/gollectd"
)

// DefaultPort for collectd is 25826
const DefaultPort = 25826

// SeriesWriter defines the interface for the destination of the data.
type SeriesWriter interface {
	WriteSeries(database, retentionPolicy string, points []influxdb.Point) (uint64, error)
}

type Server struct {
	mu sync.Mutex
	wg sync.WaitGroup

	conn *net.UDPConn

	writer      SeriesWriter
	Database    string
	typesdb     gollectd.Types
	typesdbpath string
}

func NewServer(w SeriesWriter, typesDBPath string) *Server {
	s := Server{
		writer:      w,
		typesdbpath: typesDBPath,
		typesdb:     make(gollectd.Types),
	}

	return &s
}

func ListenAndServe(s *Server, iface string) error {
	if iface == "" { // Make sure we have an address
		return errors.New("bind address required")
	} else if s.Database == "" { // Make sure they have a database
		return errors.New("database was not specified in config")
	}

	addr, err := net.ResolveUDPAddr("udp", iface)
	if err != nil {
		return fmt.Errorf("unable to resolve UDP address: %v", err)
	}

	s.typesdb, err = gollectd.TypesDBFile(s.typesdbpath)
	if err != nil {
		return fmt.Errorf("unable to parse typesDBFile: %v", err)
	}

	conn, err := net.ListenUDP("udp", addr)
	if err != nil {
		return fmt.Errorf("unable to listen on UDP: %v", err)
	}
	s.conn = conn

	s.wg.Add(1)
	go s.serve(conn)

	return nil
}

func (s *Server) serve(conn *net.UDPConn) {
	defer s.wg.Done()

	// From https://collectd.org/wiki/index.php/Binary_protocol
	//   1024 bytes (payload only, not including UDP / IP headers)
	//   In versions 4.0 through 4.7, the receive buffer has a fixed size
	//   of 1024 bytes. When longer packets are received, the trailing data
	//   is simply ignored. Since version 4.8, the buffer size can be
	//   configured. Version 5.0 will increase the default buffer size to
	//   1452 bytes (the maximum payload size when using UDP/IPv6 over
	//   Ethernet).
	buffer := make([]byte, 1452)

	for {
		n, _, err := conn.ReadFromUDP(buffer)
		if err != nil && s.conn != nil {
			log.Printf("Collectd ReadFromUDP error: %s", err)
			continue
		}
		if n > 0 {
			s.handleMessage(buffer[:n])
		}
		if s.conn == nil {
			// we closed the connection, time to go
			return
		}
	}
}

func (s *Server) handleMessage(buffer []byte) {
	packets, err := gollectd.Packets(buffer, s.typesdb)
	if err != nil {
		log.Printf("Collectd parse error: %s", err)
		return
	}

	for _, packet := range *packets {
		points := Unmarshal(&packet)
		for _, p := range points {
			_, err := s.writer.WriteSeries(s.Database, "", []influxdb.Point{p})
			if err != nil {
				log.Printf("Collectd cannot write data: %s", err)
				continue
			}
		}
	}
}

// Close shuts down the server's listeners.
func (s *Server) Close() error {
	// Notify other goroutines of shutdown.
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.conn == nil {
		return errors.New("server already closed")
	}
	s.conn.Close()
	s.conn = nil

	// Wait for all goroutines to shutdown.
	s.wg.Wait()
	log.Printf("all waitgroups finished")

	return nil
}

func Unmarshal(data *gollectd.Packet) []influxdb.Point {
	// Prefer high resolution timestamp.
	var timestamp time.Time
	if data.TimeHR > 0 {
		// TimeHR is "near" nanosecond measurement, but not exactly nanasecond time
		// Since we store time in microseconds, we round here (mostly so tests will work easier)
		sec := data.TimeHR >> 30
		// Shifting, masking, and dividing by 1 billion to get nanoseconds.
		nsec := ((data.TimeHR & 0x3FFFFFFF) << 30) / 1000 / 1000 / 1000
		timestamp = time.Unix(int64(sec), int64(nsec)).UTC().Round(time.Microsecond)
	} else {
		// If we don't have high resolution time, fall back to basic unix time
		timestamp = time.Unix(int64(data.Time), 0).UTC()
	}

	var points []influxdb.Point
	for i := range data.Values {
		name := fmt.Sprintf("%s_%s", data.Plugin, data.Values[i].Name)
		tags := make(map[string]string)
		fields := make(map[string]interface{})

		fields[name] = data.Values[i].Value

		if data.Hostname != "" {
			tags["host"] = data.Hostname
		}
		if data.PluginInstance != "" {
			tags["instance"] = data.PluginInstance
		}
		if data.Type != "" {
			tags["type"] = data.Type
		}
		if data.TypeInstance != "" {
			tags["type_instance"] = data.TypeInstance
		}
		p := influxdb.Point{
			Name:      name,
			Tags:      tags,
			Timestamp: timestamp,
			Fields:    fields,
		}

		points = append(points, p)
	}
	return points
}
<|endoftext|>


<commit_before>package geoip

import (
	"fmt"
	"math"
	"math/big"
)

// For the MMDB format, see http://maxmind.github.io/MaxMind-DB/

type valueType int

const (
	typeExtended valueType = iota
	typePointer
	typeString
	typeDouble
	typeBytes
	typeUint16
	typeUint32
	typeMap
	typeInt32
	typeUint64
	typeUint128
	typeArray
	typeContainer
	typeEnd
	typeBoolean
	typeFloat
)

func decodeType(data []byte) (valueType, bool) {
	// first 3 bits
	t := data[0] >> 5
	if t == 0 {
		// extended type
		return 7 + valueType(data[1]), true
	}
	return valueType(t), false
}

func decodeSize(data []byte, extended bool) (int, int) {
	offset := 1
	if extended {
		offset++
	}
	// grab 5 lowest bits
	val := data[0] & 0x1F
	if val < 29 {
		return int(val), offset
	}
	if val == 29 {
		// 29 + next byte
		return 29 + int(data[offset]), offset + 1
	}
	if val == 30 {
		// 285 + next 2 bytes as be uint
		return 285 + int(uint32(data[offset])<<8|uint32(data[offset+1])), offset + 2
	}
	// 31 - 65821 + next 3 bytes as be uint
	return 65821 + int(uint32(data[offset])<<16|uint32(data[offset+1])<<8|uint32(data[offset+2])), offset + 3
}

func decodeUint16(data []byte, size int) uint16 {
	val := uint16(0)
	for ii := 0; ii < size; ii++ {
		val = val<<8 | uint16(data[ii])
	}
	return val
}

func decodeUint32(data []byte, size int) uint32 {
	val := uint32(0)
	for ii := 0; ii < size; ii++ {
		val = val<<8 | uint32(data[ii])
	}
	return val
}

func decodeInt32(data []byte, size int) int32 {
	return int32(decodeUint32(data, size))
}

func decodeUint64(data []byte, size int) uint64 {
	val := uint64(0)
	for ii := 0; ii < size; ii++ {
		val = val<<8 | uint64(data[ii])
	}
	return val
}

func decodeUint128(data []byte, size int) *big.Int {
	n := new(big.Int)
	return n.SetBytes(data)
}

type decoder struct {
	data []byte
	at   int
}

func (d *decoder) curData() ([]byte, error) {
	if d.at > len(d.data) {
		return nil, fmt.Errorf("invalid data pointer %d - corrupted database?", d.at)
	}
	return d.data[d.at:], nil
}

func (d *decoder) decodeType() (valueType, int, error) {
	cur, err := d.curData()
	if err != nil {
		return 0, 0, err
	}
	t, extended := decodeType(cur)
	var size, offset int
	if t == typePointer {
		// pointers look like 001SSVVV
		base := int(cur[0])
		ss := (base >> 3) & 0x03
		vvv := base & 0x07
		offset = 2 + ss
		switch ss {
		case 0:
			size = vvv<<8 | int(cur[1])
		case 1:
			size = (vvv<<16 | int(cur[1])<<8 | int(cur[2])) + 2048
		case 2:
			size = (vvv<<24 | int(cur[1])<<16 | int(cur[2])<<8 | int(cur[3])) + 526336
		case 3:
			size = int(cur[1])<<24 | int(cur[2])<<16 | int(cur[3])<<8 | int(cur[4])
		}
	} else {
		size, offset = decodeSize(cur, extended)
	}
	d.at += offset
	return t, size, nil
}

func (d *decoder) decode() (interface{}, error) {
	t, size, err := d.decodeType()
	if err != nil {
		return nil, err
	}
	cur, err := d.curData()
	if err != nil {
		return nil, err
	}
	switch t {
	case typePointer:
		dec := &decoder{d.data, size}
		return dec.decode()
	case typeString:
		d.at += size
		return makeString(cur[:size]), nil
	case typeDouble:
		if size != 8 {
			err = fmt.Errorf("double must 8 bytes, not %d", size)
			break
		}
		d.at += size
		b := decodeUint64(cur, 8)
		return math.Float64frombits(b), nil
	case typeBytes:
		d.at += size
		// Return a copy, we don't want callers
		// alterting our internal data block
		b := make([]byte, size)
		copy(b, cur)
		return b, nil
	case typeUint16:
		if size > 2 {
			err = fmt.Errorf("size %d is too big for uint16", size)
			break
		}
		d.at += size
		return decodeUint16(cur, size), nil
	case typeUint32:
		if size > 4 {
			err = fmt.Errorf("size %d is too big for uint32", size)
			break
		}
		d.at += size
		return decodeUint32(cur, size), nil
	case typeMap:
		return d.decodeMap(size)
	case typeInt32:
		if size > 4 {
			err = fmt.Errorf("size %d is too big for int32", size)
			break
		}
		d.at += size
		return decodeInt32(cur, size), nil
	case typeUint64:
		if size > 8 {
			err = fmt.Errorf("size %d is too big for uint64", size)
			break
		}
		d.at += size
		return decodeUint64(cur, size), nil
	case typeUint128:
		if size > 16 {
			err = fmt.Errorf("size %d is too big for uint128", size)
			break
		}
		d.at += size
		if size <= 8 {
			return decodeUint64(cur, size), nil
			break
		}
		return decodeUint128(cur, size), nil
	case typeArray:
		return d.decodeArray(size)
	case typeBoolean:
		return size != 0, nil
	case typeFloat:
		if size != 4 {
			err = fmt.Errorf("float must 4 bytes, not %d", size)
			break
		}
		d.at += size
		b := decodeUint32(cur, 4)
		return math.Float32frombits(b), nil
	}
	if err == nil {
		err = fmt.Errorf("invalid data type %d", int(t))
	}
	return nil, err
}

func (d *decoder) decodeArray(count int) ([]interface{}, error) {
	var values []interface{}
	for ii := 0; ii < count; ii++ {
		v, err := d.decode()
		if err != nil {
			return nil, err
		}
		values = append(values, v)
	}
	return values, nil
}

func (d *decoder) decodeMap(count int) (map[string]interface{}, error) {
	m := make(map[string]interface{})
	for ii := 0; ii < count; ii++ {
		key, err := d.decode()
		if err != nil {
			return nil, err
		}
		ks, ok := key.(string)
		if !ok {
			return nil, fmt.Errorf("non-string map key %T", key)
		}
		value, err := d.decode()
		if err != nil {
			return nil, err
		}
		m[ks] = value
	}
	return m, nil
}
<commit_msg>Add a few decoding optimizations<commit_after>package geoip

import (
	"fmt"
	"math"
	"math/big"
)

// For the MMDB format, see http://maxmind.github.io/MaxMind-DB/

type valueType int

const (
	typeExtended valueType = iota
	typePointer
	typeString
	typeDouble
	typeBytes
	typeUint16
	typeUint32
	typeMap
	typeInt32
	typeUint64
	typeUint128
	typeArray
	typeContainer
	typeEnd
	typeBoolean
	typeFloat
)

func decodeType(data []byte) (valueType, bool) {
	// first 3 bits
	t := data[0] >> 5
	if t == 0 {
		// extended type
		return 7 + valueType(data[1]), true
	}
	return valueType(t), false
}

func decodeSize(data []byte, extended bool) (int, int) {
	offset := 1
	if extended {
		offset++
	}
	// grab 5 lowest bits
	val := data[0] & 0x1F
	if val < 29 {
		return int(val), offset
	}
	if val == 29 {
		// 29 + next byte
		return 29 + int(data[offset]), offset + 1
	}
	if val == 30 {
		// 285 + next 2 bytes as be uint
		return 285 + int(uint32(data[offset])<<8|uint32(data[offset+1])), offset + 2
	}
	// 31 - 65821 + next 3 bytes as be uint
	return 65821 + int(uint32(data[offset])<<16|uint32(data[offset+1])<<8|uint32(data[offset+2])), offset + 3
}

func decodeUint16(data []byte, size int) uint16 {
	val := uint16(0)
	for ii := 0; ii < size; ii++ {
		val = val<<8 | uint16(data[ii])
	}
	return val
}

func decodeUint32(data []byte, size int) uint32 {
	val := uint32(0)
	for ii := 0; ii < size; ii++ {
		val = val<<8 | uint32(data[ii])
	}
	return val
}

func decodeInt32(data []byte, size int) int32 {
	return int32(decodeUint32(data, size))
}

func decodeUint64(data []byte, size int) uint64 {
	val := uint64(0)
	for ii := 0; ii < size; ii++ {
		val = val<<8 | uint64(data[ii])
	}
	return val
}

func decodeUint128(data []byte, size int) *big.Int {
	n := new(big.Int)
	return n.SetBytes(data)
}

type decoder struct {
	data []byte
	at   int
}

func (d *decoder) curData() ([]byte, error) {
	if d.at > len(d.data) {
		return nil, fmt.Errorf("invalid data pointer %d - corrupted database?", d.at)
	}
	return d.data[d.at:], nil
}

func (d *decoder) decodeType() (valueType, int, error) {
	cur, err := d.curData()
	if err != nil {
		return 0, 0, err
	}
	t, extended := decodeType(cur)
	var size, offset int
	if t == typePointer {
		// pointers look like 001SSVVV
		base := int(cur[0])
		ss := (base >> 3) & 0x03
		vvv := base & 0x07
		d.at += 2 + ss
		if ss == 0 {
			return t, vvv<<8 | int(cur[1]), nil
		}
		if ss == 1 {
			return t, (vvv<<16 | int(cur[1])<<8 | int(cur[2])) + 2048, nil
		}
		if ss == 2 {
			return t, (vvv<<24 | int(cur[1])<<16 | int(cur[2])<<8 | int(cur[3])) + 526336, nil
		}
		if ss == 3 {
			return t, int(cur[1])<<24 | int(cur[2])<<16 | int(cur[3])<<8 | int(cur[4]), nil
		}
		panic("unreachable")
	}
	size, offset = decodeSize(cur, extended)
	d.at += offset
	return t, size, nil
}

func (d *decoder) decode() (interface{}, error) {
	t, size, err := d.decodeType()
	if err != nil {
		return nil, err
	}
	cur, err := d.curData()
	if err != nil {
		return nil, err
	}
	switch t {
	case typePointer:
		dec := &decoder{d.data, size}
		return dec.decode()
	case typeString:
		d.at += size
		return makeString(cur[:size]), nil
	case typeDouble:
		if size != 8 {
			err = fmt.Errorf("double must 8 bytes, not %d", size)
			break
		}
		d.at += size
		b := decodeUint64(cur, 8)
		return math.Float64frombits(b), nil
	case typeBytes:
		d.at += size
		// Return a copy, we don't want callers
		// alterting our internal data block
		b := make([]byte, size)
		copy(b, cur)
		return b, nil
	case typeUint16:
		if size > 2 {
			err = fmt.Errorf("size %d is too big for uint16", size)
			break
		}
		d.at += size
		return decodeUint16(cur, size), nil
	case typeUint32:
		if size > 4 {
			err = fmt.Errorf("size %d is too big for uint32", size)
			break
		}
		d.at += size
		return decodeUint32(cur, size), nil
	case typeMap:
		return d.decodeMap(size)
	case typeInt32:
		if size > 4 {
			err = fmt.Errorf("size %d is too big for int32", size)
			break
		}
		d.at += size
		return decodeInt32(cur, size), nil
	case typeUint64:
		if size > 8 {
			err = fmt.Errorf("size %d is too big for uint64", size)
			break
		}
		d.at += size
		return decodeUint64(cur, size), nil
	case typeUint128:
		if size > 16 {
			err = fmt.Errorf("size %d is too big for uint128", size)
			break
		}
		d.at += size
		if size <= 8 {
			return decodeUint64(cur, size), nil
			break
		}
		return decodeUint128(cur, size), nil
	case typeArray:
		return d.decodeArray(size)
	case typeBoolean:
		return size != 0, nil
	case typeFloat:
		if size != 4 {
			err = fmt.Errorf("float must 4 bytes, not %d", size)
			break
		}
		d.at += size
		b := decodeUint32(cur, 4)
		return math.Float32frombits(b), nil
	}
	if err == nil {
		err = fmt.Errorf("invalid data type %d", int(t))
	}
	return nil, err
}

func (d *decoder) decodeArray(count int) ([]interface{}, error) {
	values := make([]interface{}, count)
	for ii := 0; ii < count; ii++ {
		v, err := d.decode()
		if err != nil {
			return nil, err
		}
		values[ii] = v
	}
	return values, nil
}

// fast path for decoding strings from decodeMap
func (d *decoder) decodeString() (string, error) {
	t, size, err := d.decodeType()
	if err != nil {
		return "", err
	}
	if t == typePointer {
		dec := &decoder{d.data, size}
		return dec.decodeString()
	}
	if t != typeString {
		return "", fmt.Errorf("type %d is not string", t)
	}
	end := d.at + size
	s := makeString(d.data[d.at:end])
	d.at = end
	return s, nil
}

func (d *decoder) decodeMap(count int) (map[string]interface{}, error) {
	m := make(map[string]interface{}, count)
	for ii := 0; ii < count; ii++ {
		key, err := d.decodeString()
		if err != nil {
			return nil, err
		}
		value, err := d.decode()
		if err != nil {
			return nil, err
		}
		m[key] = value
	}
	return m, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"crypto/md5"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/brotherlogic/goserver"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	pbd "github.com/brotherlogic/discovery/proto"
	pbgh "github.com/brotherlogic/githubcard/proto"
	pb "github.com/brotherlogic/gobuildslave/proto"
	pbs "github.com/brotherlogic/goserver/proto"
	"github.com/brotherlogic/goserver/utils"
)

// Server the main server type
type Server struct {
	*goserver.GoServer
	runner *Runner
	disk   diskChecker
	jobs   map[string]*pb.JobDetails
}

func deliverCrashReport(job *runnerCommand, getter func(name string) (string, int), logger func(text string)) {
	ip, port := getter("githubcard")
	if port > 0 {
		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
		defer cancel()
		conn, err := grpc.Dial(ip+":"+strconv.Itoa(port), grpc.WithInsecure())
		if err == nil {
			defer conn.Close()
			client := pbgh.NewGithubClient(conn)
			elems := strings.Split(job.details.Spec.GetName(), "/")
			if len(job.output) > 0 {
				client.AddIssue(ctx, &pbgh.Issue{Service: elems[len(elems)-1], Title: "CRASH REPORT", Body: job.output}, grpc.FailFast(false))
			}
		}
	}
}

func (s *Server) addMessage(details *pb.JobDetails, message string) {
	for _, t := range s.runner.backgroundTasks {
		if t.details.GetSpec().Name == details.Spec.Name {
			t.output += message
		}
	}
}

func (s *Server) monitor(job *pb.JobDetails) {
	for true {
		switch job.State {
		case pb.JobDetails_ACKNOWLEDGED:
			job.StartTime = 0
			job.GetSpec().Port = 0
			job.State = pb.JobDetails_BUILDING
			s.runner.Checkout(job.GetSpec().Name)
			job.State = pb.JobDetails_BUILT
		case pb.JobDetails_BUILT:
			s.runner.Run(job)
			for job.StartTime == 0 {
				time.Sleep(waitTime)
			}
			job.State = pb.JobDetails_PENDING
		case pb.JobDetails_KILLING:
			s.runner.kill(job)
			if !isAlive(job.GetSpec()) {
				job.State = pb.JobDetails_DEAD
			}
		case pb.JobDetails_UPDATE_STARTING:
			s.runner.Update(job)
			job.State = pb.JobDetails_RUNNING
		case pb.JobDetails_PENDING:
			time.Sleep(time.Minute)
			if isAlive(job.GetSpec()) {
				job.State = pb.JobDetails_RUNNING
			} else {
				job.State = pb.JobDetails_DEAD
			}
		case pb.JobDetails_RUNNING:
			time.Sleep(waitTime)
			if !isAlive(job.GetSpec()) {
				job.TestCount++
			} else {
				job.TestCount = 0
			}
			if job.TestCount > 60 {
				s.Log(fmt.Sprintf("Killing beacuse we couldn't reach 60 times: %v", job))
				job.State = pb.JobDetails_DEAD
			}
		case pb.JobDetails_DEAD:
			job.State = pb.JobDetails_ACKNOWLEDGED
		}
	}
}

func getHash(file string) (string, error) {
	env := os.Environ()
	home := ""
	for _, s := range env {
		if strings.HasPrefix(s, "HOME=") {
			home = s[5:]
		}
	}

	gpath := home + "/gobuild"

	f, err := os.Open(strings.Replace(file, "$GOPATH", gpath, 1))
	if err != nil {
		return "", err
	}
	defer f.Close()

	h := md5.New()
	if _, err := io.Copy(h, f); err != nil {
		return "", err
	}

	return string(h.Sum(nil)), nil
}

func getIP(name string, server string) (string, int32, error) {
	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
	defer cancel()
	conn, err := grpc.Dial(utils.RegistryIP+":"+strconv.Itoa(utils.RegistryPort), grpc.WithInsecure())
	defer conn.Close()

	if err != nil {
		return "", -1, err
	}

	registry := pbd.NewDiscoveryServiceClient(conn)
	entry := pbd.RegistryEntry{Name: name, Identifier: server}
	r, err := registry.Discover(ctx, &entry, grpc.FailFast(false))

	if err != nil {
		return "", -1, err
	}

	return r.Ip, r.Port, nil
}

// updateState of the runner command
func isAlive(spec *pb.JobSpec) bool {
	elems := strings.Split(spec.Name, "/")
	if spec.GetPort() == 0 {
		dServer, dPort, err := getIP(elems[len(elems)-1], spec.Server)

		e, ok := status.FromError(err)
		if ok && e.Code() == codes.DeadlineExceeded {
			//Ignore deadline exceeds on discover
			return true
		}

		if err != nil {
			return false
		}

		spec.Host = dServer
		spec.Port = dPort
	}

	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	dConn, err := grpc.Dial(spec.Host+":"+strconv.Itoa(int(spec.Port)), grpc.WithInsecure())
	if err != nil {
		return false
	}
	defer dConn.Close()

	c := pbs.NewGoserverServiceClient(dConn)
	resp, err := c.IsAlive(ctx, &pbs.Alive{}, grpc.FailFast(false))

	if err != nil || resp.Name != elems[len(elems)-1] {
		e, ok := status.FromError(err)
		if ok && e.Code() != codes.Unavailable {
			return false
		}
	}

	return true
}

// DoRegister Registers this server
func (s Server) DoRegister(server *grpc.Server) {
	pb.RegisterGoBuildSlaveServer(server, &s)
}

// ReportHealth determines if the server is healthy
func (s Server) ReportHealth() bool {
	return true
}

// Mote promotes/demotes this server
func (s Server) Mote(master bool) error {
	return nil
}

// GetState gets the state of the server
func (s Server) GetState() []*pbs.State {
	return []*pbs.State{}
}

//Init builds the default runner framework
func Init() *Runner {
	r := &Runner{gopath: "goautobuild", m: &sync.Mutex{}, bm: &sync.Mutex{}}
	r.runner = runCommand
	go r.run()
	return r
}

func runCommand(c *runnerCommand) {
	if c == nil || c.command == nil {
		return
	}

	env := os.Environ()
	home := ""
	for _, s := range env {
		if strings.HasPrefix(s, "HOME=") {
			home = s[5:]
		}
	}

	gpath := home + "/gobuild"
	c.command.Path = strings.Replace(c.command.Path, "$GOPATH", gpath, -1)
	for i := range c.command.Args {
		c.command.Args[i] = strings.Replace(c.command.Args[i], "$GOPATH", gpath, -1)
	}

	path := fmt.Sprintf("GOPATH=" + home + "/gobuild/")
	pathbin := fmt.Sprintf("GOBIN=" + home + "/gobuild/bin/")
	found := false
	envl := os.Environ()
	for i, blah := range envl {
		if strings.HasPrefix(blah, "GOPATH") {
			envl[i] = path
			found = true
		}
		if strings.HasPrefix(blah, "GOBIN") {
			envl[i] = pathbin
			found = true
		}
	}
	if !found {
		envl = append(envl, path)
	}
	c.command.Env = envl

	out, err := c.command.StderrPipe()
	if err != nil {
		log.Fatalf("Problem getting stderr: %v", err)
	}

	if out != nil {
		scanner := bufio.NewScanner(out)
		go func() {
			for scanner != nil && scanner.Scan() {
				c.output += scanner.Text()
			}
		}()
	}

	c.command.Start()

	if !c.background {
		c.command.Wait()
		c.complete = true
	} else {
		c.details.StartTime = time.Now().Unix()
	}
}

func (diskChecker prodDiskChecker) diskUsage(path string) int64 {
	return diskUsage(path)
}

func (s *Server) rebuildLoop() {
	for true {
		time.Sleep(time.Minute * 60)

		var rebuildList []*pb.JobDetails
		var hashList []string
		for _, job := range s.runner.backgroundTasks {
			if time.Since(job.started) > time.Hour {
				rebuildList = append(rebuildList, job.details)
				hashList = append(hashList, job.hash)
			}
		}

		for i := range rebuildList {
			s.runner.Rebuild(rebuildList[i], hashList[i])
		}
	}
}

func main() {
	var quiet = flag.Bool("quiet", false, "Show all output")
	flag.Parse()

	if *quiet {
		log.SetFlags(0)
		log.SetOutput(ioutil.Discard)
	}

	s := Server{&goserver.GoServer{}, Init(), prodDiskChecker{}, make(map[string]*pb.JobDetails)}
	s.runner.getip = s.GetIP
	s.runner.logger = s.Log
	s.Register = s
	s.PrepServer()
	s.GoServer.Killme = false
	s.RegisterServingTask(s.rebuildLoop)
	s.RegisterServer("gobuildslave", false)
	err := s.Serve()
	log.Fatalf("Unable to serve: %v", err)
}
<commit_msg>Closes pipe on finish. This closes #81.<commit_after>package main

import (
	"bufio"
	"crypto/md5"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/brotherlogic/goserver"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	pbd "github.com/brotherlogic/discovery/proto"
	pbgh "github.com/brotherlogic/githubcard/proto"
	pb "github.com/brotherlogic/gobuildslave/proto"
	pbs "github.com/brotherlogic/goserver/proto"
	"github.com/brotherlogic/goserver/utils"
)

// Server the main server type
type Server struct {
	*goserver.GoServer
	runner *Runner
	disk   diskChecker
	jobs   map[string]*pb.JobDetails
}

func deliverCrashReport(job *runnerCommand, getter func(name string) (string, int), logger func(text string)) {
	ip, port := getter("githubcard")
	if port > 0 {
		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
		defer cancel()
		conn, err := grpc.Dial(ip+":"+strconv.Itoa(port), grpc.WithInsecure())
		if err == nil {
			defer conn.Close()
			client := pbgh.NewGithubClient(conn)
			elems := strings.Split(job.details.Spec.GetName(), "/")
			if len(job.output) > 0 {
				client.AddIssue(ctx, &pbgh.Issue{Service: elems[len(elems)-1], Title: "CRASH REPORT", Body: job.output}, grpc.FailFast(false))
			}
		}
	}
}

func (s *Server) addMessage(details *pb.JobDetails, message string) {
	for _, t := range s.runner.backgroundTasks {
		if t.details.GetSpec().Name == details.Spec.Name {
			t.output += message
		}
	}
}

func (s *Server) monitor(job *pb.JobDetails) {
	for true {
		switch job.State {
		case pb.JobDetails_ACKNOWLEDGED:
			job.StartTime = 0
			job.GetSpec().Port = 0
			job.State = pb.JobDetails_BUILDING
			s.runner.Checkout(job.GetSpec().Name)
			job.State = pb.JobDetails_BUILT
		case pb.JobDetails_BUILT:
			s.runner.Run(job)
			for job.StartTime == 0 {
				time.Sleep(waitTime)
			}
			job.State = pb.JobDetails_PENDING
		case pb.JobDetails_KILLING:
			s.runner.kill(job)
			if !isAlive(job.GetSpec()) {
				job.State = pb.JobDetails_DEAD
			}
		case pb.JobDetails_UPDATE_STARTING:
			s.runner.Update(job)
			job.State = pb.JobDetails_RUNNING
		case pb.JobDetails_PENDING:
			time.Sleep(time.Minute)
			if isAlive(job.GetSpec()) {
				job.State = pb.JobDetails_RUNNING
			} else {
				job.State = pb.JobDetails_DEAD
			}
		case pb.JobDetails_RUNNING:
			time.Sleep(waitTime)
			if !isAlive(job.GetSpec()) {
				job.TestCount++
			} else {
				job.TestCount = 0
			}
			if job.TestCount > 60 {
				s.Log(fmt.Sprintf("Killing beacuse we couldn't reach 60 times: %v", job))
				job.State = pb.JobDetails_DEAD
			}
		case pb.JobDetails_DEAD:
			job.State = pb.JobDetails_ACKNOWLEDGED
		}
	}
}

func getHash(file string) (string, error) {
	env := os.Environ()
	home := ""
	for _, s := range env {
		if strings.HasPrefix(s, "HOME=") {
			home = s[5:]
		}
	}

	gpath := home + "/gobuild"

	f, err := os.Open(strings.Replace(file, "$GOPATH", gpath, 1))
	if err != nil {
		return "", err
	}
	defer f.Close()

	h := md5.New()
	if _, err := io.Copy(h, f); err != nil {
		return "", err
	}

	return string(h.Sum(nil)), nil
}

func getIP(name string, server string) (string, int32, error) {
	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
	defer cancel()
	conn, err := grpc.Dial(utils.RegistryIP+":"+strconv.Itoa(utils.RegistryPort), grpc.WithInsecure())
	defer conn.Close()

	if err != nil {
		return "", -1, err
	}

	registry := pbd.NewDiscoveryServiceClient(conn)
	entry := pbd.RegistryEntry{Name: name, Identifier: server}
	r, err := registry.Discover(ctx, &entry, grpc.FailFast(false))

	if err != nil {
		return "", -1, err
	}

	return r.Ip, r.Port, nil
}

// updateState of the runner command
func isAlive(spec *pb.JobSpec) bool {
	elems := strings.Split(spec.Name, "/")
	if spec.GetPort() == 0 {
		dServer, dPort, err := getIP(elems[len(elems)-1], spec.Server)

		e, ok := status.FromError(err)
		if ok && e.Code() == codes.DeadlineExceeded {
			//Ignore deadline exceeds on discover
			return true
		}

		if err != nil {
			return false
		}

		spec.Host = dServer
		spec.Port = dPort
	}

	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	dConn, err := grpc.Dial(spec.Host+":"+strconv.Itoa(int(spec.Port)), grpc.WithInsecure())
	if err != nil {
		return false
	}
	defer dConn.Close()

	c := pbs.NewGoserverServiceClient(dConn)
	resp, err := c.IsAlive(ctx, &pbs.Alive{}, grpc.FailFast(false))

	if err != nil || resp.Name != elems[len(elems)-1] {
		e, ok := status.FromError(err)
		if ok && e.Code() != codes.Unavailable {
			return false
		}
	}

	return true
}

// DoRegister Registers this server
func (s Server) DoRegister(server *grpc.Server) {
	pb.RegisterGoBuildSlaveServer(server, &s)
}

// ReportHealth determines if the server is healthy
func (s Server) ReportHealth() bool {
	return true
}

// Mote promotes/demotes this server
func (s Server) Mote(master bool) error {
	return nil
}

// GetState gets the state of the server
func (s Server) GetState() []*pbs.State {
	return []*pbs.State{}
}

//Init builds the default runner framework
func Init() *Runner {
	r := &Runner{gopath: "goautobuild", m: &sync.Mutex{}, bm: &sync.Mutex{}}
	r.runner = runCommand
	go r.run()
	return r
}

func runCommand(c *runnerCommand) {
	if c == nil || c.command == nil {
		return
	}

	env := os.Environ()
	home := ""
	for _, s := range env {
		if strings.HasPrefix(s, "HOME=") {
			home = s[5:]
		}
	}

	gpath := home + "/gobuild"
	c.command.Path = strings.Replace(c.command.Path, "$GOPATH", gpath, -1)
	for i := range c.command.Args {
		c.command.Args[i] = strings.Replace(c.command.Args[i], "$GOPATH", gpath, -1)
	}

	path := fmt.Sprintf("GOPATH=" + home + "/gobuild/")
	pathbin := fmt.Sprintf("GOBIN=" + home + "/gobuild/bin/")
	found := false
	envl := os.Environ()
	for i, blah := range envl {
		if strings.HasPrefix(blah, "GOPATH") {
			envl[i] = path
			found = true
		}
		if strings.HasPrefix(blah, "GOBIN") {
			envl[i] = pathbin
			found = true
		}
	}
	if !found {
		envl = append(envl, path)
	}
	c.command.Env = envl

	out, err := c.command.StderrPipe()
	if err != nil {
		log.Fatalf("Problem getting stderr: %v", err)
	}

	if out != nil {
		scanner := bufio.NewScanner(out)
		go func() {
			for scanner != nil && scanner.Scan() {
				c.output += scanner.Text()
			}
			out.Close()
		}()
	}

	c.command.Start()

	if !c.background {
		c.command.Wait()
		c.complete = true
	} else {
		c.details.StartTime = time.Now().Unix()
	}
}

func (diskChecker prodDiskChecker) diskUsage(path string) int64 {
	return diskUsage(path)
}

func (s *Server) rebuildLoop() {
	for true {
		time.Sleep(time.Minute * 60)

		var rebuildList []*pb.JobDetails
		var hashList []string
		for _, job := range s.runner.backgroundTasks {
			if time.Since(job.started) > time.Hour {
				rebuildList = append(rebuildList, job.details)
				hashList = append(hashList, job.hash)
			}
		}

		for i := range rebuildList {
			s.runner.Rebuild(rebuildList[i], hashList[i])
		}
	}
}

func main() {
	var quiet = flag.Bool("quiet", false, "Show all output")
	flag.Parse()

	if *quiet {
		log.SetFlags(0)
		log.SetOutput(ioutil.Discard)
	}

	s := Server{&goserver.GoServer{}, Init(), prodDiskChecker{}, make(map[string]*pb.JobDetails)}
	s.runner.getip = s.GetIP
	s.runner.logger = s.Log
	s.Register = s
	s.PrepServer()
	s.GoServer.Killme = false
	s.RegisterServingTask(s.rebuildLoop)
	s.RegisterServer("gobuildslave", false)
	err := s.Serve()
	log.Fatalf("Unable to serve: %v", err)
}
<|endoftext|>


<commit_before>package faker

// CompanyName returns a company name
func (f Faker) CompanyName() string {
	return template("CompanyName", randomElement(f.CurrentLocale().CompanyNamesFormats), f)
}

// CompanySuffix returns a company suffix such as 'corp', 'LLC'
func (f Faker) CompanySuffix() string {
	return randomElement(f.CurrentLocale().CompanySuffixes)
}

func (f Faker) CompanyBuzzword() string {
	return "CompanyBuzzword"
}

// CompanyLogo returns a fake URL company name
func (f Faker) CompanyLogo() string {
	return randomElement(companyLogos)
}
<commit_msg>Removed BuzzWord<commit_after>package faker

// CompanyName returns a company name
func (f Faker) CompanyName() string {
	return template("CompanyName", randomElement(f.CurrentLocale().CompanyNamesFormats), f)
}

// CompanySuffix returns a company suffix such as 'corp', 'LLC'
func (f Faker) CompanySuffix() string {
	return randomElement(f.CurrentLocale().CompanySuffixes)
}

// CompanyLogo returns a fake URL company name
func (f Faker) CompanyLogo() string {
	return randomElement(companyLogos)
}
<|endoftext|>


<commit_before>package collector

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"

	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	"github.com/prometheus/client_golang/prometheus"
)

var (
	defaultIndexLabels      = []string{"cluster", "index"}
	defaultIndexLabelValues = func(clusterName string, indexName string) []string {
		return []string{clusterName, indexName}
	}
)

type indexMetric struct {
	Type   prometheus.ValueType
	Desc   *prometheus.Desc
	Value  func(indexStats IndexStatsIndexResponse) float64
	Labels func(clusterName string, indexName string) []string
}

type Indices struct {
	logger        log.Logger
	client        *http.Client
	url           *url.URL
	all           bool
	exportIndices bool

	up                prometheus.Gauge
	totalScrapes      prometheus.Counter
	jsonParseFailures prometheus.Counter

	indexMetrics []*indexMetric
}

func NewIndices(logger log.Logger, client *http.Client, url *url.URL, all bool, exportIndices bool) *Indices {
	return &Indices{
		logger:        logger,
		client:        client,
		url:           url,
		all:           all,
		exportIndices: exportIndices,

		up: prometheus.NewGauge(prometheus.GaugeOpts{
			Name: prometheus.BuildFQName(namespace, "index_stats", "up"),
			Help: "Was the last scrape of the ElasticSearch index endpoint successful.",
		}),
		totalScrapes: prometheus.NewCounter(prometheus.CounterOpts{
			Name: prometheus.BuildFQName(namespace, "index_stats", "total_scrapes"),
			Help: "Current total ElasticSearch index scrapes.",
		}),
		jsonParseFailures: prometheus.NewCounter(prometheus.CounterOpts{
			Name: prometheus.BuildFQName(namespace, "index_stats", "json_parse_failures"),
			Help: "Number of errors while parsing JSON.",
		}),

		indexMetrics: []*indexMetric{
			{
				Type: prometheus.GaugeValue,
				Desc: prometheus.NewDesc(
					prometheus.BuildFQName(namespace, "indices", "docs_primary"),
					"Count of documents which only primary shards",
					defaultIndexLabels, nil,
				),
				Value: func(indexStats IndexStatsIndexResponse) float64 {
					return float64(indexStats.Primaries.Docs.Count)
				},
				Labels: defaultIndexLabelValues,
			},
			{
				Type: prometheus.GaugeValue,
				Desc: prometheus.NewDesc(
					prometheus.BuildFQName(namespace, "indices", "store_size_bytes_primary"),
					"Current total size of stored index data in bytes which only primary shards on all nodes",
					defaultIndexLabels, nil,
				),
				Value: func(indexStats IndexStatsIndexResponse) float64 {
					return float64(indexStats.Primaries.Store.SizeInBytes)
				},
				Labels: defaultIndexLabelValues,
			},
			{
				Type: prometheus.GaugeValue,
				Desc: prometheus.NewDesc(
					prometheus.BuildFQName(namespace, "indices", "store_size_bytes_total"),
					"Current total size of stored index data in bytes which all shards on all nodes",
					defaultIndexLabels, nil,
				),
				Value: func(indexStats IndexStatsIndexResponse) float64 {
					return float64(indexStats.Total.Store.SizeInBytes)
				},
				Labels: defaultIndexLabelValues,
			},
		},
	}
}

func (i *Indices) Describe(ch chan<- *prometheus.Desc) {
	for _, metric := range i.indexMetrics {
		ch <- metric.Desc
	}
	ch <- i.up.Desc()
	ch <- i.totalScrapes.Desc()
	ch <- i.jsonParseFailures.Desc()
}

func (c *Indices) fetchAndDecodeIndexStats() (indexStatsResponse, error) {
	var isr indexStatsResponse

	u := *c.url
	u.Path = "/_all/_stats"

	res, err := c.client.Get(u.String())
	if err != nil {
		return isr, fmt.Errorf("failed to get index stats from %s://%s:%s/%s: %s",
			u.Scheme, u.Hostname(), u.Port(), u.Path, err)
	}
	defer res.Body.Close()
	if res.StatusCode != http.StatusOK {
		return isr, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode)
	}

	if err := json.NewDecoder(res.Body).Decode(&isr); err != nil {
		c.jsonParseFailures.Inc()
		return isr, err
	}
	return isr, nil
}

func (i *Indices) Collect(ch chan<- prometheus.Metric) {
	i.totalScrapes.Inc()
	defer func() {
		ch <- i.up
		ch <- i.totalScrapes
		ch <- i.jsonParseFailures
	}()

	// clusterHealth
	clusterHealth := NewClusterHealth(i.logger, i.client, i.url)
	clusterHealthResponse, err := clusterHealth.fetchAndDecodeClusterHealth()
	if err != nil {
		i.up.Set(0)
		level.Warn(i.logger).Log(
			"msg", "failed to fetch and decode cluster health",
			"err", err,
		)
		return
	}

	// indices
	indexStatsResponse, err := i.fetchAndDecodeIndexStats()
	if err != nil {
		i.up.Set(0)
		level.Warn(i.logger).Log(
			"msg", "failed to fetch and decode index stats",
			"err", err,
		)
		return
	}
	i.up.Set(1)

	// Index stats
	for indexName, indexStats := range indexStatsResponse.Indices {
		for _, metric := range i.indexMetrics {
			ch <- prometheus.MustNewConstMetric(
				metric.Desc,
				metric.Type,
				metric.Value(indexStats),
				metric.Labels(clusterHealthResponse.ClusterName, indexName)...,
			)
		}
	}
}
<commit_msg>Make index metrics optional<commit_after>package collector

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"

	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	"github.com/prometheus/client_golang/prometheus"
)

var (
	defaultIndexLabels      = []string{"cluster", "index"}
	defaultIndexLabelValues = func(clusterName string, indexName string) []string {
		return []string{clusterName, indexName}
	}
)

type indexMetric struct {
	Type   prometheus.ValueType
	Desc   *prometheus.Desc
	Value  func(indexStats IndexStatsIndexResponse) float64
	Labels func(clusterName string, indexName string) []string
}

type Indices struct {
	logger        log.Logger
	client        *http.Client
	url           *url.URL
	all           bool
	exportIndices bool

	up                prometheus.Gauge
	totalScrapes      prometheus.Counter
	jsonParseFailures prometheus.Counter

	indexMetrics []*indexMetric
}

func NewIndices(logger log.Logger, client *http.Client, url *url.URL, all bool, exportIndices bool) *Indices {
	return &Indices{
		logger:        logger,
		client:        client,
		url:           url,
		all:           all,
		exportIndices: exportIndices,

		up: prometheus.NewGauge(prometheus.GaugeOpts{
			Name: prometheus.BuildFQName(namespace, "index_stats", "up"),
			Help: "Was the last scrape of the ElasticSearch index endpoint successful.",
		}),
		totalScrapes: prometheus.NewCounter(prometheus.CounterOpts{
			Name: prometheus.BuildFQName(namespace, "index_stats", "total_scrapes"),
			Help: "Current total ElasticSearch index scrapes.",
		}),
		jsonParseFailures: prometheus.NewCounter(prometheus.CounterOpts{
			Name: prometheus.BuildFQName(namespace, "index_stats", "json_parse_failures"),
			Help: "Number of errors while parsing JSON.",
		}),

		indexMetrics: []*indexMetric{
			{
				Type: prometheus.GaugeValue,
				Desc: prometheus.NewDesc(
					prometheus.BuildFQName(namespace, "indices", "docs_primary"),
					"Count of documents which only primary shards",
					defaultIndexLabels, nil,
				),
				Value: func(indexStats IndexStatsIndexResponse) float64 {
					return float64(indexStats.Primaries.Docs.Count)
				},
				Labels: defaultIndexLabelValues,
			},
			{
				Type: prometheus.GaugeValue,
				Desc: prometheus.NewDesc(
					prometheus.BuildFQName(namespace, "indices", "store_size_bytes_primary"),
					"Current total size of stored index data in bytes which only primary shards on all nodes",
					defaultIndexLabels, nil,
				),
				Value: func(indexStats IndexStatsIndexResponse) float64 {
					return float64(indexStats.Primaries.Store.SizeInBytes)
				},
				Labels: defaultIndexLabelValues,
			},
			{
				Type: prometheus.GaugeValue,
				Desc: prometheus.NewDesc(
					prometheus.BuildFQName(namespace, "indices", "store_size_bytes_total"),
					"Current total size of stored index data in bytes which all shards on all nodes",
					defaultIndexLabels, nil,
				),
				Value: func(indexStats IndexStatsIndexResponse) float64 {
					return float64(indexStats.Total.Store.SizeInBytes)
				},
				Labels: defaultIndexLabelValues,
			},
		},
	}
}

func (i *Indices) Describe(ch chan<- *prometheus.Desc) {
	for _, metric := range i.indexMetrics {
		ch <- metric.Desc
	}
	ch <- i.up.Desc()
	ch <- i.totalScrapes.Desc()
	ch <- i.jsonParseFailures.Desc()
}

func (c *Indices) fetchAndDecodeIndexStats() (indexStatsResponse, error) {
	var isr indexStatsResponse

	if c.exportIndices {
		u := *c.url
		u.Path = "/_all/_stats"

		res, err := c.client.Get(u.String())
		if err != nil {
			return isr, fmt.Errorf("failed to get index stats from %s://%s:%s/%s: %s",
				u.Scheme, u.Hostname(), u.Port(), u.Path, err)
		}
		defer res.Body.Close()
		if res.StatusCode != http.StatusOK {
			return isr, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode)
		}

		if err := json.NewDecoder(res.Body).Decode(&isr); err != nil {
			c.jsonParseFailures.Inc()
			return isr, err
		}
	}

	return isr, nil
}

func (i *Indices) Collect(ch chan<- prometheus.Metric) {
	i.totalScrapes.Inc()
	defer func() {
		ch <- i.up
		ch <- i.totalScrapes
		ch <- i.jsonParseFailures
	}()

	// clusterHealth
	clusterHealth := NewClusterHealth(i.logger, i.client, i.url)
	clusterHealthResponse, err := clusterHealth.fetchAndDecodeClusterHealth()
	if err != nil {
		i.up.Set(0)
		level.Warn(i.logger).Log(
			"msg", "failed to fetch and decode cluster health",
			"err", err,
		)
		return
	}

	// indices
	indexStatsResponse, err := i.fetchAndDecodeIndexStats()
	if err != nil {
		i.up.Set(0)
		level.Warn(i.logger).Log(
			"msg", "failed to fetch and decode index stats",
			"err", err,
		)
		return
	}
	i.up.Set(1)

	// Index stats
	for indexName, indexStats := range indexStatsResponse.Indices {
		for _, metric := range i.indexMetrics {
			ch <- prometheus.MustNewConstMetric(
				metric.Desc,
				metric.Type,
				metric.Value(indexStats),
				metric.Labels(clusterHealthResponse.ClusterName, indexName)...,
			)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"log"

	"gopkg.in/yaml.v2"
)

var (
	err error
)

type Config struct {
	Static []StaticProject
}

type StaticProject struct {
	Name       string
	Branch     string
	Domain     string
	Subdomain  string
	GitHub     string
	Bucket     string
	Owner      string
	Repository string
}

func main() {
	data, err := ioutil.ReadFile("deployd.conf")

	if err != nil {
		log.Fatal(err)
	}

	var config Config
	err = yaml.Unmarshal(data, &config)

	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Implemented downloading static project archives from S3 and GitHub<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"

	"gopkg.in/yaml.v2"
)

var (
	err error
)

type Config struct {
	Static []StaticProject
}

type StaticProject struct {
	Name       string
	Branch     string
	Domain     string
	Subdomain  string
	GitHub     bool
	Bucket     string
	Owner      string
	Repository string
}

func main() {
	data, err := ioutil.ReadFile("deployd.conf")

	if err != nil {
		log.Fatal(err)
	}

	var config Config
	err = yaml.Unmarshal(data, &config)

	if err != nil {
		log.Fatal(err)
	}

	for _, project := range config.Static {
		/*
			path = fmt.Sprintf("/tmp/%v-%v", project.Name, project.Branch)
			dir, err := os.Stat(path)

			if path.IsDir() {
				os.Remove(path)
			}

			err = os.MkDir(path, 0700)

			if err != nil {
				log.Fatal(err)
			}
		*/

		archivePath := fmt.Sprintf("/tmp/%v-%v.zip", project.Name, project.Branch)

		if _, err := os.Stat(archivePath); err == nil {
			err := os.Remove(archivePath)

			if err != nil {
				log.Fatal(err)
			}
		}

		if err != nil {
			log.Fatal(err)
		}

		archive, err := os.Create(archivePath)
		defer archive.Close()

		if err != nil {
			log.Fatal(err)
		}

		var archiveLocation string

		if project.GitHub {
			archiveLocation = fmt.Sprintf("https://github.com/%v/%v/archive/%v.zip", project.Owner, project.Repository, project.Branch)
		} else {
			archiveLocation = fmt.Sprintf("https://s3.amazonaws.com/%v/%v-latest.zip", project.Bucket, project.Branch)
		}

		response, err := http.Get(archiveLocation)
		defer response.Body.Close()

		if err != nil {
			log.Fatal(err)
		}

		_, err = io.Copy(archive, response.Body)

		if err != nil {
			log.Fatal(err)
		}
	}
}
<|endoftext|>


<commit_before>package qb

import (
	"fmt"
	"strings"
)

// NewDialect generates a new dialect object
func NewDialect(driver string) *Dialect {
	return &Dialect{
		driver:       driver,
		query:        NewQuery(),
		bindingIndex: 0,
	}
}

// Dialect is a subset of dialect could be used for common sql queries
// it has all the common functions except multiple statements & table crudders
type Dialect struct {
	driver       string
	query        *Query
	bindingIndex int
}

func (d *Dialect) placeholder() string {
	if d.driver == "postgres" {
		d.bindingIndex++
		return fmt.Sprintf("$%d", d.bindingIndex)
	}
	return "?"
}

func (d *Dialect) placeholders(values ...interface{}) []string {
	placeholders := make([]string, len(values))
	for k := range values {
		placeholders[k] = d.placeholder()
	}
	return placeholders
}

// Reset clears query bindings and its errors
func (d *Dialect) Reset() {
	d.query = NewQuery()
	d.bindingIndex = 0
}

// Query returns the active query and resets the query.
// The query clauses and returns the sql and bindings
func (d *Dialect) Query() *Query {
	query := d.query
	d.Reset()
	return query
}

// Insert generates an "insert into %s(%s)" statement
func (d *Dialect) Insert(table string, columns ...string) *Dialect {
	clause := fmt.Sprintf("INSERT INTO %s(%s)", table, strings.Join(columns, ", "))
	d.query.AddClause(clause)
	return d
}

// Values generates "values(%s)" statement and add bindings for each value
func (d *Dialect) Values(values ...interface{}) *Dialect {
	d.query.AddBinding(values...)
	clause := fmt.Sprintf("VALUES (%s)", strings.Join(d.placeholders(values...), ", "))
	d.query.AddClause(clause)
	return d
}

// Update generates "update %s" statement
func (d *Dialect) Update(table string) *Dialect {
	clause := fmt.Sprintf("UPDATE %s", table)
	d.query.AddClause(clause)
	return d
}

// Set generates "set a = placeholder" statement for each key a and add bindings for map value
func (d *Dialect) Set(m map[string]interface{}) *Dialect {
	updates := []string{}
	for k, v := range m {
		updates = append(updates, fmt.Sprintf("%s = %s", k, d.placeholder()))
		d.query.AddBinding(v)
	}
	clause := fmt.Sprintf("SET %s", strings.Join(updates, ", "))
	d.query.AddClause(clause)
	return d
}

// Delete generates "delete" statement
func (d *Dialect) Delete(table string) *Dialect {
	d.query.AddClause(fmt.Sprintf("DELETE FROM %s", table))
	return d
}

// Select generates "select %s" statement
func (d *Dialect) Select(columns ...string) *Dialect {
	clause := fmt.Sprintf("SELECT %s", strings.Join(columns, ", "))
	d.query.AddClause(clause)
	return d
}

// From generates "from %s" statement for each table name
func (d *Dialect) From(tables ...string) *Dialect {
	d.query.AddClause(fmt.Sprintf("FROM %s", strings.Join(tables, ", ")))
	return d
}

// InnerJoin generates "inner join %s on %s" statement for each expression
func (d *Dialect) InnerJoin(table string, expressions ...string) *Dialect {
	d.query.AddClause(fmt.Sprintf("INNER JOIN %s ON %s", table, strings.Join(expressions, " ")))
	return d
}

// CrossJoin generates "cross join %s" statement for table
func (d *Dialect) CrossJoin(table string) *Dialect {
	d.query.AddClause(fmt.Sprintf("CROSS JOIN %s", table))
	return d
}

// LeftOuterJoin generates "left outer join %s on %s" statement for each expression
func (d *Dialect) LeftOuterJoin(table string, expressions ...string) *Dialect {
	d.query.AddClause(fmt.Sprintf("LEFT OUTER JOIN %s ON %s", table, strings.Join(expressions, " ")))
	return d
}

// RightOuterJoin generates "right outer join %s on %s" statement for each expression
func (d *Dialect) RightOuterJoin(table string, expressions ...string) *Dialect {
	d.query.AddClause(fmt.Sprintf("RIGHT OUTER JOIN %s ON %s", table, strings.Join(expressions, " ")))
	return d
}

// FullOuterJoin generates "full outer join %s on %s" for each expression
func (d *Dialect) FullOuterJoin(table string, expressions ...string) *Dialect {
	d.query.AddClause(fmt.Sprintf("FULL OUTER JOIN %s ON %s", table, strings.Join(expressions, " ")))
	return d
}

// Where generates "where %s" for the expression and adds bindings for each value
func (d *Dialect) Where(expression string, bindings ...interface{}) *Dialect {
	expression = strings.Replace(expression, "?", d.placeholder(), -1)
	d.query.AddClause(fmt.Sprintf("WHERE %s", expression))
	d.query.AddBinding(bindings...)
	return d
}

// OrderBy generates "order by %s" for each expression
func (d *Dialect) OrderBy(expressions ...string) *Dialect {
	d.query.AddClause(fmt.Sprintf("ORDER BY %s", strings.Join(expressions, ", ")))
	return d
}

// GroupBy generates "group by %s" for each column
func (d *Dialect) GroupBy(columns ...string) *Dialect {
	d.query.AddClause(fmt.Sprintf("GROUP BY %s", strings.Join(columns, ", ")))
	return d
}

// Having generates "having %s" for each expression
func (d *Dialect) Having(expressions ...string) *Dialect {
	d.query.AddClause(fmt.Sprintf("HAVING %s", strings.Join(expressions, ", ")))
	return d
}

// Limit generates limit %d offset %d for offset and count
func (d *Dialect) Limit(offset int, count int) *Dialect {
	d.query.AddClause(fmt.Sprintf("LIMIT %d OFFSET %d", count, offset))
	return d
}

// aggregates

// Avg function generates "avg(%s)" statement for column
func (d *Dialect) Avg(column string) string {
	return fmt.Sprintf("AVG(%s)", column)
}

// Count function generates "count(%s)" statement for column
func (d *Dialect) Count(column string) string {
	return fmt.Sprintf("COUNT(%s)", column)
}

// Sum function generates "sum(%s)" statement for column
func (d *Dialect) Sum(column string) string {
	return fmt.Sprintf("SUM(%s)", column)
}

// Min function generates "min(%s)" statement for column
func (d *Dialect) Min(column string) string {
	return fmt.Sprintf("MIN(%s)", column)
}

// Max function generates "max(%s)" statement for column
func (d *Dialect) Max(column string) string {
	return fmt.Sprintf("MAX(%s)", column)
}

// expressions

// NotIn function generates "%s not in (%s)" for key and adds bindings for each value
func (d *Dialect) NotIn(key string, values ...interface{}) string {
	d.query.AddBinding(values...)
	return fmt.Sprintf("%s NOT IN (%s)", key, strings.Join(d.placeholders(values...), ","))
}

// In function generates "%s in (%s)" for key and adds bindings for each value
func (d *Dialect) In(key string, values ...interface{}) string {
	d.query.AddBinding(values...)
	return fmt.Sprintf("%s IN (%s)", key, strings.Join(d.placeholders(values...), ","))
}

// NotEq function generates "%s != placeholder" for key and adds binding for value
func (d *Dialect) NotEq(key string, value interface{}) string {
	d.query.AddBinding(value)
	return fmt.Sprintf("%s != %s", key, d.placeholder())
}

// Eq function generates "%s = placeholder" for key and adds binding for value
func (d *Dialect) Eq(key string, value interface{}) string {
	d.query.AddBinding(value)
	return fmt.Sprintf("%s = %s", key, d.placeholder())
}

// Gt function generates "%s > placeholder" for key and adds binding for value
func (d *Dialect) Gt(key string, value interface{}) string {
	d.query.AddBinding(value)
	return fmt.Sprintf("%s > %s", key, d.placeholder())
}

// Gte function generates "%s >= placeholder" for key and adds binding for value
func (d *Dialect) Gte(key string, value interface{}) string {
	d.query.AddBinding(value)
	return fmt.Sprintf("%s >= %s", key, d.placeholder())
}

// St function generates "%s < placeholder" for key and adds binding for value
func (d *Dialect) St(key string, value interface{}) string {
	d.query.AddBinding(value)
	return fmt.Sprintf("%s < %s", key, d.placeholder())
}

// Ste function generates "%s <= placeholder" for key and adds binding for value
func (d *Dialect) Ste(key string, value interface{}) string {
	d.query.AddBinding(value)
	return fmt.Sprintf("%s <= %s", key, d.placeholder())
}

// And function generates " AND " between any number of expressions
func (d *Dialect) And(expressions ...string) string {
	return fmt.Sprintf("(%s)", strings.Join(expressions, " AND "))
}

// Or function generates " OR " between any number of expressions
func (d *Dialect) Or(expressions ...string) string {
	return strings.Join(expressions, " OR ")
}

// CreateTable generates generic CREATE TABLE statement
func (d *Dialect) CreateTable(table string, fields []string, constraints []string) *Dialect {

	d.query.AddClause(fmt.Sprintf("CREATE TABLE %s(", table))

	for k, f := range fields {
		clause := fmt.Sprintf("\t%s", f)
		if len(fields)-1 > k || len(constraints) > 0 {
			clause += ","
		}
		d.query.AddClause(clause)
	}

	for k, c := range constraints {
		constraint := fmt.Sprintf("\t%s", c)
		if len(constraints)-1 > k {
			constraint += ","
		}
		d.query.AddClause(fmt.Sprintf("%s", constraint))
	}

	d.query.AddClause(")")
	return d
}

// AlterTable generates generic ALTER TABLE statement
func (d *Dialect) AlterTable(table string) *Dialect {
	d.query.AddClause(fmt.Sprintf("ALTER TABLE %s", table))
	return d
}

// DropTable generates generic DROP TABLE statement
func (d *Dialect) DropTable(table string) *Dialect {
	d.query.AddClause(fmt.Sprintf("DROP TABLE %s", table))
	return d
}

// Add generates generic ADD COLUMN statement
func (d *Dialect) Add(colName string, colType string) *Dialect {
	d.query.AddClause(fmt.Sprintf("ADD %s %s", colName, colType))
	return d
}

// Drop generates generic DROP COLUMN statement
func (d *Dialect) Drop(colName string) *Dialect {
	d.query.AddClause(fmt.Sprintf("DROP %s", colName))
	return d
}
<commit_msg>make placeholder functions public in dialect<commit_after>package qb

import (
	"fmt"
	"strings"
)

// NewDialect generates a new dialect object
func NewDialect(driver string) *Dialect {
	return &Dialect{
		driver:       driver,
		query:        NewQuery(),
		bindingIndex: 0,
	}
}

// Dialect is a subset of dialect could be used for common sql queries
// it has all the common functions except multiple statements & table crudders
type Dialect struct {
	driver       string
	query        *Query
	bindingIndex int
}

func (d *Dialect) Placeholder() string {
	if d.driver == "postgres" {
		d.bindingIndex++
		return fmt.Sprintf("$%d", d.bindingIndex)
	}
	return "?"
}

func (d *Dialect) Placeholders(values ...interface{}) []string {
	placeholders := make([]string, len(values))
	for k := range values {
		placeholders[k] = d.Placeholder()
	}
	return placeholders
}

// Reset clears query bindings and its errors
func (d *Dialect) Reset() {
	d.query = NewQuery()
	d.bindingIndex = 0
}

// Query returns the active query and resets the query.
// The query clauses and returns the sql and bindings
func (d *Dialect) Query() *Query {
	query := d.query
	d.Reset()
	return query
}

// Insert generates an "insert into %s(%s)" statement
func (d *Dialect) Insert(table string, columns ...string) *Dialect {
	clause := fmt.Sprintf("INSERT INTO %s(%s)", table, strings.Join(columns, ", "))
	d.query.AddClause(clause)
	return d
}

// Values generates "values(%s)" statement and add bindings for each value
func (d *Dialect) Values(values ...interface{}) *Dialect {
	d.query.AddBinding(values...)
	clause := fmt.Sprintf("VALUES (%s)", strings.Join(d.Placeholders(values...), ", "))
	d.query.AddClause(clause)
	return d
}

// Update generates "update %s" statement
func (d *Dialect) Update(table string) *Dialect {
	clause := fmt.Sprintf("UPDATE %s", table)
	d.query.AddClause(clause)
	return d
}

// Set generates "set a = placeholder" statement for each key a and add bindings for map value
func (d *Dialect) Set(m map[string]interface{}) *Dialect {
	updates := []string{}
	for k, v := range m {
		updates = append(updates, fmt.Sprintf("%s = %s", k, d.Placeholder()))
		d.query.AddBinding(v)
	}
	clause := fmt.Sprintf("SET %s", strings.Join(updates, ", "))
	d.query.AddClause(clause)
	return d
}

// Delete generates "delete" statement
func (d *Dialect) Delete(table string) *Dialect {
	d.query.AddClause(fmt.Sprintf("DELETE FROM %s", table))
	return d
}

// Select generates "select %s" statement
func (d *Dialect) Select(columns ...string) *Dialect {
	clause := fmt.Sprintf("SELECT %s", strings.Join(columns, ", "))
	d.query.AddClause(clause)
	return d
}

// From generates "from %s" statement for each table name
func (d *Dialect) From(tables ...string) *Dialect {
	d.query.AddClause(fmt.Sprintf("FROM %s", strings.Join(tables, ", ")))
	return d
}

// InnerJoin generates "inner join %s on %s" statement for each expression
func (d *Dialect) InnerJoin(table string, expressions ...string) *Dialect {
	d.query.AddClause(fmt.Sprintf("INNER JOIN %s ON %s", table, strings.Join(expressions, " ")))
	return d
}

// CrossJoin generates "cross join %s" statement for table
func (d *Dialect) CrossJoin(table string) *Dialect {
	d.query.AddClause(fmt.Sprintf("CROSS JOIN %s", table))
	return d
}

// LeftOuterJoin generates "left outer join %s on %s" statement for each expression
func (d *Dialect) LeftOuterJoin(table string, expressions ...string) *Dialect {
	d.query.AddClause(fmt.Sprintf("LEFT OUTER JOIN %s ON %s", table, strings.Join(expressions, " ")))
	return d
}

// RightOuterJoin generates "right outer join %s on %s" statement for each expression
func (d *Dialect) RightOuterJoin(table string, expressions ...string) *Dialect {
	d.query.AddClause(fmt.Sprintf("RIGHT OUTER JOIN %s ON %s", table, strings.Join(expressions, " ")))
	return d
}

// FullOuterJoin generates "full outer join %s on %s" for each expression
func (d *Dialect) FullOuterJoin(table string, expressions ...string) *Dialect {
	d.query.AddClause(fmt.Sprintf("FULL OUTER JOIN %s ON %s", table, strings.Join(expressions, " ")))
	return d
}

// Where generates "where %s" for the expression and adds bindings for each value
func (d *Dialect) Where(expression string, bindings ...interface{}) *Dialect {
	expression = strings.Replace(expression, "?", d.Placeholder(), -1)
	d.query.AddClause(fmt.Sprintf("WHERE %s", expression))
	d.query.AddBinding(bindings...)
	return d
}

// OrderBy generates "order by %s" for each expression
func (d *Dialect) OrderBy(expressions ...string) *Dialect {
	d.query.AddClause(fmt.Sprintf("ORDER BY %s", strings.Join(expressions, ", ")))
	return d
}

// GroupBy generates "group by %s" for each column
func (d *Dialect) GroupBy(columns ...string) *Dialect {
	d.query.AddClause(fmt.Sprintf("GROUP BY %s", strings.Join(columns, ", ")))
	return d
}

// Having generates "having %s" for each expression
func (d *Dialect) Having(expressions ...string) *Dialect {
	d.query.AddClause(fmt.Sprintf("HAVING %s", strings.Join(expressions, ", ")))
	return d
}

// Limit generates limit %d offset %d for offset and count
func (d *Dialect) Limit(offset int, count int) *Dialect {
	d.query.AddClause(fmt.Sprintf("LIMIT %d OFFSET %d", count, offset))
	return d
}

// aggregates

// Avg function generates "avg(%s)" statement for column
func (d *Dialect) Avg(column string) string {
	return fmt.Sprintf("AVG(%s)", column)
}

// Count function generates "count(%s)" statement for column
func (d *Dialect) Count(column string) string {
	return fmt.Sprintf("COUNT(%s)", column)
}

// Sum function generates "sum(%s)" statement for column
func (d *Dialect) Sum(column string) string {
	return fmt.Sprintf("SUM(%s)", column)
}

// Min function generates "min(%s)" statement for column
func (d *Dialect) Min(column string) string {
	return fmt.Sprintf("MIN(%s)", column)
}

// Max function generates "max(%s)" statement for column
func (d *Dialect) Max(column string) string {
	return fmt.Sprintf("MAX(%s)", column)
}

// expressions

// NotIn function generates "%s not in (%s)" for key and adds bindings for each value
func (d *Dialect) NotIn(key string, values ...interface{}) string {
	d.query.AddBinding(values...)
	return fmt.Sprintf("%s NOT IN (%s)", key, strings.Join(d.Placeholders(values...), ","))
}

// In function generates "%s in (%s)" for key and adds bindings for each value
func (d *Dialect) In(key string, values ...interface{}) string {
	d.query.AddBinding(values...)
	return fmt.Sprintf("%s IN (%s)", key, strings.Join(d.Placeholders(values...), ","))
}

// NotEq function generates "%s != placeholder" for key and adds binding for value
func (d *Dialect) NotEq(key string, value interface{}) string {
	d.query.AddBinding(value)
	return fmt.Sprintf("%s != %s", key, d.Placeholder())
}

// Eq function generates "%s = placeholder" for key and adds binding for value
func (d *Dialect) Eq(key string, value interface{}) string {
	d.query.AddBinding(value)
	return fmt.Sprintf("%s = %s", key, d.Placeholder())
}

// Gt function generates "%s > placeholder" for key and adds binding for value
func (d *Dialect) Gt(key string, value interface{}) string {
	d.query.AddBinding(value)
	return fmt.Sprintf("%s > %s", key, d.Placeholder())
}

// Gte function generates "%s >= placeholder" for key and adds binding for value
func (d *Dialect) Gte(key string, value interface{}) string {
	d.query.AddBinding(value)
	return fmt.Sprintf("%s >= %s", key, d.Placeholder())
}

// St function generates "%s < placeholder" for key and adds binding for value
func (d *Dialect) St(key string, value interface{}) string {
	d.query.AddBinding(value)
	return fmt.Sprintf("%s < %s", key, d.Placeholder())
}

// Ste function generates "%s <= placeholder" for key and adds binding for value
func (d *Dialect) Ste(key string, value interface{}) string {
	d.query.AddBinding(value)
	return fmt.Sprintf("%s <= %s", key, d.Placeholder())
}

// And function generates " AND " between any number of expressions
func (d *Dialect) And(expressions ...string) string {
	return fmt.Sprintf("(%s)", strings.Join(expressions, " AND "))
}

// Or function generates " OR " between any number of expressions
func (d *Dialect) Or(expressions ...string) string {
	return strings.Join(expressions, " OR ")
}

// CreateTable generates generic CREATE TABLE statement
func (d *Dialect) CreateTable(table string, fields []string, constraints []string) *Dialect {

	d.query.AddClause(fmt.Sprintf("CREATE TABLE %s(", table))

	for k, f := range fields {
		clause := fmt.Sprintf("\t%s", f)
		if len(fields)-1 > k || len(constraints) > 0 {
			clause += ","
		}
		d.query.AddClause(clause)
	}

	for k, c := range constraints {
		constraint := fmt.Sprintf("\t%s", c)
		if len(constraints)-1 > k {
			constraint += ","
		}
		d.query.AddClause(fmt.Sprintf("%s", constraint))
	}

	d.query.AddClause(")")
	return d
}

// AlterTable generates generic ALTER TABLE statement
func (d *Dialect) AlterTable(table string) *Dialect {
	d.query.AddClause(fmt.Sprintf("ALTER TABLE %s", table))
	return d
}

// DropTable generates generic DROP TABLE statement
func (d *Dialect) DropTable(table string) *Dialect {
	d.query.AddClause(fmt.Sprintf("DROP TABLE %s", table))
	return d
}

// Add generates generic ADD COLUMN statement
func (d *Dialect) Add(colName string, colType string) *Dialect {
	d.query.AddClause(fmt.Sprintf("ADD %s %s", colName, colType))
	return d
}

// Drop generates generic DROP COLUMN statement
func (d *Dialect) Drop(colName string) *Dialect {
	d.query.AddClause(fmt.Sprintf("DROP %s", colName))
	return d
}
<|endoftext|>