<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"strconv"
	"time"

	kube_client "github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/cloudcfg"
)

const APP_VERSION = "0.1"

// The flag package provides a default help printer via -h switch
var (
	versionFlag  = flag.Bool("v", false, "Print the version number.")
	httpServer   = flag.String("h", "", "The host to connect to.")
	config       = flag.String("c", "", "Path to the config file.")
	labelQuery   = flag.String("l", "", "Label query to use for listing")
	updatePeriod = flag.Duration("u", 60*time.Second, "Update interarrival in seconds")
	portSpec     = flag.String("p", "", "The port spec, comma-separated list of <external>:<internal>,...")
	servicePort  = flag.Int("s", -1, "If positive, create and run a corresponding service on this port, only used with 'run'")
	authConfig   = flag.String("auth", os.Getenv("HOME")+"/.kubernetes_auth", "Path to the auth info file.  If missing, prompt the user")
	json         = flag.Bool("json", false, "If true, print raw JSON for responses")
	yaml         = flag.Bool("yaml", false, "If true, print raw YAML for responses")
)

func usage() {
	log.Fatal("Usage: cloudcfg -h <host> [-c config/file.json] [-p <hostPort>:<containerPort>,..., <hostPort-n>:<containerPort-n> <method> <path>")
}

// CloudCfg command line tool.
func main() {
	flag.Parse() // Scan the arguments list

	if *versionFlag {
		fmt.Println("Version:", APP_VERSION)
		os.Exit(0)
	}

	if len(flag.Args()) < 2 {
		usage()
	}
	method := flag.Arg(0)
	url := *httpServer + "/api/v1beta1" + flag.Arg(1)
	var request *http.Request
	var err error

	var printer cloudcfg.ResourcePrinter
	if *json {
		printer = &cloudcfg.IdentityPrinter{}
	} else if *yaml {
		printer = &cloudcfg.YAMLPrinter{}
	} else {
		printer = &cloudcfg.HumanReadablePrinter{}
	}

	auth, err := cloudcfg.LoadAuthInfo(*authConfig)
	if err != nil {
		log.Fatalf("Error loading auth: %#v", err)
	}

	if method == "get" || method == "list" {
		if len(*labelQuery) > 0 && method == "list" {
			url = url + "?labels=" + *labelQuery
		}
		request, err = http.NewRequest("GET", url, nil)
	} else if method == "delete" {
		request, err = http.NewRequest("DELETE", url, nil)
	} else if method == "create" {
		request, err = cloudcfg.RequestWithBody(*config, url, "POST")
	} else if method == "update" {
		request, err = cloudcfg.RequestWithBody(*config, url, "PUT")
	} else if method == "rollingupdate" {
		client := &kube_client.Client{
			Host: *httpServer,
			Auth: &auth,
		}
		cloudcfg.Update(flag.Arg(1), client, *updatePeriod)
	} else if method == "run" {
		args := flag.Args()
		if len(args) < 4 {
			log.Fatal("usage: cloudcfg -h <host> run <image> <replicas> <name>")
		}
		image := args[1]
		replicas, err := strconv.Atoi(args[2])
		name := args[3]
		if err != nil {
			log.Fatalf("Error parsing replicas: %#v", err)
		}
		err = cloudcfg.RunController(image, name, replicas, kube_client.Client{Host: *httpServer, Auth: &auth}, *portSpec, *servicePort)
		if err != nil {
			log.Fatalf("Error: %#v", err)
		}
		return
	} else if method == "stop" {
		err = cloudcfg.StopController(flag.Arg(1), kube_client.Client{Host: *httpServer, Auth: &auth})
		if err != nil {
			log.Fatalf("Error: %#v", err)
		}
		return
	} else if method == "rm" {
		err = cloudcfg.DeleteController(flag.Arg(1), kube_client.Client{Host: *httpServer, Auth: &auth})
		if err != nil {
			log.Fatalf("Error: %#v", err)
		}
		return
	} else {
		log.Fatalf("Unknown command: %s", method)
	}
	if err != nil {
		log.Fatalf("Error: %#v", err)
	}
	var body string
	body, err = cloudcfg.DoRequest(request, auth.User, auth.Password)
	if err != nil {
		log.Fatalf("Error: %#v", err)
	}
	err = printer.Print(body, os.Stdout)
	if err != nil {
		log.Fatalf("Failed to print: %#v", err)
	}
	fmt.Print("\n")
}
<commit_msg>Add some extra log info.<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"strconv"
	"time"

	kube_client "github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/cloudcfg"
)

const APP_VERSION = "0.1"

// The flag package provides a default help printer via -h switch
var (
	versionFlag  = flag.Bool("v", false, "Print the version number.")
	httpServer   = flag.String("h", "", "The host to connect to.")
	config       = flag.String("c", "", "Path to the config file.")
	labelQuery   = flag.String("l", "", "Label query to use for listing")
	updatePeriod = flag.Duration("u", 60*time.Second, "Update interarrival in seconds")
	portSpec     = flag.String("p", "", "The port spec, comma-separated list of <external>:<internal>,...")
	servicePort  = flag.Int("s", -1, "If positive, create and run a corresponding service on this port, only used with 'run'")
	authConfig   = flag.String("auth", os.Getenv("HOME")+"/.kubernetes_auth", "Path to the auth info file.  If missing, prompt the user")
	json         = flag.Bool("json", false, "If true, print raw JSON for responses")
	yaml         = flag.Bool("yaml", false, "If true, print raw YAML for responses")
)

func usage() {
	log.Fatal("Usage: cloudcfg -h <host> [-c config/file.json] [-p <hostPort>:<containerPort>,..., <hostPort-n>:<containerPort-n> <method> <path>")
}

// CloudCfg command line tool.
func main() {
	flag.Parse() // Scan the arguments list

	if *versionFlag {
		fmt.Println("Version:", APP_VERSION)
		os.Exit(0)
	}

	if len(flag.Args()) < 2 {
		usage()
	}
	method := flag.Arg(0)
	url := *httpServer + "/api/v1beta1" + flag.Arg(1)
	var request *http.Request
	var err error

	var printer cloudcfg.ResourcePrinter
	if *json {
		printer = &cloudcfg.IdentityPrinter{}
	} else if *yaml {
		printer = &cloudcfg.YAMLPrinter{}
	} else {
		printer = &cloudcfg.HumanReadablePrinter{}
	}

	auth, err := cloudcfg.LoadAuthInfo(*authConfig)
	if err != nil {
		log.Fatalf("Error loading auth: %#v", err)
	}

	if method == "get" || method == "list" {
		if len(*labelQuery) > 0 && method == "list" {
			url = url + "?labels=" + *labelQuery
		}
		request, err = http.NewRequest("GET", url, nil)
	} else if method == "delete" {
		request, err = http.NewRequest("DELETE", url, nil)
	} else if method == "create" {
		request, err = cloudcfg.RequestWithBody(*config, url, "POST")
	} else if method == "update" {
		request, err = cloudcfg.RequestWithBody(*config, url, "PUT")
	} else if method == "rollingupdate" {
		client := &kube_client.Client{
			Host: *httpServer,
			Auth: &auth,
		}
		cloudcfg.Update(flag.Arg(1), client, *updatePeriod)
	} else if method == "run" {
		args := flag.Args()
		if len(args) < 4 {
			log.Fatal("usage: cloudcfg -h <host> run <image> <replicas> <name>")
		}
		image := args[1]
		replicas, err := strconv.Atoi(args[2])
		name := args[3]
		if err != nil {
			log.Fatalf("Error parsing replicas: %#v", err)
		}
		err = cloudcfg.RunController(image, name, replicas, kube_client.Client{Host: *httpServer, Auth: &auth}, *portSpec, *servicePort)
		if err != nil {
			log.Fatalf("Error: %#v", err)
		}
		return
	} else if method == "stop" {
		err = cloudcfg.StopController(flag.Arg(1), kube_client.Client{Host: *httpServer, Auth: &auth})
		if err != nil {
			log.Fatalf("Error: %#v", err)
		}
		return
	} else if method == "rm" {
		err = cloudcfg.DeleteController(flag.Arg(1), kube_client.Client{Host: *httpServer, Auth: &auth})
		if err != nil {
			log.Fatalf("Error: %#v", err)
		}
		return
	} else {
		log.Fatalf("Unknown command: %s", method)
	}
	if err != nil {
		log.Fatalf("Error: %#v", err)
	}
	var body string
	body, err = cloudcfg.DoRequest(request, auth.User, auth.Password)
	if err != nil {
		log.Fatalf("Error: %#v %s", err, body)
	}
	err = printer.Print(body, os.Stdout)
	if err != nil {
		log.Fatalf("Failed to print: %#v", err)
	}
	fmt.Print("\n")
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"testing"

	"github.com/pinzolo/csvutil"
)

const (
	testdataDir = "../../testdata"
	tempDir     = "../../testdata/temp"
)

var bakFileRegex = regexp.MustCompile(`utf8\.\d{14}\.csv`)

func TestMain(m *testing.M) {
	//errWriter = &bytes.Buffer{}
	code := m.Run()
	defer os.Exit(code)
}

func copyTestFile() error {
	src, err := os.Open(testFilePath("utf8.csv"))
	if err != nil {
		return err
	}
	defer src.Close()

	dst, err := os.Create(tempFilePath())
	if err != nil {
		return nil
	}
	defer dst.Close()
	_, err = io.Copy(dst, src)
	return err
}

func prepareWritingTest() (func(), error) {
	if _, err := os.Stat(tempDir); err == nil {
		os.RemoveAll(tempDir)
	}
	err := os.Mkdir(tempDir, 0777)
	if err != nil {
		return nil, err
	}
	err = copyTestFile()
	f := func() {
		ferr := os.RemoveAll(tempDir)
		if ferr != nil {
			panic(ferr)
		}
	}
	return f, err
}

func testFilePath(n string) string {
	return filepath.Join(testdataDir, n)
}

func tempFilePath() string {
	return filepath.Join(filepath.Join(tempDir, "utf8.csv"))
}

func overwriteContent() ([][]string, error) {
	f, err := os.Open(tempFilePath())
	if err != nil {
		return nil, err
	}
	r, _ := csvutil.NewReader(f)
	return r.ReadAll()
}

func existsBackup() (bool, error) {
	files, err := ioutil.ReadDir(tempDir)
	if err != nil {
		return false, err
	}
	for _, f := range files {
		if bakFileRegex.MatchString(f.Name()) {
			return true, nil
		}
	}
	return false, nil
}
<commit_msg>Remove TestMain<commit_after>package main

import (
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"

	"github.com/pinzolo/csvutil"
)

const (
	testdataDir = "../../testdata"
	tempDir     = "../../testdata/temp"
)

var bakFileRegex = regexp.MustCompile(`utf8\.\d{14}\.csv`)

func copyTestFile() error {
	src, err := os.Open(testFilePath("utf8.csv"))
	if err != nil {
		return err
	}
	defer src.Close()

	dst, err := os.Create(tempFilePath())
	if err != nil {
		return nil
	}
	defer dst.Close()
	_, err = io.Copy(dst, src)
	return err
}

func prepareWritingTest() (func(), error) {
	if _, err := os.Stat(tempDir); err == nil {
		os.RemoveAll(tempDir)
	}
	err := os.Mkdir(tempDir, 0777)
	if err != nil {
		return nil, err
	}
	err = copyTestFile()
	f := func() {
		ferr := os.RemoveAll(tempDir)
		if ferr != nil {
			panic(ferr)
		}
	}
	return f, err
}

func testFilePath(n string) string {
	return filepath.Join(testdataDir, n)
}

func tempFilePath() string {
	return filepath.Join(filepath.Join(tempDir, "utf8.csv"))
}

func overwriteContent() ([][]string, error) {
	f, err := os.Open(tempFilePath())
	if err != nil {
		return nil, err
	}
	r, _ := csvutil.NewReader(f)
	return r.ReadAll()
}

func existsBackup() (bool, error) {
	files, err := ioutil.ReadDir(tempDir)
	if err != nil {
		return false, err
	}
	for _, f := range files {
		if bakFileRegex.MatchString(f.Name()) {
			return true, nil
		}
	}
	return false, nil
}
<|endoftext|>


<commit_before>package cmd

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/drud/ddev/pkg/ddevapp"
	"github.com/drud/ddev/pkg/exec"
	"github.com/drud/ddev/pkg/fileutil"
	"github.com/drud/ddev/pkg/globalconfig"
	"github.com/drud/ddev/pkg/util"
	"github.com/gobuffalo/packr/v2"
	"github.com/mattn/go-isatty"
	"github.com/spf13/cobra"
)

// addCustomCommands looks for custom command scripts in
// ~/.ddev/commands/<servicename> etc. and
// .ddev/commands/<servicename> and .ddev/commands/host
// and if it finds them adds them to Cobra's commands.
func addCustomCommands(rootCmd *cobra.Command) error {
	app, err := ddevapp.GetActiveApp("")
	if err != nil {
		return nil
	}

	sourceGlobalCommandPath := filepath.Join(globalconfig.GetGlobalDdevDir(), "commands")
	err = os.MkdirAll(sourceGlobalCommandPath, 0755)
	if err != nil {
		return nil
	}

	projectCommandPath := app.GetConfigPath("commands")
	// Make sure our target global command directory is empty
	targetGlobalCommandPath := app.GetConfigPath(".global_commands")
	_ = os.RemoveAll(targetGlobalCommandPath)

	err = fileutil.CopyDir(sourceGlobalCommandPath, targetGlobalCommandPath)
	if err != nil {
		return err
	}

	if !fileutil.FileExists(projectCommandPath) || !fileutil.IsDirectory(projectCommandPath) {
		return nil
	}

	commandsAdded := map[string]int{}
	for _, commandSet := range []string{projectCommandPath, targetGlobalCommandPath} {
		commandDirs, err := fileutil.ListFilesInDirFullPath(commandSet)
		if err != nil {
			return err
		}
		for _, serviceDirOnHost := range commandDirs {
			service := filepath.Base(serviceDirOnHost)

			// If the item isn't actually a directory, just skip it.
			if !fileutil.IsDirectory(serviceDirOnHost) {
				continue
			}
			commandFiles, err := fileutil.ListFilesInDir(serviceDirOnHost)
			if err != nil {
				return err
			}
			if runtime.GOOS == "windows" {
				windowsBashPath := util.FindWindowsBashPath()
				if windowsBashPath == "" {
					fmt.Println("Unable to find bash.exe in PATH, not loading custom commands")
					return nil
				}
			}

			for _, commandName := range commandFiles {
				// Use path.Join() for the inContainerFullPath because it'serviceDirOnHost about the path in the container, not on the
				// host; a Windows path is not useful here.
				inContainerFullPath := path.Join("/mnt/ddev_config", filepath.Base(commandSet), service, commandName)
				onHostFullPath := filepath.Join(commandSet, service, commandName)

				if strings.HasSuffix(commandName, ".example") || strings.HasPrefix(commandName, "README") || strings.HasPrefix(commandName, ".") || fileutil.IsDirectory(onHostFullPath) {
					continue
				}

				// If command has already been added, we won't work with it again.
				if _, ok := commandsAdded[commandName]; ok {
					util.Warning("not adding command %s (%s) because it was already added to project %s", commandName, onHostFullPath, app.Name)
					continue
				}

				// Any command we find will want to be executable on Linux
				_ = os.Chmod(onHostFullPath, 0755)
				if hasCR, _ := fileutil.FgrepStringInFile(onHostFullPath, "\r\n"); hasCR {
					util.Warning("command '%s' contains CRLF, please convert to Linux-style linefeeds with dos2unix or another tool, skipping %s", commandName, onHostFullPath)
					continue
				}

				directives := findDirectivesInScriptCommand(onHostFullPath)
				var description, usage, example, projectTypes, osTypes, hostBinaryExists string

				description = commandName
				if val, ok := directives["Description"]; ok {
					description = val
				}

				if val, ok := directives["Usage"]; ok {
					usage = val
				}

				if val, ok := directives["Example"]; ok {
					example = "  " + strings.ReplaceAll(val, `\n`, "\n  ")
				}

				// Init and import flags
				var flags Flags
				flags.Init(commandName, onHostFullPath)

				if val, ok := directives["Flags"]; ok {
					if err = flags.LoadFromJSON(val); err != nil {
						util.Warning("Error '%s', command '%s' contains an invalid flags definition '%s', skipping add flags of %s", err, commandName, val, onHostFullPath)
					}
				}

				// Import and handle ProjectTypes
				if val, ok := directives["ProjectTypes"]; ok {
					projectTypes = val
				}

				// If ProjectTypes is specified and we aren't of that type, skip
				if projectTypes != "" && !strings.Contains(projectTypes, app.Type) {
					continue
				}

				// Import and handle OSTypes
				if val, ok := directives["OSTypes"]; ok {
					osTypes = val
				}

				// If OSTypes is specified and we aren't this isn't a specified OS, skip
				if osTypes != "" && !strings.Contains(osTypes, runtime.GOOS) {
					continue
				}

				// Import and handle HostBinaryExists
				if val, ok := directives["HostBinaryExists"]; ok {
					hostBinaryExists = val
				}

				// If hostBinaryExists is specified it doesn't exist here, skip
				if hostBinaryExists != "" && !fileutil.FileExists(hostBinaryExists) {
					continue
				}

				// Create proper description suffix
				descSuffix := " (shell " + service + " container command)"
				if commandSet == targetGlobalCommandPath {
					descSuffix = " (global shell " + service + " container command)"
				}

				// Initialize the new command
				commandToAdd := &cobra.Command{
					Use:     usage,
					Short:   description + descSuffix,
					Example: example,
					FParseErrWhitelist: cobra.FParseErrWhitelist{
						UnknownFlags: true,
					},
				}

				// Add flags to command
				if err = flags.AssignToCommand(commandToAdd); err != nil {
					util.Warning("Error '%s' in the flags definition for command '%s', skipping %s", err, commandName, onHostFullPath)
					continue
				}

				if service == "host" {
					commandToAdd.Run = makeHostCmd(app, onHostFullPath, commandName)
				} else {
					commandToAdd.Run = makeContainerCmd(app, inContainerFullPath, commandName, service)
				}

				// Add the command and mark as added
				rootCmd.AddCommand(commandToAdd)
				commandsAdded[commandName] = 1
			}
		}
	}

	return nil
}

// makeHostCmd creates a command which will run on the host
func makeHostCmd(app *ddevapp.DdevApp, fullPath, name string) func(*cobra.Command, []string) {
	var windowsBashPath = ""
	if runtime.GOOS == "windows" {
		windowsBashPath = util.FindWindowsBashPath()
	}

	return func(cmd *cobra.Command, cobraArgs []string) {
		if app.SiteStatus() != ddevapp.SiteRunning {
			err := app.Start()
			if err != nil {
				util.Failed("Failed to start project for custom command: %v", err)
			}
		}
		app.DockerEnv()

		osArgs := []string{}
		if len(os.Args) > 2 {
			osArgs = os.Args[2:]
		}
		var err error
		// Load environment variables that may be useful for script.
		app.DockerEnv()
		if runtime.GOOS == "windows" {
			// Sadly, not sure how to have a bash interpreter without this.
			args := []string{fullPath}
			args = append(args, osArgs...)
			err = exec.RunInteractiveCommand(windowsBashPath, args)
		} else {
			err = exec.RunInteractiveCommand(fullPath, osArgs)
		}
		if err != nil {
			util.Failed("Failed to run %s %v; error=%v", name, strings.Join(osArgs, " "), err)
		}
	}
}

// makeContainerCmd creates the command which will app.Exec to a container command
func makeContainerCmd(app *ddevapp.DdevApp, fullPath, name string, service string) func(*cobra.Command, []string) {
	s := service
	if s[0:1] == "." {
		s = s[1:]
	}
	return func(cmd *cobra.Command, args []string) {
		if app.SiteStatus() != ddevapp.SiteRunning {
			err := app.Start()
			if err != nil {
				util.Failed("Failed to start project for custom command: %v", err)
			}
		}
		app.DockerEnv()

		osArgs := []string{}
		if len(os.Args) > 2 {
			osArgs = os.Args[2:]
		}
		_, _, err := app.Exec(&ddevapp.ExecOpts{
			Cmd:       fullPath + " " + strings.Join(osArgs, " "),
			Service:   s,
			Dir:       app.GetWorkingDir(s, ""),
			Tty:       isatty.IsTerminal(os.Stdin.Fd()),
			NoCapture: true,
		})

		if err != nil {
			util.Failed("Failed to run %s %v: %v", name, strings.Join(osArgs, " "), err)
		}
	}
}

// findDirectivesInScriptCommand() Returns a map of directives and their contents
// found in the named script
func findDirectivesInScriptCommand(script string) map[string]string {
	f, err := os.Open(script)
	if err != nil {
		util.Failed("Failed to open %s: %v", script, err)
	}

	// nolint errcheck
	defer f.Close()

	var directives = make(map[string]string)

	// Splits on newlines by default.
	scanner := bufio.NewScanner(f)

	for scanner.Scan() {
		line := scanner.Text()
		if strings.HasPrefix(line, "## ") && strings.Contains(line, ":") {
			line = strings.Replace(line, "## ", "", 1)
			parts := strings.SplitN(line, ":", 2)
			parts[1] = strings.Trim(parts[1], " ")
			directives[parts[0]] = parts[1]
		}
	}

	if err := scanner.Err(); err != nil {
		return nil
	}

	return directives
}

// populateExamplesCommandsHomeadditions grabs packr2 assets
// When the items in the assets directory are changed, the packr2 command
// must be run again in this directory (cmd/ddev/cmd) to update the saved
// embedded files.
// "make packr2" can be used to update the packr2 cache.
func populateExamplesCommandsHomeadditions() error {
	app, err := ddevapp.GetActiveApp("")
	if err != nil {
		return nil
	}
	box := packr.New("customcommands", "./dotddev_assets")

	list := box.List()
	for _, file := range list {
		localPath := app.GetConfigPath(file)
		sigFound, err := fileutil.FgrepStringInFile(localPath, ddevapp.DdevFileSignature)
		if sigFound || err != nil {
			content, err := box.Find(file)
			if err != nil {
				return err
			}
			err = os.MkdirAll(filepath.Dir(localPath), 0755)
			if err != nil {
				return err
			}
			err = ioutil.WriteFile(localPath, content, 0755)
			if err != nil {
				return err
			}
		}
	}

	// This brings in both the commands and the homeadditions files
	box = packr.New("global_dotddev", "./global_dotddev_assets")

	list = box.List()
	globalDdevDir := globalconfig.GetGlobalDdevDir()
	for _, file := range list {
		localPath := filepath.Join(globalDdevDir, file)
		sigFound, err := fileutil.FgrepStringInFile(localPath, ddevapp.DdevFileSignature)
		if sigFound || err != nil {
			content, err := box.Find(file)
			if err != nil {
				return err
			}
			err = os.MkdirAll(filepath.Dir(localPath), 0755)
			if err != nil {
				return err
			}
			err = ioutil.WriteFile(localPath, content, 0755)
			if err != nil {
				return err
			}
		}
	}
	return nil
}
<commit_msg>Add default usage of shell custom commands again based on filename, fixes #2633 (#2635)<commit_after>package cmd

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/drud/ddev/pkg/ddevapp"
	"github.com/drud/ddev/pkg/exec"
	"github.com/drud/ddev/pkg/fileutil"
	"github.com/drud/ddev/pkg/globalconfig"
	"github.com/drud/ddev/pkg/util"
	"github.com/gobuffalo/packr/v2"
	"github.com/mattn/go-isatty"
	"github.com/spf13/cobra"
)

// addCustomCommands looks for custom command scripts in
// ~/.ddev/commands/<servicename> etc. and
// .ddev/commands/<servicename> and .ddev/commands/host
// and if it finds them adds them to Cobra's commands.
func addCustomCommands(rootCmd *cobra.Command) error {
	app, err := ddevapp.GetActiveApp("")
	if err != nil {
		return nil
	}

	sourceGlobalCommandPath := filepath.Join(globalconfig.GetGlobalDdevDir(), "commands")
	err = os.MkdirAll(sourceGlobalCommandPath, 0755)
	if err != nil {
		return nil
	}

	projectCommandPath := app.GetConfigPath("commands")
	// Make sure our target global command directory is empty
	targetGlobalCommandPath := app.GetConfigPath(".global_commands")
	_ = os.RemoveAll(targetGlobalCommandPath)

	err = fileutil.CopyDir(sourceGlobalCommandPath, targetGlobalCommandPath)
	if err != nil {
		return err
	}

	if !fileutil.FileExists(projectCommandPath) || !fileutil.IsDirectory(projectCommandPath) {
		return nil
	}

	commandsAdded := map[string]int{}
	for _, commandSet := range []string{projectCommandPath, targetGlobalCommandPath} {
		commandDirs, err := fileutil.ListFilesInDirFullPath(commandSet)
		if err != nil {
			return err
		}
		for _, serviceDirOnHost := range commandDirs {
			service := filepath.Base(serviceDirOnHost)

			// If the item isn't actually a directory, just skip it.
			if !fileutil.IsDirectory(serviceDirOnHost) {
				continue
			}
			commandFiles, err := fileutil.ListFilesInDir(serviceDirOnHost)
			if err != nil {
				return err
			}
			if runtime.GOOS == "windows" {
				windowsBashPath := util.FindWindowsBashPath()
				if windowsBashPath == "" {
					fmt.Println("Unable to find bash.exe in PATH, not loading custom commands")
					return nil
				}
			}

			for _, commandName := range commandFiles {
				// Use path.Join() for the inContainerFullPath because it'serviceDirOnHost about the path in the container, not on the
				// host; a Windows path is not useful here.
				inContainerFullPath := path.Join("/mnt/ddev_config", filepath.Base(commandSet), service, commandName)
				onHostFullPath := filepath.Join(commandSet, service, commandName)

				if strings.HasSuffix(commandName, ".example") || strings.HasPrefix(commandName, "README") || strings.HasPrefix(commandName, ".") || fileutil.IsDirectory(onHostFullPath) {
					continue
				}

				// If command has already been added, we won't work with it again.
				if _, ok := commandsAdded[commandName]; ok {
					util.Warning("not adding command %s (%s) because it was already added to project %s", commandName, onHostFullPath, app.Name)
					continue
				}

				// Any command we find will want to be executable on Linux
				_ = os.Chmod(onHostFullPath, 0755)
				if hasCR, _ := fileutil.FgrepStringInFile(onHostFullPath, "\r\n"); hasCR {
					util.Warning("command '%s' contains CRLF, please convert to Linux-style linefeeds with dos2unix or another tool, skipping %s", commandName, onHostFullPath)
					continue
				}

				directives := findDirectivesInScriptCommand(onHostFullPath)
				var description, usage, example, projectTypes, osTypes, hostBinaryExists string

				description = commandName
				if val, ok := directives["Description"]; ok {
					description = val
				}

				usage = commandName + " [flags] [args]"
				if val, ok := directives["Usage"]; ok {
					usage = val
				}

				if val, ok := directives["Example"]; ok {
					example = "  " + strings.ReplaceAll(val, `\n`, "\n  ")
				}

				// Init and import flags
				var flags Flags
				flags.Init(commandName, onHostFullPath)

				if val, ok := directives["Flags"]; ok {
					if err = flags.LoadFromJSON(val); err != nil {
						util.Warning("Error '%s', command '%s' contains an invalid flags definition '%s', skipping add flags of %s", err, commandName, val, onHostFullPath)
					}
				}

				// Import and handle ProjectTypes
				if val, ok := directives["ProjectTypes"]; ok {
					projectTypes = val
				}

				// If ProjectTypes is specified and we aren't of that type, skip
				if projectTypes != "" && !strings.Contains(projectTypes, app.Type) {
					continue
				}

				// Import and handle OSTypes
				if val, ok := directives["OSTypes"]; ok {
					osTypes = val
				}

				// If OSTypes is specified and we aren't this isn't a specified OS, skip
				if osTypes != "" && !strings.Contains(osTypes, runtime.GOOS) {
					continue
				}

				// Import and handle HostBinaryExists
				if val, ok := directives["HostBinaryExists"]; ok {
					hostBinaryExists = val
				}

				// If hostBinaryExists is specified it doesn't exist here, skip
				if hostBinaryExists != "" && !fileutil.FileExists(hostBinaryExists) {
					continue
				}

				// Create proper description suffix
				descSuffix := " (shell " + service + " container command)"
				if commandSet == targetGlobalCommandPath {
					descSuffix = " (global shell " + service + " container command)"
				}

				// Initialize the new command
				commandToAdd := &cobra.Command{
					Use:     usage,
					Short:   description + descSuffix,
					Example: example,
					FParseErrWhitelist: cobra.FParseErrWhitelist{
						UnknownFlags: true,
					},
				}

				// Add flags to command
				if err = flags.AssignToCommand(commandToAdd); err != nil {
					util.Warning("Error '%s' in the flags definition for command '%s', skipping %s", err, commandName, onHostFullPath)
					continue
				}

				if service == "host" {
					commandToAdd.Run = makeHostCmd(app, onHostFullPath, commandName)
				} else {
					commandToAdd.Run = makeContainerCmd(app, inContainerFullPath, commandName, service)
				}

				// Add the command and mark as added
				rootCmd.AddCommand(commandToAdd)
				commandsAdded[commandName] = 1
			}
		}
	}

	return nil
}

// makeHostCmd creates a command which will run on the host
func makeHostCmd(app *ddevapp.DdevApp, fullPath, name string) func(*cobra.Command, []string) {
	var windowsBashPath = ""
	if runtime.GOOS == "windows" {
		windowsBashPath = util.FindWindowsBashPath()
	}

	return func(cmd *cobra.Command, cobraArgs []string) {
		if app.SiteStatus() != ddevapp.SiteRunning {
			err := app.Start()
			if err != nil {
				util.Failed("Failed to start project for custom command: %v", err)
			}
		}
		app.DockerEnv()

		osArgs := []string{}
		if len(os.Args) > 2 {
			osArgs = os.Args[2:]
		}
		var err error
		// Load environment variables that may be useful for script.
		app.DockerEnv()
		if runtime.GOOS == "windows" {
			// Sadly, not sure how to have a bash interpreter without this.
			args := []string{fullPath}
			args = append(args, osArgs...)
			err = exec.RunInteractiveCommand(windowsBashPath, args)
		} else {
			err = exec.RunInteractiveCommand(fullPath, osArgs)
		}
		if err != nil {
			util.Failed("Failed to run %s %v; error=%v", name, strings.Join(osArgs, " "), err)
		}
	}
}

// makeContainerCmd creates the command which will app.Exec to a container command
func makeContainerCmd(app *ddevapp.DdevApp, fullPath, name string, service string) func(*cobra.Command, []string) {
	s := service
	if s[0:1] == "." {
		s = s[1:]
	}
	return func(cmd *cobra.Command, args []string) {
		if app.SiteStatus() != ddevapp.SiteRunning {
			err := app.Start()
			if err != nil {
				util.Failed("Failed to start project for custom command: %v", err)
			}
		}
		app.DockerEnv()

		osArgs := []string{}
		if len(os.Args) > 2 {
			osArgs = os.Args[2:]
		}
		_, _, err := app.Exec(&ddevapp.ExecOpts{
			Cmd:       fullPath + " " + strings.Join(osArgs, " "),
			Service:   s,
			Dir:       app.GetWorkingDir(s, ""),
			Tty:       isatty.IsTerminal(os.Stdin.Fd()),
			NoCapture: true,
		})

		if err != nil {
			util.Failed("Failed to run %s %v: %v", name, strings.Join(osArgs, " "), err)
		}
	}
}

// findDirectivesInScriptCommand() Returns a map of directives and their contents
// found in the named script
func findDirectivesInScriptCommand(script string) map[string]string {
	f, err := os.Open(script)
	if err != nil {
		util.Failed("Failed to open %s: %v", script, err)
	}

	// nolint errcheck
	defer f.Close()

	var directives = make(map[string]string)

	// Splits on newlines by default.
	scanner := bufio.NewScanner(f)

	for scanner.Scan() {
		line := scanner.Text()
		if strings.HasPrefix(line, "## ") && strings.Contains(line, ":") {
			line = strings.Replace(line, "## ", "", 1)
			parts := strings.SplitN(line, ":", 2)
			parts[1] = strings.Trim(parts[1], " ")
			directives[parts[0]] = parts[1]
		}
	}

	if err := scanner.Err(); err != nil {
		return nil
	}

	return directives
}

// populateExamplesCommandsHomeadditions grabs packr2 assets
// When the items in the assets directory are changed, the packr2 command
// must be run again in this directory (cmd/ddev/cmd) to update the saved
// embedded files.
// "make packr2" can be used to update the packr2 cache.
func populateExamplesCommandsHomeadditions() error {
	app, err := ddevapp.GetActiveApp("")
	if err != nil {
		return nil
	}
	box := packr.New("customcommands", "./dotddev_assets")

	list := box.List()
	for _, file := range list {
		localPath := app.GetConfigPath(file)
		sigFound, err := fileutil.FgrepStringInFile(localPath, ddevapp.DdevFileSignature)
		if sigFound || err != nil {
			content, err := box.Find(file)
			if err != nil {
				return err
			}
			err = os.MkdirAll(filepath.Dir(localPath), 0755)
			if err != nil {
				return err
			}
			err = ioutil.WriteFile(localPath, content, 0755)
			if err != nil {
				return err
			}
		}
	}

	// This brings in both the commands and the homeadditions files
	box = packr.New("global_dotddev", "./global_dotddev_assets")

	list = box.List()
	globalDdevDir := globalconfig.GetGlobalDdevDir()
	for _, file := range list {
		localPath := filepath.Join(globalDdevDir, file)
		sigFound, err := fileutil.FgrepStringInFile(localPath, ddevapp.DdevFileSignature)
		if sigFound || err != nil {
			content, err := box.Find(file)
			if err != nil {
				return err
			}
			err = os.MkdirAll(filepath.Dir(localPath), 0755)
			if err != nil {
				return err
			}
			err = ioutil.WriteFile(localPath, content, 0755)
			if err != nil {
				return err
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"context"
	"testing"

	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/ingress-gce/pkg/annotations"
	"k8s.io/ingress-gce/pkg/e2e"
	"k8s.io/ingress-gce/pkg/fuzz"
	"k8s.io/ingress-gce/pkg/fuzz/features"
)

func TestNEG(t *testing.T) {
	t.Parallel()

	port80 := intstr.FromInt(80)

	ctx := context.Background()

	for _, tc := range []struct {
		desc               string
		annotations        annotations.NegAnnotation
		negExpected        bool
		numForwardingRules int
		numBackendServices int
	}{
		{
			desc:               "Create a basic NEG",
			annotations:        annotations.NegAnnotation{Ingress: true},
			negExpected:        true,
			numForwardingRules: 1,
			numBackendServices: 1,
		},
		{
			desc:               "Annotation with no NEG",
			annotations:        annotations.NegAnnotation{Ingress: false},
			negExpected:        false,
			numForwardingRules: 1,
			numBackendServices: 1,
		},
	} {
		tc := tc // Capture tc as we are running this in parallel.
		Framework.RunWithSandbox(tc.desc, t, func(t *testing.T, s *e2e.Sandbox) {
			_, err := e2e.EnsureEchoService(s, "service-1", map[string]string{
				annotations.NEGAnnotationKey: tc.annotations.String()}, v1.ProtocolTCP, 80, v1.ServiceTypeNodePort, 1)
			if err != nil {
				t.Fatalf("error ensuring echo service: %v", err)
			}
			t.Logf("Echo service ensured (%s/%s)", s.Namespace, "service-1")

			// Create the ingress
			ing := fuzz.NewIngressBuilder("", "ingress-1", "").
				DefaultBackend("service-1", port80).
				Build()
			ing, err = e2e.EnsureIngress(s, ing)
			if err != nil {
				t.Fatalf("error ensuring Ingress spec: %v", err)
			}
			t.Logf("Ingress ensured (%s/%s)", s.Namespace, ing.Name)

			ing, err = e2e.WaitForIngress(s, ing, nil)
			if err != nil {
				t.Fatalf("error waiting for Ingress to stabilize: %v", err)
			}
			t.Logf("GCLB resources created (%s/%s)", s.Namespace, ing.Name)

			// Perform whitebox testing.
			if len(ing.Status.LoadBalancer.Ingress) < 1 {
				t.Fatalf("Ingress does not have an IP: %+v", ing.Status)
			}

			vip := ing.Status.LoadBalancer.Ingress[0].IP
			t.Logf("Ingress %s/%s VIP = %s", s.Namespace, ing.Name, vip)
			gclb, err := fuzz.GCLBForVIP(context.Background(), Framework.Cloud, vip,
				fuzz.FeatureValidators(features.All))
			if err != nil {
				t.Fatalf("Error getting GCP resources for LB with IP = %q: %v", vip, err)
			}

			if err = e2e.CheckGCLB(gclb, tc.numForwardingRules, tc.numBackendServices); err != nil {
				t.Error(err)
			}

			if (len(gclb.NetworkEndpointGroup) > 0) != tc.negExpected {
				t.Errorf("Error: negExpected = %v, %d negs found for gclb %v", tc.negExpected, len(gclb.NetworkEndpointGroup), gclb)
			}

			if err := e2e.WaitForIngressDeletion(ctx, gclb, s, ing, &fuzz.GCLBDeleteOptions{}); err != nil {
				t.Errorf("e2e.WaitForIngressDeletion(..., %q, nil) = %v, want nil", ing.Name, err)
			}
		})
	}
}

func TestNEGTransition(t *testing.T) {
	t.Parallel()

	port80 := intstr.FromInt(80)

	ctx := context.Background()

	Framework.RunWithSandbox("Basic NEG Tests", t, func(t *testing.T, s *e2e.Sandbox) {

		ing := fuzz.NewIngressBuilder("", "ingress-1", "").
			DefaultBackend("service-1", port80).
			Build()

		var previousGCLBState *fuzz.GCLB

		for _, tc := range []struct {
			desc        string
			annotations annotations.NegAnnotation
			// negGC is true if a NEG should be garbage collected after applying the annotations
			negGC              bool
			numForwardingRules int
			numBackendServices int
		}{
			{
				desc:               "Using ingress only",
				annotations:        annotations.NegAnnotation{Ingress: true},
				negGC:              false,
				numForwardingRules: 1,
				numBackendServices: 1,
			},
			{
				desc:               "Disable NEG for ingress",
				annotations:        annotations.NegAnnotation{Ingress: false},
				negGC:              true,
				numForwardingRules: 1,
				numBackendServices: 1,
			},
			{
				desc:               "Re-enable NEG for ingress",
				annotations:        annotations.NegAnnotation{Ingress: true},
				negGC:              false,
				numForwardingRules: 1,
				numBackendServices: 1,
			},
			{
				desc:               "No annotations",
				annotations:        annotations.NegAnnotation{},
				negGC:              true,
				numForwardingRules: 1,
				numBackendServices: 1,
			},
		} {
			// First create the echo service, we will be adapting it throughout the basic tests
			_, err := e2e.EnsureEchoService(s, "service-1", map[string]string{
				annotations.NEGAnnotationKey: tc.annotations.String()}, v1.ProtocolTCP, 80, v1.ServiceTypeNodePort, 1)
			if err != nil {
				t.Fatalf("error ensuring echo service: %v", err)
			}
			t.Logf("Echo service ensured (%s/%s)", s.Namespace, "service-1")

			// Create the ingress
			ing, err = e2e.EnsureIngress(s, ing)
			if err != nil {
				t.Fatalf("error ensuring Ingress spec: %v", err)
			}
			t.Logf("Ingress ensured (%s/%s)", s.Namespace, ing.Name)

			ing, err = e2e.WaitForIngress(s, ing, nil)
			if err != nil {
				t.Fatalf("error waiting for Ingress to stabilize: %v", err)
			}
			t.Logf("GCLB resources created (%s/%s)", s.Namespace, ing.Name)

			// Perform whitebox testing.
			if len(ing.Status.LoadBalancer.Ingress) < 1 {
				t.Fatalf("Ingress does not have an IP: %+v", ing.Status)
			}

			vip := ing.Status.LoadBalancer.Ingress[0].IP
			t.Logf("Ingress %s/%s VIP = %s", s.Namespace, ing.Name, vip)
			gclb, err := fuzz.GCLBForVIP(context.Background(), Framework.Cloud, vip,
				fuzz.FeatureValidators(features.All))
			if err != nil {
				t.Fatalf("Error getting GCP resources for LB with IP = %q: %v", vip, err)
			}

			if err = e2e.CheckGCLB(gclb, tc.numForwardingRules, tc.numBackendServices); err != nil {
				t.Error(err)
			}

			if tc.negGC {
				if len(gclb.NetworkEndpointGroup) != 0 {
					t.Errorf("NegGC = true, expected 0 negs for gclb %v, got %d", gclb, len(gclb.NetworkEndpointGroup))
				}
				if err = e2e.WaitForNEGDeletion(ctx, s.ValidatorEnv.Cloud(), previousGCLBState, nil); err != nil {
					t.Errorf("Error waiting for NEGDeletion: %v", err)
				}
			} else {
				if len(gclb.NetworkEndpointGroup) < 1 {
					t.Errorf("Error, no NEGS associated with gclb %v, expected at least one", gclb)
				}
			}
			previousGCLBState = gclb
		}

		if ing != nil && previousGCLBState != nil {
			if err := e2e.WaitForIngressDeletion(ctx, previousGCLBState, s, ing, &fuzz.GCLBDeleteOptions{}); err != nil {
				t.Errorf("e2e.WaitForIngressDeletion(..., %q, nil) = %v, want nil", ing.Name, err)
			}
		}
	})
}
<commit_msg>NEG clusterIP e2e test<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"context"
	"testing"

	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/ingress-gce/pkg/annotations"
	"k8s.io/ingress-gce/pkg/e2e"
	"k8s.io/ingress-gce/pkg/fuzz"
	"k8s.io/ingress-gce/pkg/fuzz/features"
)

func TestNEG(t *testing.T) {
	t.Parallel()

	port80 := intstr.FromInt(80)

	ctx := context.Background()

	for _, tc := range []struct {
		desc               string
		annotations        annotations.NegAnnotation
		svcType						 v1.ServiceType
		negExpected        bool
		numForwardingRules int
		numBackendServices int
	}{
		{
			desc:               "Create a basic NEG",
			annotations:        annotations.NegAnnotation{Ingress: true},
			svcType: v1.ServiceTypeNodePort,
			negExpected:        true,
			numForwardingRules: 1,
			numBackendServices: 1,
		},
		{
			desc:               "Annotation with no NEG",
			annotations:        annotations.NegAnnotation{Ingress: false},
			negExpected:        false,
			svcType:	v1.ServiceTypeNodePort,
			numForwardingRules: 1,
			numBackendServices: 1,
		},
		{
			desc:               "Ingress can reference ClusterIP service NEG enabled",
			annotations:        annotations.NegAnnotation{Ingress: true},
			negExpected:        true,
			svcType:	v1.ServiceTypeClusterIP,
			numForwardingRules: 1,
			numBackendServices: 1,
		},
	} {
		tc := tc // Capture tc as we are running this in parallel.
		Framework.RunWithSandbox(tc.desc, t, func(t *testing.T, s *e2e.Sandbox) {
			_, err := e2e.EnsureEchoService(s, "service-1", map[string]string{
				annotations.NEGAnnotationKey: tc.annotations.String()}, v1.ProtocolTCP, 80, tc.svcType, 1)
			if err != nil {
				t.Fatalf("error ensuring echo service: %v", err)
			}
			t.Logf("Echo service ensured (%s/%s)", s.Namespace, "service-1")

			// Create the ingress
			ing := fuzz.NewIngressBuilder("", "ingress-1", "").
				DefaultBackend("service-1", port80).
				Build()
			ing, err = e2e.EnsureIngress(s, ing)
			if err != nil {
				t.Fatalf("error ensuring Ingress spec: %v", err)
			}
			t.Logf("Ingress ensured (%s/%s)", s.Namespace, ing.Name)

			ing, err = e2e.WaitForIngress(s, ing, nil)
			if err != nil {
				t.Fatalf("error waiting for Ingress to stabilize: %v", err)
			}
			t.Logf("GCLB resources created (%s/%s)", s.Namespace, ing.Name)

			// Perform whitebox testing.
			if len(ing.Status.LoadBalancer.Ingress) < 1 {
				t.Fatalf("Ingress does not have an IP: %+v", ing.Status)
			}

			vip := ing.Status.LoadBalancer.Ingress[0].IP
			t.Logf("Ingress %s/%s VIP = %s", s.Namespace, ing.Name, vip)
			gclb, err := fuzz.GCLBForVIP(context.Background(), Framework.Cloud, vip,
				fuzz.FeatureValidators(features.All))
			if err != nil {
				t.Fatalf("Error getting GCP resources for LB with IP = %q: %v", vip, err)
			}

			if err = e2e.CheckGCLB(gclb, tc.numForwardingRules, tc.numBackendServices); err != nil {
				t.Error(err)
			}

			if (len(gclb.NetworkEndpointGroup) > 0) != tc.negExpected {
				t.Errorf("Error: negExpected = %v, %d negs found for gclb %v", tc.negExpected, len(gclb.NetworkEndpointGroup), gclb)
			}

			if err := e2e.WaitForIngressDeletion(ctx, gclb, s, ing, &fuzz.GCLBDeleteOptions{}); err != nil {
				t.Errorf("e2e.WaitForIngressDeletion(..., %q, nil) = %v, want nil", ing.Name, err)
			}
		})
	}
}

func TestNEGTransition(t *testing.T) {
	t.Parallel()

	port80 := intstr.FromInt(80)

	ctx := context.Background()

	Framework.RunWithSandbox("Basic NEG Tests", t, func(t *testing.T, s *e2e.Sandbox) {

		ing := fuzz.NewIngressBuilder("", "ingress-1", "").
			DefaultBackend("service-1", port80).
			Build()

		var previousGCLBState *fuzz.GCLB

		for _, tc := range []struct {
			desc        string
			annotations annotations.NegAnnotation
			// negGC is true if a NEG should be garbage collected after applying the annotations
			negGC              bool
			numForwardingRules int
			numBackendServices int
		}{
			{
				desc:               "Using ingress only",
				annotations:        annotations.NegAnnotation{Ingress: true},
				negGC:              false,
				numForwardingRules: 1,
				numBackendServices: 1,
			},
			{
				desc:               "Disable NEG for ingress",
				annotations:        annotations.NegAnnotation{Ingress: false},
				negGC:              true,
				numForwardingRules: 1,
				numBackendServices: 1,
			},
			{
				desc:               "Re-enable NEG for ingress",
				annotations:        annotations.NegAnnotation{Ingress: true},
				negGC:              false,
				numForwardingRules: 1,
				numBackendServices: 1,
			},
			{
				desc:               "No annotations",
				annotations:        annotations.NegAnnotation{},
				negGC:              true,
				numForwardingRules: 1,
				numBackendServices: 1,
			},
		} {
			// First create the echo service, we will be adapting it throughout the basic tests
			_, err := e2e.EnsureEchoService(s, "service-1", map[string]string{
				annotations.NEGAnnotationKey: tc.annotations.String()}, v1.ProtocolTCP, 80, v1.ServiceTypeNodePort, 1)
			if err != nil {
				t.Fatalf("error ensuring echo service: %v", err)
			}
			t.Logf("Echo service ensured (%s/%s)", s.Namespace, "service-1")

			// Create the ingress
			ing, err = e2e.EnsureIngress(s, ing)
			if err != nil {
				t.Fatalf("error ensuring Ingress spec: %v", err)
			}
			t.Logf("Ingress ensured (%s/%s)", s.Namespace, ing.Name)

			ing, err = e2e.WaitForIngress(s, ing, nil)
			if err != nil {
				t.Fatalf("error waiting for Ingress to stabilize: %v", err)
			}
			t.Logf("GCLB resources created (%s/%s)", s.Namespace, ing.Name)

			// Perform whitebox testing.
			if len(ing.Status.LoadBalancer.Ingress) < 1 {
				t.Fatalf("Ingress does not have an IP: %+v", ing.Status)
			}

			vip := ing.Status.LoadBalancer.Ingress[0].IP
			t.Logf("Ingress %s/%s VIP = %s", s.Namespace, ing.Name, vip)
			gclb, err := fuzz.GCLBForVIP(context.Background(), Framework.Cloud, vip,
				fuzz.FeatureValidators(features.All))
			if err != nil {
				t.Fatalf("Error getting GCP resources for LB with IP = %q: %v", vip, err)
			}

			if err = e2e.CheckGCLB(gclb, tc.numForwardingRules, tc.numBackendServices); err != nil {
				t.Error(err)
			}

			if tc.negGC {
				if len(gclb.NetworkEndpointGroup) != 0 {
					t.Errorf("NegGC = true, expected 0 negs for gclb %v, got %d", gclb, len(gclb.NetworkEndpointGroup))
				}
				if err = e2e.WaitForNEGDeletion(ctx, s.ValidatorEnv.Cloud(), previousGCLBState, nil); err != nil {
					t.Errorf("Error waiting for NEGDeletion: %v", err)
				}
			} else {
				if len(gclb.NetworkEndpointGroup) < 1 {
					t.Errorf("Error, no NEGS associated with gclb %v, expected at least one", gclb)
				}
			}
			previousGCLBState = gclb
		}

		if ing != nil && previousGCLBState != nil {
			if err := e2e.WaitForIngressDeletion(ctx, previousGCLBState, s, ing, &fuzz.GCLBDeleteOptions{}); err != nil {
				t.Errorf("e2e.WaitForIngressDeletion(..., %q, nil) = %v, want nil", ing.Name, err)
			}
		}
	})
}
<|endoftext|>


<commit_before>package cmd

import (
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"strings"

	"github.com/exoscale/egoscale"
	"github.com/exoscale/egoscale/cmd/exo/utils"
	"github.com/spf13/cobra"
)

// vmCreateCmd represents the create command
var vmCreateCmd = &cobra.Command{
	Use:     "create <vm name>+",
	Short:   "Create and deploy a virtual machine",
	Aliases: gCreateAlias,
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) < 1 {
			return cmd.Usage()
		}

		userDataPath, err := cmd.Flags().GetString("cloud-init-file")
		if err != nil {
			return err
		}

		userData := ""

		if userDataPath != "" {
			userData, err = getUserData(userDataPath)
			if err != nil {
				return err
			}
		}

		zoneName, err := cmd.Flags().GetString("zone")
		if err != nil {
			return err
		}

		if zoneName == "" {
			zoneName = gCurrentAccount.DefaultZone
		}

		zone, err := getZoneIDByName(zoneName)
		if err != nil {
			return err
		}

		templateName, err := cmd.Flags().GetString("template")
		if err != nil {
			return err
		}

		if templateName == "" {
			templateName = gCurrentAccount.DefaultTemplate
		}

		diskSize, err := cmd.Flags().GetInt64("disk")
		if err != nil {
			return err
		}

		template, err := getTemplateByName(zone, templateName)
		if err != nil {
			return err
		}

		keypair, err := cmd.Flags().GetString("keypair")
		if err != nil {
			return err
		}

		sg, err := cmd.Flags().GetString("security-group")
		if err != nil {
			return err
		}

		sgs, err := getSecurityGroups(cs, sg)
		if err != nil {
			return err
		}

		ipv6, err := cmd.Flags().GetBool("ipv6")
		if err != nil {
			return err
		}

		privnet, err := cmd.Flags().GetString("privnet")
		if err != nil {
			return err
		}

		pvs, err := getPrivnetList(privnet, zoneName)
		if err != nil {
			return err
		}

		so, err := cmd.Flags().GetString("service-offering")
		if err != nil {
			return err
		}

		servOffering, err := getServiceOfferingByName(so)
		if err != nil {
			return err
		}

		affinitygroup, err := cmd.Flags().GetString("anti-affinity-group")
		if err != nil {
			return err
		}

		affinitygroups, err := getAffinityGroup(cs, affinitygroup)
		if err != nil {
			return err
		}

		tasks := make([]egoscale.DeployVirtualMachine, len(args))

		for i, name := range args {

			vmInfo := &egoscale.DeployVirtualMachine{
				Name:              name,
				UserData:          userData,
				ZoneID:            zone,
				TemplateID:        template.ID,
				RootDiskSize:      diskSize,
				KeyPair:           keypair,
				SecurityGroupIDs:  sgs,
				IP6:               &ipv6,
				NetworkIDs:        pvs,
				ServiceOfferingID: servOffering.ID,
				AffinityGroupIDs:  affinitygroups,
			}

			tasks[i] = *vmInfo

		}

		r, errs := createVM(tasks)
		if len(errs) > 0 {
			return errs[0]
		}

		if len(r) > 1 {
			return nil
		}

		sshinfo, err := getSSHInfo(r[0].ID.String(), ipv6)
		if err != nil {
			return err
		}

		fmt.Printf(`
What to do now?

1. Connect to the machine

> exo ssh %s
`, r[0].Name)

		printSSHConnectSTR(sshinfo)

		fmt.Printf(`
2. Put the SSH configuration into ".ssh/config"

> exo ssh %s --info
`, r[0].Name)

		printSSHInfo(sshinfo)

		fmt.Print(`
Tip of the day:
	You're the sole owner of the private key.
	Be cautious with it.
`)

		return nil
	},
}

func getCommaflag(p string) []string {
	if p == "" {
		return nil
	}

	p = strings.Trim(p, ",")
	args := strings.Split(p, ",")

	res := []string{}

	for _, arg := range args {
		if arg == "" {
			continue
		}
		res = append(res, strings.TrimSpace(arg))
	}

	return res
}

func getSecurityGroups(cs *egoscale.Client, commaParameter string) ([]egoscale.UUID, error) {

	sgs := getCommaflag(commaParameter)
	ids := make([]egoscale.UUID, len(sgs))

	for i, sg := range sgs {
		s, err := getSecurityGroupByNameOrID(sg)
		if err != nil {
			return nil, err
		}

		ids[i] = *s.ID
	}

	return ids, nil
}

func getPrivnetList(commaParameter, zoneID string) ([]egoscale.UUID, error) {

	sgs := getCommaflag(commaParameter)
	ids := make([]egoscale.UUID, len(sgs))

	for i, sg := range sgs {
		n, err := getNetworkByName(sg)
		if err != nil {
			return nil, err
		}

		ids[i] = *n.ID
	}

	return ids, nil
}

func getAffinityGroup(cs *egoscale.Client, commaParameter string) ([]egoscale.UUID, error) {
	affs := getCommaflag(commaParameter)
	ids := make([]egoscale.UUID, len(affs))

	for i, aff := range affs {
		s, err := getAffinityGroupByName(aff)

		if err != nil {
			return nil, err
		}

		ids[i] = *s.ID
	}

	return ids, nil
}

func getUserData(userDataPath string) (string, error) {
	buff, err := ioutil.ReadFile(userDataPath)
	if err != nil {
		return "", err
	}

	return base64.StdEncoding.EncodeToString(buff), nil
}

func createVM(deploys []egoscale.DeployVirtualMachine) ([]egoscale.VirtualMachine, []error) {
	isDefaultKeyPair := false
	var keyPairs *egoscale.SSHKeyPair

	var keypairs string
	if deploys[0].KeyPair == "" {
		isDefaultKeyPair = true
		fmt.Println("Creating private SSH key")
		sshKeyName, err := utils.RandStringBytes(64)
		if err != nil {
			return nil, []error{err}
		}
		keyPairs, err = createSSHKey(sshKeyName)
		if err != nil {
			r := err.(*egoscale.ErrorResponse)
			if r.ErrorCode != egoscale.ParamError && r.CSErrorCode != egoscale.InvalidParameterValueException {
				return nil, []error{err}
			}
			return nil, []error{fmt.Errorf("an SSH key with that name %q already exists, please choose a different name", sshKeyName)}
		}
		defer deleteSSHKey(keyPairs.Name) // nolint: errcheck

		keypairs = keyPairs.Name

	}

	tasks := make([]task, len(deploys))

	for i, req := range deploys {
		tasks[i].string = fmt.Sprintf("Deploying %q", req.Name)
		if keypairs != "" {
			req.KeyPair = keypairs
		}
		tasks[i].AsyncCommand = req
	}

	resps := asyncTasks(tasks)
	errors := filterErrors(resps)
	if len(errors) > 0 {
		return nil, errors
	}

	vmResp := make([]egoscale.VirtualMachine, len(resps))
	if isDefaultKeyPair {
		for i, vm := range resps {
			v := vm.resp.(*egoscale.VirtualMachine)
			vmResp[i] = *v
			saveKeyPair(keyPairs, *v.ID)
		}
	}
	return vmResp, nil
}

func saveKeyPair(keyPairs *egoscale.SSHKeyPair, vmID egoscale.UUID) {
	filePath := path.Join(gConfigFolder, "instances", vmID.String())

	if _, err := os.Stat(filePath); os.IsNotExist(err) {
		if err := os.MkdirAll(filePath, os.ModePerm); err != nil {
			log.Fatal(err)
		}
	}

	filePath = path.Join(filePath, "id_rsa")

	if _, err := os.Stat(filePath); os.IsNotExist(err) {
		if err := ioutil.WriteFile(filePath, []byte(keyPairs.PrivateKey), 0600); err != nil {
			log.Fatalf("SSH private key could not be written. %s", err)
		}
	}
}

func init() {
	vmCreateCmd.Flags().StringP("cloud-init-file", "f", "", "Deploy instance with a cloud-init file")
	vmCreateCmd.Flags().StringP("zone", "z", "", "<zone name | id | keyword> (ch-dk-2|ch-gva-2|at-vie-1|de-fra-1)")
	vmCreateCmd.Flags().StringP("template", "t", "", fmt.Sprintf("<template name | id> (default: %s)", defaultTemplate))
	vmCreateCmd.Flags().Int64P("disk", "d", 50, "<disk size>")
	vmCreateCmd.Flags().StringP("keypair", "k", "", "<ssh keys name>")
	vmCreateCmd.Flags().StringP("security-group", "s", "", "<name | id, name | id, ...>")
	vmCreateCmd.Flags().StringP("privnet", "p", "", "<name | id, name | id, ...>")
	vmCreateCmd.Flags().StringP("anti-affinity-group", "a", "", "<name | id, name | id, ...>")
	vmCreateCmd.Flags().BoolP("ipv6", "6", false, "enable ipv6")
	vmCreateCmd.Flags().StringP("service-offering", "o", "Small", "<name | id> (micro|tiny|small|medium|large|extra-large|huge|mega|titan")
	vmCmd.AddCommand(vmCreateCmd)
}
<commit_msg>exo: fix ssh keypair on vm create<commit_after>package cmd

import (
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"strings"

	"github.com/exoscale/egoscale"
	"github.com/exoscale/egoscale/cmd/exo/utils"
	"github.com/spf13/cobra"
)

// vmCreateCmd represents the create command
var vmCreateCmd = &cobra.Command{
	Use:     "create <vm name>+",
	Short:   "Create and deploy a virtual machine",
	Aliases: gCreateAlias,
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) < 1 {
			return cmd.Usage()
		}

		userDataPath, err := cmd.Flags().GetString("cloud-init-file")
		if err != nil {
			return err
		}

		userData := ""

		if userDataPath != "" {
			userData, err = getUserData(userDataPath)
			if err != nil {
				return err
			}
		}

		zoneName, err := cmd.Flags().GetString("zone")
		if err != nil {
			return err
		}

		if zoneName == "" {
			zoneName = gCurrentAccount.DefaultZone
		}

		zone, err := getZoneIDByName(zoneName)
		if err != nil {
			return err
		}

		templateName, err := cmd.Flags().GetString("template")
		if err != nil {
			return err
		}

		if templateName == "" {
			templateName = gCurrentAccount.DefaultTemplate
		}

		diskSize, err := cmd.Flags().GetInt64("disk")
		if err != nil {
			return err
		}

		template, err := getTemplateByName(zone, templateName)
		if err != nil {
			return err
		}

		keypair, err := cmd.Flags().GetString("keypair")
		if err != nil {
			return err
		}

		sg, err := cmd.Flags().GetString("security-group")
		if err != nil {
			return err
		}

		sgs, err := getSecurityGroups(cs, sg)
		if err != nil {
			return err
		}

		ipv6, err := cmd.Flags().GetBool("ipv6")
		if err != nil {
			return err
		}

		privnet, err := cmd.Flags().GetString("privnet")
		if err != nil {
			return err
		}

		pvs, err := getPrivnetList(privnet, zoneName)
		if err != nil {
			return err
		}

		so, err := cmd.Flags().GetString("service-offering")
		if err != nil {
			return err
		}

		servOffering, err := getServiceOfferingByName(so)
		if err != nil {
			return err
		}

		affinitygroup, err := cmd.Flags().GetString("anti-affinity-group")
		if err != nil {
			return err
		}

		affinitygroups, err := getAffinityGroup(cs, affinitygroup)
		if err != nil {
			return err
		}

		tasks := make([]egoscale.DeployVirtualMachine, len(args))

		for i, name := range args {

			vmInfo := &egoscale.DeployVirtualMachine{
				Name:              name,
				UserData:          userData,
				ZoneID:            zone,
				TemplateID:        template.ID,
				RootDiskSize:      diskSize,
				KeyPair:           keypair,
				SecurityGroupIDs:  sgs,
				IP6:               &ipv6,
				NetworkIDs:        pvs,
				ServiceOfferingID: servOffering.ID,
				AffinityGroupIDs:  affinitygroups,
			}

			tasks[i] = *vmInfo

		}

		r, errs := createVM(tasks)
		if len(errs) > 0 {
			return errs[0]
		}

		if len(r) > 1 {
			return nil
		}

		sshinfo, err := getSSHInfo(r[0].ID.String(), ipv6)
		if err != nil {
			return err
		}

		fmt.Printf(`
What to do now?

1. Connect to the machine

> exo ssh %s
`, r[0].Name)

		printSSHConnectSTR(sshinfo)

		fmt.Printf(`
2. Put the SSH configuration into ".ssh/config"

> exo ssh %s --info
`, r[0].Name)

		printSSHInfo(sshinfo)

		fmt.Print(`
Tip of the day:
	You're the sole owner of the private key.
	Be cautious with it.
`)

		return nil
	},
}

func getCommaflag(p string) []string {
	if p == "" {
		return nil
	}

	p = strings.Trim(p, ",")
	args := strings.Split(p, ",")

	res := []string{}

	for _, arg := range args {
		if arg == "" {
			continue
		}
		res = append(res, strings.TrimSpace(arg))
	}

	return res
}

func getSecurityGroups(cs *egoscale.Client, commaParameter string) ([]egoscale.UUID, error) {

	sgs := getCommaflag(commaParameter)
	ids := make([]egoscale.UUID, len(sgs))

	for i, sg := range sgs {
		s, err := getSecurityGroupByNameOrID(sg)
		if err != nil {
			return nil, err
		}

		ids[i] = *s.ID
	}

	return ids, nil
}

func getPrivnetList(commaParameter, zoneID string) ([]egoscale.UUID, error) {

	sgs := getCommaflag(commaParameter)
	ids := make([]egoscale.UUID, len(sgs))

	for i, sg := range sgs {
		n, err := getNetworkByName(sg)
		if err != nil {
			return nil, err
		}

		ids[i] = *n.ID
	}

	return ids, nil
}

func getAffinityGroup(cs *egoscale.Client, commaParameter string) ([]egoscale.UUID, error) {
	affs := getCommaflag(commaParameter)
	ids := make([]egoscale.UUID, len(affs))

	for i, aff := range affs {
		s, err := getAffinityGroupByName(aff)

		if err != nil {
			return nil, err
		}

		ids[i] = *s.ID
	}

	return ids, nil
}

func getUserData(userDataPath string) (string, error) {
	buff, err := ioutil.ReadFile(userDataPath)
	if err != nil {
		return "", err
	}

	return base64.StdEncoding.EncodeToString(buff), nil
}

func createVM(deploys []egoscale.DeployVirtualMachine) ([]egoscale.VirtualMachine, []error) {
	isDefaultKeyPair := false
	var keyPairs *egoscale.SSHKeyPair

	var keypairs string
	if deploys[0].KeyPair == "" {
		isDefaultKeyPair = true
		fmt.Println("Creating private SSH key")
		sshKeyName, err := utils.RandStringBytes(64)
		if err != nil {
			return nil, []error{err}
		}
		keyPairs, err = createSSHKey(sshKeyName)
		if err != nil {
			r := err.(*egoscale.ErrorResponse)
			if r.ErrorCode != egoscale.ParamError && r.CSErrorCode != egoscale.InvalidParameterValueException {
				return nil, []error{err}
			}
			return nil, []error{fmt.Errorf("an SSH key with that name %q already exists, please choose a different name", sshKeyName)}
		}
		defer deleteSSHKey(keyPairs.Name) // nolint: errcheck

		keypairs = keyPairs.Name

	}

	tasks := make([]task, len(deploys))

	for i := range deploys {
		tasks[i].string = fmt.Sprintf("Deploying %q", deploys[i].Name)
		if keypairs != "" {
			deploys[i].KeyPair = keypairs
		}
		tasks[i].AsyncCommand = deploys[i]
	}

	resps := asyncTasks(tasks)
	errors := filterErrors(resps)
	if len(errors) > 0 {
		return nil, errors
	}

	vmResp := make([]egoscale.VirtualMachine, len(resps))
	if isDefaultKeyPair {
		for i, vm := range resps {
			v := vm.resp.(*egoscale.VirtualMachine)
			vmResp[i] = *v
			saveKeyPair(keyPairs, *v.ID)
		}
	}
	return vmResp, nil
}

func saveKeyPair(keyPairs *egoscale.SSHKeyPair, vmID egoscale.UUID) {
	filePath := path.Join(gConfigFolder, "instances", vmID.String())

	if _, err := os.Stat(filePath); os.IsNotExist(err) {
		if err := os.MkdirAll(filePath, os.ModePerm); err != nil {
			log.Fatal(err)
		}
	}

	filePath = path.Join(filePath, "id_rsa")

	if _, err := os.Stat(filePath); os.IsNotExist(err) {
		if err := ioutil.WriteFile(filePath, []byte(keyPairs.PrivateKey), 0600); err != nil {
			log.Fatalf("SSH private key could not be written. %s", err)
		}
	}
}

func init() {
	vmCreateCmd.Flags().StringP("cloud-init-file", "f", "", "Deploy instance with a cloud-init file")
	vmCreateCmd.Flags().StringP("zone", "z", "", "<zone name | id | keyword> (ch-dk-2|ch-gva-2|at-vie-1|de-fra-1)")
	vmCreateCmd.Flags().StringP("template", "t", "", fmt.Sprintf("<template name | id> (default: %s)", defaultTemplate))
	vmCreateCmd.Flags().Int64P("disk", "d", 50, "<disk size>")
	vmCreateCmd.Flags().StringP("keypair", "k", "", "<ssh keys name>")
	vmCreateCmd.Flags().StringP("security-group", "s", "", "<name | id, name | id, ...>")
	vmCreateCmd.Flags().StringP("privnet", "p", "", "<name | id, name | id, ...>")
	vmCreateCmd.Flags().StringP("anti-affinity-group", "a", "", "<name | id, name | id, ...>")
	vmCreateCmd.Flags().BoolP("ipv6", "6", false, "enable ipv6")
	vmCreateCmd.Flags().StringP("service-offering", "o", "Small", "<name | id> (micro|tiny|small|medium|large|extra-large|huge|mega|titan")
	vmCmd.AddCommand(vmCreateCmd)
}
<|endoftext|>


<commit_before>package prove

import (
	"bytes"
	"fmt"
	"log"
	"runtime"
	"strings"
	"time"

	"github.com/nelhage/taktician/ptn"
	"github.com/nelhage/taktician/tak"
)

type evaluation int8

const (
	evalUnknown evaluation = iota
	evalTrue
	evalFalse
)

const (
	flagIrreversible = 1 << iota
	flagExpanded     = 1 << iota
)

const inf = ^uint64(0)

type node struct {
	parent          *node
	position        *tak.Position
	proof, disproof uint64
	move            tak.Move

	value evaluation
	flags int32

	children []*node
}

func (n *node) expanded() bool {
	return n.flags&flagExpanded != 0
}

func (n *node) depth() int {
	d := 0
	for n.parent != nil {
		n = n.parent
		d += 1
	}
	return d
}

type prover struct {
	stats struct {
		nodes     uint64
		proved    uint64
		disproved uint64
		dropped   uint64
	}
	player tak.Color
	root   *node
}

func prove(pos *tak.Position) {
	p := prover{
		player: pos.ToMove(),
	}
	p.prove(pos)
}

const kProgressFrequency = 10000

func (p *prover) prove(pos *tak.Position) {
	start := time.Now()
	p.stats.nodes += 1
	p.root = &node{
		position: pos,
		parent:   nil,
	}
	p.evaluate(p.root)
	p.setNumbers(p.root)
	var i uint64
	current := p.root
	for p.root.proof != 0 && p.root.disproof != 0 {
		i++
		next := p.selectMostProving(current)

		if i%kProgressFrequency == 0 {
			var stats runtime.MemStats
			runtime.ReadMemStats(&stats)
			log.Printf("time=%s nodes=%d live=%d done=%d/%d/%d root=(%d, %d) heap=%d",
				time.Since(start),
				p.stats.nodes,
				p.stats.nodes-(p.stats.proved+p.stats.disproved+p.stats.dropped),
				p.stats.proved,
				p.stats.disproved,
				p.stats.dropped,
				p.root.proof,
				p.root.disproof,
				stats.HeapAlloc,
			)
			/*
				log.Printf("  children=%s", formatChildren(p.root.children))
				log.Printf("  line=%s", formatLine(next))
			*/

		}

		p.expand(next)
		current = p.updateAncestors(next)
	}
	log.Printf("Done in %s, nodes=%d proof=%d disproof=%d",
		time.Since(start),
		p.stats.nodes,
		p.root.proof,
		p.root.disproof,
	)
}

func (p *prover) checkRepetition(n *node) bool {
	if (n.flags & flagIrreversible) != 0 {
		return false
	}
	count := 1
	walk := n.parent
	for walk != nil && (walk.flags&flagIrreversible) == 0 && count < 3 {
		if walk.position.Equal(n.position) {
			count += 1
		}
		walk = walk.parent
	}
	return count == 3
}

func (p *prover) evaluate(node *node) {
	if over, who := node.position.GameOver(); over {
		if who == p.player {
			node.value = evalTrue
		} else {
			node.value = evalFalse
		}
	} else {
		if p.checkRepetition(node) {
			node.value = evalFalse
		} else {
			node.value = evalUnknown
		}

	}
}

func (p *prover) setNumbers(node *node) {
	if node.expanded() {
		if p.andNode(node) {
			node.proof = 0
			node.disproof = inf
			for _, c := range node.children {
				node.proof += c.proof
				if c.disproof < node.disproof {
					node.disproof = c.disproof
				}
			}
			if node.disproof == 0 {
				node.proof = inf
			}
		} else {
			node.proof = inf
			node.disproof = 0
			for _, c := range node.children {
				node.disproof += c.disproof
				if c.proof < node.proof {
					node.proof = c.proof
				}
			}
			if node.proof == 0 {
				node.disproof = inf
			}
		}
	} else {
		switch node.value {
		case evalTrue:
			node.proof = 0
			node.disproof = inf
		case evalFalse:
			node.proof = inf
			node.disproof = 0
		case evalUnknown:
			node.proof = 1
			node.disproof = 1
		}
	}
}

func formatChildren(children []*node) string {
	var buf bytes.Buffer
	for _, c := range children {
		fmt.Fprintf(&buf, "(%d, %d) ", c.proof, c.disproof)
	}
	return buf.String()
}

func formatLine(node *node) string {
	var bits []string
	for node != nil && node.parent != nil {
		bits = append(bits, fmt.Sprintf("%s@(%d, %d)",
			ptn.FormatMove(node.move), node.proof, node.disproof))
		node = node.parent
	}
	for i := 0; i < len(bits)/2; i += 1 {
		bits[i], bits[len(bits)-1-i] = bits[len(bits)-1-i], bits[i]
	}
	return strings.Join(bits, " ")
}

func (p *prover) selectMostProving(current *node) *node {
	for current.expanded() {
		var child *node
		if p.andNode(current) {
			for _, c := range current.children {
				if c.disproof == current.disproof {
					child = c
					break
				}
			}

		} else {
			for _, c := range current.children {
				if c.proof == current.proof {
					child = c
					break
				}
			}
		}
		if child == nil {
			var ty string
			if p.andNode(current) {
				ty = "AND"
			} else {
				ty = "OR"
			}
			log.Printf("consistency error depth=%d type=%s proof=%d disproof=%d",
				current.depth(),
				ty,
				current.proof,
				current.disproof,
			)
			log.Printf("children: %s", formatChildren(current.children))
			panic("consistency error")
		}
		current = child
	}
	return current
}

func (p *prover) andNode(n *node) bool {
	return n.position.ToMove() != p.player
}

func (p *prover) expand(n *node) {
	var buffer [30]tak.Move
	allMoves := n.position.AllMoves(buffer[:])
	for _, m := range allMoves {
		cn, e := n.position.Move(m)
		if e != nil {
			continue
		}
		p.stats.nodes += 1
		child := &node{
			position: cn,
			parent:   n,
			move:     m,
		}

		dx, dy := m.Dest()
		reversible := m.IsSlide() && n.position.Top(int(dx), int(dy)).Kind() != tak.Standing
		if !reversible {
			child.flags |= flagIrreversible
		}
		p.evaluate(child)
		p.setNumbers(child)
		n.children = append(n.children, child)
		if (p.andNode(n) && child.proof == 0) || (!p.andNode(n) && child.disproof == 0) {
			break
		}
	}
	n.flags |= flagExpanded
}

func (p *prover) updateAncestors(node *node) *node {
	for true {
		oldproof := node.proof
		olddisproof := node.disproof
		p.setNumbers(node)
		if node.proof == 0 || node.disproof == 0 {
			if node.proof == 0 {
				p.stats.proved += 1
				if !p.andNode(node) {
					p.stats.dropped += uint64(len(node.children) - 1)
				}
			} else {
				p.stats.disproved += 1
				if p.andNode(node) {
					p.stats.dropped += uint64(len(node.children) - 1)
				}
			}
			node.children = nil
		} else if node.proof == oldproof && node.disproof == olddisproof {
			return node
		}

		if node.parent == nil {
			return node
		}
		node = node.parent
	}
	return node
}
<commit_msg>trim a field<commit_after>package prove

import (
	"bytes"
	"fmt"
	"log"
	"runtime"
	"time"

	"github.com/nelhage/taktician/tak"
)

type evaluation int8

const (
	evalUnknown evaluation = iota
	evalTrue
	evalFalse
)

const (
	flagIrreversible = 1 << iota
	flagExpanded     = 1 << iota
)

const inf = ^uint64(0)

type node struct {
	parent          *node
	position        *tak.Position
	proof, disproof uint64

	value evaluation
	flags int32

	children []*node
}

func (n *node) expanded() bool {
	return n.flags&flagExpanded != 0
}

func (n *node) depth() int {
	d := 0
	for n.parent != nil {
		n = n.parent
		d += 1
	}
	return d
}

type prover struct {
	stats struct {
		nodes     uint64
		proved    uint64
		disproved uint64
		dropped   uint64
	}
	player tak.Color
	root   *node
}

func prove(pos *tak.Position) {
	p := prover{
		player: pos.ToMove(),
	}
	p.prove(pos)
}

const kProgressFrequency = 10000

func (p *prover) prove(pos *tak.Position) {
	start := time.Now()
	p.stats.nodes += 1
	p.root = &node{
		position: pos,
		parent:   nil,
	}
	p.evaluate(p.root)
	p.setNumbers(p.root)
	var i uint64
	current := p.root
	for p.root.proof != 0 && p.root.disproof != 0 {
		i++
		next := p.selectMostProving(current)

		if i%kProgressFrequency == 0 {
			var stats runtime.MemStats
			runtime.ReadMemStats(&stats)
			log.Printf("time=%s nodes=%d live=%d done=%d/%d/%d root=(%d, %d) heap=%d",
				time.Since(start),
				p.stats.nodes,
				p.stats.nodes-(p.stats.proved+p.stats.disproved+p.stats.dropped),
				p.stats.proved,
				p.stats.disproved,
				p.stats.dropped,
				p.root.proof,
				p.root.disproof,
				stats.HeapAlloc,
			)
			/*
				log.Printf("  children=%s", formatChildren(p.root.children))
			*/

		}

		p.expand(next)
		current = p.updateAncestors(next)
	}
	log.Printf("Done in %s, nodes=%d proof=%d disproof=%d",
		time.Since(start),
		p.stats.nodes,
		p.root.proof,
		p.root.disproof,
	)
}

func (p *prover) checkRepetition(n *node) bool {
	if (n.flags & flagIrreversible) != 0 {
		return false
	}
	count := 1
	walk := n.parent
	for walk != nil && (walk.flags&flagIrreversible) == 0 && count < 3 {
		if walk.position.Equal(n.position) {
			count += 1
		}
		walk = walk.parent
	}
	return count == 3
}

func (p *prover) evaluate(node *node) {
	if over, who := node.position.GameOver(); over {
		if who == p.player {
			node.value = evalTrue
		} else {
			node.value = evalFalse
		}
	} else {
		if p.checkRepetition(node) {
			node.value = evalFalse
		} else {
			node.value = evalUnknown
		}

	}
}

func (p *prover) setNumbers(node *node) {
	if node.expanded() {
		if p.andNode(node) {
			node.proof = 0
			node.disproof = inf
			for _, c := range node.children {
				node.proof += c.proof
				if c.disproof < node.disproof {
					node.disproof = c.disproof
				}
			}
			if node.disproof == 0 {
				node.proof = inf
			}
		} else {
			node.proof = inf
			node.disproof = 0
			for _, c := range node.children {
				node.disproof += c.disproof
				if c.proof < node.proof {
					node.proof = c.proof
				}
			}
			if node.proof == 0 {
				node.disproof = inf
			}
		}
	} else {
		switch node.value {
		case evalTrue:
			node.proof = 0
			node.disproof = inf
		case evalFalse:
			node.proof = inf
			node.disproof = 0
		case evalUnknown:
			node.proof = 1
			node.disproof = 1
		}
	}
}

func formatChildren(children []*node) string {
	var buf bytes.Buffer
	for _, c := range children {
		fmt.Fprintf(&buf, "(%d, %d) ", c.proof, c.disproof)
	}
	return buf.String()
}

func (p *prover) selectMostProving(current *node) *node {
	for current.expanded() {
		var child *node
		if p.andNode(current) {
			for _, c := range current.children {
				if c.disproof == current.disproof {
					child = c
					break
				}
			}

		} else {
			for _, c := range current.children {
				if c.proof == current.proof {
					child = c
					break
				}
			}
		}
		if child == nil {
			var ty string
			if p.andNode(current) {
				ty = "AND"
			} else {
				ty = "OR"
			}
			log.Printf("consistency error depth=%d type=%s proof=%d disproof=%d",
				current.depth(),
				ty,
				current.proof,
				current.disproof,
			)
			log.Printf("children: %s", formatChildren(current.children))
			panic("consistency error")
		}
		current = child
	}
	return current
}

func (p *prover) andNode(n *node) bool {
	return n.position.ToMove() != p.player
}

func (p *prover) expand(n *node) {
	var buffer [30]tak.Move
	allMoves := n.position.AllMoves(buffer[:])
	for _, m := range allMoves {
		cn, e := n.position.Move(m)
		if e != nil {
			continue
		}
		p.stats.nodes += 1
		child := &node{
			position: cn,
			parent:   n,
		}

		dx, dy := m.Dest()
		reversible := m.IsSlide() && n.position.Top(int(dx), int(dy)).Kind() != tak.Standing
		if !reversible {
			child.flags |= flagIrreversible
		}
		p.evaluate(child)
		p.setNumbers(child)
		n.children = append(n.children, child)
		if (p.andNode(n) && child.proof == 0) || (!p.andNode(n) && child.disproof == 0) {
			break
		}
	}
	n.flags |= flagExpanded
}

func (p *prover) updateAncestors(node *node) *node {
	for true {
		oldproof := node.proof
		olddisproof := node.disproof
		p.setNumbers(node)
		if node.proof == 0 || node.disproof == 0 {
			if node.proof == 0 {
				p.stats.proved += 1
				if !p.andNode(node) {
					p.stats.dropped += uint64(len(node.children) - 1)
				}
			} else {
				p.stats.disproved += 1
				if p.andNode(node) {
					p.stats.dropped += uint64(len(node.children) - 1)
				}
			}
			node.children = nil
		} else if node.proof == oldproof && node.disproof == olddisproof {
			return node
		}

		if node.parent == nil {
			return node
		}
		node = node.parent
	}
	return node
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"github.com/siddontang/ledisdb/server"
	"os"
	"os/signal"
	"runtime"
	"syscall"
)

var configFile = flag.String("config", "/etc/ledis.json", "ledisdb config file")
var dbName = flag.String("db_name", "", "select a db to use, it will overwrite the config's db name")

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	flag.Parse()

	if len(*configFile) == 0 {
		println("must use a config file")
		return
	}

	cfg, err := server.NewConfigWithFile(*configFile)
	if err != nil {
		println(err.Error())
		return
	}

	if len(*dbName) > 0 {
		cfg.DB.Name = *dbName
	}

	var app *server.App
	app, err = server.NewApp(cfg)
	if err != nil {
		println(err.Error())
		return
	}

	sc := make(chan os.Signal, 1)
	signal.Notify(sc,
		syscall.SIGHUP,
		syscall.SIGINT,
		syscall.SIGTERM,
		syscall.SIGQUIT)

	go func() {
		<-sc

		app.Close()
	}()

	app.Run()
}
<commit_msg>add net http pprof<commit_after>package main

import (
	"flag"
	"github.com/siddontang/ledisdb/server"
	"log"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"runtime"
	"syscall"
)

var configFile = flag.String("config", "/etc/ledis.json", "ledisdb config file")
var dbName = flag.String("db_name", "", "select a db to use, it will overwrite the config's db name")

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	flag.Parse()

	if len(*configFile) == 0 {
		println("must use a config file")
		return
	}

	cfg, err := server.NewConfigWithFile(*configFile)
	if err != nil {
		println(err.Error())
		return
	}

	if len(*dbName) > 0 {
		cfg.DB.Name = *dbName
	}

	var app *server.App
	app, err = server.NewApp(cfg)
	if err != nil {
		println(err.Error())
		return
	}

	sc := make(chan os.Signal, 1)
	signal.Notify(sc,
		syscall.SIGHUP,
		syscall.SIGINT,
		syscall.SIGTERM,
		syscall.SIGQUIT)

	go func() {
		<-sc

		app.Close()
	}()

	go func() {
		log.Println(http.ListenAndServe("localhost:6060", nil))
	}()

	app.Run()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	log "github.com/sirupsen/logrus"
	"github.com/skeema/mybase"
	"github.com/skeema/tengo"
)

func init() {
	summary := "Update the filesystem representation of schemas and tables"
	desc := `Updates the existing filesystem representation of the schemas and tables on a DB
instance. Use this command when changes have been applied to the database
without using skeema, and the filesystem representation needs to be updated to
reflect those changes.

You may optionally pass an environment name as a CLI option. This will affect
which section of .skeema config files is used for processing. For example,
running ` + "`" + `skeema pull staging` + "`" + ` will apply config directives from the
[staging] section of config files, as well as any sectionless directives at the
top of the file. If no environment name is supplied, the default is
"production".`

	cmd := mybase.NewCommand("pull", summary, desc, PullHandler)
	cmd.AddOption(mybase.BoolOption("include-auto-inc", 0, false, "Include starting auto-inc values in new table files, and update in existing files"))
	cmd.AddOption(mybase.BoolOption("normalize", 0, true, "Reformat *.sql files to match SHOW CREATE TABLE"))
	cmd.AddArg("environment", "production", false)
	CommandSuite.AddSubCommand(cmd)
}

// PullHandler is the handler method for `skeema pull`
func PullHandler(cfg *mybase.Config) error {
	AddGlobalConfigFiles(cfg)
	dir, err := NewDir(".", cfg)
	if err != nil {
		return err
	}

	var errCount int

	for _, t := range dir.Targets() {
		if t.Err != nil {
			log.Errorf("Skipping %s:", t.Dir)
			log.Errorf("    %s\n", t.Err)
			errCount++
			continue
		}

		log.Infof("Updating %s to reflect %s %s", t.Dir, t.Instance, t.SchemaFromDir.Name)

		// If schema doesn't exist on instance, remove the corresponding dir
		if t.SchemaFromInstance == nil {
			if err := t.Dir.Delete(); err != nil {
				return fmt.Errorf("Unable to delete directory %s: %s", t.Dir, err)
			}
			log.Infof("Deleted directory %s -- schema no longer exists\n", t.Dir)
			continue
		}

		diff := tengo.NewSchemaDiff(t.SchemaFromDir, t.SchemaFromInstance)

		// Handle changes in schema's default character set and/or collation by
		// persisting changes to the dir's option file. File operation errors here
		// are just surfaced as warnings.
		if diff.SchemaDDL != "" {
			instCharSet, instCollation, err := t.Instance.DefaultCharSetAndCollation()
			if err != nil {
				return err
			}
			optionFile, err := t.Dir.OptionFile()
			if err != nil {
				log.Warnf("Unable to update character set and/or collation for %s/.skeema: %s", t.Dir, err)
			} else if optionFile == nil {
				log.Warnf("Unable to update character set and/or collation for %s/.skeema: cannot read file", t.Dir)
			} else {
				if instCharSet != t.SchemaFromInstance.CharSet {
					optionFile.SetOptionValue("", "default-character-set", t.SchemaFromInstance.CharSet)
				} else {
					optionFile.UnsetOptionValue("", "default-character-set")
				}
				if instCollation != t.SchemaFromInstance.Collation {
					optionFile.SetOptionValue("", "default-collation", t.SchemaFromInstance.Collation)
				} else {
					optionFile.UnsetOptionValue("", "default-collation")
				}
				if err = optionFile.Write(true); err != nil {
					log.Warnf("Unable to update character set and/or collation for %s: %s", optionFile.Path(), err)
				} else {
					log.Infof("Wrote %s -- updated schema-level default-character-set and default-collation", optionFile.Path())
				}
			}
		}

		// We're permissive of unsafe operations here since we don't ever actually
		// execute the generated statement! We just examine its type.
		mods := tengo.StatementModifiers{
			AllowUnsafe: true,
		}
		// pull command updates next auto-increment value for existing table always
		// if requested, or only if previously present in file otherwise
		if t.Dir.Config.GetBool("include-auto-inc") {
			mods.NextAutoInc = tengo.NextAutoIncAlways
		} else {
			mods.NextAutoInc = tengo.NextAutoIncIfAlready
		}
		mods.IgnoreTable, err = t.Dir.Config.GetRegexp("ignore-table")
		if err != nil {
			return err
		}

		for _, td := range diff.TableDiffs {
			stmt, stmtErr := td.Statement(mods)
			// Errors are fatal, except for UnsupportedDiffError which we can safely
			// ignore (since pull doesn't actually run ALTERs; it just needs to know
			// which tables were altered)
			if stmtErr != nil && !tengo.IsUnsupportedDiff(stmtErr) {
				return stmtErr
			}
			// skip if mods caused the diff to be a no-op; if it's an ALTER, treat it
			// as an unchanged table so that --normalize logic still runs
			if stmt == "" && stmtErr == nil {
				if td.Type == tengo.TableDiffAlter {
					diff.SameTables = append(diff.SameTables, td.To)
				}
				continue
			}

			// For DROP TABLE, we're deleting corresponding table file; vs other
			// types we're updating/rewriting the file.
			if td.Type == tengo.TableDiffDrop {
				sf := SQLFile{
					Dir:      t.Dir,
					FileName: fmt.Sprintf("%s.sql", td.From.Name),
				}
				if err := sf.Delete(); err != nil {
					return fmt.Errorf("Unable to delete %s: %s", sf.Path(), err)
				}
				log.Infof("Deleted %s -- table no longer exists", sf.Path())
				continue
			}

			var reason string
			sf := SQLFile{
				Dir:      t.Dir,
				FileName: fmt.Sprintf("%s.sql", td.To.Name),
				Contents: stmt,
			}

			// For ALTER TABLE, we don't care about the ALTER statement, but we do
			// need to get the corresponding CREATE TABLE and process auto-inc properly
			if td.Type == tengo.TableDiffAlter {
				sf.Contents = td.To.CreateStatement
				if td.To.HasAutoIncrement() && !t.Dir.Config.GetBool("include-auto-inc") && td.From.NextAutoIncrement <= 1 {
					sf.Contents, _ = tengo.ParseCreateAutoInc(sf.Contents)
				}
				reason = "updated file to reflect table alterations"
				if tengo.IsUnsupportedDiff(stmtErr) {
					log.Warnf("Table %s uses unsupported features", td.To.Name)
					DebugLogUnsupportedDiff(stmtErr.(*tengo.UnsupportedDiffError))
				}
			} else if _, hadErr := t.SQLFileErrors[sf.Path()]; hadErr {
				// SQL files with syntax errors will result in TableDiffCreate since the
				// temp schema will be missing the table
				reason = "updated file to replace invalid SQL"
			} else {
				reason = "new table"
			}

			length, err := sf.Write()
			if err != nil {
				return fmt.Errorf("Unable to write to %s: %s", sf.Path(), err)
			}
			log.Infof("Wrote %s (%d bytes) -- %s", sf.Path(), length, reason)
		}

		if dir.Config.GetBool("normalize") {
			for _, table := range diff.SameTables {
				if mods.IgnoreTable != nil && mods.IgnoreTable.MatchString(table.Name) {
					continue
				}
				sf := SQLFile{
					Dir:      t.Dir,
					FileName: fmt.Sprintf("%s.sql", table.Name),
				}
				if _, err := sf.Read(); err != nil {
					return err
				}
				for _, warning := range sf.Warnings {
					log.Debug(warning)
				}
				newContents := table.CreateStatement
				if table.HasAutoIncrement() && !t.Dir.Config.GetBool("include-auto-inc") && t.SchemaFromDir.Table(table.Name).NextAutoIncrement <= 1 {
					newContents, _ = tengo.ParseCreateAutoInc(newContents)
				}
				if sf.Contents != newContents {
					sf.Contents = newContents
					var length int
					if length, err = sf.Write(); err != nil {
						return fmt.Errorf("Unable to write to %s: %s", sf.Path(), err)
					}
					log.Infof("Wrote %s (%d bytes) -- updated file to normalize format", sf.Path(), length)
				}
			}
		}

		os.Stderr.WriteString("\n")
	}

	if err := findNewSchemas(dir); err != nil {
		return err
	}

	if errCount == 0 {
		return nil
	}
	var plural string
	if errCount > 1 {
		plural = "s"
	}
	return NewExitValue(CodePartialError, "Skipped %d operation%s due to error%s", errCount, plural, plural)
}

func findNewSchemas(dir *Dir) error {
	subdirs, err := dir.Subdirs()
	if err != nil {
		return err
	}

	if dir.HasHost() && !dir.HasSchema() {
		instance, err := dir.FirstInstance()
		if err != nil || instance == nil {
			return err
		}

		subdirHasSchema := make(map[string]bool)
		for _, subdir := range subdirs {
			// We only want to evaluate subdirs that explicitly define the schema option
			// in that subdir's .skeema file, vs inheriting it from a parent dir.
			if !subdir.HasSchema() {
				continue
			}

			// If a subdir's schema is set to "*", it maps to all schemas on the
			// instance, so no sense in trying to detect "new" schemas
			if subdir.Config.Get("schema") == "*" {
				return nil
			}

			schemaNames, err := subdir.SchemaNames(instance)
			if err != nil {
				return err
			}
			for _, name := range schemaNames {
				subdirHasSchema[name] = true
			}
		}

		// Compare dirs to schemas, UNLESS subdirs exist but don't actually map to schemas directly
		if len(subdirHasSchema) > 0 || len(subdirs) == 0 {
			var instCharSet, instCollation string
			schemaNames, err := instance.SchemaNames()
			if err != nil {
				return err
			}
			for _, name := range schemaNames {
				// If no existing subdir maps to the schema, we need to create and populate new dir
				if !subdirHasSchema[name] {
					s, err := instance.Schema(name)
					if err != nil {
						return err
					}
					// Lazily fetch instance charset and collation upon first need
					if instCharSet == "" {
						instCharSet, instCollation, err = instance.DefaultCharSetAndCollation()
						if err != nil {
							return err
						}
					}
					// use same logic from init command
					if err := PopulateSchemaDir(s, dir, true, instCharSet != s.CharSet, instCollation != s.Collation); err != nil {
						return err
					}
				}
			}

			// If we did a schema-to-subdir comparison, no need to continue recursion
			// even if there are additional levels of subdirs
			return nil
		}
	}

	for _, subdir := range subdirs {
		if subdir.BaseName()[0] != '.' {
			err := findNewSchemas(subdir)
			if err != nil {
				return err
			}
		}
	}

	return nil
}
<commit_msg>pull: de-dupe operations by table, so file only updated once<commit_after>package main

import (
	"fmt"
	"os"

	log "github.com/sirupsen/logrus"
	"github.com/skeema/mybase"
	"github.com/skeema/tengo"
)

func init() {
	summary := "Update the filesystem representation of schemas and tables"
	desc := `Updates the existing filesystem representation of the schemas and tables on a DB
instance. Use this command when changes have been applied to the database
without using skeema, and the filesystem representation needs to be updated to
reflect those changes.

You may optionally pass an environment name as a CLI option. This will affect
which section of .skeema config files is used for processing. For example,
running ` + "`" + `skeema pull staging` + "`" + ` will apply config directives from the
[staging] section of config files, as well as any sectionless directives at the
top of the file. If no environment name is supplied, the default is
"production".`

	cmd := mybase.NewCommand("pull", summary, desc, PullHandler)
	cmd.AddOption(mybase.BoolOption("include-auto-inc", 0, false, "Include starting auto-inc values in new table files, and update in existing files"))
	cmd.AddOption(mybase.BoolOption("normalize", 0, true, "Reformat *.sql files to match SHOW CREATE TABLE"))
	cmd.AddArg("environment", "production", false)
	CommandSuite.AddSubCommand(cmd)
}

// PullHandler is the handler method for `skeema pull`
func PullHandler(cfg *mybase.Config) error {
	AddGlobalConfigFiles(cfg)
	dir, err := NewDir(".", cfg)
	if err != nil {
		return err
	}

	var errCount int

	for _, t := range dir.Targets() {
		if t.Err != nil {
			log.Errorf("Skipping %s:", t.Dir)
			log.Errorf("    %s\n", t.Err)
			errCount++
			continue
		}

		log.Infof("Updating %s to reflect %s %s", t.Dir, t.Instance, t.SchemaFromDir.Name)

		// If schema doesn't exist on instance, remove the corresponding dir
		if t.SchemaFromInstance == nil {
			if err := t.Dir.Delete(); err != nil {
				return fmt.Errorf("Unable to delete directory %s: %s", t.Dir, err)
			}
			log.Infof("Deleted directory %s -- schema no longer exists\n", t.Dir)
			continue
		}

		diff := tengo.NewSchemaDiff(t.SchemaFromDir, t.SchemaFromInstance)

		// Handle changes in schema's default character set and/or collation by
		// persisting changes to the dir's option file. File operation errors here
		// are just surfaced as warnings.
		if diff.SchemaDDL != "" {
			instCharSet, instCollation, err := t.Instance.DefaultCharSetAndCollation()
			if err != nil {
				return err
			}
			optionFile, err := t.Dir.OptionFile()
			if err != nil {
				log.Warnf("Unable to update character set and/or collation for %s/.skeema: %s", t.Dir, err)
			} else if optionFile == nil {
				log.Warnf("Unable to update character set and/or collation for %s/.skeema: cannot read file", t.Dir)
			} else {
				if instCharSet != t.SchemaFromInstance.CharSet {
					optionFile.SetOptionValue("", "default-character-set", t.SchemaFromInstance.CharSet)
				} else {
					optionFile.UnsetOptionValue("", "default-character-set")
				}
				if instCollation != t.SchemaFromInstance.Collation {
					optionFile.SetOptionValue("", "default-collation", t.SchemaFromInstance.Collation)
				} else {
					optionFile.UnsetOptionValue("", "default-collation")
				}
				if err = optionFile.Write(true); err != nil {
					log.Warnf("Unable to update character set and/or collation for %s: %s", optionFile.Path(), err)
				} else {
					log.Infof("Wrote %s -- updated schema-level default-character-set and default-collation", optionFile.Path())
				}
			}
		}

		// We're permissive of unsafe operations here since we don't ever actually
		// execute the generated statement! We just examine its type.
		mods := tengo.StatementModifiers{
			AllowUnsafe: true,
		}
		// pull command updates next auto-increment value for existing table always
		// if requested, or only if previously present in file otherwise
		if t.Dir.Config.GetBool("include-auto-inc") {
			mods.NextAutoInc = tengo.NextAutoIncAlways
		} else {
			mods.NextAutoInc = tengo.NextAutoIncIfAlready
		}
		mods.IgnoreTable, err = t.Dir.Config.GetRegexp("ignore-table")
		if err != nil {
			return err
		}

		// Track which table names have already been seen, to handle cases where the
		// same table shows up in two different TableDiffs. This can happen because
		// Tengo avoids generating single ALTERs that both drop and add foreign keys
		// in the same statement.
		alreadySeen := make(map[string]bool)

		for _, td := range diff.TableDiffs {
			if td.To != nil && alreadySeen[td.To.Name] {
				continue
			}

			stmt, stmtErr := td.Statement(mods)
			// Errors are fatal, except for UnsupportedDiffError which we can safely
			// ignore (since pull doesn't actually run ALTERs; it just needs to know
			// which tables were altered)
			if stmtErr != nil && !tengo.IsUnsupportedDiff(stmtErr) {
				return stmtErr
			}
			// skip if mods caused the diff to be a no-op; if it's an ALTER, treat it
			// as an unchanged table so that --normalize logic still runs
			if stmt == "" && stmtErr == nil {
				if td.Type == tengo.TableDiffAlter {
					diff.SameTables = append(diff.SameTables, td.To)
				}
				continue
			}

			// For DROP TABLE, we're deleting corresponding table file; vs other
			// types we're updating/rewriting the file.
			if td.Type == tengo.TableDiffDrop {
				sf := SQLFile{
					Dir:      t.Dir,
					FileName: fmt.Sprintf("%s.sql", td.From.Name),
				}
				if err := sf.Delete(); err != nil {
					return fmt.Errorf("Unable to delete %s: %s", sf.Path(), err)
				}
				log.Infof("Deleted %s -- table no longer exists", sf.Path())
				continue
			}

			var reason string
			sf := SQLFile{
				Dir:      t.Dir,
				FileName: fmt.Sprintf("%s.sql", td.To.Name),
				Contents: stmt,
			}

			// For ALTER TABLE, we don't care about the ALTER statement, but we do
			// need to get the corresponding CREATE TABLE and process auto-inc properly
			if td.Type == tengo.TableDiffAlter {
				sf.Contents = td.To.CreateStatement
				if td.To.HasAutoIncrement() && !t.Dir.Config.GetBool("include-auto-inc") && td.From.NextAutoIncrement <= 1 {
					sf.Contents, _ = tengo.ParseCreateAutoInc(sf.Contents)
				}
				reason = "updated file to reflect table alterations"
				if tengo.IsUnsupportedDiff(stmtErr) {
					log.Warnf("Table %s uses unsupported features", td.To.Name)
					DebugLogUnsupportedDiff(stmtErr.(*tengo.UnsupportedDiffError))
				}
			} else if _, hadErr := t.SQLFileErrors[sf.Path()]; hadErr {
				// SQL files with syntax errors will result in TableDiffCreate since the
				// temp schema will be missing the table
				reason = "updated file to replace invalid SQL"
			} else {
				reason = "new table"
			}

			length, err := sf.Write()
			if err != nil {
				return fmt.Errorf("Unable to write to %s: %s", sf.Path(), err)
			}
			log.Infof("Wrote %s (%d bytes) -- %s", sf.Path(), length, reason)
			alreadySeen[td.To.Name] = true
		}

		if dir.Config.GetBool("normalize") {
			for _, table := range diff.SameTables {
				if mods.IgnoreTable != nil && mods.IgnoreTable.MatchString(table.Name) {
					continue
				}
				sf := SQLFile{
					Dir:      t.Dir,
					FileName: fmt.Sprintf("%s.sql", table.Name),
				}
				if _, err := sf.Read(); err != nil {
					return err
				}
				for _, warning := range sf.Warnings {
					log.Debug(warning)
				}
				newContents := table.CreateStatement
				if table.HasAutoIncrement() && !t.Dir.Config.GetBool("include-auto-inc") && t.SchemaFromDir.Table(table.Name).NextAutoIncrement <= 1 {
					newContents, _ = tengo.ParseCreateAutoInc(newContents)
				}
				if sf.Contents != newContents {
					sf.Contents = newContents
					var length int
					if length, err = sf.Write(); err != nil {
						return fmt.Errorf("Unable to write to %s: %s", sf.Path(), err)
					}
					log.Infof("Wrote %s (%d bytes) -- updated file to normalize format", sf.Path(), length)
				}
			}
		}

		os.Stderr.WriteString("\n")
	}

	if err := findNewSchemas(dir); err != nil {
		return err
	}

	if errCount == 0 {
		return nil
	}
	var plural string
	if errCount > 1 {
		plural = "s"
	}
	return NewExitValue(CodePartialError, "Skipped %d operation%s due to error%s", errCount, plural, plural)
}

func findNewSchemas(dir *Dir) error {
	subdirs, err := dir.Subdirs()
	if err != nil {
		return err
	}

	if dir.HasHost() && !dir.HasSchema() {
		instance, err := dir.FirstInstance()
		if err != nil || instance == nil {
			return err
		}

		subdirHasSchema := make(map[string]bool)
		for _, subdir := range subdirs {
			// We only want to evaluate subdirs that explicitly define the schema option
			// in that subdir's .skeema file, vs inheriting it from a parent dir.
			if !subdir.HasSchema() {
				continue
			}

			// If a subdir's schema is set to "*", it maps to all schemas on the
			// instance, so no sense in trying to detect "new" schemas
			if subdir.Config.Get("schema") == "*" {
				return nil
			}

			schemaNames, err := subdir.SchemaNames(instance)
			if err != nil {
				return err
			}
			for _, name := range schemaNames {
				subdirHasSchema[name] = true
			}
		}

		// Compare dirs to schemas, UNLESS subdirs exist but don't actually map to schemas directly
		if len(subdirHasSchema) > 0 || len(subdirs) == 0 {
			var instCharSet, instCollation string
			schemaNames, err := instance.SchemaNames()
			if err != nil {
				return err
			}
			for _, name := range schemaNames {
				// If no existing subdir maps to the schema, we need to create and populate new dir
				if !subdirHasSchema[name] {
					s, err := instance.Schema(name)
					if err != nil {
						return err
					}
					// Lazily fetch instance charset and collation upon first need
					if instCharSet == "" {
						instCharSet, instCollation, err = instance.DefaultCharSetAndCollation()
						if err != nil {
							return err
						}
					}
					// use same logic from init command
					if err := PopulateSchemaDir(s, dir, true, instCharSet != s.CharSet, instCollation != s.Collation); err != nil {
						return err
					}
				}
			}

			// If we did a schema-to-subdir comparison, no need to continue recursion
			// even if there are additional levels of subdirs
			return nil
		}
	}

	for _, subdir := range subdirs {
		if subdir.BaseName()[0] != '.' {
			err := findNewSchemas(subdir)
			if err != nil {
				return err
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/mistifyio/mistify-agent/log"
	"github.com/mistifyio/mistify-dhcp"
	"os"
)

func main() {
	conf, err := dhcp.GetConfig()
	if err != nil {
		log.Error(err)
		os.Exit(1)
	}

	server := dhcp.NewServer(conf)
	server.Run()
}
<commit_msg>[MIST-359] Moved main.go to logrus<commit_after>package main

import (
	log "github.com/Sirupsen/logrus"
	"github.com/mistifyio/mistify-dhcp"
)

func main() {
	log.SetFormatter(&log.JSONFormatter{})

	conf, err := dhcp.GetConfig()
	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
			"func":  "dhcp.GetConfig",
		}).Fatal(err)
	}

	server := dhcp.NewServer(conf)
	server.Run()
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"context"
	"errors"
	"fmt"
	"os"

	"github.com/sirupsen/logrus"

	"github.com/GoogleContainerTools/skaffold/cmd/skaffold/app"
)

func main() {
	if err := app.Run(os.Stdout, os.Stderr); err != nil {
		if errors.Is(err, context.Canceled) {
			logrus.Debugln(fmt.Errorf("ignore error since context is cancelled: %w", err))
		} else {
			logrus.Fatal(err)
		}
	}
}
<commit_msg>Simplify code (#4060)<commit_after>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"context"
	"errors"
	"os"

	"github.com/sirupsen/logrus"

	"github.com/GoogleContainerTools/skaffold/cmd/skaffold/app"
)

func main() {
	if err := app.Run(os.Stdout, os.Stderr); err != nil {
		if errors.Is(err, context.Canceled) {
			logrus.Debugln("ignore error since context is cancelled:", err)
		} else {
			logrus.Fatal(err)
		}
	}
}
<|endoftext|>