<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package windows

import (
	"context"
	"fmt"
	"time"

	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/kubernetes/test/e2e/framework"
	e2ekubelet "k8s.io/kubernetes/test/e2e/framework/kubelet"
	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
	imageutils "k8s.io/kubernetes/test/utils/image"

	"github.com/onsi/ginkgo"
)

var _ = SIGDescribe("[Feature:Windows] Kubelet-Stats [Serial]", func() {
	f := framework.NewDefaultFramework("kubelet-stats-test-windows-serial")

	ginkgo.Describe("Kubelet stats collection for Windows nodes", func() {
		ginkgo.Context("when running 10 pods", func() {
			// 10 seconds is the default scrape timeout for metrics-server and kube-prometheus
			ginkgo.It("should return within 10 seconds", func() {

				ginkgo.By("Selecting a Windows node")
				targetNode, err := findWindowsNode(f)
				framework.ExpectNoError(err, "Error finding Windows node")
				framework.Logf("Using node: %v", targetNode.Name)

				ginkgo.By("Scheduling 10 pods")
				powershellImage := imageutils.GetConfig(imageutils.BusyBox)
				pods := newKubeletStatsTestPods(10, powershellImage, targetNode.Name)
				f.PodClient().CreateBatch(pods)

				ginkgo.By("Waiting up to 3 minutes for pods to be running")
				timeout := 3 * time.Minute
				e2epod.WaitForPodsRunningReady(f.ClientSet, f.Namespace.Name, 10, 0, timeout, make(map[string]string))

				ginkgo.By("Getting kubelet stats 5 times and checking average duration")
				iterations := 5
				var totalDurationMs int64

				for i := 0; i < iterations; i++ {
					start := time.Now()
					nodeStats, err := e2ekubelet.GetStatsSummary(f.ClientSet, targetNode.Name)
					duration := time.Since(start)
					totalDurationMs += duration.Milliseconds()

					framework.ExpectNoError(err, "Error getting kubelet stats")

					// Perform some basic sanity checks on retrieved stats for pods in this test's namespace
					statsChecked := 0
					for _, podStats := range nodeStats.Pods {
						if podStats.PodRef.Namespace != f.Namespace.Name {
							continue
						}
						statsChecked = statsChecked + 1

						framework.ExpectEqual(*podStats.CPU.UsageCoreNanoSeconds > 0, true, "Pod stats should not report 0 cpu usage")
						framework.ExpectEqual(*podStats.Memory.WorkingSetBytes > 0, true, "Pod stats should not report 0 bytes for memory working set ")

						for _, containerStats := range podStats.Containers {
							framework.ExpectEqual(containerStats.Logs != nil, true, "Pod stats should have container log stats")
							framework.ExpectEqual(*containerStats.Logs.AvailableBytes > 0, true, "container log stats should not report 0 bytes for AvailableBytes")
						}
					}
					framework.ExpectEqual(statsChecked, 10, "Should find stats for 10 pods in kubelet stats")

					time.Sleep(5 * time.Second)
				}

				avgDurationMs := totalDurationMs / int64(iterations)

				durationMatch := avgDurationMs <= time.Duration(10*time.Second).Milliseconds()
				framework.Logf("Getting kubelet stats for node %v took an average of %v milliseconds over %v iterations", targetNode.Name, avgDurationMs, iterations)
				framework.ExpectEqual(durationMatch, true, "Collecting kubelet stats should not take longer than 10 seconds")
			})
		})
	})
})
var _ = SIGDescribe("[Feature:Windows] Kubelet-Stats", func() {
	f := framework.NewDefaultFramework("kubelet-stats-test-windows")

	ginkgo.Describe("Kubelet stats collection for Windows nodes", func() {
		ginkgo.Context("when running 3 pods", func() {
			// 10 seconds is the default scrape timeout for metrics-server and kube-prometheus
			ginkgo.It("should return within 10 seconds", func() {

				ginkgo.By("Selecting a Windows node")
				targetNode, err := findWindowsNode(f)
				framework.ExpectNoError(err, "Error finding Windows node")
				framework.Logf("Using node: %v", targetNode.Name)

				ginkgo.By("Scheduling 3 pods")
				powershellImage := imageutils.GetConfig(imageutils.BusyBox)
				pods := newKubeletStatsTestPods(3, powershellImage, targetNode.Name)
				f.PodClient().CreateBatch(pods)

				ginkgo.By("Waiting up to 3 minutes for pods to be running")
				timeout := 3 * time.Minute
				e2epod.WaitForPodsRunningReady(f.ClientSet, f.Namespace.Name, 3, 0, timeout, make(map[string]string))

				ginkgo.By("Getting kubelet stats 1 time")
				iterations := 1
				var totalDurationMs int64

				for i := 0; i < iterations; i++ {
					start := time.Now()
					nodeStats, err := e2ekubelet.GetStatsSummary(f.ClientSet, targetNode.Name)
					duration := time.Since(start)
					totalDurationMs += duration.Milliseconds()

					framework.ExpectNoError(err, "Error getting kubelet stats")

					// Perform some basic sanity checks on retrieved stats for pods in this test's namespace
					statsChecked := 0
					for _, podStats := range nodeStats.Pods {
						if podStats.PodRef.Namespace != f.Namespace.Name {
							continue
						}
						statsChecked = statsChecked + 1

						framework.ExpectEqual(*podStats.CPU.UsageCoreNanoSeconds > 0, true, "Pod stats should not report 0 cpu usage")
						framework.ExpectEqual(*podStats.Memory.WorkingSetBytes > 0, true, "Pod stats should not report 0 bytes for memory working set ")

						for _, containerStats := range podStats.Containers {
							framework.ExpectEqual(containerStats.Logs != nil, true, "Pod stats should have container log stats")
							framework.ExpectEqual(*containerStats.Logs.AvailableBytes > 0, true, "container log stats should not report 0 bytes for AvailableBytes")
						}
					}
					framework.ExpectEqual(statsChecked, 3, "Should find stats for 10 pods in kubelet stats")

					time.Sleep(5 * time.Second)
				}

				avgDurationMs := totalDurationMs / int64(iterations)

				durationMatch := avgDurationMs <= time.Duration(10*time.Second).Milliseconds()
				framework.Logf("Getting kubelet stats for node %v took an average of %v milliseconds over %v iterations", targetNode.Name, avgDurationMs, iterations)
				framework.ExpectEqual(durationMatch, true, "Collecting kubelet stats should not take longer than 10 seconds")
			})
		})
	})
})

// findWindowsNode finds a Windows node that is Ready and Schedulable
func findWindowsNode(f *framework.Framework) (v1.Node, error) {
	selector := labels.Set{"kubernetes.io/os": "windows"}.AsSelector()
	nodeList, err := f.ClientSet.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{LabelSelector: selector.String()})

	if err != nil {
		return v1.Node{}, err
	}

	var targetNode v1.Node
	foundNode := false
	for _, n := range nodeList.Items {
		if e2enode.IsNodeReady(&n) && e2enode.IsNodeSchedulable(&n) {
			targetNode = n
			foundNode = true
			break
		}
	}

	if foundNode == false {
		e2eskipper.Skipf("Could not find and ready and schedulable Windows nodes")
	}

	return targetNode, nil
}

// newKubeletStatsTestPods creates a list of pods (specification) for test.
func newKubeletStatsTestPods(numPods int, image imageutils.Config, nodeName string) []*v1.Pod {
	var pods []*v1.Pod

	for i := 0; i < numPods; i++ {
		podName := "statscollectiontest-" + string(uuid.NewUUID())
		pod := v1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name: fmt.Sprintf("%s-%d", podName, i),
				Labels: map[string]string{
					"name":    podName,
					"testapp": "stats-collection",
				},
			},
			Spec: v1.PodSpec{
				Containers: []v1.Container{
					{
						Image: image.GetE2EImage(),
						Name:  "stat-container",
						Command: []string{
							"powershell.exe",
							"-Command",
							"sleep -Seconds 600",
						},
					},
				},
				InitContainers: []v1.Container{
					{
						Image: image.GetE2EImage(),
						Name:  "init-container",
						Command: []string{
							"powershell.exe",
							"-Command",
							"sleep -Seconds 1",
						},
					},
				},
				NodeName: nodeName,
			},
		}

		pods = append(pods, &pod)
	}

	return pods
}
<commit_msg>Add check for network stats to e2e tests<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package windows

import (
	"context"
	"fmt"
	"time"

	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/kubernetes/test/e2e/framework"
	e2ekubelet "k8s.io/kubernetes/test/e2e/framework/kubelet"
	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
	imageutils "k8s.io/kubernetes/test/utils/image"

	"github.com/onsi/ginkgo"
)

var _ = SIGDescribe("[Feature:Windows] Kubelet-Stats [Serial]", func() {
	f := framework.NewDefaultFramework("kubelet-stats-test-windows-serial")

	ginkgo.Describe("Kubelet stats collection for Windows nodes", func() {
		ginkgo.Context("when running 10 pods", func() {
			// 10 seconds is the default scrape timeout for metrics-server and kube-prometheus
			ginkgo.It("should return within 10 seconds", func() {

				ginkgo.By("Selecting a Windows node")
				targetNode, err := findWindowsNode(f)
				framework.ExpectNoError(err, "Error finding Windows node")
				framework.Logf("Using node: %v", targetNode.Name)

				ginkgo.By("Scheduling 10 pods")
				powershellImage := imageutils.GetConfig(imageutils.BusyBox)
				pods := newKubeletStatsTestPods(10, powershellImage, targetNode.Name)
				f.PodClient().CreateBatch(pods)

				ginkgo.By("Waiting up to 3 minutes for pods to be running")
				timeout := 3 * time.Minute
				e2epod.WaitForPodsRunningReady(f.ClientSet, f.Namespace.Name, 10, 0, timeout, make(map[string]string))

				ginkgo.By("Getting kubelet stats 5 times and checking average duration")
				iterations := 5
				var totalDurationMs int64

				for i := 0; i < iterations; i++ {
					start := time.Now()
					nodeStats, err := e2ekubelet.GetStatsSummary(f.ClientSet, targetNode.Name)
					duration := time.Since(start)
					totalDurationMs += duration.Milliseconds()

					framework.ExpectNoError(err, "Error getting kubelet stats")

					// Perform some basic sanity checks on retrieved stats for pods in this test's namespace
					statsChecked := 0
					for _, podStats := range nodeStats.Pods {
						if podStats.PodRef.Namespace != f.Namespace.Name {
							continue
						}
						statsChecked = statsChecked + 1

						framework.ExpectEqual(*podStats.CPU.UsageCoreNanoSeconds > 0, true, "Pod stats should not report 0 cpu usage")
						framework.ExpectEqual(*podStats.Memory.WorkingSetBytes > 0, true, "Pod stats should not report 0 bytes for memory working set ")

						framework.ExpectEqual(podStats.Network != nil, true, "Pod stats should report network stats")
						framework.ExpectEqual(podStats.Network.Name != "", true, "Pod stats should report network name")
						framework.ExpectEqual(*podStats.Network.TxBytes > 0, true, "Pod stats should report network Tx stats")
						framework.ExpectEqual(len(podStats.Network.Interfaces) > 0, true, "Pod Stats should report individual interfaces stats")

						for _, containerStats := range podStats.Containers {
							framework.ExpectEqual(containerStats.Logs != nil, true, "Pod stats should have container log stats")
							framework.ExpectEqual(*containerStats.Logs.AvailableBytes > 0, true, "container log stats should not report 0 bytes for AvailableBytes")
						}
					}
					framework.ExpectEqual(statsChecked, 10, "Should find stats for 10 pods in kubelet stats")

					time.Sleep(5 * time.Second)
				}

				avgDurationMs := totalDurationMs / int64(iterations)

				durationMatch := avgDurationMs <= time.Duration(10*time.Second).Milliseconds()
				framework.Logf("Getting kubelet stats for node %v took an average of %v milliseconds over %v iterations", targetNode.Name, avgDurationMs, iterations)
				framework.ExpectEqual(durationMatch, true, "Collecting kubelet stats should not take longer than 10 seconds")
			})
		})
	})
})
var _ = SIGDescribe("[Feature:Windows] Kubelet-Stats", func() {
	f := framework.NewDefaultFramework("kubelet-stats-test-windows")

	ginkgo.Describe("Kubelet stats collection for Windows nodes", func() {
		ginkgo.Context("when running 3 pods", func() {
			// 10 seconds is the default scrape timeout for metrics-server and kube-prometheus
			ginkgo.It("should return within 10 seconds", func() {

				ginkgo.By("Selecting a Windows node")
				targetNode, err := findWindowsNode(f)
				framework.ExpectNoError(err, "Error finding Windows node")
				framework.Logf("Using node: %v", targetNode.Name)

				ginkgo.By("Scheduling 3 pods")
				powershellImage := imageutils.GetConfig(imageutils.BusyBox)
				pods := newKubeletStatsTestPods(3, powershellImage, targetNode.Name)
				f.PodClient().CreateBatch(pods)

				ginkgo.By("Waiting up to 3 minutes for pods to be running")
				timeout := 3 * time.Minute
				e2epod.WaitForPodsRunningReady(f.ClientSet, f.Namespace.Name, 3, 0, timeout, make(map[string]string))

				ginkgo.By("Getting kubelet stats 1 time")
				iterations := 1
				var totalDurationMs int64

				for i := 0; i < iterations; i++ {
					start := time.Now()
					nodeStats, err := e2ekubelet.GetStatsSummary(f.ClientSet, targetNode.Name)
					duration := time.Since(start)
					totalDurationMs += duration.Milliseconds()

					framework.ExpectNoError(err, "Error getting kubelet stats")

					// Perform some basic sanity checks on retrieved stats for pods in this test's namespace
					statsChecked := 0
					for _, podStats := range nodeStats.Pods {
						if podStats.PodRef.Namespace != f.Namespace.Name {
							continue
						}
						statsChecked = statsChecked + 1

						framework.ExpectEqual(*podStats.CPU.UsageCoreNanoSeconds > 0, true, "Pod stats should not report 0 cpu usage")
						framework.ExpectEqual(*podStats.Memory.WorkingSetBytes > 0, true, "Pod stats should not report 0 bytes for memory working set ")

						for _, containerStats := range podStats.Containers {
							framework.ExpectEqual(containerStats.Logs != nil, true, "Pod stats should have container log stats")
							framework.ExpectEqual(*containerStats.Logs.AvailableBytes > 0, true, "container log stats should not report 0 bytes for AvailableBytes")
						}
					}
					framework.ExpectEqual(statsChecked, 3, "Should find stats for 10 pods in kubelet stats")

					time.Sleep(5 * time.Second)
				}

				avgDurationMs := totalDurationMs / int64(iterations)

				durationMatch := avgDurationMs <= time.Duration(10*time.Second).Milliseconds()
				framework.Logf("Getting kubelet stats for node %v took an average of %v milliseconds over %v iterations", targetNode.Name, avgDurationMs, iterations)
				framework.ExpectEqual(durationMatch, true, "Collecting kubelet stats should not take longer than 10 seconds")
			})
		})
	})
})

// findWindowsNode finds a Windows node that is Ready and Schedulable
func findWindowsNode(f *framework.Framework) (v1.Node, error) {
	selector := labels.Set{"kubernetes.io/os": "windows"}.AsSelector()
	nodeList, err := f.ClientSet.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{LabelSelector: selector.String()})

	if err != nil {
		return v1.Node{}, err
	}

	var targetNode v1.Node
	foundNode := false
	for _, n := range nodeList.Items {
		if e2enode.IsNodeReady(&n) && e2enode.IsNodeSchedulable(&n) {
			targetNode = n
			foundNode = true
			break
		}
	}

	if foundNode == false {
		e2eskipper.Skipf("Could not find and ready and schedulable Windows nodes")
	}

	return targetNode, nil
}

// newKubeletStatsTestPods creates a list of pods (specification) for test.
func newKubeletStatsTestPods(numPods int, image imageutils.Config, nodeName string) []*v1.Pod {
	var pods []*v1.Pod

	for i := 0; i < numPods; i++ {
		podName := "statscollectiontest-" + string(uuid.NewUUID())
		pod := v1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name: fmt.Sprintf("%s-%d", podName, i),
				Labels: map[string]string{
					"name":    podName,
					"testapp": "stats-collection",
				},
			},
			Spec: v1.PodSpec{
				Containers: []v1.Container{
					{
						Image: image.GetE2EImage(),
						Name:  "stat-container",
						Command: []string{
							"powershell.exe",
							"-Command",
							"sleep -Seconds 600",
						},
					},
				},
				InitContainers: []v1.Container{
					{
						Image: image.GetE2EImage(),
						Name:  "init-container",
						Command: []string{
							"powershell.exe",
							"-Command",
							"sleep -Seconds 1",
						},
					},
				},
				NodeName: nodeName,
			},
		}

		pods = append(pods, &pod)
	}

	return pods
}
<|endoftext|>


<commit_before>package prj

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"regexp"
	"runtime"
	"strings"
)

var project *Project

type Project struct {
	rootFolder string
	// Global  GOPATH
	ggopath string
}

func GetProject() (*Project, error) {
	if project == nil {
		project = &Project{}
		gdir, gerr := Git("rev-parse --git-dir")
		gdir = strings.TrimSpace(gdir)
		// fmt.Printf("ko '%s' '%s'", gdir, gerr)
		if gerr != nil {
			return nil, gerr
		}
		if gdir != ".git" {
			project.rootFolder = gdir[:len(gdir)-5]
		} else {
			// fmt.Printf("ok")
			project.rootFolder = wd
		}
		project.ggopath = os.Getenv("GOPATH")
	}
	// fmt.Printf("prf '%s'", project.rootFolder)
	// fmt.Printf("prf '%s'", project.ggopath)
	return project, nil
}

func (p *Project) RootFolder() string {
	return p.rootFolder
}

// Inspired by https://github.com/ghthor/journal/blob/0bd4968a4f9841befdd0dde96b2096e6c930e74c/git/git.go

var gitPath string
var goPath string
var wd string

func init() {
	gitPath = getPathForExe("git")
	goPath = getPathForExe("go")
	var err error
	wd, err = os.Getwd()
	if err != nil {
		log.Fatal("Working directory not accessible")
	}
}

func getPathForExe(exe string) string {
	var err error
	var path = ""
	if path, err = exec.LookPath(exe); err != nil {
		aliases := ""
		if runtime.GOOS == "windows" {
			aliases, err = execcmd("doskey", "/macros")
		} else {
			aliases, err = execcmd("alias", "")
		}
		r := regexp.MustCompile(`(?m)^` + exe + `=(.*)\s+[\$%@\*].*$`)
		sm := r.FindAllStringSubmatch(aliases, 1)
		if len(sm) != 1 || len(sm[0]) != 2 {
			log.Fatalf("Unable to find '%s' path in aliases '%s'", exe)
		}
		return sm[0][1]
	}
	if runtime.GOOS == "windows" {
		if strings.HasSuffix(path, ".bat") {
			bat, err := ioutil.ReadFile(path)
			if err != nil {
				log.Fatalf("Unable to read '%s' for '%s'", path, exe)
			}
			bats := string(bat)
			r := regexp.MustCompile(`(?m)^\s*?(.*)\s+[\$%@\*].*$`)
			sm := r.FindAllStringSubmatch(bats, 1)
			if len(sm) != 1 || len(sm[0]) != 2 {
				log.Fatalf("Unable to find '%s' path in file '%s'", exe, path)
			}
			return sm[0][1]
		}
	}
	if path == "" {
		log.Fatalf("Unable to get path for '%s'", exe)
	}
	return path
}

// Construct an *exec.Cmd for `git {args}` with a workingDirectory
func Git(cmd string) (string, error) {
	return execcmd(gitPath, cmd)
}
func Golang(cmd string) (string, error) {
	os.Setenv("GOPATH", project.rootFolder+`/deps`)
	os.Setenv("GOBIN", project.rootFolder+`/bin`)
	return execcmd(goPath, cmd)
}

func execcmd(exe, cmd string) (string, error) {
	args := strings.Split(cmd, " ")
	c := exec.Command(exe, args...)
	c.Dir = project.rootFolder
	var bout bytes.Buffer
	c.Stdout = &bout
	var berr bytes.Buffer
	c.Stderr = &berr
	err := c.Run()
	if err != nil {
		return bout.String(), fmt.Errorf("Unable to run '%s %s' in '%s': err '%s'\n'%s'", exe, cmd, wd, err.Error(), berr.String())
	} else if berr.String() != "" {
		return bout.String(), fmt.Errorf("Warning on run '%s %s' in '%s': '%s'", exe, cmd, wd, berr.String())
	}
	return bout.String(), nil
}
<commit_msg>project.go: uses Debug to print exec commands<commit_after>package prj

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"regexp"
	"runtime"
	"strings"
)

var project *Project
var Debug bool

type Project struct {
	rootFolder string
	// Global  GOPATH
	ggopath string
}

func GetProject() (*Project, error) {
	if project == nil {
		project = &Project{}
		gdir, gerr := Git("rev-parse --git-dir")
		gdir = strings.TrimSpace(gdir)
		// fmt.Printf("ko '%s' '%s'", gdir, gerr)
		if gerr != nil {
			return nil, gerr
		}
		if gdir != ".git" {
			project.rootFolder = gdir[:len(gdir)-5]
		} else {
			// fmt.Printf("ok")
			project.rootFolder = wd
		}
		project.ggopath = os.Getenv("GOPATH")
	}
	// fmt.Printf("prf '%s'", project.rootFolder)
	// fmt.Printf("prf '%s'", project.ggopath)
	return project, nil
}

func (p *Project) RootFolder() string {
	return p.rootFolder
}

// Inspired by https://github.com/ghthor/journal/blob/0bd4968a4f9841befdd0dde96b2096e6c930e74c/git/git.go

var gitPath string
var goPath string
var wd string

func init() {
	gitPath = getPathForExe("git")
	goPath = getPathForExe("go")
	var err error
	wd, err = os.Getwd()
	if err != nil {
		log.Fatal("Working directory not accessible")
	}
}

func getPathForExe(exe string) string {
	var err error
	var path = ""
	if path, err = exec.LookPath(exe); err != nil {
		aliases := ""
		if runtime.GOOS == "windows" {
			aliases, err = execcmd("doskey", "/macros")
		} else {
			aliases, err = execcmd("alias", "")
		}
		r := regexp.MustCompile(`(?m)^` + exe + `=(.*)\s+[\$%@\*].*$`)
		sm := r.FindAllStringSubmatch(aliases, 1)
		if len(sm) != 1 || len(sm[0]) != 2 {
			log.Fatalf("Unable to find '%s' path in aliases '%s'", exe)
		}
		return sm[0][1]
	}
	if runtime.GOOS == "windows" {
		if strings.HasSuffix(path, ".bat") {
			bat, err := ioutil.ReadFile(path)
			if err != nil {
				log.Fatalf("Unable to read '%s' for '%s'", path, exe)
			}
			bats := string(bat)
			r := regexp.MustCompile(`(?m)^\s*?(.*)\s+[\$%@\*].*$`)
			sm := r.FindAllStringSubmatch(bats, 1)
			if len(sm) != 1 || len(sm[0]) != 2 {
				log.Fatalf("Unable to find '%s' path in file '%s'", exe, path)
			}
			return sm[0][1]
		}
	}
	if path == "" {
		log.Fatalf("Unable to get path for '%s'", exe)
	}
	return path
}

// Construct an *exec.Cmd for `git {args}` with a workingDirectory
func Git(cmd string) (string, error) {
	return execcmd(gitPath, cmd)
}
func Golang(cmd string) (string, error) {
	os.Setenv("GOPATH", project.rootFolder+`/deps`)
	os.Setenv("GOBIN", project.rootFolder+`/bin`)
	return execcmd(goPath, cmd)
}

func execcmd(exe, cmd string) (string, error) {
	if Debug {
		fmt.Printf("%s %s\n", exe, cmd)
	}
	args := strings.Split(cmd, " ")
	c := exec.Command(exe, args...)
	c.Dir = project.rootFolder
	var bout bytes.Buffer
	c.Stdout = &bout
	var berr bytes.Buffer
	c.Stderr = &berr
	err := c.Run()
	if err != nil {
		return bout.String(), fmt.Errorf("Unable to run '%s %s' in '%s': err '%s'\n'%s'", exe, cmd, wd, err.Error(), berr.String())
	} else if berr.String() != "" {
		return bout.String(), fmt.Errorf("Warning on run '%s %s' in '%s': '%s'", exe, cmd, wd, berr.String())
	}
	return bout.String(), nil
}
<|endoftext|>


<commit_before>package main

import (
  "path/filepath"
  "os"
  "flag"
  "fmt"
  "net/http"
  "io"
  "io/ioutil"
  "strings"
)

func WalkFunc(path string, info os.FileInfo, err error) error {
	blacklist := []string{".bzr", ".cvs", ".git", ".hg", ".svn"}
	if contains(path, blacklist){
		fmt.Printf("Skipping version control dir: %s\n", path)
		return filepath.SkipDir
	} else {
		inf, err := os.Open(path)
		defer inf.Close();
		if (err!=nil) { inf.Close(); return err; }
		readStart := io.LimitReader(inf, 512);
		data, err := ioutil.ReadAll(readStart);
		fileType := http.DetectContentType(data);
		if strings.Contains(fileType, "text/plain"){
			fmt.Printf("Trimming: %v\n", path)
		} else {
			fmt.Printf("Skipping file of type: %v: %v\n", fileType, path)
		}
	}
	return nil
}

func contains(x string, a []string) bool {
	for _, e := range(a) {
		if (x==e) { return true; }
	}
	return false;
}

func main() {
	flag.Parse()
	root := flag.Arg(0)
	err := filepath.Walk(root, WalkFunc)
	fmt.Printf("filepath.Walk() returned %v\n", err)
}
<commit_msg>Prettifying the print out.<commit_after>package main

import (
  "path/filepath"
  "os"
  "flag"
  "fmt"
  "net/http"
  "io"
  "io/ioutil"
  "strings"
)

func WalkFunc(path string, info os.FileInfo, err error) error {
	blacklist := []string{".bzr", ".cvs", ".git", ".hg", ".svn"}
	if contains(path, blacklist){
		fmt.Printf("Skipping version control dir: %s\n", path)
		return filepath.SkipDir
	} else {
		inf, err := os.Open(path)
		defer inf.Close();
		if (err!=nil) { inf.Close(); return err; }
		readStart := io.LimitReader(inf, 512);
		data, err := ioutil.ReadAll(readStart);
		fileType := http.DetectContentType(data);
		if strings.Contains(fileType, "text/plain"){
			fmt.Printf("Trimming: %v\n", path)
		} else {
			fmt.Printf("Skipping file of type '%v': %v\n", fileType, path)
		}
	}
	return nil
}

func contains(x string, a []string) bool {
	for _, e := range(a) {
		if (x==e) { return true; }
	}
	return false;
}

func main() {
	flag.Parse()
	root := flag.Arg(0)
	err := filepath.Walk(root, WalkFunc)
	fmt.Printf("filepath.Walk() returned %v\n", err)
}
<|endoftext|>


<commit_before>package images

import (
	"fmt"
	"time"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"

	kapierrs "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/wait"

	exutil "github.com/openshift/origin/test/extended/util"
)

var _ = g.Describe("[Conformance][Area:Networking][Feature:Router]", func() {
	defer g.GinkgoRecover()
	var (
		configPath = exutil.FixturePath("testdata", "reencrypt-serving-cert.yaml")
		oc         = exutil.NewCLI("router-reencrypt", exutil.KubeConfigPath())

		ip, ns string
	)

	g.BeforeEach(func() {
		svc, err := oc.AdminKubeClient().Core().Services("default").Get("router", metav1.GetOptions{})
		if kapierrs.IsNotFound(err) {
			g.Skip("no router installed on the cluster")
			return
		}
		o.Expect(err).NotTo(o.HaveOccurred())
		ip = svc.Spec.ClusterIP
		ns = oc.KubeFramework().Namespace.Name
	})

	g.Describe("The HAProxy router", func() {
		g.It("should support reencrypt to services backed by a serving certificate automatically", func() {
			routerURL := fmt.Sprintf("https://%s", ip)

			execPodName := exutil.CreateExecPodOrFail(oc.AdminKubeClient().Core(), ns, "execpod")
			defer func() { oc.AdminKubeClient().Core().Pods(ns).Delete(execPodName, metav1.NewDeleteOptions(1)) }()
			g.By(fmt.Sprintf("deploying a service using a reencrypt route without a destinationCACertificate"))
			err := oc.Run("create").Args("-f", configPath).Execute()
			o.Expect(err).NotTo(o.HaveOccurred())

			var hostname string
			err = wait.Poll(time.Second, changeTimeoutSeconds*time.Second, func() (bool, error) {
				route, err := oc.RouteClient().Route().Routes(ns).Get("serving-cert", metav1.GetOptions{})
				if err != nil {
					return false, err
				}
				if len(route.Status.Ingress) == 0 || len(route.Status.Ingress[0].Host) == 0 {
					return false, nil
				}
				hostname = route.Status.Ingress[0].Host
				return true, nil
			})
			o.Expect(err).NotTo(o.HaveOccurred())

			// don't assume the router is available via external DNS, because of complexity
			err = waitForRouterOKResponseExec(ns, execPodName, routerURL, hostname, changeTimeoutSeconds)
			o.Expect(err).NotTo(o.HaveOccurred())
		})
	})
})
<commit_msg>Reencrypt routes are failing now<commit_after>package images

import (
	"fmt"
	"time"

	g "github.com/onsi/ginkgo"
	o "github.com/onsi/gomega"

	kapierrs "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/wait"

	routeclientset "github.com/openshift/client-go/route/clientset/versioned"
	exutil "github.com/openshift/origin/test/extended/util"
)

var _ = g.Describe("[Conformance][Area:Networking][Feature:Router]", func() {
	defer g.GinkgoRecover()
	var (
		configPath = exutil.FixturePath("testdata", "reencrypt-serving-cert.yaml")
		oc         *exutil.CLI

		ip, ns string
	)

	// this hook must be registered before the framework namespace teardown
	// hook
	g.AfterEach(func() {
		if g.CurrentGinkgoTestDescription().Failed {
			client := routeclientset.NewForConfigOrDie(oc.AdminConfig()).Route().Routes(ns)
			if routes, _ := client.List(metav1.ListOptions{}); routes != nil {
				outputIngress(routes.Items...)
			}
			exutil.DumpPodLogsStartingWithInNamespace("router", "default", oc.AsAdmin())
		}
	})

	oc = exutil.NewCLI("router-reencrypt", exutil.KubeConfigPath())

	g.BeforeEach(func() {
		svc, err := oc.AdminKubeClient().Core().Services("default").Get("router", metav1.GetOptions{})
		if kapierrs.IsNotFound(err) {
			g.Skip("no router installed on the cluster")
			return
		}
		o.Expect(err).NotTo(o.HaveOccurred())
		ip = svc.Spec.ClusterIP
		ns = oc.KubeFramework().Namespace.Name
	})

	g.Describe("The HAProxy router", func() {
		g.It("should support reencrypt to services backed by a serving certificate automatically", func() {
			routerURL := fmt.Sprintf("https://%s", ip)

			execPodName := exutil.CreateExecPodOrFail(oc.AdminKubeClient().Core(), ns, "execpod")
			defer func() { oc.AdminKubeClient().Core().Pods(ns).Delete(execPodName, metav1.NewDeleteOptions(1)) }()
			g.By(fmt.Sprintf("deploying a service using a reencrypt route without a destinationCACertificate"))
			err := oc.Run("create").Args("-f", configPath).Execute()
			o.Expect(err).NotTo(o.HaveOccurred())

			var hostname string
			err = wait.Poll(time.Second, changeTimeoutSeconds*time.Second, func() (bool, error) {
				route, err := oc.RouteClient().Route().Routes(ns).Get("serving-cert", metav1.GetOptions{})
				if err != nil {
					return false, err
				}
				if len(route.Status.Ingress) == 0 || len(route.Status.Ingress[0].Host) == 0 {
					return false, nil
				}
				hostname = route.Status.Ingress[0].Host
				return true, nil
			})
			o.Expect(err).NotTo(o.HaveOccurred())

			// don't assume the router is available via external DNS, because of complexity
			err = waitForRouterOKResponseExec(ns, execPodName, routerURL, hostname, changeTimeoutSeconds)
			o.Expect(err).NotTo(o.HaveOccurred())
		})
	})
})
<|endoftext|>


<commit_before>// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storage

import (
	"context"
	"database/sql"

	"github.com/lib/pq"
	"github.com/matrix-org/dendrite/common"
	"github.com/matrix-org/dendrite/roomserver/types"
)

const eventStateKeysSchema = `
-- Numeric versions of the event "state_key"s. State keys tend to be reused so
-- assigning each string a numeric ID should reduce the amount of data that
-- needs to be stored and fetched from the database.
-- It also means that many operations can work with int64 arrays rather than
-- string arrays which may help reduce GC pressure.
-- Well known state keys are pre-assigned numeric IDs:
--   1 -> "" (the empty string)
-- Other state keys are automatically assigned numeric IDs starting from 2**16.
-- This leaves room to add more pre-assigned numeric IDs and clearly separates
-- the automatically assigned IDs from the pre-assigned IDs.
CREATE SEQUENCE IF NOT EXISTS roomserver_event_state_key_nid_seq START 65536;
CREATE TABLE IF NOT EXISTS roomserver_event_state_keys (
    -- Local numeric ID for the state key.
    event_state_key_nid BIGINT PRIMARY KEY DEFAULT nextval('roomserver_event_state_key_nid_seq'),
    event_state_key TEXT NOT NULL CONSTRAINT roomserver_event_state_key_unique UNIQUE
);
INSERT INTO roomserver_event_state_keys (event_state_key_nid, event_state_key) VALUES
    (1, '') ON CONFLICT DO NOTHING;
`

// Same as insertEventTypeNIDSQL
const insertEventStateKeyNIDSQL = "" +
	"INSERT INTO roomserver_event_state_keys (event_state_key) VALUES ($1)" +
	" ON CONFLICT ON CONSTRAINT roomserver_event_state_key_unique" +
	" DO NOTHING RETURNING (event_state_key_nid)"

const selectEventStateKeyNIDSQL = "" +
	"SELECT event_state_key_nid FROM roomserver_event_state_keys" +
	" WHERE event_state_key = $1"

// Bulk lookup from string state key to numeric ID for that state key.
// Takes an array of strings as the query parameter.
const bulkSelectEventStateKeyNIDSQL = "" +
	"SELECT event_state_key, event_state_key_nid FROM roomserver_event_state_keys" +
	" WHERE event_state_key = ANY($1)"

// Bulk lookup from numeric ID to string state key for that state key.
// Takes an array of strings as the query parameter.
const bulkSelectEventStateKeySQL = "" +
	"SELECT event_state_key, event_state_key_nid FROM roomserver_event_state_keys" +
	" WHERE event_state_key_nid = ANY($1)"

type eventStateKeyStatements struct {
	insertEventStateKeyNIDStmt     *sql.Stmt
	selectEventStateKeyNIDStmt     *sql.Stmt
	bulkSelectEventStateKeyNIDStmt *sql.Stmt
	bulkSelectEventStateKeyStmt    *sql.Stmt
}

func (s *eventStateKeyStatements) prepare(db *sql.DB) (err error) {
	_, err = db.Exec(eventStateKeysSchema)
	if err != nil {
		return
	}
	return statementList{
		{&s.insertEventStateKeyNIDStmt, insertEventStateKeyNIDSQL},
		{&s.selectEventStateKeyNIDStmt, selectEventStateKeyNIDSQL},
		{&s.bulkSelectEventStateKeyNIDStmt, bulkSelectEventStateKeyNIDSQL},
		{&s.bulkSelectEventStateKeyStmt, bulkSelectEventStateKeySQL},
	}.prepare(db)
}

func (s *eventStateKeyStatements) insertEventStateKeyNID(
	ctx context.Context, txn *sql.Tx, eventStateKey string,
) (types.EventStateKeyNID, error) {
	var eventStateKeyNID int64
	stmt := common.TxStmt(txn, s.insertEventStateKeyNIDStmt)
	err := stmt.QueryRowContext(ctx, eventStateKey).Scan(&eventStateKeyNID)
	return types.EventStateKeyNID(eventStateKeyNID), err
}

func (s *eventStateKeyStatements) selectEventStateKeyNID(
	ctx context.Context, txn *sql.Tx, eventStateKey string,
) (types.EventStateKeyNID, error) {
	var eventStateKeyNID int64
	stmt := common.TxStmt(txn, s.selectEventStateKeyNIDStmt)
	err := stmt.QueryRowContext(ctx, eventStateKey).Scan(&eventStateKeyNID)
	return types.EventStateKeyNID(eventStateKeyNID), err
}

func (s *eventStateKeyStatements) bulkSelectEventStateKeyNID(
	ctx context.Context, eventStateKeys []string,
) (map[string]types.EventStateKeyNID, error) {
	rows, err := s.bulkSelectEventStateKeyNIDStmt.QueryContext(
		ctx, pq.StringArray(eventStateKeys),
	)
	if err != nil {
		return nil, err
	}
	defer rows.Close() // nolint: errcheck

	result := make(map[string]types.EventStateKeyNID, len(eventStateKeys))
	for rows.Next() {
		var stateKey string
		var stateKeyNID int64
		if err := rows.Scan(&stateKey, &stateKeyNID); err != nil {
			return nil, err
		}
		result[stateKey] = types.EventStateKeyNID(stateKeyNID)
	}
	return result, nil
}

func (s *eventStateKeyStatements) bulkSelectEventStateKey(
	ctx context.Context, eventStateKeyNIDs []types.EventStateKeyNID,
) (map[types.EventStateKeyNID]string, error) {
	var nIDs pq.Int64Array
	for i := range eventStateKeyNIDs {
		nIDs[i] = int64(eventStateKeyNIDs[i])
	}
	rows, err := s.bulkSelectEventStateKeyStmt.QueryContext(ctx, nIDs)
	if err != nil {
		return nil, err
	}
	defer rows.Close() // nolint: errcheck

	result := make(map[types.EventStateKeyNID]string, len(eventStateKeyNIDs))
	for rows.Next() {
		var stateKey string
		var stateKeyNID int64
		if err := rows.Scan(&stateKey, &stateKeyNID); err != nil {
			return nil, err
		}
		result[types.EventStateKeyNID(stateKeyNID)] = stateKey
	}
	return result, nil
}
<commit_msg>Prevent index out of bounds error (#503)<commit_after>// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storage

import (
	"context"
	"database/sql"

	"github.com/lib/pq"
	"github.com/matrix-org/dendrite/common"
	"github.com/matrix-org/dendrite/roomserver/types"
)

const eventStateKeysSchema = `
-- Numeric versions of the event "state_key"s. State keys tend to be reused so
-- assigning each string a numeric ID should reduce the amount of data that
-- needs to be stored and fetched from the database.
-- It also means that many operations can work with int64 arrays rather than
-- string arrays which may help reduce GC pressure.
-- Well known state keys are pre-assigned numeric IDs:
--   1 -> "" (the empty string)
-- Other state keys are automatically assigned numeric IDs starting from 2**16.
-- This leaves room to add more pre-assigned numeric IDs and clearly separates
-- the automatically assigned IDs from the pre-assigned IDs.
CREATE SEQUENCE IF NOT EXISTS roomserver_event_state_key_nid_seq START 65536;
CREATE TABLE IF NOT EXISTS roomserver_event_state_keys (
    -- Local numeric ID for the state key.
    event_state_key_nid BIGINT PRIMARY KEY DEFAULT nextval('roomserver_event_state_key_nid_seq'),
    event_state_key TEXT NOT NULL CONSTRAINT roomserver_event_state_key_unique UNIQUE
);
INSERT INTO roomserver_event_state_keys (event_state_key_nid, event_state_key) VALUES
    (1, '') ON CONFLICT DO NOTHING;
`

// Same as insertEventTypeNIDSQL
const insertEventStateKeyNIDSQL = "" +
	"INSERT INTO roomserver_event_state_keys (event_state_key) VALUES ($1)" +
	" ON CONFLICT ON CONSTRAINT roomserver_event_state_key_unique" +
	" DO NOTHING RETURNING (event_state_key_nid)"

const selectEventStateKeyNIDSQL = "" +
	"SELECT event_state_key_nid FROM roomserver_event_state_keys" +
	" WHERE event_state_key = $1"

// Bulk lookup from string state key to numeric ID for that state key.
// Takes an array of strings as the query parameter.
const bulkSelectEventStateKeyNIDSQL = "" +
	"SELECT event_state_key, event_state_key_nid FROM roomserver_event_state_keys" +
	" WHERE event_state_key = ANY($1)"

// Bulk lookup from numeric ID to string state key for that state key.
// Takes an array of strings as the query parameter.
const bulkSelectEventStateKeySQL = "" +
	"SELECT event_state_key, event_state_key_nid FROM roomserver_event_state_keys" +
	" WHERE event_state_key_nid = ANY($1)"

type eventStateKeyStatements struct {
	insertEventStateKeyNIDStmt     *sql.Stmt
	selectEventStateKeyNIDStmt     *sql.Stmt
	bulkSelectEventStateKeyNIDStmt *sql.Stmt
	bulkSelectEventStateKeyStmt    *sql.Stmt
}

func (s *eventStateKeyStatements) prepare(db *sql.DB) (err error) {
	_, err = db.Exec(eventStateKeysSchema)
	if err != nil {
		return
	}
	return statementList{
		{&s.insertEventStateKeyNIDStmt, insertEventStateKeyNIDSQL},
		{&s.selectEventStateKeyNIDStmt, selectEventStateKeyNIDSQL},
		{&s.bulkSelectEventStateKeyNIDStmt, bulkSelectEventStateKeyNIDSQL},
		{&s.bulkSelectEventStateKeyStmt, bulkSelectEventStateKeySQL},
	}.prepare(db)
}

func (s *eventStateKeyStatements) insertEventStateKeyNID(
	ctx context.Context, txn *sql.Tx, eventStateKey string,
) (types.EventStateKeyNID, error) {
	var eventStateKeyNID int64
	stmt := common.TxStmt(txn, s.insertEventStateKeyNIDStmt)
	err := stmt.QueryRowContext(ctx, eventStateKey).Scan(&eventStateKeyNID)
	return types.EventStateKeyNID(eventStateKeyNID), err
}

func (s *eventStateKeyStatements) selectEventStateKeyNID(
	ctx context.Context, txn *sql.Tx, eventStateKey string,
) (types.EventStateKeyNID, error) {
	var eventStateKeyNID int64
	stmt := common.TxStmt(txn, s.selectEventStateKeyNIDStmt)
	err := stmt.QueryRowContext(ctx, eventStateKey).Scan(&eventStateKeyNID)
	return types.EventStateKeyNID(eventStateKeyNID), err
}

func (s *eventStateKeyStatements) bulkSelectEventStateKeyNID(
	ctx context.Context, eventStateKeys []string,
) (map[string]types.EventStateKeyNID, error) {
	rows, err := s.bulkSelectEventStateKeyNIDStmt.QueryContext(
		ctx, pq.StringArray(eventStateKeys),
	)
	if err != nil {
		return nil, err
	}
	defer rows.Close() // nolint: errcheck

	result := make(map[string]types.EventStateKeyNID, len(eventStateKeys))
	for rows.Next() {
		var stateKey string
		var stateKeyNID int64
		if err := rows.Scan(&stateKey, &stateKeyNID); err != nil {
			return nil, err
		}
		result[stateKey] = types.EventStateKeyNID(stateKeyNID)
	}
	return result, nil
}

func (s *eventStateKeyStatements) bulkSelectEventStateKey(
	ctx context.Context, eventStateKeyNIDs []types.EventStateKeyNID,
) (map[types.EventStateKeyNID]string, error) {
	nIDs := make(pq.Int64Array, len(eventStateKeyNIDs))
	for i := range eventStateKeyNIDs {
		nIDs[i] = int64(eventStateKeyNIDs[i])
	}
	rows, err := s.bulkSelectEventStateKeyStmt.QueryContext(ctx, nIDs)
	if err != nil {
		return nil, err
	}
	defer rows.Close() // nolint: errcheck

	result := make(map[types.EventStateKeyNID]string, len(eventStateKeyNIDs))
	for rows.Next() {
		var stateKey string
		var stateKeyNID int64
		if err := rows.Scan(&stateKey, &stateKeyNID); err != nil {
			return nil, err
		}
		result[types.EventStateKeyNID(stateKeyNID)] = stateKey
	}
	return result, nil
}
<|endoftext|>


<commit_before>// Copyright 2019 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cli

import (
	"bufio"
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"strings"
	"time"

	"github.com/golang/protobuf/jsonpb"
	"github.com/golang/protobuf/proto"
	"github.com/maruel/subcommands"
	"golang.org/x/sync/errgroup"
	"google.golang.org/genproto/protobuf/field_mask"
	"google.golang.org/protobuf/encoding/protojson"

	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/common/cli"
	"go.chromium.org/luci/common/data/text"
	"go.chromium.org/luci/common/data/text/indented"
	"go.chromium.org/luci/common/errors"

	"go.chromium.org/luci/resultdb/pbutil"
	pb "go.chromium.org/luci/resultdb/proto/v1"
)

func cmdQuery(p Params) *subcommands.Command {
	return &subcommands.Command{
		UsageLine: `query [flags] [INVOCATION_ID]...`,
		ShortDesc: "query results",
		LongDesc: text.Doc(`
			Query results.

			Most users will be interested only in results of test variants that had
			unexpected results. This can be achieved by passing -u flag.
			This significantly reduces output size and latency.

			If no invocation ids are specified on the command line, read them from
			stdin separated by newline. Example:
			  bb chromium/ci/linux-rel -status failure -inv -10 | rdb query
		`),
		CommandRun: func() subcommands.CommandRun {
			r := &queryRun{}
			r.RegisterGlobalFlags(p)
			r.RegisterJSONFlag(text.Doc(`
				Print results in JSON format separated by newline.
				One result takes exactly one line. Result object properties are invocationId
				and one of
					testResult: luci.resultdb.v1.TestResult message.
					testExoneration: luci.resultdb.v1.TestExoneration message.
					invocation: luci.resultdb.v1.Invocation message.
			`))

			r.Flags.IntVar(&r.limit, "n", 0, text.Doc(`
				Print up to n results. If 0, then unlimited.
				Invocations do not count as results.
			`))

			r.Flags.BoolVar(&r.unexpected, "u", false, text.Doc(`
				Print only test results of test variants that have unexpected results.
				For example, if a test variant expected PASS and had results FAIL, FAIL,
				PASS, then print all of them.
				This signficantly reduces output size and latency.
			`))

			r.Flags.StringVar(&r.testID, "test", "", text.Doc(`
				A regular expression for test id. Implicitly wrapped with ^ and $.

				Example: ninja://chrome/test:browser_tests/.+
			`))

			r.Flags.BoolVar(&r.merge, "merge", false, text.Doc(`
				Merge results of the invocations, as if they were included into one
				invocation.
				Useful when the invocations are a part of one computation, e.g. shards
				of a test.
			`))

			r.Flags.StringVar(&r.trFields, "tr-fields", "", text.Doc(`
				Test result fields to include in the response. Fields should be passed
				as a comma separated string to match the JSON encoding schema of FieldMask.
				Test result names will always be included even if "name" is not a part
				of the fields.
			`))
			return r
		},
	}
}

type queryRun struct {
	baseCommandRun

	limit      int
	unexpected bool
	testID     string
	merge      bool
	trFields   string
	invIDs     []string

	// TODO(crbug.com/1021849): add flag -artifact-dir
	// TODO(crbug.com/1021849): add flag -artifact-name
}

func (r *queryRun) parseArgs(args []string) error {
	r.invIDs = args
	if len(r.invIDs) == 0 {
		var err error
		if r.invIDs, err = readStdin(); err != nil {
			return err
		}
	}

	for _, id := range r.invIDs {
		if err := pbutil.ValidateInvocationID(id); err != nil {
			return errors.Annotate(err, "invocation id %q", id).Err()
		}
	}

	if r.limit < 0 {
		return errors.Reason("-n must be non-negative").Err()
	}

	// TODO(crbug.com/1021849): improve validation.
	return nil
}

func (r *queryRun) Run(a subcommands.Application, args []string, env subcommands.Env) int {
	ctx := cli.GetContext(a, r, env)

	if err := r.parseArgs(args); err != nil {
		return r.done(err)
	}

	if err := r.initClients(ctx, auth.SilentLogin); err != nil {
		return r.done(err)
	}

	return r.done(r.queryAndPrint(ctx, r.invIDs))
}

// readStdin reads all lines from os.Stdin.
func readStdin() ([]string, error) {
	// This context is used only to cancel the goroutine below.
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	go func() {
		select {
		case <-time.After(time.Second):
			fmt.Fprintln(os.Stderr, "expecting invocation ids on the command line or stdin...")
		case <-ctx.Done():
		}
	}()

	var ret []string
	stdin := bufio.NewReader(os.Stdin)
	for {
		line, err := stdin.ReadString('\n')
		if err == io.EOF {
			return ret, nil
		}
		if err != nil {
			return nil, err
		}
		ret = append(ret, strings.TrimSuffix(line, "\n"))
		cancel() // do not print the warning since we got something.
	}
}

type resultItem struct {
	invocationID string
	result       proto.Message
}

// queryAndPrint queries results and prints them.
func (r *queryRun) queryAndPrint(ctx context.Context, invIDs []string) error {
	eg, ctx := errgroup.WithContext(ctx)
	resultC := make(chan resultItem)

	for _, id := range invIDs {
		id := id
		eg.Go(func() error {
			return r.fetchInvocation(ctx, id, resultC)
		})
	}

	trMask := &field_mask.FieldMask{}
	if r.trFields != "" {
		if err := protojson.Unmarshal([]byte(fmt.Sprintf(`"%s"`, r.trFields)), trMask); err != nil {
			return errors.Annotate(err, "tr-fields").Err()
		}
	}

	// Fetch items into resultC.
	if r.merge {
		eg.Go(func() error {
			return r.fetchItems(ctx, invIDs, trMask, resultItem{}, resultC)
		})
	} else {
		for _, id := range invIDs {
			id := id
			tmpl := resultItem{invocationID: id}
			eg.Go(func() error {
				return r.fetchItems(ctx, []string{id}, trMask, tmpl, resultC)
			})
		}
	}

	// Wait for fetchers to finish and close resultC.
	errC := make(chan error)
	go func() {
		err := eg.Wait()
		close(resultC)
		errC <- err
	}()

	r.printProto(resultC, r.json)
	return <-errC
}

// fetchInvocation fetches an invocation.
func (r *queryRun) fetchInvocation(ctx context.Context, invID string, dest chan<- resultItem) error {
	res, err := r.resultdb.GetInvocation(ctx, &pb.GetInvocationRequest{Name: pbutil.InvocationName(invID)})
	if err != nil {
		return err
	}
	dest <- resultItem{invocationID: invID, result: res}
	return nil
}

// fetchItems fetches test results and exonerations from the specified invocations.
func (r *queryRun) fetchItems(ctx context.Context, invIDs []string, trMask *field_mask.FieldMask, resultItemTemplate resultItem, dest chan<- resultItem) error {
	invNames := make([]string, len(invIDs))
	for i, id := range invIDs {
		invNames[i] = pbutil.InvocationName(id)
	}

	// Prepare a test result request.
	trReq := &pb.QueryTestResultsRequest{
		Invocations: invNames,
		Predicate:   &pb.TestResultPredicate{TestIdRegexp: r.testID},
		PageSize:    int32(r.limit),
		ReadMask:    trMask,
	}
	if r.unexpected {
		trReq.Predicate.Expectancy = pb.TestResultPredicate_VARIANTS_WITH_UNEXPECTED_RESULTS
	}

	// Prepare a test exoneration request.
	teReq := &pb.QueryTestExonerationsRequest{
		Invocations: invNames,
		Predicate: &pb.TestExonerationPredicate{
			TestIdRegexp: r.testID,
		},
		PageSize: int32(r.limit),
	}

	// Query for results.
	msgC := make(chan proto.Message)
	errC := make(chan error, 1)
	queryCtx, cancelQuery := context.WithCancel(ctx)
	defer cancelQuery()
	go func() {
		defer close(msgC)
		errC <- pbutil.Query(queryCtx, msgC, r.resultdb, trReq, teReq)
	}()

	// Send findings to the destination channel.
	count := 0
	reachedLimit := false
	for m := range msgC {
		if r.limit > 0 && count > r.limit {
			reachedLimit = true
			cancelQuery()
			break
		}
		count++

		item := resultItemTemplate
		item.result = m
		select {
		case dest <- item:
		case <-ctx.Done():
			return ctx.Err()
		}
	}

	// Return the query error.
	err := <-errC
	if reachedLimit && err == context.Canceled {
		err = nil
	}
	return err
}

// printProto prints results in JSON or TextProto format to stdout.
// Each result takes exactly one line and is followed by newline.
//
// The printed JSON supports streaming, and is easy to parse by languages (Python)
// that cannot parse an arbitrary sequence of JSON values.
func (r *queryRun) printProto(resultC <-chan resultItem, printJSON bool) {
	enc := json.NewEncoder(os.Stdout)
	ind := &indented.Writer{
		Writer:    os.Stdout,
		UseSpaces: true,
	}
	for res := range resultC {
		var key string
		switch res.result.(type) {
		case *pb.TestResult:
			key = "testResult"
		case *pb.TestExoneration:
			key = "testExoneration"
		case *pb.Invocation:
			key = "invocation"
		default:
			panic(fmt.Sprintf("unexpected result type %T", res.result))
		}

		if printJSON {
			obj := map[string]interface{}{
				key: json.RawMessage(msgToJSON(res.result)),
			}
			if !r.merge {
				obj["invocationId"] = res.invocationID
			}
			enc.Encode(obj) // prints \n in the end
		} else {
			fmt.Fprintf(ind, "%s:\n", key)
			ind.Level++
			proto.MarshalText(ind, res.result)
			ind.Level--
			fmt.Fprintln(ind)
		}
	}
}

func msgToJSON(msg proto.Message) []byte {
	buf := &bytes.Buffer{}
	m := jsonpb.Marshaler{}
	if err := m.Marshal(buf, msg); err != nil {
		panic(fmt.Sprintf("failed to marshal protobuf message %q in memory", msg))
	}
	return buf.Bytes()
}
<commit_msg>resultdb: use google.golang.org/protobuf<commit_after>// Copyright 2019 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cli

import (
	"bufio"
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"strings"
	"time"

	"github.com/golang/protobuf/jsonpb"
	"github.com/golang/protobuf/proto"
	"github.com/maruel/subcommands"
	"golang.org/x/sync/errgroup"
	"google.golang.org/protobuf/encoding/protojson"
	"google.golang.org/protobuf/types/known/fieldmaskpb"

	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/common/cli"
	"go.chromium.org/luci/common/data/text"
	"go.chromium.org/luci/common/data/text/indented"
	"go.chromium.org/luci/common/errors"

	"go.chromium.org/luci/resultdb/pbutil"
	pb "go.chromium.org/luci/resultdb/proto/v1"
)

func cmdQuery(p Params) *subcommands.Command {
	return &subcommands.Command{
		UsageLine: `query [flags] [INVOCATION_ID]...`,
		ShortDesc: "query results",
		LongDesc: text.Doc(`
			Query results.

			Most users will be interested only in results of test variants that had
			unexpected results. This can be achieved by passing -u flag.
			This significantly reduces output size and latency.

			If no invocation ids are specified on the command line, read them from
			stdin separated by newline. Example:
			  bb chromium/ci/linux-rel -status failure -inv -10 | rdb query
		`),
		CommandRun: func() subcommands.CommandRun {
			r := &queryRun{}
			r.RegisterGlobalFlags(p)
			r.RegisterJSONFlag(text.Doc(`
				Print results in JSON format separated by newline.
				One result takes exactly one line. Result object properties are invocationId
				and one of
					testResult: luci.resultdb.v1.TestResult message.
					testExoneration: luci.resultdb.v1.TestExoneration message.
					invocation: luci.resultdb.v1.Invocation message.
			`))

			r.Flags.IntVar(&r.limit, "n", 0, text.Doc(`
				Print up to n results. If 0, then unlimited.
				Invocations do not count as results.
			`))

			r.Flags.BoolVar(&r.unexpected, "u", false, text.Doc(`
				Print only test results of test variants that have unexpected results.
				For example, if a test variant expected PASS and had results FAIL, FAIL,
				PASS, then print all of them.
				This signficantly reduces output size and latency.
			`))

			r.Flags.StringVar(&r.testID, "test", "", text.Doc(`
				A regular expression for test id. Implicitly wrapped with ^ and $.

				Example: ninja://chrome/test:browser_tests/.+
			`))

			r.Flags.BoolVar(&r.merge, "merge", false, text.Doc(`
				Merge results of the invocations, as if they were included into one
				invocation.
				Useful when the invocations are a part of one computation, e.g. shards
				of a test.
			`))

			r.Flags.StringVar(&r.trFields, "tr-fields", "", text.Doc(`
				Test result fields to include in the response. Fields should be passed
				as a comma separated string to match the JSON encoding schema of FieldMask.
				Test result names will always be included even if "name" is not a part
				of the fields.
			`))
			return r
		},
	}
}

type queryRun struct {
	baseCommandRun

	limit      int
	unexpected bool
	testID     string
	merge      bool
	trFields   string
	invIDs     []string

	// TODO(crbug.com/1021849): add flag -artifact-dir
	// TODO(crbug.com/1021849): add flag -artifact-name
}

func (r *queryRun) parseArgs(args []string) error {
	r.invIDs = args
	if len(r.invIDs) == 0 {
		var err error
		if r.invIDs, err = readStdin(); err != nil {
			return err
		}
	}

	for _, id := range r.invIDs {
		if err := pbutil.ValidateInvocationID(id); err != nil {
			return errors.Annotate(err, "invocation id %q", id).Err()
		}
	}

	if r.limit < 0 {
		return errors.Reason("-n must be non-negative").Err()
	}

	// TODO(crbug.com/1021849): improve validation.
	return nil
}

func (r *queryRun) Run(a subcommands.Application, args []string, env subcommands.Env) int {
	ctx := cli.GetContext(a, r, env)

	if err := r.parseArgs(args); err != nil {
		return r.done(err)
	}

	if err := r.initClients(ctx, auth.SilentLogin); err != nil {
		return r.done(err)
	}

	return r.done(r.queryAndPrint(ctx, r.invIDs))
}

// readStdin reads all lines from os.Stdin.
func readStdin() ([]string, error) {
	// This context is used only to cancel the goroutine below.
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	go func() {
		select {
		case <-time.After(time.Second):
			fmt.Fprintln(os.Stderr, "expecting invocation ids on the command line or stdin...")
		case <-ctx.Done():
		}
	}()

	var ret []string
	stdin := bufio.NewReader(os.Stdin)
	for {
		line, err := stdin.ReadString('\n')
		if err == io.EOF {
			return ret, nil
		}
		if err != nil {
			return nil, err
		}
		ret = append(ret, strings.TrimSuffix(line, "\n"))
		cancel() // do not print the warning since we got something.
	}
}

type resultItem struct {
	invocationID string
	result       proto.Message
}

// queryAndPrint queries results and prints them.
func (r *queryRun) queryAndPrint(ctx context.Context, invIDs []string) error {
	eg, ctx := errgroup.WithContext(ctx)
	resultC := make(chan resultItem)

	for _, id := range invIDs {
		id := id
		eg.Go(func() error {
			return r.fetchInvocation(ctx, id, resultC)
		})
	}

	trMask := &fieldmaskpb.FieldMask{}
	if r.trFields != "" {
		if err := protojson.Unmarshal([]byte(fmt.Sprintf(`"%s"`, r.trFields)), trMask); err != nil {
			return errors.Annotate(err, "tr-fields").Err()
		}
	}

	// Fetch items into resultC.
	if r.merge {
		eg.Go(func() error {
			return r.fetchItems(ctx, invIDs, trMask, resultItem{}, resultC)
		})
	} else {
		for _, id := range invIDs {
			id := id
			tmpl := resultItem{invocationID: id}
			eg.Go(func() error {
				return r.fetchItems(ctx, []string{id}, trMask, tmpl, resultC)
			})
		}
	}

	// Wait for fetchers to finish and close resultC.
	errC := make(chan error)
	go func() {
		err := eg.Wait()
		close(resultC)
		errC <- err
	}()

	r.printProto(resultC, r.json)
	return <-errC
}

// fetchInvocation fetches an invocation.
func (r *queryRun) fetchInvocation(ctx context.Context, invID string, dest chan<- resultItem) error {
	res, err := r.resultdb.GetInvocation(ctx, &pb.GetInvocationRequest{Name: pbutil.InvocationName(invID)})
	if err != nil {
		return err
	}
	dest <- resultItem{invocationID: invID, result: res}
	return nil
}

// fetchItems fetches test results and exonerations from the specified invocations.
func (r *queryRun) fetchItems(ctx context.Context, invIDs []string, trMask *fieldmaskpb.FieldMask, resultItemTemplate resultItem, dest chan<- resultItem) error {
	invNames := make([]string, len(invIDs))
	for i, id := range invIDs {
		invNames[i] = pbutil.InvocationName(id)
	}

	// Prepare a test result request.
	trReq := &pb.QueryTestResultsRequest{
		Invocations: invNames,
		Predicate:   &pb.TestResultPredicate{TestIdRegexp: r.testID},
		PageSize:    int32(r.limit),
		ReadMask:    trMask,
	}
	if r.unexpected {
		trReq.Predicate.Expectancy = pb.TestResultPredicate_VARIANTS_WITH_UNEXPECTED_RESULTS
	}

	// Prepare a test exoneration request.
	teReq := &pb.QueryTestExonerationsRequest{
		Invocations: invNames,
		Predicate: &pb.TestExonerationPredicate{
			TestIdRegexp: r.testID,
		},
		PageSize: int32(r.limit),
	}

	// Query for results.
	msgC := make(chan proto.Message)
	errC := make(chan error, 1)
	queryCtx, cancelQuery := context.WithCancel(ctx)
	defer cancelQuery()
	go func() {
		defer close(msgC)
		errC <- pbutil.Query(queryCtx, msgC, r.resultdb, trReq, teReq)
	}()

	// Send findings to the destination channel.
	count := 0
	reachedLimit := false
	for m := range msgC {
		if r.limit > 0 && count > r.limit {
			reachedLimit = true
			cancelQuery()
			break
		}
		count++

		item := resultItemTemplate
		item.result = m
		select {
		case dest <- item:
		case <-ctx.Done():
			return ctx.Err()
		}
	}

	// Return the query error.
	err := <-errC
	if reachedLimit && err == context.Canceled {
		err = nil
	}
	return err
}

// printProto prints results in JSON or TextProto format to stdout.
// Each result takes exactly one line and is followed by newline.
//
// The printed JSON supports streaming, and is easy to parse by languages (Python)
// that cannot parse an arbitrary sequence of JSON values.
func (r *queryRun) printProto(resultC <-chan resultItem, printJSON bool) {
	enc := json.NewEncoder(os.Stdout)
	ind := &indented.Writer{
		Writer:    os.Stdout,
		UseSpaces: true,
	}
	for res := range resultC {
		var key string
		switch res.result.(type) {
		case *pb.TestResult:
			key = "testResult"
		case *pb.TestExoneration:
			key = "testExoneration"
		case *pb.Invocation:
			key = "invocation"
		default:
			panic(fmt.Sprintf("unexpected result type %T", res.result))
		}

		if printJSON {
			obj := map[string]interface{}{
				key: json.RawMessage(msgToJSON(res.result)),
			}
			if !r.merge {
				obj["invocationId"] = res.invocationID
			}
			enc.Encode(obj) // prints \n in the end
		} else {
			fmt.Fprintf(ind, "%s:\n", key)
			ind.Level++
			proto.MarshalText(ind, res.result)
			ind.Level--
			fmt.Fprintln(ind)
		}
	}
}

func msgToJSON(msg proto.Message) []byte {
	buf := &bytes.Buffer{}
	m := jsonpb.Marshaler{}
	if err := m.Marshal(buf, msg); err != nil {
		panic(fmt.Sprintf("failed to marshal protobuf message %q in memory", msg))
	}
	return buf.Bytes()
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"time"

	"github.com/ChubbsSolutions/urbano/objects"
	"github.com/codegangsta/cli"
	"github.com/mailgun/mailgun-go"
	"github.com/ttacon/chalk"
)

//Define constants
const version string = "0.1"
const author string = "Chubbs Solutions"
const email string = "urbano@chubbs.solutions"
const appName string = "urbano"
const appDescription string = "Get a fresh Urban  Dictionary word in your inbox"

//MailgunPublicAPIKey key for the mail service.
var MailgunPublicAPIKey = os.Getenv("MAILGUN_PUBLIC_API_KEY")

//MailgunPrivateAPIKey key for the mail service.
var MailgunDomain = os.Getenv("MAILGUN_DOMAIN")

//Evaluate options on main
func main() {
	t := time.Now()
	app := cli.NewApp()
	app.Name = appName
	app.Usage = appDescription
	app.Email = email
	app.Author = author
	app.Version = version
	app.Commands = []cli.Command{
		{
			Name:      "send",
			ShortName: "s",
			Usage:     "Get and Send a new word by email",
			Action: func(c *cli.Context) {

				word, err := getNewWord()
				if err != nil {
					fmt.Println(chalk.Red, "Error getting the word")
					os.Exit(-1)
				}

				err = displayNumbers(word)
				if err != nil {
					fmt.Println(chalk.Red, err)
					os.Exit(-1)
				}

				if len(c.Args()) == 1 {
					recipient := c.Args()[0]
					subject := fmt.Sprintf("Urban Dictionary Word of the day for %s", t.Format("Jan 02, 2006"))
					err = emailWord(word, subject, recipient)
					if err != nil {
						fmt.Println(chalk.Red, err)
						os.Exit(-1)
					}

				}
			},
		},
		{
			Name:      "display",
			ShortName: "d",
			Usage:     "Display a new word",
			Action: func(c *cli.Context) {
				word, err := getNewWord()
				if err != nil {
					fmt.Println(chalk.Red, err)
					os.Exit(-1)
				}

				err = displayNumbers(word)
				if err != nil {
					fmt.Println(chalk.Red, err)
					os.Exit(-1)
				}
			},
		},
	}

	app.Run(os.Args)

}

//getNewWord gets a random UD word
func getNewWord() (objects.WordData, error) {
	var UDURL = "http://api.urbandictionary.com/v0/random"
	wd := objects.WordDataSlice{}
	var word objects.WordData

	resp, err := http.Get(UDURL)
	if err != nil {
		return word, err
	}
	defer resp.Body.Close()

	data, _ := ioutil.ReadAll(resp.Body)
	if err != nil {
		return word, err
	}

	err = json.Unmarshal([]byte(string(data)), &wd)
	if err != nil {
		return word, err
	}

	tu := 0

	for _, element := range wd.List {
		if element.ThumbsUp > tu {
			word = element
		}
	}
	return word, nil
}

func displayNumbers(word objects.WordData) error {
	fmt.Println(word.Word)
	fmt.Print(chalk.Cyan, "Word of the day: ", word.Word, "   ++Thumbs Up: ", word.ThumbsUp, "   --Thumbs Down: ", word.ThumbsDown, "\n\n")
	fmt.Print(chalk.Green, word.Definition, "\n\n")
	fmt.Print(chalk.Blue, "Example: ", word.Example, "\n\n")
	fmt.Print(chalk.Yellow, "Courtesy of ", word.Author, "\n\n\n")
	fmt.Print(chalk.Black, "Brought to you by Chubbs Solutions.")
	return nil
}

func emailWord(word objects.WordData, subject, recipient string) error {

	if MailgunPublicAPIKey == "" || MailgunDomain == "" {
		return errors.New("Please set the MAILGUN_PUBLIC_API_KEY and MAILGUN_DOMAIN variables.")
	}

	sender := fmt.Sprintf("donotreply@%s", MailgunDomain)

	body := fmt.Sprintf("Word of the day: %s   ++Thumbs Up: %v   --Thumbs Down: %v\n\n", word.Word, word.ThumbsUp, word.ThumbsDown)
	body += fmt.Sprintf("%s\n\n", word.Definition)
	body += fmt.Sprintf("Example: %s\n\n", word.Example)
	body += fmt.Sprintf("Courtesy of %s\n\n\n", word.Author)
	body += fmt.Sprintf("Brought to you by Chubbs Solutions.")

	gun := mailgun.NewMailgun(MailgunDomain, MailgunPublicAPIKey, "")
	m := mailgun.NewMessage(sender, subject, body, recipient)

	_, _, err := gun.Send(m)
	if err != nil {
		return err
	}
	// fmt.Printf("Response ID: %s\n", id)
	// fmt.Printf("Message from server: %s\n", response)

	return nil
}
<commit_msg>Remove extra print.<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"time"

	"github.com/ChubbsSolutions/urbano/objects"
	"github.com/codegangsta/cli"
	"github.com/mailgun/mailgun-go"
	"github.com/ttacon/chalk"
)

//Define constants
const version string = "0.1"
const author string = "Chubbs Solutions"
const email string = "urbano@chubbs.solutions"
const appName string = "urbano"
const appDescription string = "Get a fresh Urban  Dictionary word in your inbox"

//MailgunPublicAPIKey key for the mail service.
var MailgunPublicAPIKey = os.Getenv("MAILGUN_PUBLIC_API_KEY")

//MailgunPrivateAPIKey key for the mail service.
var MailgunDomain = os.Getenv("MAILGUN_DOMAIN")

//Evaluate options on main
func main() {
	t := time.Now()
	app := cli.NewApp()
	app.Name = appName
	app.Usage = appDescription
	app.Email = email
	app.Author = author
	app.Version = version
	app.Commands = []cli.Command{
		{
			Name:      "send",
			ShortName: "s",
			Usage:     "Get and Send a new word by email",
			Action: func(c *cli.Context) {

				word, err := getNewWord()
				if err != nil {
					fmt.Println(chalk.Red, "Error getting the word")
					os.Exit(-1)
				}

				err = displayNumbers(word)
				if err != nil {
					fmt.Println(chalk.Red, err)
					os.Exit(-1)
				}

				if len(c.Args()) == 1 {
					recipient := c.Args()[0]
					subject := fmt.Sprintf("Urban Dictionary Word of the day for %s", t.Format("Jan 02, 2006"))
					err = emailWord(word, subject, recipient)
					if err != nil {
						fmt.Println(chalk.Red, err)
						os.Exit(-1)
					}

				}
			},
		},
		{
			Name:      "display",
			ShortName: "d",
			Usage:     "Display a new word",
			Action: func(c *cli.Context) {
				word, err := getNewWord()
				if err != nil {
					fmt.Println(chalk.Red, err)
					os.Exit(-1)
				}

				err = displayNumbers(word)
				if err != nil {
					fmt.Println(chalk.Red, err)
					os.Exit(-1)
				}
			},
		},
	}

	app.Run(os.Args)

}

//getNewWord gets a random UD word
func getNewWord() (objects.WordData, error) {
	var UDURL = "http://api.urbandictionary.com/v0/random"
	wd := objects.WordDataSlice{}
	var word objects.WordData

	resp, err := http.Get(UDURL)
	if err != nil {
		return word, err
	}
	defer resp.Body.Close()

	data, _ := ioutil.ReadAll(resp.Body)
	if err != nil {
		return word, err
	}

	err = json.Unmarshal([]byte(string(data)), &wd)
	if err != nil {
		return word, err
	}

	tu := 0

	for _, element := range wd.List {
		if element.ThumbsUp > tu {
			word = element
		}
	}
	return word, nil
}

func displayNumbers(word objects.WordData) error {

	fmt.Print(chalk.Cyan, "Word of the day: ", word.Word, "   ++Thumbs Up: ", word.ThumbsUp, "   --Thumbs Down: ", word.ThumbsDown, "\n\n")
	fmt.Print(chalk.Green, word.Definition, "\n\n")
	fmt.Print(chalk.Blue, "Example: ", word.Example, "\n\n")
	fmt.Print(chalk.Yellow, "Courtesy of ", word.Author, "\n\n\n")
	fmt.Print(chalk.Black, "Brought to you by Chubbs Solutions.")
	return nil
}

func emailWord(word objects.WordData, subject, recipient string) error {

	if MailgunPublicAPIKey == "" || MailgunDomain == "" {
		return errors.New("Please set the MAILGUN_PUBLIC_API_KEY and MAILGUN_DOMAIN variables.")
	}

	sender := fmt.Sprintf("donotreply@%s", MailgunDomain)

	body := fmt.Sprintf("Word of the day: %s   ++Thumbs Up: %v   --Thumbs Down: %v\n\n", word.Word, word.ThumbsUp, word.ThumbsDown)
	body += fmt.Sprintf("%s\n\n", word.Definition)
	body += fmt.Sprintf("Example: %s\n\n", word.Example)
	body += fmt.Sprintf("Courtesy of %s\n\n\n", word.Author)
	body += fmt.Sprintf("Brought to you by Chubbs Solutions.")

	gun := mailgun.NewMailgun(MailgunDomain, MailgunPublicAPIKey, "")
	m := mailgun.NewMessage(sender, subject, body, recipient)

	_, _, err := gun.Send(m)
	if err != nil {
		return err
	}
	// fmt.Printf("Response ID: %s\n", id)
	// fmt.Printf("Message from server: %s\n", response)

	return nil
}
<|endoftext|>


<commit_before>package brands

import (
	//"encoding/json"
	"fmt"
	"github.com/Financial-Times/base-ft-rw-app-go/baseftrwapp"
	"github.com/Financial-Times/brands-rw-neo4j/brands"
	"github.com/Financial-Times/neo-utils-go/neoutils"
	"github.com/jmcvetta/neoism"
	"github.com/stretchr/testify/assert"
	"os"
	"testing"
)

var validSimpleBrand = brands.Brand{
	UUID:           "0c63a9bf-6fc4-49d0-809b-7bc3dc8b8ec9",
	PrefLabel:      "validSimpleBrand",
	Strapline:      "Keeping it simple",
	Description:    "This brand has no parent but otherwise has valid values for all fields",
	DescriptionXML: "This brand has no parent but otherwise has valid values for all fields",
	ImageURL:       "http://media.ft.com/validSimpleBrand.png",
}

var validParentBrand = brands.Brand{
	UUID:           "d851e146-e889-43f3-8f4c-269da9bb0298",
	PrefLabel:      "validParentBrand",
	Strapline:      "Keeping it in the family",
	Description:    "This brand has is a parent",
	DescriptionXML: "This brand has is a parent",
	ImageURL:       "http://media.ft.com/validParentBrand.png",
}

var validChildBrand = brands.Brand{
	UUID:           "a806e270-edbc-423f-b8db-d21ae90e06c8",
	ParentUUID:     "d851e146-e889-43f3-8f4c-269da9bb0298",
	PrefLabel:      "validChildBrand",
	Strapline:      "I live in one family",
	Description:    "This brand has a parent and valid values for all fields",
	DescriptionXML: "This brand has a parent and valid values for all fields",
	ImageURL:       "http://media.ft.com/validChildBrand.png",
}

func TestSimpleBrand(t *testing.T) {
	err := getBrandRWDriver(t).Write(validSimpleBrand)
	assert.NoError(t, err)
	readAndCompare(&validSimpleBrand, nil, nil, t)
	cleanUp(validSimpleBrand.UUID, t)
}

func TestSimpleBrandAsParent(t *testing.T) {
	err := getBrandRWDriver(t).Write(validParentBrand)
	assert.NoError(t, err)
	err = getBrandRWDriver(t).Write(validChildBrand)
	assert.NoError(t, err)
	readAndCompare(&validChildBrand, &validParentBrand, nil, t)
	cleanUp(validChildBrand.UUID, t)
	cleanUp(validParentBrand.UUID, t)
}

func TestConnectivityCheck(t *testing.T) {
	driver := getBrandRWDriver(t)
	err := driver.Check()
	assert.NoError(t, err)
}

func readAndCompare(source *brands.Brand, parent *brands.Brand, children []*brands.Brand, t *testing.T) {
	brand, found, err := getBrandDriver(t).Read(source.UUID)
	expected := makeBrand(source, parent, children, t)
	assert.NoError(t, err)
	assert.True(t, found)
	assert.NotEmpty(t, brand)
	fmt.Printf("**Made %v+\n\n**Found %v+\n", expected, brand)
	if brand.Parent != nil {
		fmt.Printf("\n**Made.Parent %v+\n\n**Found.Parent %v+\n", *expected.Parent, *brand.Parent)
	}
	for _, child := range brand.Children {
		fmt.Printf("brand.child %v+\n", *child)
	}
	assert.EqualValues(t, expected, brand)
}

func makeBrand(source *brands.Brand, parent *brands.Brand, children []*brands.Brand, t *testing.T) (brand Brand) {
	brand.Thing = makeThing(source, t)
	brand.PrefLabel = source.PrefLabel
	brand.Strapline = source.Strapline
	brand.Description = source.Description
	brand.DescriptionXML = source.DescriptionXML
	brand.ImageURL = source.ImageURL
	if parent == nil {
		brand.Parent = nil
	} else {
		parentBrand := makeThing(parent, t)
		brand.Parent = &parentBrand
	}
	childrenBrands := make([]*Thing, len(children))
	for idx := range children {
		child := makeThing(children[idx], t)
		childrenBrands[idx] = &child
	}
	brand.Children = childrenBrands
	return brand
}

func makeThing(source *brands.Brand, t *testing.T) Thing {
	thing := Thing{}
	thing.ID = "http://api.ft.com/things/" + source.UUID
	thing.APIURL = "http://test.api.ft.com/brands/" + source.UUID
	thing.Types = []string{"http://www.ft.com/ontology/product/Brand"}
	thing.PrefLabel = source.PrefLabel
	return thing
}

func getBrandRWDriver(t *testing.T) (service baseftrwapp.Service) {
	url := os.Getenv("NEO4J_TEST_URL")
	if url == "" {
		url = "http://localhost:7474/db/data"
	}
	db, err := neoism.Connect(url)
	assert.NoError(t, err, "Error setting up connection to %s", url)
	return brands.NewCypherBrandsService(neoutils.StringerDb{db}, db)
}

func getBrandDriver(t *testing.T) CypherDriver {
	url := os.Getenv("NEO4J_TEST_URL")
	if url == "" {
		url = "http://localhost:7474/db/data"
	}
	db, err := neoism.Connect(url)
	assert.NoError(t, err, "Error setting up connection to %s", url)
	return NewCypherDriver(db, "test")
}

func cleanUp(uuid string, t *testing.T) {
	found, err := getBrandRWDriver(t).Delete(uuid)
	assert.True(t, found, "Unable to delete brand with uuid %s", uuid)
	assert.NoError(t, err, "Error deleting brand with uuid %s", uuid)
}
<commit_msg>update of test cases<commit_after>package brands

import (
	//"encoding/json"
	"fmt"
	"github.com/Financial-Times/base-ft-rw-app-go/baseftrwapp"
	"github.com/Financial-Times/brands-rw-neo4j/brands"
	"github.com/Financial-Times/neo-utils-go/neoutils"
	"github.com/jmcvetta/neoism"
	"github.com/stretchr/testify/assert"
	"os"
	"testing"
)

var validSimpleBrand = brands.Brand{
	UUID:           "0c63a9bf-6fc4-49d0-809b-7bc3dc8b8ec9",
	PrefLabel:      "validSimpleBrand",
	Strapline:      "Keeping it simple",
	Description:    "This brand has no parent but otherwise has valid values for all fields",
	DescriptionXML: "This brand has no parent but otherwise has valid values for all fields",
	ImageURL:       "http://media.ft.com/validSimpleBrand.png",
}

var grandDaddy = brands.Brand{
	UUID:      "d05b48ae-933e-4b2f-afcb-99b19bdceaf3",
	PrefLabel: "The daddy of all brands",
}

var parentBrand = brands.Brand{
	UUID:           "d851e146-e889-43f3-8f4c-269da9bb0298",
	PrefLabel:      "parentBrand",
	Strapline:      "Keeping it in the family",
	Description:    "This brand has is a parent",
	DescriptionXML: "This brand has is a parent",
	ImageURL:       "http://media.ft.com/parentBrand.png",
}

var childBrand1 = brands.Brand{
	UUID:           "a806e270-edbc-423f-b8db-d21ae90e06c8",
	ParentUUID:     "d851e146-e889-43f3-8f4c-269da9bb0298",
	PrefLabel:      "childBrand1",
	Strapline:      "I live in one family",
	Description:    "This brand has a parent and valid values for all fields",
	DescriptionXML: "This brand has a parent and valid values for all fields",
	ImageURL:       "http://media.ft.com/childBrand1.png",
}

func TestSimpleBrand(t *testing.T) {
	err := getBrandRWDriver(t).Write(validSimpleBrand)
	assert.NoError(t, err)
	readAndCompare(&validSimpleBrand, nil, nil, t)
	cleanUp(validSimpleBrand.UUID, t)
}

func TestSimpleBrandAsParent(t *testing.T) {
	err := getBrandRWDriver(t).Write(parentBrand)
	assert.NoError(t, err)
	err = getBrandRWDriver(t).Write(childBrand1)
	assert.NoError(t, err)
	readAndCompare(&childBrand1, &parentBrand, nil, t)
	cleanUp(childBrand1.UUID, t)
	cleanUp(parentBrand.UUID, t)
}

func TestConnectivityCheck(t *testing.T) {
	driver := getBrandRWDriver(t)
	err := driver.Check()
	assert.NoError(t, err)
}

func readAndCompare(source *brands.Brand, parent *brands.Brand, children []*brands.Brand, t *testing.T) {
	brand, found, err := getBrandDriver(t).Read(source.UUID)
	expected := makeBrand(source, parent, children, t)
	assert.NoError(t, err)
	assert.True(t, found)
	assert.NotEmpty(t, brand)
	fmt.Printf("**Made %v+\n\n**Found %v+\n", expected, brand)
	if brand.Parent != nil {
		fmt.Printf("\n**Made.Parent %v+\n\n**Found.Parent %v+\n", *expected.Parent, *brand.Parent)
	}
	for _, child := range brand.Children {
		fmt.Printf("brand.child %v+\n", *child)
	}
	assert.EqualValues(t, expected, brand)
}

func makeBrand(source *brands.Brand, parent *brands.Brand, children []*brands.Brand, t *testing.T) (brand Brand) {
	brand.Thing = makeThing(source, t)
	brand.PrefLabel = source.PrefLabel
	brand.Strapline = source.Strapline
	brand.Description = source.Description
	brand.DescriptionXML = source.DescriptionXML
	brand.ImageURL = source.ImageURL
	if parent == nil {
		brand.Parent = nil
	} else {
		parentBrand := makeThing(parent, t)
		brand.Parent = &parentBrand
	}
	childrenBrands := make([]*Thing, len(children))
	for idx := range children {
		child := makeThing(children[idx], t)
		childrenBrands[idx] = &child
	}
	brand.Children = childrenBrands
	return brand
}

func makeThing(source *brands.Brand, t *testing.T) Thing {
	thing := Thing{}
	thing.ID = "http://api.ft.com/things/" + source.UUID
	thing.APIURL = "http://test.api.ft.com/brands/" + source.UUID
	thing.Types = []string{"http://www.ft.com/ontology/product/Brand"}
	thing.PrefLabel = source.PrefLabel
	return thing
}

func getBrandRWDriver(t *testing.T) (service baseftrwapp.Service) {
	url := os.Getenv("NEO4J_TEST_URL")
	if url == "" {
		url = "http://localhost:7474/db/data"
	}
	db, err := neoism.Connect(url)
	assert.NoError(t, err, "Error setting up connection to %s", url)
	return brands.NewCypherBrandsService(neoutils.StringerDb{db}, db)
}

func getBrandDriver(t *testing.T) CypherDriver {
	url := os.Getenv("NEO4J_TEST_URL")
	if url == "" {
		url = "http://localhost:7474/db/data"
	}
	db, err := neoism.Connect(url)
	assert.NoError(t, err, "Error setting up connection to %s", url)
	return NewCypherDriver(db, "test")
}

func cleanUp(uuid string, t *testing.T) {
	found, err := getBrandRWDriver(t).Delete(uuid)
	assert.True(t, found, "Unable to delete brand with uuid %s", uuid)
	assert.NoError(t, err, "Error deleting brand with uuid %s", uuid)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"

	"github.com/gorilla/mux"

	"github.com/lxc/lxd/lxd/daemon"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/version"
)

// DevLxdServer creates an http.Server capable of handling requests against the
// /dev/lxd Unix socket endpoint created inside VMs.
func devLxdServer(d *Daemon) *http.Server {
	return &http.Server{
		Handler: devLxdAPI(d),
	}
}

type devLxdResponse struct {
	content any
	code    int
	ctype   string
}

func okResponse(ct any, ctype string) *devLxdResponse {
	return &devLxdResponse{ct, http.StatusOK, ctype}
}

type devLxdHandler struct {
	path string

	/*
	 * This API will have to be changed slightly when we decide to support
	 * websocket events upgrading, but since we don't have events on the
	 * server side right now either, I went the simple route to avoid
	 * needless noise.
	 */
	f func(d *Daemon, w http.ResponseWriter, r *http.Request) *devLxdResponse
}

var devlxdConfigGet = devLxdHandler{"/1.0/config", func(d *Daemon, w http.ResponseWriter, r *http.Request) *devLxdResponse {
	data, err := ioutil.ReadFile("instance-data")
	if err != nil {
		return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
	}

	var instance instancetype.VMAgentData

	err = json.Unmarshal(data, &instance)
	if err != nil {
		return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
	}

	filtered := []string{}
	for k := range instance.Config {
		if strings.HasPrefix(k, "user.") || strings.HasPrefix(k, "cloud-init.") {
			filtered = append(filtered, fmt.Sprintf("/1.0/config/%s", k))
		}
	}
	return okResponse(filtered, "json")
}}

var devlxdConfigKeyGet = devLxdHandler{"/1.0/config/{key}", func(d *Daemon, w http.ResponseWriter, r *http.Request) *devLxdResponse {
	key, err := url.PathUnescape(mux.Vars(r)["key"])
	if err != nil {
		return &devLxdResponse{"bad request", http.StatusBadRequest, "raw"}
	}

	if !strings.HasPrefix(key, "user.") && !strings.HasPrefix(key, "cloud-init.") {
		return &devLxdResponse{"not authorized", http.StatusForbidden, "raw"}
	}

	data, err := ioutil.ReadFile("instance-data")
	if err != nil {
		return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
	}

	var instance instancetype.VMAgentData

	err = json.Unmarshal(data, &instance)
	if err != nil {
		return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
	}

	value, ok := instance.Config[key]
	if !ok {
		return &devLxdResponse{"not found", http.StatusNotFound, "raw"}
	}

	return okResponse(value, "raw")
}}

var devlxdMetadataGet = devLxdHandler{"/1.0/meta-data", func(d *Daemon, w http.ResponseWriter, r *http.Request) *devLxdResponse {
	data, err := ioutil.ReadFile("instance-data")
	if err != nil {
		return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
	}

	var instance instancetype.VMAgentData

	err = json.Unmarshal(data, &instance)
	if err != nil {
		return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
	}

	value := instance.Config["user.meta-data"]
	return okResponse(fmt.Sprintf("#cloud-config\ninstance-id: %s\nlocal-hostname: %s\n%s", instance.CloudInitID, instance.Name, value), "raw")
}}

var devLxdEventsGet = devLxdHandler{"/1.0/events", func(d *Daemon, w http.ResponseWriter, r *http.Request) *devLxdResponse {
	err := eventsGet(d, r).Render(w)
	if err != nil {
		return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
	}

	return okResponse("", "raw")
}}

var devlxdAPIGet = devLxdHandler{"/1.0", func(d *Daemon, w http.ResponseWriter, r *http.Request) *devLxdResponse {
	data, err := ioutil.ReadFile("instance-data")
	if err != nil {
		return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
	}

	var instance instancetype.VMAgentData

	err = json.Unmarshal(data, &instance)
	if err != nil {
		return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
	}

	return okResponse(shared.Jmap{"api_version": version.APIVersion, "location": instance.Location}, "json")
}}

var devlxdDevicesGet = devLxdHandler{"/1.0/devices", func(d *Daemon, w http.ResponseWriter, r *http.Request) *devLxdResponse {
	data, err := ioutil.ReadFile("instance-data")
	if err != nil {
		return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
	}

	var instance instancetype.VMAgentData

	err = json.Unmarshal(data, &instance)
	if err != nil {
		return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
	}

	return okResponse(instance.Devices, "json")
}}

var handlers = []devLxdHandler{
	{"/", func(d *Daemon, w http.ResponseWriter, r *http.Request) *devLxdResponse {
		return okResponse([]string{"/1.0"}, "json")
	}},
	devlxdAPIGet,
	devlxdConfigGet,
	devlxdConfigKeyGet,
	devlxdMetadataGet,
	devLxdEventsGet,
	devlxdDevicesGet,
}

func hoistReq(f func(*Daemon, http.ResponseWriter, *http.Request) *devLxdResponse, d *Daemon) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		resp := f(d, w, r)
		if resp.code != http.StatusOK {
			http.Error(w, fmt.Sprintf("%s", resp.content), resp.code)
		} else if resp.ctype == "json" {
			w.Header().Set("Content-Type", "application/json")

			var debugLogger logger.Logger
			if daemon.Debug {
				debugLogger = logger.Logger(logger.Log)
			}

			_ = util.WriteJSON(w, resp.content, debugLogger)
		} else if resp.ctype != "websocket" {
			w.Header().Set("Content-Type", "application/octet-stream")
			_, _ = fmt.Fprint(w, resp.content.(string))
		}
	}
}

func devLxdAPI(d *Daemon) http.Handler {
	m := mux.NewRouter()
	m.UseEncodedPath() // Allow encoded values in path segments.

	for _, handler := range handlers {
		m.HandleFunc(handler.path, hoistReq(handler.f, d))
	}

	return m
}

// Create a new net.Listener bound to the unix socket of the devlxd endpoint.
func createDevLxdlListener(dir string) (net.Listener, error) {
	path := filepath.Join(dir, "lxd", "sock")

	err := os.MkdirAll(filepath.Dir(path), 0755)
	if err != nil {
		return nil, err
	}

	// If this socket exists, that means a previous LXD instance died and
	// didn't clean up. We assume that such LXD instance is actually dead
	// if we get this far, since localCreateListener() tries to connect to
	// the actual lxd socket to make sure that it is actually dead. So, it
	// is safe to remove it here without any checks.
	//
	// Also, it would be nice to SO_REUSEADDR here so we don't have to
	// delete the socket, but we can't:
	//   http://stackoverflow.com/questions/15716302/so-reuseaddr-and-af-unix
	//
	// Note that this will force clients to reconnect when LXD is restarted.
	err = socketUnixRemoveStale(path)
	if err != nil {
		return nil, err
	}

	listener, err := socketUnixListen(path)
	if err != nil {
		return nil, err
	}

	err = socketUnixSetPermissions(path, 0600)
	if err != nil {
		_ = listener.Close()
		return nil, err
	}

	return listener, nil
}

// Remove any stale socket file at the given path.
func socketUnixRemoveStale(path string) error {
	// If there's no socket file at all, there's nothing to do.
	if !shared.PathExists(path) {
		return nil
	}

	logger.Debugf("Detected stale unix socket, deleting")
	err := os.Remove(path)
	if err != nil {
		return fmt.Errorf("could not delete stale local socket: %w", err)
	}

	return nil
}

// Change the file mode of the given unix socket file.
func socketUnixSetPermissions(path string, mode os.FileMode) error {
	err := os.Chmod(path, mode)
	if err != nil {
		return fmt.Errorf("cannot set permissions on local socket: %w", err)
	}

	return nil
}

// Bind to the given unix socket path.
func socketUnixListen(path string) (net.Listener, error) {
	addr, err := net.ResolveUnixAddr("unix", path)
	if err != nil {
		return nil, fmt.Errorf("cannot resolve socket address: %w", err)
	}

	listener, err := net.ListenUnix("unix", addr)
	if err != nil {
		return nil, fmt.Errorf("cannot bind socket: %w", err)
	}

	return listener, err
}
<commit_msg>lxd-agent: Use vsock interface for devlxd<commit_after>package main

import (
	"fmt"
	"net"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"

	"github.com/gorilla/mux"

	"github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxd/daemon"
	"github.com/lxc/lxd/lxd/device/config"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/logger"
)

// DevLxdServer creates an http.Server capable of handling requests against the
// /dev/lxd Unix socket endpoint created inside VMs.
func devLxdServer(d *Daemon) *http.Server {
	return &http.Server{
		Handler: devLxdAPI(d),
	}
}

type devLxdResponse struct {
	content any
	code    int
	ctype   string
}

func okResponse(ct any, ctype string) *devLxdResponse {
	return &devLxdResponse{ct, http.StatusOK, ctype}
}

type devLxdHandler struct {
	path string

	/*
	 * This API will have to be changed slightly when we decide to support
	 * websocket events upgrading, but since we don't have events on the
	 * server side right now either, I went the simple route to avoid
	 * needless noise.
	 */
	f func(d *Daemon, w http.ResponseWriter, r *http.Request) *devLxdResponse
}

func getVsockClient(d *Daemon) (lxd.InstanceServer, error) {
	// Try connecting to LXD server.
	client, err := getClient(int(d.serverCID), int(d.serverPort), d.serverCertificate)
	if err != nil {
		return nil, err
	}

	server, err := lxd.ConnectLXDHTTP(nil, client)
	if err != nil {
		return nil, err
	}

	return server, nil
}

var devlxdConfigGet = devLxdHandler{"/1.0/config", func(d *Daemon, w http.ResponseWriter, r *http.Request) *devLxdResponse {
	client, err := getVsockClient(d)
	if err != nil {
		return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
	}

	defer client.Disconnect()

	resp, _, err := client.RawQuery("GET", "/1.0/config", nil, "")
	if err != nil {
		return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
	}

	var config []string

	err = resp.MetadataAsStruct(&config)
	if err != nil {
		return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
	}

	filtered := []string{}
	for _, k := range config {
		if strings.HasPrefix(k, "/1.0/config/user.") || strings.HasPrefix(k, "/1.0/config/cloud-init.") {
			filtered = append(filtered, k)
		}
	}
	return okResponse(filtered, "json")
}}

var devlxdConfigKeyGet = devLxdHandler{"/1.0/config/{key}", func(d *Daemon, w http.ResponseWriter, r *http.Request) *devLxdResponse {
	key, err := url.PathUnescape(mux.Vars(r)["key"])
	if err != nil {
		return &devLxdResponse{"bad request", http.StatusBadRequest, "raw"}
	}

	if !strings.HasPrefix(key, "user.") && !strings.HasPrefix(key, "cloud-init.") {
		return &devLxdResponse{"not authorized", http.StatusForbidden, "raw"}
	}

	client, err := getVsockClient(d)
	if err != nil {
		return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
	}

	defer client.Disconnect()

	resp, _, err := client.RawQuery("GET", fmt.Sprintf("/1.0/config/%s", key), nil, "")
	if err != nil {
		return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
	}

	var value string

	err = resp.MetadataAsStruct(&value)
	if err != nil {
		return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
	}

	return okResponse(value, "raw")
}}

var devlxdMetadataGet = devLxdHandler{"/1.0/meta-data", func(d *Daemon, w http.ResponseWriter, r *http.Request) *devLxdResponse {
	client, err := getVsockClient(d)
	if err != nil {
		return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
	}

	defer client.Disconnect()

	resp, _, err := client.RawQuery("GET", "/1.0/meta-data", nil, "")
	if err != nil {
		return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
	}

	var metaData string

	err = resp.MetadataAsStruct(&metaData)
	if err != nil {
		return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
	}

	return okResponse(metaData, "raw")
}}

var devLxdEventsGet = devLxdHandler{"/1.0/events", func(d *Daemon, w http.ResponseWriter, r *http.Request) *devLxdResponse {
	err := eventsGet(d, r).Render(w)
	if err != nil {
		return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
	}

	return okResponse("", "raw")
}}

var devlxdAPIGet = devLxdHandler{"/1.0", func(d *Daemon, w http.ResponseWriter, r *http.Request) *devLxdResponse {
	client, err := getVsockClient(d)
	if err != nil {
		return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
	}

	defer client.Disconnect()

	resp, _, err := client.RawQuery("GET", "/1.0", nil, "")
	if err != nil {
		return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
	}

	var instanceData api.VsockServerGet

	err = resp.MetadataAsStruct(&instanceData)
	if err != nil {
		return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
	}

	return okResponse(instanceData, "json")
}}

var devlxdDevicesGet = devLxdHandler{"/1.0/devices", func(d *Daemon, w http.ResponseWriter, r *http.Request) *devLxdResponse {
	client, err := getVsockClient(d)
	if err != nil {
		return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
	}

	defer client.Disconnect()

	resp, _, err := client.RawQuery("GET", "/1.0/devices", nil, "")
	if err != nil {
		return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
	}

	var devices config.Devices

	err = resp.MetadataAsStruct(&devices)
	if err != nil {
		return &devLxdResponse{"internal server error", http.StatusInternalServerError, "raw"}
	}

	return okResponse(devices, "json")
}}

var handlers = []devLxdHandler{
	{"/", func(d *Daemon, w http.ResponseWriter, r *http.Request) *devLxdResponse {
		return okResponse([]string{"/1.0"}, "json")
	}},
	devlxdAPIGet,
	devlxdConfigGet,
	devlxdConfigKeyGet,
	devlxdMetadataGet,
	devLxdEventsGet,
	devlxdDevicesGet,
}

func hoistReq(f func(*Daemon, http.ResponseWriter, *http.Request) *devLxdResponse, d *Daemon) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		resp := f(d, w, r)
		if resp.code != http.StatusOK {
			http.Error(w, fmt.Sprintf("%s", resp.content), resp.code)
		} else if resp.ctype == "json" {
			w.Header().Set("Content-Type", "application/json")

			var debugLogger logger.Logger
			if daemon.Debug {
				debugLogger = logger.Logger(logger.Log)
			}

			_ = util.WriteJSON(w, resp.content, debugLogger)
		} else if resp.ctype != "websocket" {
			w.Header().Set("Content-Type", "application/octet-stream")
			_, _ = fmt.Fprint(w, resp.content.(string))
		}
	}
}

func devLxdAPI(d *Daemon) http.Handler {
	m := mux.NewRouter()
	m.UseEncodedPath() // Allow encoded values in path segments.

	for _, handler := range handlers {
		m.HandleFunc(handler.path, hoistReq(handler.f, d))
	}

	return m
}

// Create a new net.Listener bound to the unix socket of the devlxd endpoint.
func createDevLxdlListener(dir string) (net.Listener, error) {
	path := filepath.Join(dir, "lxd", "sock")

	err := os.MkdirAll(filepath.Dir(path), 0755)
	if err != nil {
		return nil, err
	}

	// If this socket exists, that means a previous LXD instance died and
	// didn't clean up. We assume that such LXD instance is actually dead
	// if we get this far, since localCreateListener() tries to connect to
	// the actual lxd socket to make sure that it is actually dead. So, it
	// is safe to remove it here without any checks.
	//
	// Also, it would be nice to SO_REUSEADDR here so we don't have to
	// delete the socket, but we can't:
	//   http://stackoverflow.com/questions/15716302/so-reuseaddr-and-af-unix
	//
	// Note that this will force clients to reconnect when LXD is restarted.
	err = socketUnixRemoveStale(path)
	if err != nil {
		return nil, err
	}

	listener, err := socketUnixListen(path)
	if err != nil {
		return nil, err
	}

	err = socketUnixSetPermissions(path, 0600)
	if err != nil {
		_ = listener.Close()
		return nil, err
	}

	return listener, nil
}

// Remove any stale socket file at the given path.
func socketUnixRemoveStale(path string) error {
	// If there's no socket file at all, there's nothing to do.
	if !shared.PathExists(path) {
		return nil
	}

	logger.Debugf("Detected stale unix socket, deleting")
	err := os.Remove(path)
	if err != nil {
		return fmt.Errorf("could not delete stale local socket: %w", err)
	}

	return nil
}

// Change the file mode of the given unix socket file.
func socketUnixSetPermissions(path string, mode os.FileMode) error {
	err := os.Chmod(path, mode)
	if err != nil {
		return fmt.Errorf("cannot set permissions on local socket: %w", err)
	}

	return nil
}

// Bind to the given unix socket path.
func socketUnixListen(path string) (net.Listener, error) {
	addr, err := net.ResolveUnixAddr("unix", path)
	if err != nil {
		return nil, fmt.Errorf("cannot resolve socket address: %w", err)
	}

	listener, err := net.ListenUnix("unix", addr)
	if err != nil {
		return nil, fmt.Errorf("cannot bind socket: %w", err)
	}

	return listener, err
}
<|endoftext|>