<commit_before>package server

import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/pkg/stringid"
	"github.com/kubernetes-incubator/ocid/oci"
	"github.com/kubernetes-incubator/ocid/utils"
	"github.com/opencontainers/runtime-tools/generate"
	"golang.org/x/net/context"
	pb "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
)

type sandbox struct {
	id         string
	name       string
	logDir     string
	labels     map[string]string
	containers oci.Store
}

const (
	podInfraRootfs      = "/var/lib/ocid/graph/vfs/pause"
	podDefaultNamespace = "default"
)

func (s *sandbox) addContainer(c *oci.Container) {
	s.containers.Add(c.Name(), c)
}

func (s *sandbox) getContainer(name string) *oci.Container {
	return s.containers.Get(name)
}

func (s *sandbox) removeContainer(c *oci.Container) {
	s.containers.Delete(c.Name())
}

func (s *Server) generatePodIDandName(name, namespace string) (string, string, error) {
	var (
		err error
		id  = stringid.GenerateNonCryptoID()
	)
	if namespace == "" {
		namespace = podDefaultNamespace
	}
	if name, err = s.reservePodName(id, namespace+"-"+name); err != nil {
		return "", "", err
	}
	return id, name, err
}

// RunPodSandbox creates and runs a pod-level sandbox.
func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest) (*pb.RunPodSandboxResponse, error) {
	// process req.Name
	name := req.GetConfig().GetMetadata().GetName()
	if name == "" {
		return nil, fmt.Errorf("PodSandboxConfig.Name should not be empty")
	}

	namespace := req.GetConfig().GetMetadata().GetNamespace()

	var err error
	id, name, err := s.generatePodIDandName(name, namespace)
	if err != nil {
		return nil, err
	}
	podSandboxDir := filepath.Join(s.sandboxDir, id)
	if _, err = os.Stat(podSandboxDir); err == nil {
		return nil, fmt.Errorf("pod sandbox (%s) already exists", podSandboxDir)
	}

	if err = os.MkdirAll(podSandboxDir, 0755); err != nil {
		return nil, err
	}

	defer func() {
		if err != nil {
			if err2 := os.RemoveAll(podSandboxDir); err2 != nil {
				logrus.Warnf("couldn't cleanup podSandboxDir %s: %v", podSandboxDir, err2)
			}
		}
	}()

	// creates a spec Generator with the default spec.
	g := generate.New()

	podInfraRootfs := filepath.Join(s.root, "graph/vfs/pause")
	// setup defaults for the pod sandbox
	g.SetRootPath(filepath.Join(podInfraRootfs, "rootfs"))
	g.SetRootReadonly(true)
	g.SetProcessArgs([]string{"/pause"})

	// set hostname
	hostname := req.GetConfig().GetHostname()
	if hostname != "" {
		g.SetHostname(hostname)
	}

	// set log directory
	logDir := req.GetConfig().GetLogDirectory()
	if logDir == "" {
		logDir = fmt.Sprintf("/var/log/ocid/pods/%s", id)
	}

	// set DNS options
	dnsServers := req.GetConfig().GetDnsOptions().GetServers()
	dnsSearches := req.GetConfig().GetDnsOptions().GetSearches()
	resolvPath := fmt.Sprintf("%s/resolv.conf", podSandboxDir)
	err = parseDNSOptions(dnsServers, dnsSearches, resolvPath)
	if err != nil {
		err1 := removeFile(resolvPath)
		if err1 != nil {
			err = err1
			return nil, fmt.Errorf("%v; failed to remove %s: %v", err, resolvPath, err1)
		}
		return nil, err
	}

	g.AddBindMount(resolvPath, "/etc/resolv.conf", "ro")

	// add labels
	labels := req.GetConfig().GetLabels()
	labelsJSON, err := json.Marshal(labels)
	if err != nil {
		return nil, err
	}
	g.AddAnnotation("ocid/labels", string(labelsJSON))
	g.AddAnnotation("ocid/log_path", logDir)
	g.AddAnnotation("ocid/name", name)
	containerName := name + "-infra"
	g.AddAnnotation("ocid/container_name", containerName)
	s.addSandbox(&sandbox{
		id:         id,
		name:       name,
		logDir:     logDir,
		labels:     labels,
		containers: oci.NewMemoryStore(),
	})

	annotations := req.GetConfig().GetAnnotations()
	for k, v := range annotations {
		g.AddAnnotation(k, v)
	}

	// setup cgroup settings
	cgroupParent := req.GetConfig().GetLinux().GetCgroupParent()
	if cgroupParent != "" {
		g.SetLinuxCgroupsPath(cgroupParent)
	}

	// set up namespaces
	if req.GetConfig().GetLinux().GetNamespaceOptions().GetHostNetwork() {
		err = g.RemoveLinuxNamespace("network")
		if err != nil {
			return nil, err
		}
	}

	if req.GetConfig().GetLinux().GetNamespaceOptions().GetHostPid() {
		err = g.RemoveLinuxNamespace("pid")
		if err != nil {
			return nil, err
		}
	}

	if req.GetConfig().GetLinux().GetNamespaceOptions().GetHostIpc() {
		err = g.RemoveLinuxNamespace("ipc")
		if err != nil {
			return nil, err
		}
	}

	err = g.SaveToFile(filepath.Join(podSandboxDir, "config.json"))
	if err != nil {
		return nil, err
	}

	if _, err = os.Stat(podInfraRootfs); err != nil {
		if os.IsNotExist(err) {
			// TODO: Replace by rootfs creation API when it is ready
			if err = utils.CreateFakeRootfs(podInfraRootfs, "docker://kubernetes/pause"); err != nil {
				return nil, err
			}
		} else {
			return nil, err
		}
	}

	container, err := oci.NewContainer(containerName, podSandboxDir, podSandboxDir, labels, id, false)
	if err != nil {
		return nil, err
	}

	if err = s.runtime.CreateContainer(container); err != nil {
		return nil, err
	}

	if err = s.runtime.UpdateStatus(container); err != nil {
		return nil, err
	}

	// setup the network
	podNamespace := ""
	netnsPath, err := container.NetNsPath()
	if err != nil {
		return nil, err
	}
	if err = s.netPlugin.SetUpPod(netnsPath, podNamespace, id, containerName); err != nil {
		return nil, fmt.Errorf("failed to create network for container %s in sandbox %s: %v", containerName, id, err)
	}

	if err = s.runtime.StartContainer(container); err != nil {
		return nil, err
	}

	s.addContainer(container)

	if err = s.podIDIndex.Add(id); err != nil {
		return nil, err
	}

	if err = s.runtime.UpdateStatus(container); err != nil {
		return nil, err
	}

	return &pb.RunPodSandboxResponse{PodSandboxId: &id}, nil
}

// StopPodSandbox stops the sandbox. If there are any running containers in the
// sandbox, they should be force terminated.
func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxRequest) (*pb.StopPodSandboxResponse, error) {
	sbID := req.PodSandboxId
	if *sbID == "" {
		return nil, fmt.Errorf("PodSandboxId should not be empty")
	}
	sb := s.getSandbox(*sbID)
	if sb == nil {
		return nil, fmt.Errorf("specified sandbox not found: %s", *sbID)
	}

	podInfraContainer := sb.name + "-infra"
	for _, c := range sb.containers.List() {
		if podInfraContainer == c.Name() {
			podNamespace := ""
			netnsPath, err := c.NetNsPath()
			if err != nil {
				return nil, err
			}
			if err := s.netPlugin.TearDownPod(netnsPath, podNamespace, *sbID, podInfraContainer); err != nil {
				return nil, fmt.Errorf("failed to destroy network for container %s in sandbox %s: %v", c.Name(), *sbID, err)
			}
		}
		cStatus := s.runtime.ContainerStatus(c)
		if cStatus.Status != "stopped" {
			if err := s.runtime.StopContainer(c); err != nil {
				return nil, fmt.Errorf("failed to stop container %s in sandbox %s: %v", c.Name(), *sbID, err)
			}
		}
	}

	return &pb.StopPodSandboxResponse{}, nil
}

// RemovePodSandbox deletes the sandbox. If there are any running containers in the
// sandbox, they should be force deleted.
func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxRequest) (*pb.RemovePodSandboxResponse, error) {
	sbID := req.PodSandboxId
	if *sbID == "" {
		return nil, fmt.Errorf("PodSandboxId should not be empty")
	}
	sb := s.getSandbox(*sbID)
	if sb == nil {
		return nil, fmt.Errorf("specified sandbox not found: %s", *sbID)
	}

	podInfraContainer := sb.name + "-infra"

	// Delete all the containers in the sandbox
	for _, c := range sb.containers.List() {
		if err := s.runtime.DeleteContainer(c); err != nil {
			return nil, fmt.Errorf("failed to delete container %s in sandbox %s: %v", c.Name(), *sbID, err)
		}
		if podInfraContainer == c.Name() {
			continue
		}
		containerDir := filepath.Join(s.runtime.ContainerDir(), c.Name())
		if err := os.RemoveAll(containerDir); err != nil {
			return nil, fmt.Errorf("failed to remove container %s directory: %v", c.Name(), err)
		}
	}

	// Remove the files related to the sandbox
	podSandboxDir := filepath.Join(s.sandboxDir, *sbID)
	if err := os.RemoveAll(podSandboxDir); err != nil {
		return nil, fmt.Errorf("failed to remove sandbox %s directory: %v", *sbID, err)
	}

	return &pb.RemovePodSandboxResponse{}, nil
}

// PodSandboxStatus returns the Status of the PodSandbox.
func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusRequest) (*pb.PodSandboxStatusResponse, error) {
	sbID := req.PodSandboxId
	if *sbID == "" {
		return nil, fmt.Errorf("PodSandboxId should not be empty")
	}
	sb := s.getSandbox(*sbID)
	if sb == nil {
		return nil, fmt.Errorf("specified sandbox not found: %s", *sbID)
	}

	podInfraContainerName := sb.name + "-infra"
	podInfraContainer := sb.getContainer(podInfraContainerName)
	if err := s.runtime.UpdateStatus(podInfraContainer); err != nil {
		return nil, err
	}

	cState := s.runtime.ContainerStatus(podInfraContainer)
	created := cState.Created.Unix()

	netNsPath, err := podInfraContainer.NetNsPath()
	if err != nil {
		return nil, err
	}
	podNamespace := ""
	ip, err := s.netPlugin.GetContainerNetworkStatus(netNsPath, podNamespace, *sbID, podInfraContainerName)
	if err != nil {
		// ignore the error on network status
		ip = ""
	}

	rStatus := pb.PodSandBoxState_NOTREADY
	if cState.Status == ContainerStateRunning {
		rStatus = pb.PodSandBoxState_READY
	}

	return &pb.PodSandboxStatusResponse{
		Status: &pb.PodSandboxStatus{
			Id:        sbID,
			CreatedAt: int64Ptr(created),
			Linux: &pb.LinuxPodSandboxStatus{
				Namespaces: &pb.Namespace{
					Network: sPtr(netNsPath),
				},
			},
			Network: &pb.PodSandboxNetworkStatus{Ip: &ip},
			State:   &rStatus,
		},
	}, nil
}

// ListPodSandbox returns a list of SandBoxes.
func (s *Server) ListPodSandbox(context.Context, *pb.ListPodSandboxRequest) (*pb.ListPodSandboxResponse, error) {
	var pods []*pb.PodSandbox
	for _, sb := range s.state.sandboxes {
		podInfraContainerName := sb.name + "-infra"
		podInfraContainer := sb.getContainer(podInfraContainerName)
		if err := s.runtime.UpdateStatus(podInfraContainer); err != nil {
			return nil, err
		}
		cState := s.runtime.ContainerStatus(podInfraContainer)
		created := cState.Created.Unix()
		rStatus := pb.PodSandBoxState_NOTREADY
		if cState.Status == ContainerStateRunning {
			rStatus = pb.PodSandBoxState_READY
		}

		pod := &pb.PodSandbox{
			Id:        &sb.id,
			CreatedAt: int64Ptr(created),
			State:     &rStatus,
		}

		pods = append(pods, pod)
	}

	return &pb.ListPodSandboxResponse{
		Items: pods,
	}, nil
}
<commit_msg>Allow specifying pod IDs by unique prefixes<commit_after>package server

import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/pkg/stringid"
	"github.com/kubernetes-incubator/ocid/oci"
	"github.com/kubernetes-incubator/ocid/utils"
	"github.com/opencontainers/runtime-tools/generate"
	"golang.org/x/net/context"
	pb "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
)

type sandbox struct {
	id         string
	name       string
	logDir     string
	labels     map[string]string
	containers oci.Store
}

const (
	podInfraRootfs      = "/var/lib/ocid/graph/vfs/pause"
	podDefaultNamespace = "default"
)

func (s *sandbox) addContainer(c *oci.Container) {
	s.containers.Add(c.Name(), c)
}

func (s *sandbox) getContainer(name string) *oci.Container {
	return s.containers.Get(name)
}

func (s *sandbox) removeContainer(c *oci.Container) {
	s.containers.Delete(c.Name())
}

func (s *Server) generatePodIDandName(name, namespace string) (string, string, error) {
	var (
		err error
		id  = stringid.GenerateNonCryptoID()
	)
	if namespace == "" {
		namespace = podDefaultNamespace
	}
	if name, err = s.reservePodName(id, namespace+"-"+name); err != nil {
		return "", "", err
	}
	return id, name, err
}

// RunPodSandbox creates and runs a pod-level sandbox.
func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest) (*pb.RunPodSandboxResponse, error) {
	// process req.Name
	name := req.GetConfig().GetMetadata().GetName()
	if name == "" {
		return nil, fmt.Errorf("PodSandboxConfig.Name should not be empty")
	}

	namespace := req.GetConfig().GetMetadata().GetNamespace()

	var err error
	id, name, err := s.generatePodIDandName(name, namespace)
	if err != nil {
		return nil, err
	}
	podSandboxDir := filepath.Join(s.sandboxDir, id)
	if _, err = os.Stat(podSandboxDir); err == nil {
		return nil, fmt.Errorf("pod sandbox (%s) already exists", podSandboxDir)
	}

	if err = os.MkdirAll(podSandboxDir, 0755); err != nil {
		return nil, err
	}

	defer func() {
		if err != nil {
			if err2 := os.RemoveAll(podSandboxDir); err2 != nil {
				logrus.Warnf("couldn't cleanup podSandboxDir %s: %v", podSandboxDir, err2)
			}
		}
	}()

	// creates a spec Generator with the default spec.
	g := generate.New()

	podInfraRootfs := filepath.Join(s.root, "graph/vfs/pause")
	// setup defaults for the pod sandbox
	g.SetRootPath(filepath.Join(podInfraRootfs, "rootfs"))
	g.SetRootReadonly(true)
	g.SetProcessArgs([]string{"/pause"})

	// set hostname
	hostname := req.GetConfig().GetHostname()
	if hostname != "" {
		g.SetHostname(hostname)
	}

	// set log directory
	logDir := req.GetConfig().GetLogDirectory()
	if logDir == "" {
		logDir = fmt.Sprintf("/var/log/ocid/pods/%s", id)
	}

	// set DNS options
	dnsServers := req.GetConfig().GetDnsOptions().GetServers()
	dnsSearches := req.GetConfig().GetDnsOptions().GetSearches()
	resolvPath := fmt.Sprintf("%s/resolv.conf", podSandboxDir)
	err = parseDNSOptions(dnsServers, dnsSearches, resolvPath)
	if err != nil {
		err1 := removeFile(resolvPath)
		if err1 != nil {
			err = err1
			return nil, fmt.Errorf("%v; failed to remove %s: %v", err, resolvPath, err1)
		}
		return nil, err
	}

	g.AddBindMount(resolvPath, "/etc/resolv.conf", "ro")

	// add labels
	labels := req.GetConfig().GetLabels()
	labelsJSON, err := json.Marshal(labels)
	if err != nil {
		return nil, err
	}
	g.AddAnnotation("ocid/labels", string(labelsJSON))
	g.AddAnnotation("ocid/log_path", logDir)
	g.AddAnnotation("ocid/name", name)
	containerName := name + "-infra"
	g.AddAnnotation("ocid/container_name", containerName)
	s.addSandbox(&sandbox{
		id:         id,
		name:       name,
		logDir:     logDir,
		labels:     labels,
		containers: oci.NewMemoryStore(),
	})

	annotations := req.GetConfig().GetAnnotations()
	for k, v := range annotations {
		g.AddAnnotation(k, v)
	}

	// setup cgroup settings
	cgroupParent := req.GetConfig().GetLinux().GetCgroupParent()
	if cgroupParent != "" {
		g.SetLinuxCgroupsPath(cgroupParent)
	}

	// set up namespaces
	if req.GetConfig().GetLinux().GetNamespaceOptions().GetHostNetwork() {
		err = g.RemoveLinuxNamespace("network")
		if err != nil {
			return nil, err
		}
	}

	if req.GetConfig().GetLinux().GetNamespaceOptions().GetHostPid() {
		err = g.RemoveLinuxNamespace("pid")
		if err != nil {
			return nil, err
		}
	}

	if req.GetConfig().GetLinux().GetNamespaceOptions().GetHostIpc() {
		err = g.RemoveLinuxNamespace("ipc")
		if err != nil {
			return nil, err
		}
	}

	err = g.SaveToFile(filepath.Join(podSandboxDir, "config.json"))
	if err != nil {
		return nil, err
	}

	if _, err = os.Stat(podInfraRootfs); err != nil {
		if os.IsNotExist(err) {
			// TODO: Replace by rootfs creation API when it is ready
			if err = utils.CreateFakeRootfs(podInfraRootfs, "docker://kubernetes/pause"); err != nil {
				return nil, err
			}
		} else {
			return nil, err
		}
	}

	container, err := oci.NewContainer(containerName, podSandboxDir, podSandboxDir, labels, id, false)
	if err != nil {
		return nil, err
	}

	if err = s.runtime.CreateContainer(container); err != nil {
		return nil, err
	}

	if err = s.runtime.UpdateStatus(container); err != nil {
		return nil, err
	}

	// setup the network
	podNamespace := ""
	netnsPath, err := container.NetNsPath()
	if err != nil {
		return nil, err
	}
	if err = s.netPlugin.SetUpPod(netnsPath, podNamespace, id, containerName); err != nil {
		return nil, fmt.Errorf("failed to create network for container %s in sandbox %s: %v", containerName, id, err)
	}

	if err = s.runtime.StartContainer(container); err != nil {
		return nil, err
	}

	s.addContainer(container)

	if err = s.podIDIndex.Add(id); err != nil {
		return nil, err
	}

	if err = s.runtime.UpdateStatus(container); err != nil {
		return nil, err
	}

	return &pb.RunPodSandboxResponse{PodSandboxId: &id}, nil
}

// StopPodSandbox stops the sandbox. If there are any running containers in the
// sandbox, they should be force terminated.
func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxRequest) (*pb.StopPodSandboxResponse, error) {
	sbID := req.GetPodSandboxId()
	if sbID == "" {
		return nil, fmt.Errorf("PodSandboxId should not be empty")
	}

	sandboxID, err := s.podIDIndex.Get(sbID)
	if err != nil {
		return nil, fmt.Errorf("PodSandbox with ID starting with %s not found: %v", sbID, err)
	}

	sb := s.getSandbox(sandboxID)
	if sb == nil {
		return nil, fmt.Errorf("specified sandbox not found: %s", sandboxID)
	}

	podInfraContainer := sb.name + "-infra"
	for _, c := range sb.containers.List() {
		if podInfraContainer == c.Name() {
			podNamespace := ""
			netnsPath, err := c.NetNsPath()
			if err != nil {
				return nil, err
			}
			if err := s.netPlugin.TearDownPod(netnsPath, podNamespace, sandboxID, podInfraContainer); err != nil {
				return nil, fmt.Errorf("failed to destroy network for container %s in sandbox %s: %v", c.Name(), sandboxID, err)
			}
		}
		cStatus := s.runtime.ContainerStatus(c)
		if cStatus.Status != "stopped" {
			if err := s.runtime.StopContainer(c); err != nil {
				return nil, fmt.Errorf("failed to stop container %s in sandbox %s: %v", c.Name(), sandboxID, err)
			}
		}
	}

	return &pb.StopPodSandboxResponse{}, nil
}

// RemovePodSandbox deletes the sandbox. If there are any running containers in the
// sandbox, they should be force deleted.
func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxRequest) (*pb.RemovePodSandboxResponse, error) {
	sbID := req.GetPodSandboxId()
	if sbID == "" {
		return nil, fmt.Errorf("PodSandboxId should not be empty")
	}

	sandboxID, err := s.podIDIndex.Get(sbID)
	if err != nil {
		return nil, fmt.Errorf("PodSandbox with ID starting with %s not found: %v", sbID, err)
	}

	sb := s.getSandbox(sandboxID)
	if sb == nil {
		return nil, fmt.Errorf("specified sandbox not found: %s", sandboxID)
	}

	podInfraContainer := sb.name + "-infra"

	// Delete all the containers in the sandbox
	for _, c := range sb.containers.List() {
		if err := s.runtime.DeleteContainer(c); err != nil {
			return nil, fmt.Errorf("failed to delete container %s in sandbox %s: %v", c.Name(), sandboxID, err)
		}
		if podInfraContainer == c.Name() {
			continue
		}
		containerDir := filepath.Join(s.runtime.ContainerDir(), c.Name())
		if err := os.RemoveAll(containerDir); err != nil {
			return nil, fmt.Errorf("failed to remove container %s directory: %v", c.Name(), err)
		}
	}

	// Remove the files related to the sandbox
	podSandboxDir := filepath.Join(s.sandboxDir, sandboxID)
	if err := os.RemoveAll(podSandboxDir); err != nil {
		return nil, fmt.Errorf("failed to remove sandbox %s directory: %v", sandboxID, err)
	}

	return &pb.RemovePodSandboxResponse{}, nil
}

// PodSandboxStatus returns the Status of the PodSandbox.
func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusRequest) (*pb.PodSandboxStatusResponse, error) {
	sbID := req.GetPodSandboxId()
	if sbID == "" {
		return nil, fmt.Errorf("PodSandboxId should not be empty")
	}

	sandboxID, err := s.podIDIndex.Get(sbID)
	if err != nil {
		return nil, fmt.Errorf("PodSandbox with ID starting with %s not found: %v", sbID, err)
	}

	sb := s.getSandbox(sandboxID)
	if sb == nil {
		return nil, fmt.Errorf("specified sandbox not found: %s", sandboxID)
	}

	podInfraContainerName := sb.name + "-infra"
	podInfraContainer := sb.getContainer(podInfraContainerName)
	if err = s.runtime.UpdateStatus(podInfraContainer); err != nil {
		return nil, err
	}

	cState := s.runtime.ContainerStatus(podInfraContainer)
	created := cState.Created.Unix()

	netNsPath, err := podInfraContainer.NetNsPath()
	if err != nil {
		return nil, err
	}
	podNamespace := ""
	ip, err := s.netPlugin.GetContainerNetworkStatus(netNsPath, podNamespace, sandboxID, podInfraContainerName)
	if err != nil {
		// ignore the error on network status
		ip = ""
	}

	rStatus := pb.PodSandBoxState_NOTREADY
	if cState.Status == ContainerStateRunning {
		rStatus = pb.PodSandBoxState_READY
	}

	return &pb.PodSandboxStatusResponse{
		Status: &pb.PodSandboxStatus{
			Id:        &sbID,
			CreatedAt: int64Ptr(created),
			Linux: &pb.LinuxPodSandboxStatus{
				Namespaces: &pb.Namespace{
					Network: sPtr(netNsPath),
				},
			},
			Network: &pb.PodSandboxNetworkStatus{Ip: &ip},
			State:   &rStatus,
		},
	}, nil
}

// ListPodSandbox returns a list of SandBoxes.
func (s *Server) ListPodSandbox(context.Context, *pb.ListPodSandboxRequest) (*pb.ListPodSandboxResponse, error) {
	var pods []*pb.PodSandbox
	for _, sb := range s.state.sandboxes {
		podInfraContainerName := sb.name + "-infra"
		podInfraContainer := sb.getContainer(podInfraContainerName)
		if err := s.runtime.UpdateStatus(podInfraContainer); err != nil {
			return nil, err
		}
		cState := s.runtime.ContainerStatus(podInfraContainer)
		created := cState.Created.Unix()
		rStatus := pb.PodSandBoxState_NOTREADY
		if cState.Status == ContainerStateRunning {
			rStatus = pb.PodSandBoxState_READY
		}

		pod := &pb.PodSandbox{
			Id:        &sb.id,
			CreatedAt: int64Ptr(created),
			State:     &rStatus,
		}

		pods = append(pods, pod)
	}

	return &pb.ListPodSandboxResponse{
		Items: pods,
	}, nil
}
<|endoftext|>


<commit_before>package testing

import (
	"fmt"
	"log"
)

// T is the interface that mimics the standard library *testing.T.
//
// In unit tests you can just pass a *testing.T struct. At runtime, outside
// of tests, you can pass in a RuntimeT struct from this package.
type T interface {
	Error(args ...interface{})
	Errorf(format string, args ...interface{})
	Fatal(args ...interface{})
	Fatalf(format string, args ...interface{})
	Fail()
	FailNow()
	Failed() bool
	Log(args ...interface{})
	Logf(format string, args ...interface{})
}

// RuntimeT implements T and can be instantiated and run at runtime to
// mimic *testing.T behavior. Unlike *testing.T, this will simply panic
// for calls to Fatal. For calls to Error, you'll have to check the errors
// list to determine whether to exit yourself.
type RuntimeT struct {
	failed bool
}

func (t *RuntimeT) Error(args ...interface{}) {
	log.Println(fmt.Sprintln(args...))
	t.Fail()
}

func (t *RuntimeT) Errorf(format string, args ...interface{}) {
	log.Println(fmt.Sprintf(format, args...))
	t.Fail()
}

func (t *RuntimeT) Fatal(args ...interface{}) {
	log.Println(fmt.Sprintln(args...))
	t.FailNow()
}

func (t *RuntimeT) Fatalf(format string, args ...interface{}) {
	log.Println(fmt.Sprintf(format, args...))
	t.FailNow()
}

func (t *RuntimeT) Fail() {
	t.failed = true
}

func (t *RuntimeT) FailNow() {
	panic("testing.T failed, see logs for output (if any)")
}

func (t *RuntimeT) Failed() bool {
	return t.failed
}

func (t *RuntimeT) Log(args ...interface{}) {
	log.Println(fmt.Sprintln(args...))
}

func (t *RuntimeT) Logf(format string, args ...interface{}) {
	log.Println(fmt.Sprintf(format, args...))
}
<commit_msg>Add Skip and Name methods<commit_after>package testing

import (
	"fmt"
	"log"
)

// T is the interface that mimics the standard library *testing.T.
//
// In unit tests you can just pass a *testing.T struct. At runtime, outside
// of tests, you can pass in a RuntimeT struct from this package.
type T interface {
	Error(args ...interface{})
	Errorf(format string, args ...interface{})
	Fail()
	FailNow()
	Failed() bool
	Fatal(args ...interface{})
	Fatalf(format string, args ...interface{})
	Log(args ...interface{})
	Logf(format string, args ...interface{})
	Name() string
	Skip(args ...interface{})
	SkipNow()
	Skipf(format string, args ...interface{})
	Skipped() bool
}

// RuntimeT implements T and can be instantiated and run at runtime to
// mimic *testing.T behavior. Unlike *testing.T, this will simply panic
// for calls to Fatal. For calls to Error, you'll have to check the errors
// list to determine whether to exit yourself. Name and Skip methods are
// unimplemented noops.
type RuntimeT struct {
	failed bool
}

func (t *RuntimeT) Error(args ...interface{}) {
	log.Println(fmt.Sprintln(args...))
	t.Fail()
}

func (t *RuntimeT) Errorf(format string, args ...interface{}) {
	log.Println(fmt.Sprintf(format, args...))
	t.Fail()
}

func (t *RuntimeT) Fatal(args ...interface{}) {
	log.Println(fmt.Sprintln(args...))
	t.FailNow()
}

func (t *RuntimeT) Fatalf(format string, args ...interface{}) {
	log.Println(fmt.Sprintf(format, args...))
	t.FailNow()
}

func (t *RuntimeT) Fail() {
	t.failed = true
}

func (t *RuntimeT) FailNow() {
	panic("testing.T failed, see logs for output (if any)")
}

func (t *RuntimeT) Failed() bool {
	return t.failed
}

func (t *RuntimeT) Log(args ...interface{}) {
	log.Println(fmt.Sprintln(args...))
}

func (t *RuntimeT) Logf(format string, args ...interface{}) {
	log.Println(fmt.Sprintf(format, args...))
}

func (t *RuntimeT) Name() string                             { return "" }
func (t *RuntimeT) Skip(args ...interface{})                 {}
func (t *RuntimeT) SkipNow()                                 {}
func (t *RuntimeT) Skipf(format string, args ...interface{}) {}
func (t *RuntimeT) Skipped() bool                            { return false }
<|endoftext|>


<commit_before>package service

import (
	"context"
	"errors"
	"fmt"
	"log"
	"net/url"
	"os"
	"time"

	"github.com/aandryashin/selenoid/config"
	"github.com/docker/engine-api/client"
	"github.com/docker/engine-api/types"
	"github.com/docker/engine-api/types/container"
	"github.com/docker/engine-api/types/network"
	"github.com/docker/go-connections/nat"
)

type Docker struct {
	Client  *client.Client
	Service *config.Browser
}

func (docker *Docker) StartWithCancel() (*url.URL, func(), error) {
	port, err := nat.NewPort("tcp", docker.Service.Port)
	if err != nil {
		return nil, nil, err
	}
	ctx := context.Background()
	log.Println("Creating Docker container", docker.Service.Image, "...")
	resp, err := docker.Client.ContainerCreate(ctx,
		&container.Config{
			Hostname:     "localhost",
			Image:        docker.Service.Image.(string),
			ExposedPorts: map[nat.Port]struct{}{port: struct{}{}},
		},
		&container.HostConfig{
			AutoRemove: true,
			PortBindings: nat.PortMap{
				port: []nat.PortBinding{nat.PortBinding{HostIP: "0.0.0.0"}},
			},
			ShmSize:    268435456,
			Privileged: true,
		},
		&network.NetworkingConfig{}, "")
	if err != nil {
		log.Println("error creating container:", err)
		return nil, nil, err
	}
	log.Println("Starting container...")
	err = docker.Client.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{})
	if err != nil {
		log.Println("error starting container:", err)
		return nil, nil, err
	}
	log.Printf("Container %s started\n", resp.ID)
	stat, err := docker.Client.ContainerInspect(ctx, resp.ID)
	if err != nil {
		log.Printf("unable to inspect container %s: %s\n", resp.ID, err)
		return nil, nil, err
	}
	_, ok := stat.NetworkSettings.Ports[port]
	if !ok {
		err := errors.New(fmt.Sprintf("no bingings available for %v...\n", port))
		log.Println(err)
		return nil, nil, err
	}
	if len(stat.NetworkSettings.Ports[port]) != 1 {
		err := errors.New(fmt.Sprintf("error: wrong number of port bindings"))
		log.Println(err)
		return nil, nil, err
	}
	addr := stat.NetworkSettings.Ports[port][0]
	_, err = os.Stat(".dockerenv")
	if err == nil {
		addr.HostIP = "172.17.0.1"
	}
	log.Println(addr.HostIP, addr.HostPort)
	host := fmt.Sprintf("http://%s:%s%s", addr.HostIP, addr.HostPort, docker.Service.Path)
	s := time.Now()
	err = wait(host, 10*time.Second)
	if err != nil {
		log.Println(err)
		stop(ctx, docker.Client, resp.ID)
		return nil, nil, err
	}
	log.Println(time.Since(s))
	u, _ := url.Parse(host)
	log.Println("proxying requests to:", host)
	return u, func() { stop(ctx, docker.Client, resp.ID) }, nil
}

func stop(ctx context.Context, cli *client.Client, id string) {
	fmt.Println("Stopping container", id)
	err := cli.ContainerStop(ctx, id, nil)
	if err != nil {
		log.Println("error: unable to stop container", id, err)
		return
	}
	cli.ContainerWait(ctx, id)
	fmt.Printf("Container %s stopped\n", id)
	err = cli.ContainerRemove(ctx, id, types.ContainerRemoveOptions{Force: true})
	if err != nil {
		fmt.Println("error: unable to remove container", id, err)
		return
	}
	fmt.Printf("Container %s removed\n", id)
}
<commit_msg>Minor refactoring and fixes<commit_after>package service

import (
	"context"
	"fmt"
	"log"
	"net/url"
	"os"
	"time"

	"github.com/aandryashin/selenoid/config"
	"github.com/docker/engine-api/client"
	"github.com/docker/engine-api/types"
	"github.com/docker/engine-api/types/container"
	"github.com/docker/engine-api/types/network"
	"github.com/docker/go-connections/nat"
)

type Docker struct {
	Client  *client.Client
	Service *config.Browser
}

func (docker *Docker) StartWithCancel() (*url.URL, func(), error) {
	port, err := nat.NewPort("tcp", docker.Service.Port)
	if err != nil {
		return nil, nil, err
	}
	ctx := context.Background()
	log.Println("Creating Docker container", docker.Service.Image, "...")
	resp, err := docker.Client.ContainerCreate(ctx,
		&container.Config{
			Hostname:     "localhost",
			Image:        docker.Service.Image.(string),
			ExposedPorts: map[nat.Port]struct{}{port: struct{}{}},
		},
		&container.HostConfig{
			AutoRemove: true,
			PortBindings: nat.PortMap{
				port: []nat.PortBinding{nat.PortBinding{HostIP: "0.0.0.0"}},
			},
			ShmSize:    268435456,
			Privileged: true,
		},
		&network.NetworkingConfig{}, "")
	if err != nil {
		return nil, nil, fmt.Errorf("error creating container: %v", err)
	}
	log.Println("Starting container...")
	err = docker.Client.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{})
	if err != nil {
		return nil, nil, fmt.Errorf("error starting container: %v", err)
	}
	log.Printf("Container %s started\n", resp.ID)
	stat, err := docker.Client.ContainerInspect(ctx, resp.ID)
	if err != nil {
		stop(ctx, docker.Client, resp.ID)
		return nil, nil, fmt.Errorf("unable to inspect container %s: %s\n", resp.ID, err)
	}
	_, ok := stat.NetworkSettings.Ports[port]
	if !ok {
		stop(ctx, docker.Client, resp.ID)
		return nil, nil, fmt.Errorf("no bingings available for %v...\n", port)
	}
	if len(stat.NetworkSettings.Ports[port]) != 1 {
		stop(ctx, docker.Client, resp.ID)
		return nil, nil, fmt.Errorf("error: wrong number of port bindings")
	}
	addr := stat.NetworkSettings.Ports[port][0]
	_, err = os.Stat(".dockerenv")
	if err == nil {
		addr.HostIP = "172.17.0.1"
	}
	host := fmt.Sprintf("http://%s:%s%s", addr.HostIP, addr.HostPort, docker.Service.Path)
	s := time.Now()
	err = wait(host, 10*time.Second)
	if err != nil {
		stop(ctx, docker.Client, resp.ID)
		return nil, nil, err
	}
	log.Println(time.Since(s))
	u, _ := url.Parse(host)
	log.Println("proxying requests to:", host)
	return u, func() { stop(ctx, docker.Client, resp.ID) }, nil
}

func stop(ctx context.Context, cli *client.Client, id string) {
	fmt.Println("Stopping container", id)
	err := cli.ContainerStop(ctx, id, nil)
	if err != nil {
		log.Println("error: unable to stop container", id, err)
		return
	}
	cli.ContainerWait(ctx, id)
	fmt.Printf("Container %s stopped\n", id)
	err = cli.ContainerRemove(ctx, id, types.ContainerRemoveOptions{Force: true})
	if err != nil {
		fmt.Println("error: unable to remove container", id, err)
		return
	}
	fmt.Printf("Container %s removed\n", id)
}
<|endoftext|>


<commit_before>package service

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/aosfather/bingo"
	"github.com/aosfather/bingo/utils"
	"github.com/go-redis/redis"
)

/**
  搜索实现
  通过倒排实现关键信息的实现
  规则：
   1、原始内容使用hashmap存储，对象【ID，Content】，键值 indexname，二级key根据md5 对象转json字符串
   2、针对原始内容带的标签，key value，生成set，名称为 indexname_key_value的形式，set内容放 通过内容md5出来的键值
   3、搜索的时候，根据传递的搜索条件 key，value 数组，对找到的set（形如indexname_key_value ），实行找交集
   4、根据交集的结果二级key，并从indexname的hashmap中获取json内容

*/

type Field struct {
	Key   string `json:"key"`
	Value string `json:"value"`
}

type PageSearchResult struct {
	Id    string `json:"uuid"` //查询的请求id
	Index int64  `json:"page"` //页码
	Data  []TargetObject
}

type TargetObject struct {
	Id   string          `json:"id"`
	Data json.RawMessage `json:"data"`
}
type SourceObject struct {
	TargetObject
	Fields map[string]string `json:"fields"`
}

type FieldType byte
const (
	FT_TEXT FieldType =11  //文本类型
	FT_NUMBER FieldType =9 //数字
	FT_ENUM   FieldType =8 //枚举
	FT_ID FieldType =7  //id唯一标识
	FT_DATE FieldType =6 //日期
)
//索引的元数据信息
type IndexMeta struct {
	Name string  //索引名称
	Fields []FieldMeta //字段
}

type FieldMeta struct {
	Name string //字段名称
	Type FieldType //类型

}

type SearchEngine struct {
	indexs   map[string]*searchIndex
	client   *redis.Client
	logger   utils.Log
	pageSize int64
	pageLife int64 //分钟
}

func (this *SearchEngine) Init(context *bingo.ApplicationContext) {
	fmt.Println("init .....")
	db, err := strconv.Atoi(context.GetPropertyFromConfig("service.search.db"))
	if err != nil {
		db = 0
	}

	size, err := strconv.Atoi(context.GetPropertyFromConfig("service.search.pagesize"))
	if err != nil {
		size = 20 //默认大小20条
	}
	this.pageSize = int64(size)

	life, err := strconv.Atoi(context.GetPropertyFromConfig("service.search.pagelife"))
	if err != nil {
		life = 10 //默认时间10分钟
	}
	this.pageLife = int64(life)

	this.client = redis.NewClient(&redis.Options{
		Addr:     context.GetPropertyFromConfig("service.search.redis"),
		Password: "", // no password set
		DB:       db,
	})
	fmt.Println(context.GetPropertyFromConfig("service.search.redis"))
	this.indexs = make(map[string]*searchIndex)
	this.logger = context.GetLog("bingo_search")
}

//创建索引
func (this *SearchEngine) CreateIndex(name string) *searchIndex {
	if name != "" {
		index := this.indexs[name]
		if index == nil {
			index = &searchIndex{name, this}
			this.indexs[name] = index
		}
		return index
	}

	return nil
}

//清除索引，将整个索引的数据摧毁
func (this *SearchEngine)FlushIndex(name string) {
	//1、删除存放的数据
	this.client.Del(name)

	//2、删除所有的索引key
	keys,err:=this.client.Keys(name+"_*").Result()
	if err!=nil {
		this.logger.Debug("get index keys error:%s",err.Error())
	}

	if keys!=nil && len(keys)>0 {
		this.client.Del(keys...)
	}
}

//加载和刷新数据
func (this *SearchEngine) LoadSource(name string, obj *SourceObject) {
	if name==""||obj==nil {
		return
	}

	index := this.CreateIndex(name)
	if index != nil {
		index.LoadObject(obj)
	}

}

//删除数据
func (this *SearchEngine)RemoveSource(name string, obj *SourceObject){
   if name!=""&&obj!=nil{
	   index := this.CreateIndex(name)
	   if index != nil {
		   index.RemoveObject(obj)
	   }
   }
}

//按页获取数据
func (this *SearchEngine) FetchByPage(request string, page int64) *PageSearchResult {
	if request != "" {
		//获取request的name
		index := strings.Index(request, ":")
		if index < 0 {
			this.logger.Error("pagerequest's index name not found !")
			return nil //找不到对应的索引类型
		}

		name := request[0:index]
		if page <= 0 {
			page = 1
		}
		startIndex := (page - 1) * this.pageSize //+ 1 //从1开始计数
		endIndex := page*this.pageSize - 1

		keys, err := this.client.LRange(request, startIndex, endIndex).Result()
		if err != nil || len(keys) == 0 {
			this.logger.Debug("no content by page!")
			return nil
		}
		go this.client.Expire(request, time.Duration(this.pageLife)*time.Minute) //更新重置失效时间
		return &PageSearchResult{request, page, this.fetch(name, keys...)}
	}

	return nil
}

func (this *SearchEngine) createRequst(key string, keys ...string) {
	//    key:= getSearchRequestUuid(name)
	var datas []interface{}

	for _, v := range keys {
		datas = append(datas, v)
	}

	this.client.LPush(key, datas...)
	this.client.Expire(key, time.Duration(this.pageLife)*time.Minute) //指定x分钟后失效
	//	return key
}

//获取内容
func (this *SearchEngine) fetch(name string, keys ...string) []TargetObject {
	datas, err1 := this.client.HMGet(name, keys...).Result()
	if err1 == nil && len(datas) > 0 {

		var targets []TargetObject

		for _, v := range datas {
			if v != nil {
				t := TargetObject{}
				json.Unmarshal([]byte(fmt.Sprintf("%v", v)), &t)
				targets = append(targets, t)
			}
		}

		return targets

	} else {
		this.logger.Error("get data by index error!%s", err1.Error())
	}

	return nil
}

func (this *SearchEngine) Search(name string, input ...Field) *PageSearchResult {
	if name != "" {
		index := this.indexs[name]
		if index != nil {
			r, data := index.Search(input...)
			return &PageSearchResult{r, 1, data}

		}
		this.logger.Info("not found index %s", name)
	}

	return nil
}

type searchIndex struct {
	name   string
	engine *SearchEngine
}

//搜索信息
func (this *searchIndex) Search(input ...Field) (string, []TargetObject) {
	//生成索引搜索请求号
	requestkey := getSearchRequestUuid(this.name)

	//搜索索引
	var searchkeys []string
	var tmpkeys []string
	for _, f := range input {
		//处理并集
		v := f.Value
		arrays := strings.Split(v, "|")
		if len(arrays) > 1 {
			skey := requestkey + ":" + f.Key
			var subkeys []string
			for _, subkey := range arrays {
				subkeys = append(subkeys, this.buildTheKeyByItem(f.Key, subkey))
			}

			//取并集，将结果存入临时的key中，用于后续取交集用
			this.engine.client.SUnionStore(skey, subkeys...)
			tmpkeys = append(tmpkeys, skey) //放入临时组中用于使用后删除
			searchkeys = append(searchkeys, skey)
		} else {
			searchkeys = append(searchkeys, this.buildTheKey(f))
		}

	}
	defer this.deleteTempkeys(tmpkeys) //删除临时创建的key
	//取交集
	result := this.engine.client.SInter(searchkeys...)
	targetkeys, err := result.Result()
	if err != nil {
		this.engine.logger.Error("inter key error!%s", err.Error())
		return "", nil
	}

	//取出第一页的key
	size := len(targetkeys)
	if size > 0 {
		//生成request
		this.engine.createRequst(requestkey, targetkeys...)

		var query []string
		if size > int(this.engine.pageSize) {
			//写入到列表中
			query = targetkeys[0:this.engine.pageSize]
		} else {
			query = targetkeys
		}

		//根据最后的id，从data中取出第一页的元素的元素
		return requestkey, this.engine.fetch(this.name, query...)
	}

	return "", nil
}

//删除临时创建的key
func (this *searchIndex) deleteTempkeys(keys []string) {
	if keys != nil && len(keys) > 0 {
		this.engine.client.Del(keys...)
	}
}

//刷新索引，加载信息到存储中
func (this *searchIndex) LoadObject(obj *SourceObject) {
	data, _ := json.Marshal(obj)
	key:=obj.Id
	//如果没有指明对象id，则使用md5作为数据的唯一标识
	if key=="" {
		key= getMd5str(string(data))
	}

	//1、放入数据到目标集合中
	this.engine.client.HSet(this.name, key, string(data))

	//2、根据field存储到各个对应的索引中
	for k, v := range obj.Fields {
		this.engine.client.SAdd(this.buildTheKeyByItem(k, v), key)
	}

}

//删除数据：删除数据及索引存的值
func (this *searchIndex)RemoveObject(obj *SourceObject) {
	data, _ := json.Marshal(obj)
	key:=obj.Id
	if key=="" {
		key=getMd5str(string(data))
	}
	//删除数据
	this.engine.client.HDel(this.name,key)
	//删除索引里的记录
	for k,v:=range obj.Fields {
		this.engine.client.SRem(this.buildTheKeyByItem(k, v), key)
	}

}

func (this *searchIndex) buildTheKey(f Field) string {
	return this.buildTheKeyByItem(f.Key, f.Value)
}

func (this *searchIndex) buildTheKeyByItem(key, value string) string {
	return fmt.Sprintf("%s_%s_%s", this.name, key, value)
}

func getMd5str(value string) string {
	data := []byte(value)
	has := md5.Sum(data)
	md5str1 := fmt.Sprintf("%x", has) //将[]byte转成16进制

	return md5str1

}

func getSearchRequestUuid(prefix string) string {
	return fmt.Sprintf("%s:%d", prefix, time.Now().UnixNano())
}
<commit_msg>新增从增加使用索引做数组<commit_after>package service

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/aosfather/bingo"
	"github.com/aosfather/bingo/utils"
	"github.com/go-redis/redis"
)

/**
  搜索实现
  通过倒排实现关键信息的实现
  规则：
   1、原始内容使用hashmap存储，对象【ID，Content】，键值 indexname，二级key根据md5 对象转json字符串
   2、针对原始内容带的标签，key value，生成set，名称为 indexname_key_value的形式，set内容放 通过内容md5出来的键值
   3、搜索的时候，根据传递的搜索条件 key，value 数组，对找到的set（形如indexname_key_value ），实行找交集
   4、根据交集的结果二级key，并从indexname的hashmap中获取json内容

*/

type Field struct {
	Key   string `json:"key"`
	Value string `json:"value"`
}

type PageSearchResult struct {
	Id    string `json:"uuid"` //查询的请求id
	Index int64  `json:"page"` //页码
	Data  []TargetObject
}

type TargetObject struct {
	Id   string          `json:"id"`
	Data json.RawMessage `json:"data"`
}
type SourceObject struct {
	TargetObject
	Fields map[string][]string `json:"fields"`
}

type FieldType byte
const (
	FT_TEXT FieldType =11  //文本类型
	FT_NUMBER FieldType =9 //数字
	FT_ENUM   FieldType =8 //枚举
	FT_ID FieldType =7  //id唯一标识
	FT_DATE FieldType =6 //日期
)
//索引的元数据信息
type IndexMeta struct {
	Name string  //索引名称
	Fields []FieldMeta //字段
}

type FieldMeta struct {
	Name string //字段名称
	Type FieldType //类型

}

type SearchEngine struct {
	indexs   map[string]*searchIndex
	client   *redis.Client
	logger   utils.Log
	pageSize int64
	pageLife int64 //分钟
}

func (this *SearchEngine) Init(context *bingo.ApplicationContext) {
	fmt.Println("init .....")
	db, err := strconv.Atoi(context.GetPropertyFromConfig("service.search.db"))
	if err != nil {
		db = 0
	}

	size, err := strconv.Atoi(context.GetPropertyFromConfig("service.search.pagesize"))
	if err != nil {
		size = 20 //默认大小20条
	}
	this.pageSize = int64(size)

	life, err := strconv.Atoi(context.GetPropertyFromConfig("service.search.pagelife"))
	if err != nil {
		life = 10 //默认时间10分钟
	}
	this.pageLife = int64(life)

	this.client = redis.NewClient(&redis.Options{
		Addr:     context.GetPropertyFromConfig("service.search.redis"),
		Password: "", // no password set
		DB:       db,
	})
	fmt.Println(context.GetPropertyFromConfig("service.search.redis"))
	this.indexs = make(map[string]*searchIndex)
	this.logger = context.GetLog("bingo_search")
}

//创建索引
func (this *SearchEngine) CreateIndex(name string) *searchIndex {
	if name != "" {
		index := this.indexs[name]
		if index == nil {
			index = &searchIndex{name, this}
			this.indexs[name] = index
		}
		return index
	}

	return nil
}

//清除索引，将整个索引的数据摧毁
func (this *SearchEngine)FlushIndex(name string) {
	//1、删除存放的数据
	this.client.Del(name)

	//2、删除所有的索引key
	keys,err:=this.client.Keys(name+"_*").Result()
	if err!=nil {
		this.logger.Debug("get index keys error:%s",err.Error())
	}

	if keys!=nil && len(keys)>0 {
		this.client.Del(keys...)
	}
}

//加载和刷新数据
func (this *SearchEngine) LoadSource(name string, obj *SourceObject) {
	if name==""||obj==nil {
		return
	}

	index := this.CreateIndex(name)
	if index != nil {
		index.LoadObject(obj)
	}

}

//删除数据
func (this *SearchEngine)RemoveSource(name string, obj *SourceObject){
   if name!=""&&obj!=nil{
	   index := this.CreateIndex(name)
	   if index != nil {
		   index.RemoveObject(obj)
	   }
   }
}

//按页获取数据
func (this *SearchEngine) FetchByPage(request string, page int64) *PageSearchResult {
	if request != "" {
		//获取request的name
		index := strings.Index(request, ":")
		if index < 0 {
			this.logger.Error("pagerequest's index name not found !")
			return nil //找不到对应的索引类型
		}

		name := request[0:index]
		if page <= 0 {
			page = 1
		}
		startIndex := (page - 1) * this.pageSize //+ 1 //从1开始计数
		endIndex := page*this.pageSize - 1

		keys, err := this.client.LRange(request, startIndex, endIndex).Result()
		if err != nil || len(keys) == 0 {
			this.logger.Debug("no content by page!")
			return nil
		}
		go this.client.Expire(request, time.Duration(this.pageLife)*time.Minute) //更新重置失效时间
		return &PageSearchResult{request, page, this.fetch(name, keys...)}
	}

	return nil
}

func (this *SearchEngine) createRequst(key string, keys ...string) {
	//    key:= getSearchRequestUuid(name)
	var datas []interface{}

	for _, v := range keys {
		datas = append(datas, v)
	}

	this.client.LPush(key, datas...)
	this.client.Expire(key, time.Duration(this.pageLife)*time.Minute) //指定x分钟后失效
	//	return key
}

//获取内容
func (this *SearchEngine) fetch(name string, keys ...string) []TargetObject {
	datas, err1 := this.client.HMGet(name, keys...).Result()
	if err1 == nil && len(datas) > 0 {

		var targets []TargetObject

		for _, v := range datas {
			if v != nil {
				t := TargetObject{}
				json.Unmarshal([]byte(fmt.Sprintf("%v", v)), &t)
				targets = append(targets, t)
			}
		}

		return targets

	} else {
		this.logger.Error("get data by index error!%s", err1.Error())
	}

	return nil
}

func (this *SearchEngine) Search(name string, input ...Field) *PageSearchResult {
	if name != "" {
		index := this.indexs[name]
		if index != nil {
			r, data := index.Search(input...)
			return &PageSearchResult{r, 1, data}

		}
		this.logger.Info("not found index %s", name)
	}

	return nil
}

type searchIndex struct {
	name   string
	engine *SearchEngine
}

//搜索信息
func (this *searchIndex) Search(input ...Field) (string, []TargetObject) {
	//生成索引搜索请求号
	requestkey := getSearchRequestUuid(this.name)

	//搜索索引
	var searchkeys []string
	var tmpkeys []string
	for _, f := range input {
		//处理并集
		v := f.Value
		arrays := strings.Split(v, "|")
		if len(arrays) > 1 {
			skey := requestkey + ":" + f.Key
			var subkeys []string
			for _, subkey := range arrays {
				subkeys = append(subkeys, this.buildTheKeyByItem(f.Key, subkey))
			}

			//取并集，将结果存入临时的key中，用于后续取交集用
			this.engine.client.SUnionStore(skey, subkeys...)
			tmpkeys = append(tmpkeys, skey) //放入临时组中用于使用后删除
			searchkeys = append(searchkeys, skey)
		} else {
			searchkeys = append(searchkeys, this.buildTheKey(f))
		}

	}
	defer this.deleteTempkeys(tmpkeys) //删除临时创建的key
	//取交集
	result := this.engine.client.SInter(searchkeys...)
	targetkeys, err := result.Result()
	if err != nil {
		this.engine.logger.Error("inter key error!%s", err.Error())
		return "", nil
	}

	//取出第一页的key
	size := len(targetkeys)
	if size > 0 {
		//生成request
		this.engine.createRequst(requestkey, targetkeys...)

		var query []string
		if size > int(this.engine.pageSize) {
			//写入到列表中
			query = targetkeys[0:this.engine.pageSize]
		} else {
			query = targetkeys
		}

		//根据最后的id，从data中取出第一页的元素的元素
		return requestkey, this.engine.fetch(this.name, query...)
	}

	return "", nil
}

//删除临时创建的key
func (this *searchIndex) deleteTempkeys(keys []string) {
	if keys != nil && len(keys) > 0 {
		this.engine.client.Del(keys...)
	}
}

//刷新索引，加载信息到存储中
func (this *searchIndex) LoadObject(obj *SourceObject) {
	data, _ := json.Marshal(obj)
	key:=obj.Id
	//如果没有指明对象id，则使用md5作为数据的唯一标识
	if key=="" {
		key= getMd5str(string(data))
	}

	//1、放入数据到目标集合中
	this.engine.client.HSet(this.name, key, string(data))

	//2、根据field存储到各个对应的索引中
	for k, v := range obj.Fields {
		for _,vkey:=range v{
			this.engine.client.SAdd(this.buildTheKeyByItem(k, vkey), key)
		}

	}

}

//删除数据：删除数据及索引存的值
func (this *searchIndex)RemoveObject(obj *SourceObject) {
	data, _ := json.Marshal(obj)
	key:=obj.Id
	if key=="" {
		key=getMd5str(string(data))
	}
	//删除数据
	this.engine.client.HDel(this.name,key)
	//删除索引里的记录
	for k,v:=range obj.Fields {
		for _,vkey:=range v {
			this.engine.client.SRem(this.buildTheKeyByItem(k, vkey), key)
		}
	}

}

func (this *searchIndex) buildTheKey(f Field) string {
	return this.buildTheKeyByItem(f.Key, f.Value)
}

func (this *searchIndex) buildTheKeyByItem(key, value string) string {
	return fmt.Sprintf("%s_%s_%s", this.name, key, value)
}

func getMd5str(value string) string {
	data := []byte(value)
	has := md5.Sum(data)
	md5str1 := fmt.Sprintf("%x", has) //将[]byte转成16进制

	return md5str1

}

func getSearchRequestUuid(prefix string) string {
	return fmt.Sprintf("%s:%d", prefix, time.Now().UnixNano())
}
<|endoftext|>


<commit_before>// Package nlp enables natural language processing through functions around
// tokenization and stemming.
package nlp

import (
	"database/sql/driver"
	"encoding/csv"
	"errors"
	"regexp"
	"strings"

	"github.com/dchest/stemmer/porter2"
	"github.com/itsabot/abot/core/log"
)

// StructuredInput is generated by Ava and sent to plugins as a helper tool.
// Additional fields should be added, covering Times, Places, etc. to
// make plugin development even easier. Note that right now People is unused.
type StructuredInput struct {
	Commands StringSlice
	Objects  StringSlice

	// TODO
	// People   StringSlice
	// Places   StringSlice
	// Times    []time.Time
}

// SIT is a Structured Input Type. It corresponds to either a Command or an
// Object with additional Structured Input Types to be added later.
type SIT int

// TokenizeSentence returns a sentence broken into tokens. Tokens are individual
// words as well as punctuation. For example, "Hi! How are you?" becomes
// []string{"Hi", "!", "How", "are", "you", "?"}
func TokenizeSentence(sent string) []string {
	tokens := []string{}
	for _, w := range strings.Fields(sent) {
		found := []int{}
		for i, r := range w {
			switch r {
			case '\'', '"', ',', '.', ':', ';', '!', '?':
				found = append(found, i)
			}
		}
		if len(found) == 0 {
			tokens = append(tokens, w)
			continue
		}
		for i, j := range found {
			if j > 0 {
				tokens = append(tokens, w[:j])
			}
			tokens = append(tokens, string(w[j]))
			if i+1 == len(found) {
				tokens = append(tokens, w[j+1:])
			}
		}
	}
	log.Debug("found tokens", tokens)
	return tokens
}

// StemTokens returns the porter2 (snowball) stems for each token passed into
// it.
func StemTokens(tokens []string) []string {
	eng := porter2.Stemmer
	stems := []string{}
	for _, w := range tokens {
		if len(w) == 1 {
			switch w {
			case "'", "\"", ",", ".", ":", ";", "!", "?":
				continue
			}
		}
		w = strings.ToLower(w)
		stems = append(stems, eng.Stem(w))
	}
	return stems
}

// StringSlice replaces []string, adding custom sql support for arrays in lieu
// of pq.
type StringSlice []string

// QuoteEscapeRegex replaces escaped quotes except if it is preceded by a
// literal backslash, e.g. "\\" should translate to a quoted element whose value
// is \
var QuoteEscapeRegex = regexp.MustCompile(`([^\\]([\\]{2})*)\\"`)

// Scan converts to a slice of strings. See:
// http://www.postgresql.org/docs/9.1/static/arrays.html#ARRAYS-IO
func (s *StringSlice) Scan(src interface{}) error {
	asBytes, ok := src.([]byte)
	if !ok {
		return error(errors.New("scan source was not []bytes"))
	}
	str := string(asBytes)
	str = QuoteEscapeRegex.ReplaceAllString(str, `$1""`)
	str = strings.Replace(str, `\\`, `\`, -1)
	str = str[1 : len(str)-1]
	csvReader := csv.NewReader(strings.NewReader(str))
	slice, err := csvReader.Read()
	if err != nil && err.Error() != "EOF" {
		return err
	}
	*s = StringSlice(slice)
	return nil
}

// Value converts to a slice of strings. See:
// http://www.postgresql.org/docs/9.1/static/arrays.html#ARRAYS-IO
func (s StringSlice) Value() (driver.Value, error) {
	// string escapes.
	// \ => \\\
	// " => \"
	for i, elem := range s {
		s[i] = `"` + strings.Replace(strings.Replace(elem, `\`, `\\\`, -1), `"`, `\"`, -1) + `"`
	}
	return "{" + strings.Join(s, ",") + "}", nil
}

// Last safely returns the last item in a StringSlice, which is most often the
// target of a pronoun, e.g. (In "Where is that?", "that" will most often refer
// to the last Object named in the previous sentence.
func (s StringSlice) Last() string {
	if len(s) == 0 {
		return ""
	}
	return s[len(s)-1]
}

// String converts a StringSlice into a string with each word separated by
// spaces.
func (s StringSlice) String() string {
	if len(s) == 0 {
		return ""
	}
	var ss string
	for _, w := range s {
		ss += " " + w
	}
	return ss[1:]
}

// StringSlice converts a StringSlice into a []string.
func (s StringSlice) StringSlice() []string {
	ss := []string{}
	for _, tmp := range s {
		if len(tmp) <= 2 {
			continue
		}
		ss = append(ss, tmp)
	}
	return ss
}

// Map converts a StringSlice into a map to check quickly if words exist within
// it.
func (s StringSlice) Map() map[string]struct{} {
	m := map[string]struct{}{}
	for _, w := range s {
		m[w] = struct{}{}
	}
	return m
}

/*
// TODO with addContext
const (
	CommandI SIT = iota + 1
	PersonI
	ObjectI
)

// Pronouns converts pronouns to the type of object it represents. This will be
// useful for adding context into user messages. For example, when a user says,
// "buy that", Ava should know "that" refers to an Object and is most likely a
// reference to the most recent object discussed.
var Pronouns map[string]SIT = map[string]SIT{
	"me":   PersonI,
	"us":   PersonI,
	"you":  PersonI,
	"him":  PersonI,
	"her":  PersonI,
	"them": PersonI,
	"it":   ObjectI,
	"that": ObjectI,

	// Ultimately Place and Time would be nice-to-have in a structured
	// input, but they don't outweigh the cost of training a full NER on
	// each new plugin at this point. Additional thought should be given as
	// to how this can be enabled more simply than requiring training an ML
	// plugin.
	// "there": PlaceI,
	// "then":  TimeI,
}
*/
<commit_msg>Fix tokenization bugs<commit_after>// Package nlp enables natural language processing through functions around
// tokenization and stemming.
package nlp

import (
	"database/sql/driver"
	"encoding/csv"
	"errors"
	"regexp"
	"strings"

	"github.com/dchest/stemmer/porter2"
	"github.com/itsabot/abot/core/log"
)

// StructuredInput is generated by Ava and sent to plugins as a helper tool.
// Additional fields should be added, covering Times, Places, etc. to
// make plugin development even easier. Note that right now People is unused.
type StructuredInput struct {
	Commands StringSlice
	Objects  StringSlice

	// TODO
	// People   StringSlice
	// Places   StringSlice
	// Times    []time.Time
}

// SIT is a Structured Input Type. It corresponds to either a Command or an
// Object with additional Structured Input Types to be added later.
type SIT int

// TokenizeSentence returns a sentence broken into tokens. Tokens are individual
// words as well as punctuation. For example, "Hi! How are you?" becomes
// []string{"Hi", "!", "How", "are", "you", "?"}
func TokenizeSentence(sent string) []string {
	tokens := []string{}
	for _, w := range strings.Fields(sent) {
		found := []int{}
		for i, r := range w {
			switch r {
			case '\'', '"', ':', ';', '!', '?':
				found = append(found, i)

			// Handle case of currencies and fractional percents.
			case '.', ',':
				if i+1 < len(w) {
					switch w[i+1] {
					case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
						continue
					}
				}
				found = append(found, i)
				i++
			}
		}
		if len(found) == 0 {
			tokens = append(tokens, w)
			continue
		}
		for i, j := range found {
			// If the token marker is not the first character in the
			// sentence, then include all characters leading up to
			// the prior found token.
			if j > 0 {
				if i == 0 {
					tokens = append(tokens, w[:j])
				} else if i-1 < len(found) {
					// Handle case where multiple tokens are
					// found in the same word.
					tokens = append(tokens, w[found[i-1]+1:j])
				}
			}

			// Append the token marker itself
			tokens = append(tokens, string(w[j]))

			// If we're on the last token marker, append all
			// remaining parts of the word.
			if i+1 == len(found) {
				tokens = append(tokens, w[j+1:])
			}
		}
	}
	log.Debug("found tokens", tokens)
	return tokens
}

// StemTokens returns the porter2 (snowball) stems for each token passed into
// it.
func StemTokens(tokens []string) []string {
	eng := porter2.Stemmer
	stems := []string{}
	for _, w := range tokens {
		if len(w) == 1 {
			switch w {
			case "'", "\"", ",", ".", ":", ";", "!", "?":
				continue
			}
		}
		w = strings.ToLower(w)
		stems = append(stems, eng.Stem(w))
	}
	return stems
}

// StringSlice replaces []string, adding custom sql support for arrays in lieu
// of pq.
type StringSlice []string

// QuoteEscapeRegex replaces escaped quotes except if it is preceded by a
// literal backslash, e.g. "\\" should translate to a quoted element whose value
// is \
var QuoteEscapeRegex = regexp.MustCompile(`([^\\]([\\]{2})*)\\"`)

// Scan converts to a slice of strings. See:
// http://www.postgresql.org/docs/9.1/static/arrays.html#ARRAYS-IO
func (s *StringSlice) Scan(src interface{}) error {
	asBytes, ok := src.([]byte)
	if !ok {
		return error(errors.New("scan source was not []bytes"))
	}
	str := string(asBytes)
	str = QuoteEscapeRegex.ReplaceAllString(str, `$1""`)
	str = strings.Replace(str, `\\`, `\`, -1)
	str = str[1 : len(str)-1]
	csvReader := csv.NewReader(strings.NewReader(str))
	slice, err := csvReader.Read()
	if err != nil && err.Error() != "EOF" {
		return err
	}
	*s = StringSlice(slice)
	return nil
}

// Value converts to a slice of strings. See:
// http://www.postgresql.org/docs/9.1/static/arrays.html#ARRAYS-IO
func (s StringSlice) Value() (driver.Value, error) {
	// string escapes.
	// \ => \\\
	// " => \"
	for i, elem := range s {
		s[i] = `"` + strings.Replace(strings.Replace(elem, `\`, `\\\`, -1), `"`, `\"`, -1) + `"`
	}
	return "{" + strings.Join(s, ",") + "}", nil
}

// Last safely returns the last item in a StringSlice, which is most often the
// target of a pronoun, e.g. (In "Where is that?", "that" will most often refer
// to the last Object named in the previous sentence.
func (s StringSlice) Last() string {
	if len(s) == 0 {
		return ""
	}
	return s[len(s)-1]
}

// String converts a StringSlice into a string with each word separated by
// spaces.
func (s StringSlice) String() string {
	if len(s) == 0 {
		return ""
	}
	var ss string
	for _, w := range s {
		ss += " " + w
	}
	return ss[1:]
}

// StringSlice converts a StringSlice into a []string.
func (s StringSlice) StringSlice() []string {
	ss := []string{}
	for _, tmp := range s {
		if len(tmp) <= 2 {
			continue
		}
		ss = append(ss, tmp)
	}
	return ss
}

// Map converts a StringSlice into a map to check quickly if words exist within
// it.
func (s StringSlice) Map() map[string]struct{} {
	m := map[string]struct{}{}
	for _, w := range s {
		m[w] = struct{}{}
	}
	return m
}

/*
// TODO with addContext
const (
	CommandI SIT = iota + 1
	PersonI
	ObjectI
)

// Pronouns converts pronouns to the type of object it represents. This will be
// useful for adding context into user messages. For example, when a user says,
// "buy that", Ava should know "that" refers to an Object and is most likely a
// reference to the most recent object discussed.
var Pronouns map[string]SIT = map[string]SIT{
	"me":   PersonI,
	"us":   PersonI,
	"you":  PersonI,
	"him":  PersonI,
	"her":  PersonI,
	"them": PersonI,
	"it":   ObjectI,
	"that": ObjectI,

	// Ultimately Place and Time would be nice-to-have in a structured
	// input, but they don't outweigh the cost of training a full NER on
	// each new plugin at this point. Additional thought should be given as
	// to how this can be enabled more simply than requiring training an ML
	// plugin.
	// "there": PlaceI,
	// "then":  TimeI,
}
*/
<|endoftext|>


<commit_before>// Copyright 2017 The nats-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package framework

import (
	"bufio"
	"context"
	"fmt"
	"io"
	"strconv"
	"strings"
	"time"

	log "github.com/sirupsen/logrus"
	appsv1 "k8s.io/api/apps/v1"
	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/kubernetes"

	natsclient "github.com/nats-io/nats-operator/pkg/client/clientset/versioned"
	"github.com/nats-io/nats-operator/pkg/constants"
	"github.com/nats-io/nats-operator/pkg/features"
	kubernetesutil "github.com/nats-io/nats-operator/pkg/util/kubernetes"
)

const (
	// natsOperatorDeploymentName is the name of the nats-operator deployment.
	natsOperatorDeploymentName = "nats-operator"

	// natsOperatorPodName is the name of the nats-operator pod.
	natsOperatorPodName = "nats-operator"

	// natsOperatorE2ePodName is the name of the nats-operator-e2e pod.
	natsOperatorE2ePodName = "nats-operator-e2e"

	// podReadinessTimeout is the maximum amount of time we wait
	// for the nats-operator / nats-operator-e2e pods to be
	// running and ready.
	podReadinessTimeout = 5 * time.Minute
)

// ClusterFeature represents a feature that can be enabled or disabled
// on the target Kubernetes cluster.
type ClusterFeature string

const (
	// TokenRequest represents the "TokenRequest" feature.
	TokenRequest = ClusterFeature("TokenRequest")

	// ShareProcessNamespace represents the "ShareProcessNamespace" feature.
	ShareProcessNamespace = ClusterFeature("ShareProcessNamespace")
)

// Framework encapsulates the configuration for the current run, and
// provides helper methods to be used during testing.
type Framework struct {
	// ClusterFeatures is a map indicating whether specific
	// cluster features have been detected in the target cluster.
	ClusterFeatures map[ClusterFeature]bool

	// FeatureMap is the map containing features and their status for the current instance of the end-to-end test suite.
	FeatureMap features.FeatureMap

	// KubeClient is an interface to the Kubernetes base APIs.
	KubeClient kubernetes.Interface

	// Namespace is the namespace in which we are running.
	Namespace string

	// NatsClient is an interface to the nats.io/v1alpha2 API.
	NatsClient natsclient.Interface
}

// New returns a new instance of the testing framework.
func New(featureMap features.FeatureMap, kubeconfig, namespace string) *Framework {
	// Assume that all features are disabled until we do feature detection.
	cf := map[ClusterFeature]bool{
		ShareProcessNamespace: false,
		TokenRequest:          false,
	}
	// Override the namespace if nats-operator is deployed in the cluster-scoped mode.
	if featureMap.IsEnabled(features.ClusterScoped) {
		namespace = constants.KubernetesNamespaceNatsIO
	}
	config := kubernetesutil.MustNewKubeConfig(kubeconfig)
	kubeClient := kubernetesutil.MustNewKubeClientFromConfig(config)
	natsClient := kubernetesutil.MustNewNatsClientFromConfig(config)
	return &Framework{
		ClusterFeatures: cf,
		FeatureMap:      featureMap,
		KubeClient:      kubeClient,
		Namespace:       namespace,
		NatsClient:      natsClient,
	}
}

// Cleanup deletes the nats-operator deployment and the nats-operator-e2e pod, ignoring errors.
func (f *Framework) Cleanup() {
	if err := f.KubeClient.CoreV1().Pods(f.Namespace).Delete(natsOperatorE2ePodName, &metav1.DeleteOptions{}); err != nil {
		log.Warnf("failed to delete the %q pod: %v", natsOperatorE2ePodName, err)
	}
}

// FeatureDetect performs feature detection on the target Kubernetes cluster.
func (f *Framework) FeatureDetect() {
	v, err := f.KubeClient.Discovery().ServerVersion()
	if err != nil {
		return
	}
	major, err := strconv.Atoi(strings.TrimSuffix(v.Major, "+"))
	if err != nil {
		return
	}
	minor, err := strconv.Atoi(strings.TrimSuffix(v.Minor, "+"))
	if err != nil {
		return
	}
	// The features we want to detect can only be expected in 1.12+ clusters.
	if major == 0 || major == 1 && minor < 12 {
		return
	}

	// Kubernetes 1.12 has support for PID namespace sharing
	// enabled by default, so no more detection is necessary.
	f.ClusterFeatures[ShareProcessNamespace] = true

	// Detect whether the TokenRequest API is active by performing
	// a GET request to the "/token" subresource of the "default"
	// service account.
	if _, err := f.KubeClient.CoreV1().RESTClient().Get().Resource("serviceaccounts").Namespace(f.Namespace).Name("default").SubResource("token").DoRaw(); err != nil {
		if errors.IsMethodNotSupported(err) {
			// We've got a "405 METHOD NOT ALLOWED" response instead of a "404 NOT FOUND".
			// This means that the "/token" subresource is
			// indeed enabled, and it is enough to
			// conclude that the feature is supported in
			// the current cluster.
			f.ClusterFeatures[TokenRequest] = true
		}
	}
}

// WaitForNatsOperator waits for the nats-operator deployment to have at least one available replica.
func (f *Framework) WaitForNatsOperator() error {
	// Create a "fake" pod object containing the expected
	// namespace and name, as WaitUntilPodReady expects a pod
	// instance.
	ctx, fn := context.WithTimeout(context.Background(), podReadinessTimeout)
	defer fn()
	return kubernetesutil.WaitUntilDeploymentCondition(ctx, f.KubeClient, f.Namespace, natsOperatorDeploymentName, func(event watch.Event) (bool, error) {
		switch event.Type {
		case watch.Error:
			return false, fmt.Errorf("got event of type error: %+v", event.Object)
		case watch.Deleted:
			return false, fmt.Errorf("deployment \"%s/%s\" has been deleted", f.Namespace, natsOperatorDeploymentName)
		default:
			deployment := event.Object.(*appsv1.Deployment)
			return deployment.Status.AvailableReplicas >= 1, nil
		}
	})
}

// WaitForNatsOperatorE2ePodTermination waits for the nats-operator
// pod to be running and ready.
// It then starts streaming logs and returns the pod's exit code, or
// an error if any error was found during the process.
func (f *Framework) WaitForNatsOperatorE2ePodTermination() (int, error) {
	// Create a "fake" pod object containing the expected
	// namespace and name, as WaitUntilPodReady expects a pod
	// instance.
	ctx, fn := context.WithTimeout(context.Background(), podReadinessTimeout)
	defer fn()
	err := kubernetesutil.WaitUntilPodReady(ctx, f.KubeClient.CoreV1(), &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: f.Namespace,
			Name:      natsOperatorE2ePodName,
		},
	})
	if err != nil {
		return -1, err
	}

	// Start streaming logs for the nats-operator-e2e until we
	// receive io.EOF.
	req := f.KubeClient.CoreV1().Pods(f.Namespace).GetLogs(natsOperatorE2ePodName, &v1.PodLogOptions{
		Follow: true,
	})
	r, err := req.Stream()
	if err != nil {
		return -1, err
	}
	defer r.Close()
	b := bufio.NewReader(r)
	for {
		// Read a single line from the logs, and output it.
		l, err := b.ReadString('\n')
		if err != nil && err != io.EOF {
			return -1, err
		}
		if err == io.EOF {
			break
		}
		fmt.Print(l)
	}

	// Grab the first (and single) container's exit code so we can
	// use it as our own exit code.
	pod, err := f.KubeClient.CoreV1().Pods(f.Namespace).Get(natsOperatorE2ePodName, metav1.GetOptions{})
	if err != nil {
		return -1, err
	}
	return int(pod.Status.ContainerStatuses[0].State.Terminated.ExitCode), nil
}
<commit_msg>Wait for pod termination<commit_after>// Copyright 2017 The nats-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package framework

import (
	"bufio"
	"context"
	"fmt"
	"io"
	"strconv"
	"strings"
	"time"

	log "github.com/sirupsen/logrus"
	appsv1 "k8s.io/api/apps/v1"
	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/kubernetes"

	natsclient "github.com/nats-io/nats-operator/pkg/client/clientset/versioned"
	"github.com/nats-io/nats-operator/pkg/constants"
	"github.com/nats-io/nats-operator/pkg/features"
	kubernetesutil "github.com/nats-io/nats-operator/pkg/util/kubernetes"
)

const (
	// natsOperatorDeploymentName is the name of the nats-operator deployment.
	natsOperatorDeploymentName = "nats-operator"

	// natsOperatorPodName is the name of the nats-operator pod.
	natsOperatorPodName = "nats-operator"

	// natsOperatorE2ePodName is the name of the nats-operator-e2e pod.
	natsOperatorE2ePodName = "nats-operator-e2e"

	// podReadinessTimeout is the maximum amount of time we wait
	// for the nats-operator / nats-operator-e2e pods to be
	// running and ready.
	podReadinessTimeout = 5 * time.Minute
)

// ClusterFeature represents a feature that can be enabled or disabled
// on the target Kubernetes cluster.
type ClusterFeature string

const (
	// TokenRequest represents the "TokenRequest" feature.
	TokenRequest = ClusterFeature("TokenRequest")

	// ShareProcessNamespace represents the "ShareProcessNamespace" feature.
	ShareProcessNamespace = ClusterFeature("ShareProcessNamespace")
)

// Framework encapsulates the configuration for the current run, and
// provides helper methods to be used during testing.
type Framework struct {
	// ClusterFeatures is a map indicating whether specific
	// cluster features have been detected in the target cluster.
	ClusterFeatures map[ClusterFeature]bool

	// FeatureMap is the map containing features and their status for the current instance of the end-to-end test suite.
	FeatureMap features.FeatureMap

	// KubeClient is an interface to the Kubernetes base APIs.
	KubeClient kubernetes.Interface

	// Namespace is the namespace in which we are running.
	Namespace string

	// NatsClient is an interface to the nats.io/v1alpha2 API.
	NatsClient natsclient.Interface
}

// New returns a new instance of the testing framework.
func New(featureMap features.FeatureMap, kubeconfig, namespace string) *Framework {
	// Assume that all features are disabled until we do feature detection.
	cf := map[ClusterFeature]bool{
		ShareProcessNamespace: false,
		TokenRequest:          false,
	}
	// Override the namespace if nats-operator is deployed in the cluster-scoped mode.
	if featureMap.IsEnabled(features.ClusterScoped) {
		namespace = constants.KubernetesNamespaceNatsIO
	}
	config := kubernetesutil.MustNewKubeConfig(kubeconfig)
	kubeClient := kubernetesutil.MustNewKubeClientFromConfig(config)
	natsClient := kubernetesutil.MustNewNatsClientFromConfig(config)
	return &Framework{
		ClusterFeatures: cf,
		FeatureMap:      featureMap,
		KubeClient:      kubeClient,
		Namespace:       namespace,
		NatsClient:      natsClient,
	}
}

// Cleanup deletes the nats-operator deployment and the nats-operator-e2e pod, ignoring errors.
func (f *Framework) Cleanup() {
	if err := f.KubeClient.CoreV1().Pods(f.Namespace).Delete(natsOperatorE2ePodName, &metav1.DeleteOptions{}); err != nil {
		log.Warnf("failed to delete the %q pod: %v", natsOperatorE2ePodName, err)
	}
}

// FeatureDetect performs feature detection on the target Kubernetes cluster.
func (f *Framework) FeatureDetect() {
	v, err := f.KubeClient.Discovery().ServerVersion()
	if err != nil {
		return
	}
	major, err := strconv.Atoi(strings.TrimSuffix(v.Major, "+"))
	if err != nil {
		return
	}
	minor, err := strconv.Atoi(strings.TrimSuffix(v.Minor, "+"))
	if err != nil {
		return
	}
	// The features we want to detect can only be expected in 1.12+ clusters.
	if major == 0 || major == 1 && minor < 12 {
		return
	}

	// Kubernetes 1.12 has support for PID namespace sharing
	// enabled by default, so no more detection is necessary.
	f.ClusterFeatures[ShareProcessNamespace] = true

	// Detect whether the TokenRequest API is active by performing
	// a GET request to the "/token" subresource of the "default"
	// service account.
	if _, err := f.KubeClient.CoreV1().RESTClient().Get().Resource("serviceaccounts").Namespace(f.Namespace).Name("default").SubResource("token").DoRaw(); err != nil {
		if errors.IsMethodNotSupported(err) {
			// We've got a "405 METHOD NOT ALLOWED" response instead of a "404 NOT FOUND".
			// This means that the "/token" subresource is
			// indeed enabled, and it is enough to
			// conclude that the feature is supported in
			// the current cluster.
			f.ClusterFeatures[TokenRequest] = true
		}
	}
}

// WaitForNatsOperator waits for the nats-operator deployment to have at least one available replica.
func (f *Framework) WaitForNatsOperator() error {
	// Create a "fake" pod object containing the expected
	// namespace and name, as WaitUntilPodReady expects a pod
	// instance.
	ctx, fn := context.WithTimeout(context.Background(), podReadinessTimeout)
	defer fn()
	return kubernetesutil.WaitUntilDeploymentCondition(ctx, f.KubeClient, f.Namespace, natsOperatorDeploymentName, func(event watch.Event) (bool, error) {
		switch event.Type {
		case watch.Error:
			return false, fmt.Errorf("got event of type error: %+v", event.Object)
		case watch.Deleted:
			return false, fmt.Errorf("deployment \"%s/%s\" has been deleted", f.Namespace, natsOperatorDeploymentName)
		default:
			deployment := event.Object.(*appsv1.Deployment)
			return deployment.Status.AvailableReplicas >= 1, nil
		}
	})
}

// WaitForNatsOperatorE2ePodTermination waits for the nats-operator
// pod to be running and ready.
// It then starts streaming logs and returns the pod's exit code, or
// an error if any error was found during the process.
func (f *Framework) WaitForNatsOperatorE2ePodTermination() (int, error) {
	// Create a "fake" pod object containing the expected
	// namespace and name, as WaitUntilPodReady expects a pod
	// instance.
	ctx, fn := context.WithTimeout(context.Background(), podReadinessTimeout)
	defer fn()
	err := kubernetesutil.WaitUntilPodReady(ctx, f.KubeClient.CoreV1(), &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: f.Namespace,
			Name:      natsOperatorE2ePodName,
		},
	})
	if err != nil {
		return -1, err
	}

	// Start streaming logs for the nats-operator-e2e until we
	// receive io.EOF.
	req := f.KubeClient.CoreV1().Pods(f.Namespace).GetLogs(natsOperatorE2ePodName, &v1.PodLogOptions{
		Follow: true,
	})
	r, err := req.Stream()
	if err != nil {
		return -1, err
	}
	defer r.Close()
	b := bufio.NewReader(r)
	for {
		// Read a single line from the logs, and output it.
		l, err := b.ReadString('\n')
		if err != nil && err != io.EOF {
			return -1, err
		}
		if err == io.EOF {
			break
		}
		fmt.Print(l)
	}

	// Grab the first (and single) container's exit code so we can
	// use it as our own exit code. Wait for termination.
	ctx2, fn2 := context.WithTimeout(context.Background(), podReadinessTimeout)
	defer fn2()
	exitCode := int(-1)
	err = kubernetesutil.WaitUntilPodCondition(ctx2, f.KubeClient.CoreV1(), &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: f.Namespace,
			Name:      natsOperatorE2ePodName,
		},
	}, func(event watch.Event) (bool, error) {
		switch event.Type {
		case watch.Error:
			return false, fmt.Errorf("got event of type error: %+v", event.Object)
		case watch.Deleted:
			pod := event.Object.(*v1.Pod)
			return false, fmt.Errorf("pod %q has been deleted", kubernetesutil.ResourceKey(pod))
		default:
			pod := event.Object.(*v1.Pod)
			if t := pod.Status.ContainerStatuses[0].State.Terminated; t != nil {
				exitCode = int(t.ExitCode)
				return true, nil
			}
			return false, nil
		}
	})
	if err != nil {
		return -1, err
	}
	return exitCode, nil
}
<|endoftext|>


<commit_before>package activedir

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"strings"
	"time"

	"github.com/TomOnTime/utfutil"
	"github.com/miekg/dns/dnsutil"

	"github.com/StackExchange/dnscontrol/models"
	"github.com/StackExchange/dnscontrol/providers/diff"
)

const zoneDumpFilenamePrefix = "adzonedump"

type RecordConfigJson struct {
	Name string `json:"hostname"`
	Type string `json:"recordtype"`
	Data string `json:"recorddata"`
	TTL  uint32 `json:"timetolive"`
}

// GetDomainCorrections gets existing records, diffs them against existing, and returns corrections.
func (c *adProvider) GetDomainCorrections(dc *models.DomainConfig) ([]*models.Correction, error) {

	// Read foundRecords:
	foundRecords, err := c.getExistingRecords(dc.Name)
	if err != nil {
		return nil, fmt.Errorf("c.getExistingRecords(%v) failed: %v", dc.Name, err)
	}

	// Read expectedRecords:
	//expectedRecords := make([]*models.RecordConfig, len(dc.Records))
	expectedRecords := make([]diff.Record, len(dc.Records))
	for i, r := range dc.Records {
		if r.TTL == 0 {
			r.TTL = models.DefaultTTL
		}
		expectedRecords[i] = r
	}

	// Convert to []diff.Records and compare:
	foundDiffRecords := make([]diff.Record, 0, len(foundRecords))
	for _, rec := range foundRecords {
		foundDiffRecords = append(foundDiffRecords, rec)
	}

	_, create, _, mod := diff.IncrementalDiff(foundDiffRecords, expectedRecords)
	// NOTE(tlim): This provider does not delete records.  If
	// you need to delete a record, either delete it manually
	// or see providers/activedir/doc.md for implementation tips.

	// Generate changes.
	corrections := []*models.Correction{}
	for _, d := range create {
		corrections = append(corrections, c.createRec(dc.Name, d.Desired.(*models.RecordConfig))...)
	}
	for _, m := range mod {
		corrections = append(corrections, c.modifyRec(dc.Name, m))
	}
	return corrections, nil

}

// zoneDumpFilename returns the filename to use to write or read
// an activedirectory zone dump for a particular domain.
func zoneDumpFilename(domainname string) string {
	return zoneDumpFilenamePrefix + "." + domainname + ".json"
}

// readZoneDump reads a pre-existing zone dump from adzonedump.*.json.
func (c *adProvider) readZoneDump(domainname string) ([]byte, error) {
	// File not found is considered an error.
	dat, err := utfutil.ReadFile(zoneDumpFilename(domainname), utfutil.WINDOWS)
	if err != nil {
		fmt.Println("Powershell to generate zone dump:")
		fmt.Println(c.generatePowerShellZoneDump(domainname))
	}
	return dat, err
}

// powerShellLogCommand logs to flagPsLog that a PowerShell command is going to be run.
func powerShellLogCommand(command string) error {
	return logHelper(fmt.Sprintf("# %s\r\n%s\r\n", time.Now().UTC(), strings.TrimSpace(command)))
}

// powerShellLogOutput logs to flagPsLog that a PowerShell command is going to be run.
func powerShellLogOutput(s string) error {
	return logHelper(fmt.Sprintf("OUTPUT: START\r\n%s\r\nOUTPUT: END\r\n", s))
}

// powerShellLogErr logs that a PowerShell command had an error.
func powerShellLogErr(e error) error {
	err := logHelper(fmt.Sprintf("ERROR: %v\r\r", e)) //Log error to powershell.log
	if err != nil {
		return err //Bubble up error created in logHelper
	}
	return e //Bubble up original error
}

func logHelper(s string) error {
	logfile, err := os.OpenFile(*flagPsLog, os.O_APPEND|os.O_RDWR|os.O_CREATE, 0660)
	if err != nil {
		return fmt.Errorf("ERROR: Can not create/append to %#v: %v\n", *flagPsLog, err)
	}
	_, err = fmt.Fprintln(logfile, s)
	if err != nil {
		return fmt.Errorf("ERROR: Append to %#v failed: %v\n", *flagPsLog, err)
	}
	if logfile.Close() != nil {
		return fmt.Errorf("ERROR: Closing %#v failed: %v\n", *flagPsLog, err)
	}
	return nil
}

// powerShellRecord records that a PowerShell command should be executed later.
func powerShellRecord(command string) error {
	recordfile, err := os.OpenFile(*flagPsFuture, os.O_APPEND|os.O_RDWR|os.O_CREATE, 0660)
	if err != nil {
		return fmt.Errorf("ERROR: Can not create/append to %#v: %v\n", *flagPsFuture, err)
	}
	_, err = recordfile.WriteString(command)
	if err != nil {
		return fmt.Errorf("ERROR: Append to %#v failed: %v\n", *flagPsFuture, err)
	}
	return recordfile.Close()
}

func (c *adProvider) getExistingRecords(domainname string) ([]*models.RecordConfig, error) {
	//log.Printf("getExistingRecords(%s)\n", domainname)

	// Get the JSON either from adzonedump or by running a PowerShell script.
	data, err := c.getRecords(domainname)
	if err != nil {
		return nil, fmt.Errorf("getRecords failed on %#v: %v\n", domainname, err)
	}

	var recs []*RecordConfigJson
	err = json.Unmarshal(data, &recs)
	if err != nil {
		return nil, fmt.Errorf("json.Unmarshal failed on %#v: %v\n", domainname, err)
	}

	result := make([]*models.RecordConfig, 0, len(recs))
	for i := range recs {
		t, err := recs[i].unpackRecord(domainname)
		if err == nil {
			result = append(result, t)
		}
	}

	return result, nil
}

func (r *RecordConfigJson) unpackRecord(origin string) (*models.RecordConfig, error) {
	rc := models.RecordConfig{}

	rc.Name = strings.ToLower(r.Name)
	rc.NameFQDN = dnsutil.AddOrigin(rc.Name, origin)
	rc.Type = r.Type
	rc.TTL = r.TTL

	switch rc.Type {
	case "A":
		rc.Target = r.Data
	case "CNAME":
		rc.Target = strings.ToLower(r.Data)
	case "AAAA", "MX", "NAPTR", "NS", "SOA", "SRV":
		return nil, fmt.Errorf("Unimplemented: %v", r.Type)
	default:
		log.Fatalf("Unhandled models.RecordConfigJson type: %v (%v)\n", rc.Type, r)
	}

	return &rc, nil
}

// powerShellDump runs a PowerShell command to get a dump of all records in a DNS zone.
func (c *adProvider) generatePowerShellZoneDump(domainname string) string {
	cmd_txt := `@("REPLACE_WITH_ZONE") | %{
Get-DnsServerResourceRecord -ComputerName REPLACE_WITH_COMPUTER_NAME -ZoneName $_ | select hostname,recordtype,@{n="timestamp";e={$_.timestamp.tostring()}},@{n="timetolive";e={$_.timetolive.totalseconds}},@{n="recorddata";e={($_.recorddata.ipv4address,$_.recorddata.ipv6address,$_.recorddata.HostNameAlias,"other_record" -ne $null)[0]-as [string]}} | ConvertTo-Json > REPLACE_WITH_FILENAMEPREFIX.REPLACE_WITH_ZONE.json
}`
	cmd_txt = strings.Replace(cmd_txt, "REPLACE_WITH_ZONE", domainname, -1)
	cmd_txt = strings.Replace(cmd_txt, "REPLACE_WITH_COMPUTER_NAME", c.adServer, -1)
	cmd_txt = strings.Replace(cmd_txt, "REPLACE_WITH_FILENAMEPREFIX", zoneDumpFilenamePrefix, -1)

	return cmd_txt
}

// generatePowerShellCreate generates PowerShell commands to ADD a record.
func (c *adProvider) generatePowerShellCreate(domainname string, rec *models.RecordConfig) string {

	content := rec.Target

	text := "\r\n" // Skip a line.
	text += fmt.Sprintf("Add-DnsServerResourceRecord%s", rec.Type)
	text += fmt.Sprintf(` -ComputerName "%s"`, c.adServer)
	text += fmt.Sprintf(` -ZoneName "%s"`, domainname)
	text += fmt.Sprintf(` -Name "%s"`, rec.Name)
	switch rec.Type {
	case "CNAME":
		text += fmt.Sprintf(` -HostNameAlias "%s"`, content)
	case "A":
		text += fmt.Sprintf(` -IPv4Address "%s"`, content)
	case "NS":
		text = fmt.Sprintf("\r\n"+`echo "Skipping NS update (%v %v)"`+"\r\n", rec.Name, rec.Target)
	default:
		panic(fmt.Errorf("ERROR: generatePowerShellCreate() does not yet handle recType=%s recName=%#v content=%#v)\n", rec.Type, rec.Name, content))
	}
	text += "\r\n"

	return text
}

// generatePowerShellModify generates PowerShell commands to MODIFY a record.
func (c *adProvider) generatePowerShellModify(domainname, recName, recType, oldContent, newContent string, oldTTL, newTTL uint32) string {

	var queryField, queryContent string

	switch recType {
	case "A":
		queryField = "IPv4address"
		queryContent = `"` + oldContent + `"`
	case "CNAME":
		queryField = "HostNameAlias"
		queryContent = `"` + oldContent + `"`
	default:
		panic(fmt.Errorf("ERROR: generatePowerShellModify() does not yet handle recType=%s recName=%#v content=(%#v, %#v)\n", recType, recName, oldContent, newContent))
	}

	text := "\r\n" // Skip a line.
	text += fmt.Sprintf(`echo "MODIFY %s %s %s old=%s new=%s"`, recName, domainname, recType, oldContent, newContent)
	text += "\r\n"

	text += "$OldObj = Get-DnsServerResourceRecord"
	text += fmt.Sprintf(` -ComputerName "%s"`, c.adServer)
	text += fmt.Sprintf(` -ZoneName "%s"`, domainname)
	text += fmt.Sprintf(` -Name "%s"`, recName)
	text += fmt.Sprintf(` -RRType "%s"`, recType)
	text += fmt.Sprintf(" |  Where-Object {$_.RecordData.%s -eq %s -and $_.HostName -eq \"%s\"}", queryField, queryContent, recName)
	text += "\r\n"
	text += `if($OldObj.Length -ne $null){ throw "Error, multiple results for Get-DnsServerResourceRecord" }`
	text += "\r\n"

	text += "$NewObj = $OldObj.Clone()"
	text += "\r\n"

	if oldContent != newContent {
		text += fmt.Sprintf(`$NewObj.RecordData.%s = "%s"`, queryField, newContent)
		text += "\r\n"
	}

	if oldTTL != newTTL {
		text += fmt.Sprintf(`$NewObj.TimeToLive = New-TimeSpan -Seconds %d`, newTTL)
		text += "\r\n"
	}

	text += "Set-DnsServerResourceRecord"
	text += fmt.Sprintf(` -ComputerName "%s"`, c.adServer)
	text += fmt.Sprintf(` -ZoneName "%s"`, domainname)
	text += fmt.Sprintf(` -NewInputObject $NewObj -OldInputObject $OldObj`)
	text += "\r\n"

	return text
}

func (c *adProvider) createRec(domainname string, rec *models.RecordConfig) []*models.Correction {
	arr := []*models.Correction{
		{
			Msg: fmt.Sprintf("CREATE record: %s %s ttl(%d) %s", rec.Name, rec.Type, rec.TTL, rec.Target),
			F: func() error {
				return powerShellDoCommand(c.generatePowerShellCreate(domainname, rec))
			}},
	}
	return arr
}

func (c *adProvider) modifyRec(domainname string, m diff.Correlation) *models.Correction {

	old, rec := m.Existing.(*models.RecordConfig), m.Desired.(*models.RecordConfig)
	oldContent := old.GetContent()
	newContent := rec.GetContent()

	return &models.Correction{
		Msg: m.String(),
		F: func() error {
			return powerShellDoCommand(c.generatePowerShellModify(domainname, rec.Name, rec.Type, oldContent, newContent, old.TTL, rec.TTL))
		},
	}
}
<commit_msg>active directory delete command (#6)<commit_after>package activedir

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"strings"
	"time"

	"github.com/TomOnTime/utfutil"
	"github.com/miekg/dns/dnsutil"

	"github.com/StackExchange/dnscontrol/models"
	"github.com/StackExchange/dnscontrol/providers/diff"
)

const zoneDumpFilenamePrefix = "adzonedump"

type RecordConfigJson struct {
	Name string `json:"hostname"`
	Type string `json:"recordtype"`
	Data string `json:"recorddata"`
	TTL  uint32 `json:"timetolive"`
}

// GetDomainCorrections gets existing records, diffs them against existing, and returns corrections.
func (c *adProvider) GetDomainCorrections(dc *models.DomainConfig) ([]*models.Correction, error) {

	// Read foundRecords:
	foundRecords, err := c.getExistingRecords(dc.Name)
	if err != nil {
		return nil, fmt.Errorf("c.getExistingRecords(%v) failed: %v", dc.Name, err)
	}

	// Read expectedRecords:
	//expectedRecords := make([]*models.RecordConfig, len(dc.Records))
	expectedRecords := make([]diff.Record, len(dc.Records))
	for i, r := range dc.Records {
		if r.TTL == 0 {
			r.TTL = models.DefaultTTL
		}
		expectedRecords[i] = r
	}

	// Convert to []diff.Records and compare:
	foundDiffRecords := make([]diff.Record, 0, len(foundRecords))
	for _, rec := range foundRecords {
		foundDiffRecords = append(foundDiffRecords, rec)
	}

	_, creates, dels, modifications := diff.IncrementalDiff(foundDiffRecords, expectedRecords)
	// NOTE(tlim): This provider does not delete records.  If
	// you need to delete a record, either delete it manually
	// or see providers/activedir/doc.md for implementation tips.

	// Generate changes.
	corrections := []*models.Correction{}
	for _, del := range dels {
		if dc.KeepUnknown {
			break
		}
		corrections = append(corrections, c.deleteRec(dc.Name, del.Existing.(*models.RecordConfig)))
	}
	for _, cre := range creates {
		corrections = append(corrections, c.createRec(dc.Name, cre.Desired.(*models.RecordConfig))...)
	}
	for _, m := range modifications {
		corrections = append(corrections, c.modifyRec(dc.Name, m))
	}
	return corrections, nil

}

// zoneDumpFilename returns the filename to use to write or read
// an activedirectory zone dump for a particular domain.
func zoneDumpFilename(domainname string) string {
	return zoneDumpFilenamePrefix + "." + domainname + ".json"
}

// readZoneDump reads a pre-existing zone dump from adzonedump.*.json.
func (c *adProvider) readZoneDump(domainname string) ([]byte, error) {
	// File not found is considered an error.
	dat, err := utfutil.ReadFile(zoneDumpFilename(domainname), utfutil.WINDOWS)
	if err != nil {
		fmt.Println("Powershell to generate zone dump:")
		fmt.Println(c.generatePowerShellZoneDump(domainname))
	}
	return dat, err
}

// powerShellLogCommand logs to flagPsLog that a PowerShell command is going to be run.
func powerShellLogCommand(command string) error {
	return logHelper(fmt.Sprintf("# %s\r\n%s\r\n", time.Now().UTC(), strings.TrimSpace(command)))
}

// powerShellLogOutput logs to flagPsLog that a PowerShell command is going to be run.
func powerShellLogOutput(s string) error {
	return logHelper(fmt.Sprintf("OUTPUT: START\r\n%s\r\nOUTPUT: END\r\n", s))
}

// powerShellLogErr logs that a PowerShell command had an error.
func powerShellLogErr(e error) error {
	err := logHelper(fmt.Sprintf("ERROR: %v\r\r", e)) //Log error to powershell.log
	if err != nil {
		return err //Bubble up error created in logHelper
	}
	return e //Bubble up original error
}

func logHelper(s string) error {
	logfile, err := os.OpenFile(*flagPsLog, os.O_APPEND|os.O_RDWR|os.O_CREATE, 0660)
	if err != nil {
		return fmt.Errorf("ERROR: Can not create/append to %#v: %v\n", *flagPsLog, err)
	}
	_, err = fmt.Fprintln(logfile, s)
	if err != nil {
		return fmt.Errorf("ERROR: Append to %#v failed: %v\n", *flagPsLog, err)
	}
	if logfile.Close() != nil {
		return fmt.Errorf("ERROR: Closing %#v failed: %v\n", *flagPsLog, err)
	}
	return nil
}

// powerShellRecord records that a PowerShell command should be executed later.
func powerShellRecord(command string) error {
	recordfile, err := os.OpenFile(*flagPsFuture, os.O_APPEND|os.O_RDWR|os.O_CREATE, 0660)
	if err != nil {
		return fmt.Errorf("ERROR: Can not create/append to %#v: %v\n", *flagPsFuture, err)
	}
	_, err = recordfile.WriteString(command)
	if err != nil {
		return fmt.Errorf("ERROR: Append to %#v failed: %v\n", *flagPsFuture, err)
	}
	return recordfile.Close()
}

func (c *adProvider) getExistingRecords(domainname string) ([]*models.RecordConfig, error) {
	//log.Printf("getExistingRecords(%s)\n", domainname)

	// Get the JSON either from adzonedump or by running a PowerShell script.
	data, err := c.getRecords(domainname)
	if err != nil {
		return nil, fmt.Errorf("getRecords failed on %#v: %v\n", domainname, err)
	}

	var recs []*RecordConfigJson
	err = json.Unmarshal(data, &recs)
	if err != nil {
		return nil, fmt.Errorf("json.Unmarshal failed on %#v: %v\n", domainname, err)
	}

	result := make([]*models.RecordConfig, 0, len(recs))
	for i := range recs {
		t, err := recs[i].unpackRecord(domainname)
		if err == nil {
			result = append(result, t)
		}
	}

	return result, nil
}

func (r *RecordConfigJson) unpackRecord(origin string) (*models.RecordConfig, error) {
	rc := models.RecordConfig{}

	rc.Name = strings.ToLower(r.Name)
	rc.NameFQDN = dnsutil.AddOrigin(rc.Name, origin)
	rc.Type = r.Type
	rc.TTL = r.TTL

	switch rc.Type {
	case "A":
		rc.Target = r.Data
	case "CNAME":
		rc.Target = strings.ToLower(r.Data)
	case "AAAA", "MX", "NAPTR", "NS", "SOA", "SRV":
		return nil, fmt.Errorf("Unimplemented: %v", r.Type)
	default:
		log.Fatalf("Unhandled models.RecordConfigJson type: %v (%v)\n", rc.Type, r)
	}

	return &rc, nil
}

// powerShellDump runs a PowerShell command to get a dump of all records in a DNS zone.
func (c *adProvider) generatePowerShellZoneDump(domainname string) string {
	cmd_txt := `@("REPLACE_WITH_ZONE") | %{
Get-DnsServerResourceRecord -ComputerName REPLACE_WITH_COMPUTER_NAME -ZoneName $_ | select hostname,recordtype,@{n="timestamp";e={$_.timestamp.tostring()}},@{n="timetolive";e={$_.timetolive.totalseconds}},@{n="recorddata";e={($_.recorddata.ipv4address,$_.recorddata.ipv6address,$_.recorddata.HostNameAlias,"other_record" -ne $null)[0]-as [string]}} | ConvertTo-Json > REPLACE_WITH_FILENAMEPREFIX.REPLACE_WITH_ZONE.json
}`
	cmd_txt = strings.Replace(cmd_txt, "REPLACE_WITH_ZONE", domainname, -1)
	cmd_txt = strings.Replace(cmd_txt, "REPLACE_WITH_COMPUTER_NAME", c.adServer, -1)
	cmd_txt = strings.Replace(cmd_txt, "REPLACE_WITH_FILENAMEPREFIX", zoneDumpFilenamePrefix, -1)

	return cmd_txt
}

// generatePowerShellCreate generates PowerShell commands to ADD a record.
func (c *adProvider) generatePowerShellCreate(domainname string, rec *models.RecordConfig) string {

	content := rec.Target

	text := "\r\n" // Skip a line.
	text += fmt.Sprintf("Add-DnsServerResourceRecord%s", rec.Type)
	text += fmt.Sprintf(` -ComputerName "%s"`, c.adServer)
	text += fmt.Sprintf(` -ZoneName "%s"`, domainname)
	text += fmt.Sprintf(` -Name "%s"`, rec.Name)
	switch rec.Type {
	case "CNAME":
		text += fmt.Sprintf(` -HostNameAlias "%s"`, content)
	case "A":
		text += fmt.Sprintf(` -IPv4Address "%s"`, content)
	case "NS":
		text = fmt.Sprintf("\r\n"+`echo "Skipping NS update (%v %v)"`+"\r\n", rec.Name, rec.Target)
	default:
		panic(fmt.Errorf("ERROR: generatePowerShellCreate() does not yet handle recType=%s recName=%#v content=%#v)\n", rec.Type, rec.Name, content))
	}
	text += "\r\n"

	return text
}

// generatePowerShellModify generates PowerShell commands to MODIFY a record.
func (c *adProvider) generatePowerShellModify(domainname, recName, recType, oldContent, newContent string, oldTTL, newTTL uint32) string {

	var queryField, queryContent string

	switch recType {
	case "A":
		queryField = "IPv4address"
		queryContent = `"` + oldContent + `"`
	case "CNAME":
		queryField = "HostNameAlias"
		queryContent = `"` + oldContent + `"`
	default:
		panic(fmt.Errorf("ERROR: generatePowerShellModify() does not yet handle recType=%s recName=%#v content=(%#v, %#v)\n", recType, recName, oldContent, newContent))
	}

	text := "\r\n" // Skip a line.
	text += fmt.Sprintf(`echo "MODIFY %s %s %s old=%s new=%s"`, recName, domainname, recType, oldContent, newContent)
	text += "\r\n"

	text += "$OldObj = Get-DnsServerResourceRecord"
	text += fmt.Sprintf(` -ComputerName "%s"`, c.adServer)
	text += fmt.Sprintf(` -ZoneName "%s"`, domainname)
	text += fmt.Sprintf(` -Name "%s"`, recName)
	text += fmt.Sprintf(` -RRType "%s"`, recType)
	text += fmt.Sprintf(" |  Where-Object {$_.RecordData.%s -eq %s -and $_.HostName -eq \"%s\"}", queryField, queryContent, recName)
	text += "\r\n"
	text += `if($OldObj.Length -ne $null){ throw "Error, multiple results for Get-DnsServerResourceRecord" }`
	text += "\r\n"

	text += "$NewObj = $OldObj.Clone()"
	text += "\r\n"

	if oldContent != newContent {
		text += fmt.Sprintf(`$NewObj.RecordData.%s = "%s"`, queryField, newContent)
		text += "\r\n"
	}

	if oldTTL != newTTL {
		text += fmt.Sprintf(`$NewObj.TimeToLive = New-TimeSpan -Seconds %d`, newTTL)
		text += "\r\n"
	}

	text += "Set-DnsServerResourceRecord"
	text += fmt.Sprintf(` -ComputerName "%s"`, c.adServer)
	text += fmt.Sprintf(` -ZoneName "%s"`, domainname)
	text += fmt.Sprintf(` -NewInputObject $NewObj -OldInputObject $OldObj`)
	text += "\r\n"

	return text
}

func (c *adProvider) generatePowerShellDelete(domainname, recName, recType, content string) string {
	text := fmt.Sprintf(`echo "DELETE %s %s %s"`, recType, recName, content)
	text += "\r\n"
	text += `# Remove-DnsServerResourceRecord -Force -ComputerName "%s" -ZoneName "%s" -Name "%s" -RRType "%s" -RecordData "%s" -WhatIf`
	text += "\r\n"
	return fmt.Sprintf(text, c.adServer, domainname, recName, recType, content)
}

func (c *adProvider) createRec(domainname string, rec *models.RecordConfig) []*models.Correction {
	arr := []*models.Correction{
		{
			Msg: fmt.Sprintf("CREATE record: %s %s ttl(%d) %s", rec.Name, rec.Type, rec.TTL, rec.Target),
			F: func() error {
				return powerShellDoCommand(c.generatePowerShellCreate(domainname, rec))
			}},
	}
	return arr
}

func (c *adProvider) modifyRec(domainname string, m diff.Correlation) *models.Correction {

	old, rec := m.Existing.(*models.RecordConfig), m.Desired.(*models.RecordConfig)
	oldContent := old.GetContent()
	newContent := rec.GetContent()

	return &models.Correction{
		Msg: m.String(),
		F: func() error {
			return powerShellDoCommand(c.generatePowerShellModify(domainname, rec.Name, rec.Type, oldContent, newContent, old.TTL, rec.TTL))
		},
	}
}

func (c *adProvider) deleteRec(domainname string, rec *models.RecordConfig) *models.Correction {
	return &models.Correction{
		Msg: fmt.Sprintf("DELETE record: %s %s ttl(%d) %s", rec.Name, rec.Type, rec.TTL, rec.Target),
		F: func() error {
			return powerShellDoCommand(c.generatePowerShellDelete(domainname, rec.Name, rec.Type, rec.Target))
		},
	}
}
<|endoftext|>