<commit_before>package server

import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/pkg/stringid"
	"github.com/kubernetes-incubator/ocid/oci"
	"github.com/kubernetes-incubator/ocid/utils"
	"github.com/opencontainers/runtime-tools/generate"
	"golang.org/x/net/context"
	pb "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
)

type sandbox struct {
	id         string
	name       string
	logDir     string
	labels     map[string]string
	containers oci.Store
}

const (
	podInfraRootfs      = "/var/lib/ocid/graph/vfs/pause"
	podDefaultNamespace = "default"
)

func (s *sandbox) addContainer(c *oci.Container) {
	s.containers.Add(c.Name(), c)
}

func (s *sandbox) getContainer(name string) *oci.Container {
	return s.containers.Get(name)
}

func (s *sandbox) removeContainer(c *oci.Container) {
	s.containers.Delete(c.Name())
}

func (s *Server) generatePodIDandName(name, namespace string) (string, string, error) {
	var (
		err error
		id  = stringid.GenerateNonCryptoID()
	)
	if namespace == "" {
		namespace = podDefaultNamespace
	}
	if name, err = s.reservePodName(id, namespace+"-"+name); err != nil {
		return "", "", err
	}
	return id, name, err
}

// RunPodSandbox creates and runs a pod-level sandbox.
func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest) (*pb.RunPodSandboxResponse, error) {
	// process req.Name
	name := req.GetConfig().GetMetadata().GetName()
	if name == "" {
		return nil, fmt.Errorf("PodSandboxConfig.Name should not be empty")
	}

	namespace := req.GetConfig().GetMetadata().GetNamespace()

	var err error
	id, name, err := s.generatePodIDandName(name, namespace)
	if err != nil {
		return nil, err
	}
	podSandboxDir := filepath.Join(s.sandboxDir, id)
	if _, err = os.Stat(podSandboxDir); err == nil {
		return nil, fmt.Errorf("pod sandbox (%s) already exists", podSandboxDir)
	}

	if err = os.MkdirAll(podSandboxDir, 0755); err != nil {
		return nil, err
	}

	defer func() {
		if err != nil {
			if err2 := os.RemoveAll(podSandboxDir); err2 != nil {
				logrus.Warnf("couldn't cleanup podSandboxDir %s: %v", podSandboxDir, err2)
			}
		}
	}()

	// creates a spec Generator with the default spec.
	g := generate.New()

	podInfraRootfs := filepath.Join(s.root, "graph/vfs/pause")
	// setup defaults for the pod sandbox
	g.SetRootPath(filepath.Join(podInfraRootfs, "rootfs"))
	g.SetRootReadonly(true)
	g.SetProcessArgs([]string{"/pause"})

	// set hostname
	hostname := req.GetConfig().GetHostname()
	if hostname != "" {
		g.SetHostname(hostname)
	}

	// set log directory
	logDir := req.GetConfig().GetLogDirectory()
	if logDir == "" {
		logDir = fmt.Sprintf("/var/log/ocid/pods/%s", id)
	}

	// set DNS options
	dnsServers := req.GetConfig().GetDnsOptions().GetServers()
	dnsSearches := req.GetConfig().GetDnsOptions().GetSearches()
	resolvPath := fmt.Sprintf("%s/resolv.conf", podSandboxDir)
	err = parseDNSOptions(dnsServers, dnsSearches, resolvPath)
	if err != nil {
		err1 := removeFile(resolvPath)
		if err1 != nil {
			err = err1
			return nil, fmt.Errorf("%v; failed to remove %s: %v", err, resolvPath, err1)
		}
		return nil, err
	}

	g.AddBindMount(resolvPath, "/etc/resolv.conf", "ro")

	// add labels
	labels := req.GetConfig().GetLabels()
	labelsJSON, err := json.Marshal(labels)
	if err != nil {
		return nil, err
	}
	g.AddAnnotation("ocid/labels", string(labelsJSON))
	g.AddAnnotation("ocid/log_path", logDir)
	g.AddAnnotation("ocid/name", name)
	containerName := name + "-infra"
	g.AddAnnotation("ocid/container_name", containerName)
	s.addSandbox(&sandbox{
		id:         id,
		name:       name,
		logDir:     logDir,
		labels:     labels,
		containers: oci.NewMemoryStore(),
	})

	annotations := req.GetConfig().GetAnnotations()
	for k, v := range annotations {
		g.AddAnnotation(k, v)
	}

	// setup cgroup settings
	cgroupParent := req.GetConfig().GetLinux().GetCgroupParent()
	if cgroupParent != "" {
		g.SetLinuxCgroupsPath(cgroupParent)
	}

	// set up namespaces
	if req.GetConfig().GetLinux().GetNamespaceOptions().GetHostNetwork() {
		err = g.RemoveLinuxNamespace("network")
		if err != nil {
			return nil, err
		}
	}

	if req.GetConfig().GetLinux().GetNamespaceOptions().GetHostPid() {
		err = g.RemoveLinuxNamespace("pid")
		if err != nil {
			return nil, err
		}
	}

	if req.GetConfig().GetLinux().GetNamespaceOptions().GetHostIpc() {
		err = g.RemoveLinuxNamespace("ipc")
		if err != nil {
			return nil, err
		}
	}

	err = g.SaveToFile(filepath.Join(podSandboxDir, "config.json"))
	if err != nil {
		return nil, err
	}

	if _, err = os.Stat(podInfraRootfs); err != nil {
		if os.IsNotExist(err) {
			// TODO: Replace by rootfs creation API when it is ready
			if err = utils.CreateFakeRootfs(podInfraRootfs, "docker://kubernetes/pause"); err != nil {
				return nil, err
			}
		} else {
			return nil, err
		}
	}

	container, err := oci.NewContainer(containerName, podSandboxDir, podSandboxDir, labels, id, false)
	if err != nil {
		return nil, err
	}

	if err = s.runtime.CreateContainer(container); err != nil {
		return nil, err
	}

	if err = s.runtime.UpdateStatus(container); err != nil {
		return nil, err
	}

	// setup the network
	podNamespace := ""
	netnsPath, err := container.NetNsPath()
	if err != nil {
		return nil, err
	}
	if err = s.netPlugin.SetUpPod(netnsPath, podNamespace, id, containerName); err != nil {
		return nil, fmt.Errorf("failed to create network for container %s in sandbox %s: %v", containerName, id, err)
	}

	if err = s.runtime.StartContainer(container); err != nil {
		return nil, err
	}

	s.addContainer(container)

	if err = s.podIDIndex.Add(id); err != nil {
		return nil, err
	}

	if err = s.runtime.UpdateStatus(container); err != nil {
		return nil, err
	}

	return &pb.RunPodSandboxResponse{PodSandboxId: &id}, nil
}

// StopPodSandbox stops the sandbox. If there are any running containers in the
// sandbox, they should be force terminated.
func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxRequest) (*pb.StopPodSandboxResponse, error) {
	sbID := req.PodSandboxId
	if *sbID == "" {
		return nil, fmt.Errorf("PodSandboxId should not be empty")
	}
	sb := s.getSandbox(*sbID)
	if sb == nil {
		return nil, fmt.Errorf("specified sandbox not found: %s", *sbID)
	}

	podInfraContainer := sb.name + "-infra"
	for _, c := range sb.containers.List() {
		if podInfraContainer == c.Name() {
			podNamespace := ""
			netnsPath, err := c.NetNsPath()
			if err != nil {
				return nil, err
			}
			if err := s.netPlugin.TearDownPod(netnsPath, podNamespace, *sbID, podInfraContainer); err != nil {
				return nil, fmt.Errorf("failed to destroy network for container %s in sandbox %s: %v", c.Name(), *sbID, err)
			}
		}
		cStatus := s.runtime.ContainerStatus(c)
		if cStatus.Status != "stopped" {
			if err := s.runtime.StopContainer(c); err != nil {
				return nil, fmt.Errorf("failed to stop container %s in sandbox %s: %v", c.Name(), *sbID, err)
			}
		}
	}

	return &pb.StopPodSandboxResponse{}, nil
}

// RemovePodSandbox deletes the sandbox. If there are any running containers in the
// sandbox, they should be force deleted.
func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxRequest) (*pb.RemovePodSandboxResponse, error) {
	sbID := req.PodSandboxId
	if *sbID == "" {
		return nil, fmt.Errorf("PodSandboxId should not be empty")
	}
	sb := s.getSandbox(*sbID)
	if sb == nil {
		return nil, fmt.Errorf("specified sandbox not found: %s", *sbID)
	}

	podInfraContainer := sb.name + "-infra"

	// Delete all the containers in the sandbox
	for _, c := range sb.containers.List() {
		if err := s.runtime.DeleteContainer(c); err != nil {
			return nil, fmt.Errorf("failed to delete container %s in sandbox %s: %v", c.Name(), *sbID, err)
		}
		if podInfraContainer == c.Name() {
			continue
		}
		containerDir := filepath.Join(s.runtime.ContainerDir(), c.Name())
		if err := os.RemoveAll(containerDir); err != nil {
			return nil, fmt.Errorf("failed to remove container %s directory: %v", c.Name(), err)
		}
	}

	// Remove the files related to the sandbox
	podSandboxDir := filepath.Join(s.sandboxDir, *sbID)
	if err := os.RemoveAll(podSandboxDir); err != nil {
		return nil, fmt.Errorf("failed to remove sandbox %s directory: %v", *sbID, err)
	}

	return &pb.RemovePodSandboxResponse{}, nil
}

// PodSandboxStatus returns the Status of the PodSandbox.
func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusRequest) (*pb.PodSandboxStatusResponse, error) {
	sbID := req.PodSandboxId
	if *sbID == "" {
		return nil, fmt.Errorf("PodSandboxId should not be empty")
	}
	sb := s.getSandbox(*sbID)
	if sb == nil {
		return nil, fmt.Errorf("specified sandbox not found: %s", *sbID)
	}

	podInfraContainerName := sb.name + "-infra"
	podInfraContainer := sb.getContainer(podInfraContainerName)

	cState := s.runtime.ContainerStatus(podInfraContainer)
	created := cState.Created.Unix()

	netNsPath, err := podInfraContainer.NetNsPath()
	if err != nil {
		return nil, err
	}
	podNamespace := ""
	ip, err := s.netPlugin.GetContainerNetworkStatus(netNsPath, podNamespace, *sbID, podInfraContainerName)
	if err != nil {
		// ignore the error on network status
		ip = ""
	}

	rStatus := pb.PodSandBoxState_NOTREADY
	if cState.Status == "running" {
		rStatus = pb.PodSandBoxState_READY
	}

	return &pb.PodSandboxStatusResponse{
		Status: &pb.PodSandboxStatus{
			Id:        sbID,
			CreatedAt: int64Ptr(created),
			Linux: &pb.LinuxPodSandboxStatus{
				Namespaces: &pb.Namespace{
					Network: sPtr(netNsPath),
				},
			},
			Network: &pb.PodSandboxNetworkStatus{Ip: &ip},
			State:   &rStatus,
		},
	}, nil
}

// ListPodSandbox returns a list of SandBox.
func (s *Server) ListPodSandbox(context.Context, *pb.ListPodSandboxRequest) (*pb.ListPodSandboxResponse, error) {
	return nil, nil
}
<commit_msg>Add server impl of list pod sandboxes<commit_after>package server

import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/pkg/stringid"
	"github.com/kubernetes-incubator/ocid/oci"
	"github.com/kubernetes-incubator/ocid/utils"
	"github.com/opencontainers/runtime-tools/generate"
	"golang.org/x/net/context"
	pb "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
)

type sandbox struct {
	id         string
	name       string
	logDir     string
	labels     map[string]string
	containers oci.Store
}

const (
	podInfraRootfs      = "/var/lib/ocid/graph/vfs/pause"
	podDefaultNamespace = "default"
)

func (s *sandbox) addContainer(c *oci.Container) {
	s.containers.Add(c.Name(), c)
}

func (s *sandbox) getContainer(name string) *oci.Container {
	return s.containers.Get(name)
}

func (s *sandbox) removeContainer(c *oci.Container) {
	s.containers.Delete(c.Name())
}

func (s *Server) generatePodIDandName(name, namespace string) (string, string, error) {
	var (
		err error
		id  = stringid.GenerateNonCryptoID()
	)
	if namespace == "" {
		namespace = podDefaultNamespace
	}
	if name, err = s.reservePodName(id, namespace+"-"+name); err != nil {
		return "", "", err
	}
	return id, name, err
}

// RunPodSandbox creates and runs a pod-level sandbox.
func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest) (*pb.RunPodSandboxResponse, error) {
	// process req.Name
	name := req.GetConfig().GetMetadata().GetName()
	if name == "" {
		return nil, fmt.Errorf("PodSandboxConfig.Name should not be empty")
	}

	namespace := req.GetConfig().GetMetadata().GetNamespace()

	var err error
	id, name, err := s.generatePodIDandName(name, namespace)
	if err != nil {
		return nil, err
	}
	podSandboxDir := filepath.Join(s.sandboxDir, id)
	if _, err = os.Stat(podSandboxDir); err == nil {
		return nil, fmt.Errorf("pod sandbox (%s) already exists", podSandboxDir)
	}

	if err = os.MkdirAll(podSandboxDir, 0755); err != nil {
		return nil, err
	}

	defer func() {
		if err != nil {
			if err2 := os.RemoveAll(podSandboxDir); err2 != nil {
				logrus.Warnf("couldn't cleanup podSandboxDir %s: %v", podSandboxDir, err2)
			}
		}
	}()

	// creates a spec Generator with the default spec.
	g := generate.New()

	podInfraRootfs := filepath.Join(s.root, "graph/vfs/pause")
	// setup defaults for the pod sandbox
	g.SetRootPath(filepath.Join(podInfraRootfs, "rootfs"))
	g.SetRootReadonly(true)
	g.SetProcessArgs([]string{"/pause"})

	// set hostname
	hostname := req.GetConfig().GetHostname()
	if hostname != "" {
		g.SetHostname(hostname)
	}

	// set log directory
	logDir := req.GetConfig().GetLogDirectory()
	if logDir == "" {
		logDir = fmt.Sprintf("/var/log/ocid/pods/%s", id)
	}

	// set DNS options
	dnsServers := req.GetConfig().GetDnsOptions().GetServers()
	dnsSearches := req.GetConfig().GetDnsOptions().GetSearches()
	resolvPath := fmt.Sprintf("%s/resolv.conf", podSandboxDir)
	err = parseDNSOptions(dnsServers, dnsSearches, resolvPath)
	if err != nil {
		err1 := removeFile(resolvPath)
		if err1 != nil {
			err = err1
			return nil, fmt.Errorf("%v; failed to remove %s: %v", err, resolvPath, err1)
		}
		return nil, err
	}

	g.AddBindMount(resolvPath, "/etc/resolv.conf", "ro")

	// add labels
	labels := req.GetConfig().GetLabels()
	labelsJSON, err := json.Marshal(labels)
	if err != nil {
		return nil, err
	}
	g.AddAnnotation("ocid/labels", string(labelsJSON))
	g.AddAnnotation("ocid/log_path", logDir)
	g.AddAnnotation("ocid/name", name)
	containerName := name + "-infra"
	g.AddAnnotation("ocid/container_name", containerName)
	s.addSandbox(&sandbox{
		id:         id,
		name:       name,
		logDir:     logDir,
		labels:     labels,
		containers: oci.NewMemoryStore(),
	})

	annotations := req.GetConfig().GetAnnotations()
	for k, v := range annotations {
		g.AddAnnotation(k, v)
	}

	// setup cgroup settings
	cgroupParent := req.GetConfig().GetLinux().GetCgroupParent()
	if cgroupParent != "" {
		g.SetLinuxCgroupsPath(cgroupParent)
	}

	// set up namespaces
	if req.GetConfig().GetLinux().GetNamespaceOptions().GetHostNetwork() {
		err = g.RemoveLinuxNamespace("network")
		if err != nil {
			return nil, err
		}
	}

	if req.GetConfig().GetLinux().GetNamespaceOptions().GetHostPid() {
		err = g.RemoveLinuxNamespace("pid")
		if err != nil {
			return nil, err
		}
	}

	if req.GetConfig().GetLinux().GetNamespaceOptions().GetHostIpc() {
		err = g.RemoveLinuxNamespace("ipc")
		if err != nil {
			return nil, err
		}
	}

	err = g.SaveToFile(filepath.Join(podSandboxDir, "config.json"))
	if err != nil {
		return nil, err
	}

	if _, err = os.Stat(podInfraRootfs); err != nil {
		if os.IsNotExist(err) {
			// TODO: Replace by rootfs creation API when it is ready
			if err = utils.CreateFakeRootfs(podInfraRootfs, "docker://kubernetes/pause"); err != nil {
				return nil, err
			}
		} else {
			return nil, err
		}
	}

	container, err := oci.NewContainer(containerName, podSandboxDir, podSandboxDir, labels, id, false)
	if err != nil {
		return nil, err
	}

	if err = s.runtime.CreateContainer(container); err != nil {
		return nil, err
	}

	if err = s.runtime.UpdateStatus(container); err != nil {
		return nil, err
	}

	// setup the network
	podNamespace := ""
	netnsPath, err := container.NetNsPath()
	if err != nil {
		return nil, err
	}
	if err = s.netPlugin.SetUpPod(netnsPath, podNamespace, id, containerName); err != nil {
		return nil, fmt.Errorf("failed to create network for container %s in sandbox %s: %v", containerName, id, err)
	}

	if err = s.runtime.StartContainer(container); err != nil {
		return nil, err
	}

	s.addContainer(container)

	if err = s.podIDIndex.Add(id); err != nil {
		return nil, err
	}

	if err = s.runtime.UpdateStatus(container); err != nil {
		return nil, err
	}

	return &pb.RunPodSandboxResponse{PodSandboxId: &id}, nil
}

// StopPodSandbox stops the sandbox. If there are any running containers in the
// sandbox, they should be force terminated.
func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxRequest) (*pb.StopPodSandboxResponse, error) {
	sbID := req.PodSandboxId
	if *sbID == "" {
		return nil, fmt.Errorf("PodSandboxId should not be empty")
	}
	sb := s.getSandbox(*sbID)
	if sb == nil {
		return nil, fmt.Errorf("specified sandbox not found: %s", *sbID)
	}

	podInfraContainer := sb.name + "-infra"
	for _, c := range sb.containers.List() {
		if podInfraContainer == c.Name() {
			podNamespace := ""
			netnsPath, err := c.NetNsPath()
			if err != nil {
				return nil, err
			}
			if err := s.netPlugin.TearDownPod(netnsPath, podNamespace, *sbID, podInfraContainer); err != nil {
				return nil, fmt.Errorf("failed to destroy network for container %s in sandbox %s: %v", c.Name(), *sbID, err)
			}
		}
		cStatus := s.runtime.ContainerStatus(c)
		if cStatus.Status != "stopped" {
			if err := s.runtime.StopContainer(c); err != nil {
				return nil, fmt.Errorf("failed to stop container %s in sandbox %s: %v", c.Name(), *sbID, err)
			}
		}
	}

	return &pb.StopPodSandboxResponse{}, nil
}

// RemovePodSandbox deletes the sandbox. If there are any running containers in the
// sandbox, they should be force deleted.
func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxRequest) (*pb.RemovePodSandboxResponse, error) {
	sbID := req.PodSandboxId
	if *sbID == "" {
		return nil, fmt.Errorf("PodSandboxId should not be empty")
	}
	sb := s.getSandbox(*sbID)
	if sb == nil {
		return nil, fmt.Errorf("specified sandbox not found: %s", *sbID)
	}

	podInfraContainer := sb.name + "-infra"

	// Delete all the containers in the sandbox
	for _, c := range sb.containers.List() {
		if err := s.runtime.DeleteContainer(c); err != nil {
			return nil, fmt.Errorf("failed to delete container %s in sandbox %s: %v", c.Name(), *sbID, err)
		}
		if podInfraContainer == c.Name() {
			continue
		}
		containerDir := filepath.Join(s.runtime.ContainerDir(), c.Name())
		if err := os.RemoveAll(containerDir); err != nil {
			return nil, fmt.Errorf("failed to remove container %s directory: %v", c.Name(), err)
		}
	}

	// Remove the files related to the sandbox
	podSandboxDir := filepath.Join(s.sandboxDir, *sbID)
	if err := os.RemoveAll(podSandboxDir); err != nil {
		return nil, fmt.Errorf("failed to remove sandbox %s directory: %v", *sbID, err)
	}

	return &pb.RemovePodSandboxResponse{}, nil
}

// PodSandboxStatus returns the Status of the PodSandbox.
func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusRequest) (*pb.PodSandboxStatusResponse, error) {
	sbID := req.PodSandboxId
	if *sbID == "" {
		return nil, fmt.Errorf("PodSandboxId should not be empty")
	}
	sb := s.getSandbox(*sbID)
	if sb == nil {
		return nil, fmt.Errorf("specified sandbox not found: %s", *sbID)
	}

	podInfraContainerName := sb.name + "-infra"
	podInfraContainer := sb.getContainer(podInfraContainerName)
	if err := s.runtime.UpdateStatus(podInfraContainer); err != nil {
		return nil, err
	}

	cState := s.runtime.ContainerStatus(podInfraContainer)
	created := cState.Created.Unix()

	netNsPath, err := podInfraContainer.NetNsPath()
	if err != nil {
		return nil, err
	}
	podNamespace := ""
	ip, err := s.netPlugin.GetContainerNetworkStatus(netNsPath, podNamespace, *sbID, podInfraContainerName)
	if err != nil {
		// ignore the error on network status
		ip = ""
	}

	rStatus := pb.PodSandBoxState_NOTREADY
	if cState.Status == ContainerStateRunning {
		rStatus = pb.PodSandBoxState_READY
	}

	return &pb.PodSandboxStatusResponse{
		Status: &pb.PodSandboxStatus{
			Id:        sbID,
			CreatedAt: int64Ptr(created),
			Linux: &pb.LinuxPodSandboxStatus{
				Namespaces: &pb.Namespace{
					Network: sPtr(netNsPath),
				},
			},
			Network: &pb.PodSandboxNetworkStatus{Ip: &ip},
			State:   &rStatus,
		},
	}, nil
}

// ListPodSandbox returns a list of SandBoxes.
func (s *Server) ListPodSandbox(context.Context, *pb.ListPodSandboxRequest) (*pb.ListPodSandboxResponse, error) {
	var pods []*pb.PodSandbox
	for _, sb := range s.state.sandboxes {
		podInfraContainerName := sb.name + "-infra"
		podInfraContainer := sb.getContainer(podInfraContainerName)
		if err := s.runtime.UpdateStatus(podInfraContainer); err != nil {
			return nil, err
		}
		cState := s.runtime.ContainerStatus(podInfraContainer)
		created := cState.Created.Unix()
		rStatus := pb.PodSandBoxState_NOTREADY
		if cState.Status == ContainerStateRunning {
			rStatus = pb.PodSandBoxState_READY
		}

		pod := &pb.PodSandbox{
			Id:        &sb.id,
			CreatedAt: int64Ptr(created),
			State:     &rStatus,
		}

		pods = append(pods, pod)
	}

	return &pb.ListPodSandboxResponse{
		Items: pods,
	}, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Timo Savola. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package packetchan

import (
	"context"

	"github.com/tsavola/contextack"
	"github.com/tsavola/gate/packet"
)

type forwardAck struct{}

// ForwardDoneAck can be used with contextack.WithAck to subscribe to Forward
// exit acknowledgement.
var ForwardDoneAck forwardAck

// Forward packets from source to destination until shutdown is triggered (the
// channel is closed).
func Forward(ctx context.Context, destination chan<- packet.Buf, source <-chan packet.Buf) {
	defer contextack.Ack(ctx, ForwardDoneAck)

	var p packet.Buf

	for {
		var (
			input  <-chan packet.Buf
			output chan<- packet.Buf
			ok     bool
		)

		if p == nil {
			input = source
		} else {
			output = destination
		}

		select {
		case p, ok = <-input:
			if !ok {
				// EOF
				return
			}

		case output <- p:
			// ok

		case <-ctx.Done():
			return
		}
	}
}
<commit_msg>packetchan: doc fix<commit_after>// Copyright (c) 2017 Timo Savola. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package packetchan

import (
	"context"

	"github.com/tsavola/contextack"
	"github.com/tsavola/gate/packet"
)

type forwardAck struct{}

// ForwardDoneAck can be used with contextack.WithAck to subscribe to Forward
// call cancellation acknowledgement.
var ForwardDoneAck forwardAck

// Forward packets from source to destination until the context is canceled.
func Forward(ctx context.Context, destination chan<- packet.Buf, source <-chan packet.Buf) {
	defer contextack.Ack(ctx, ForwardDoneAck)

	var p packet.Buf

	for {
		var (
			input  <-chan packet.Buf
			output chan<- packet.Buf
			ok     bool
		)

		if p == nil {
			input = source
		} else {
			output = destination
		}

		select {
		case p, ok = <-input:
			if !ok {
				// EOF
				return
			}

		case output <- p:
			// ok

		case <-ctx.Done():
			return
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/dustin/gomemcached"
	"github.com/dustin/gomemcached/client"
)

var heartFreq = flag.Duration("heartbeat", 10*time.Second,
	"Heartbeat frequency")
var reconcileFreq = flag.Duration("reconcile", 24*time.Hour,
	"Reconciliation frequency")
var staleNodeFreq = flag.Duration("staleNodeCheck", 5*time.Minute,
	"How frequently to check for stale nodes.")
var staleNodeLimit = flag.Duration("staleNodeLimit", 15*time.Minute,
	"How long until we clean up nodes for being too stale")
var nodeCleanCount = flag.Int("nodeCleanCount", 1000,
	"How many blobs to clean up from a dead node per period")

type AboutNode struct {
	Addr     string    `json:"addr"`
	Type     string    `json:"type"`
	Time     time.Time `json:"time"`
	BindAddr string    `json:"bindaddr"`
	Hash     string    `json:"hash"`
}

type PeriodicJob struct {
	period time.Duration
	f      func() error
}

var periodicJobs = map[string]*PeriodicJob{
	"checkStaleNodes": &PeriodicJob{
		time.Minute * 5,
		checkStaleNodes,
	},
}

func adjustPeriodicJobs() error {
	periodicJobs["checkStaleNodes"].period = *staleNodeFreq
	return nil
}

func getNodeAddress(sid string) (string, error) {
	sidkey := "/" + sid
	aboutSid := AboutNode{}
	err := couchbase.Get(sidkey, &aboutSid)
	if err != nil {
		return "", err
	}
	if strings.HasPrefix(aboutSid.BindAddr, ":") {
		return aboutSid.Addr + aboutSid.BindAddr, nil
	}
	return aboutSid.BindAddr, nil
}

type JobMarker struct {
	Node    string    `json:"node"`
	Started time.Time `json:"started"`
	Type    string    `json:"type"`
}

// Run a named task if we know one hasn't in the last t seconds.
func runNamedGlobalTask(name string, t time.Duration, f func() error) bool {
	key := "/@" + name

	jm := JobMarker{
		Node:    serverId,
		Started: time.Now(),
		Type:    "job",
	}

	err := couchbase.Do(key, func(mc *memcached.Client, vb uint16) error {
		data, err := json.Marshal(&jm)
		if err != nil {
			log.Fatalf("Can't jsonify a JobMarker: %v", err)
		}
		resp, err := mc.Add(vb, key, 0, int(t.Seconds()), data)
		if err != nil {
			return err
		}
		if resp.Status != gomemcached.SUCCESS {
			return fmt.Errorf("Wanted success, got %v", resp.Status)
		}
		return nil
	})

	if err == nil {
		err = f()
		if err != nil {
			log.Printf("Error running periodic task %#v: %v", name, err)
		}
	}

	return true
}

func heartbeat() {
	for {
		u, err := url.Parse(*couchbaseServer)
		c, err := net.Dial("tcp", u.Host)
		localAddr := ""
		if err == nil {
			localAddr = strings.Split(c.LocalAddr().String(), ":")[0]
			c.Close()
		}

		aboutMe := AboutNode{
			Addr:     localAddr,
			Type:     "storage",
			Time:     time.Now().UTC(),
			BindAddr: *bindAddr,
			Hash:     *hashType,
		}

		err = couchbase.Set("/"+serverId, aboutMe)
		if err != nil {
			log.Printf("Failed to record a heartbeat: %v", err)
		}
		time.Sleep(*heartFreq)
	}
}

func reconcile() error {
	explen := getHash().Size() * 2
	return filepath.Walk(*root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !info.IsDir() && !strings.HasPrefix(info.Name(), "tmp") &&
			len(info.Name()) == explen {
			// I can do way more efficient stuff than this.
			recordBlobOwnership(info.Name(), info.Size())
			return err
		}
		return nil
	})
}

func reconcileLoop() {
	if *reconcileFreq == 0 {
		return
	}
	for {
		err := reconcile()
		if err != nil {
			log.Printf("Error in reconciliation loop: %v", err)
		}
		time.Sleep(*reconcileFreq)
	}
}

func removeBlobOwnershipRecord(h, node string) {
	log.Printf("Cleaning up %v from %v", h, node)

	k := "/" + h
	err := couchbase.Do(k, func(mc *memcached.Client, vb uint16) error {
		_, err := mc.CAS(vb, k, func(in []byte) []byte {
			ownership := BlobOwnership{}
			err := json.Unmarshal(in, &ownership)
			if err == nil {
				delete(ownership.Nodes, node)
			} else {
				return nil
			}

			rv, err := json.Marshal(&ownership)
			if err != nil {
				log.Fatalf("Error marshaling blob ownership: %v", err)
			}
			return rv
		}, 0)
		return err
	})
	if err != nil {
		log.Printf("Error cleaning %v from %v", node, h)
	}
}

func cleanupNode(node string) {
	log.Printf("Cleaning up node %v", node)
	vres, err := couchbase.View("cbfs", "node_blobs",
		map[string]interface{}{
			"key":    `"` + node + `"`,
			"limit":  *nodeCleanCount,
			"reduce": false,
			"stale":  false,
		})
	if err != nil {
		log.Printf("Error executing node_blobs view: %v", err)
		return
	}
	foundRows := 0
	for _, r := range vres.Rows {
		removeBlobOwnershipRecord(r.ID[1:], node)
		foundRows++
	}
	if foundRows == 0 && len(vres.Errors) == 0 {
		log.Printf("Removing node record: %v", node)
		err = couchbase.Delete("/" + node)
		if err != nil {
			log.Printf("Error deleting %v node record: %v", node, err)
		}
	}
}

func checkStaleNodes() error {
	log.Printf("Checking stale nodes")
	vres, err := couchbase.View("cbfs", "nodes", map[string]interface{}{
		"stale": false})
	if err != nil {
		return err
	}
	for _, r := range vres.Rows {
		ks, ok := r.Key.(string)
		if !ok {
			log.Printf("Wrong key type returned from view: %#v", r)
			continue
		}
		t, err := time.Parse(time.RFC3339Nano, ks)
		if err != nil {
			log.Printf("Error parsing time from %v", r)
			continue
		}
		d := time.Since(t)
		node := r.ID[1:]

		if d > *staleNodeLimit {
			if node == serverId {
				log.Printf("Would've cleaned up myself after %v",
					d)
				continue
			}
			log.Printf("  Node %v missed heartbeat schedule: %v", node, d)
			go cleanupNode(node)
		} else {
			log.Printf("%v is ok at %v", node, d)
		}
	}
	return nil
}

func runPeriodicJob(name string, job *PeriodicJob) {
	for {
		if runNamedGlobalTask(name, job.period, job.f) {
			log.Printf("Attempted job %v", name)
		} else {
			log.Printf("Didn't run job %v", name)
		}
		time.Sleep(job.period + time.Second)
	}
}

func runPeriodicJobs() {
	for n, j := range periodicJobs {
		go runPeriodicJob(n, j)
	}
}
<commit_msg>Changing node from "storage" to "node"<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/dustin/gomemcached"
	"github.com/dustin/gomemcached/client"
)

var heartFreq = flag.Duration("heartbeat", 10*time.Second,
	"Heartbeat frequency")
var reconcileFreq = flag.Duration("reconcile", 24*time.Hour,
	"Reconciliation frequency")
var staleNodeFreq = flag.Duration("staleNodeCheck", 5*time.Minute,
	"How frequently to check for stale nodes.")
var staleNodeLimit = flag.Duration("staleNodeLimit", 15*time.Minute,
	"How long until we clean up nodes for being too stale")
var nodeCleanCount = flag.Int("nodeCleanCount", 1000,
	"How many blobs to clean up from a dead node per period")

type AboutNode struct {
	Addr     string    `json:"addr"`
	Type     string    `json:"type"`
	Time     time.Time `json:"time"`
	BindAddr string    `json:"bindaddr"`
	Hash     string    `json:"hash"`
}

type PeriodicJob struct {
	period time.Duration
	f      func() error
}

var periodicJobs = map[string]*PeriodicJob{
	"checkStaleNodes": &PeriodicJob{
		time.Minute * 5,
		checkStaleNodes,
	},
}

func adjustPeriodicJobs() error {
	periodicJobs["checkStaleNodes"].period = *staleNodeFreq
	return nil
}

func getNodeAddress(sid string) (string, error) {
	sidkey := "/" + sid
	aboutSid := AboutNode{}
	err := couchbase.Get(sidkey, &aboutSid)
	if err != nil {
		return "", err
	}
	if strings.HasPrefix(aboutSid.BindAddr, ":") {
		return aboutSid.Addr + aboutSid.BindAddr, nil
	}
	return aboutSid.BindAddr, nil
}

type JobMarker struct {
	Node    string    `json:"node"`
	Started time.Time `json:"started"`
	Type    string    `json:"type"`
}

// Run a named task if we know one hasn't in the last t seconds.
func runNamedGlobalTask(name string, t time.Duration, f func() error) bool {
	key := "/@" + name

	jm := JobMarker{
		Node:    serverId,
		Started: time.Now(),
		Type:    "job",
	}

	err := couchbase.Do(key, func(mc *memcached.Client, vb uint16) error {
		data, err := json.Marshal(&jm)
		if err != nil {
			log.Fatalf("Can't jsonify a JobMarker: %v", err)
		}
		resp, err := mc.Add(vb, key, 0, int(t.Seconds()), data)
		if err != nil {
			return err
		}
		if resp.Status != gomemcached.SUCCESS {
			return fmt.Errorf("Wanted success, got %v", resp.Status)
		}
		return nil
	})

	if err == nil {
		err = f()
		if err != nil {
			log.Printf("Error running periodic task %#v: %v", name, err)
		}
	}

	return true
}

func heartbeat() {
	for {
		u, err := url.Parse(*couchbaseServer)
		c, err := net.Dial("tcp", u.Host)
		localAddr := ""
		if err == nil {
			localAddr = strings.Split(c.LocalAddr().String(), ":")[0]
			c.Close()
		}

		aboutMe := AboutNode{
			Addr:     localAddr,
			Type:     "node",
			Time:     time.Now().UTC(),
			BindAddr: *bindAddr,
			Hash:     *hashType,
		}

		err = couchbase.Set("/"+serverId, aboutMe)
		if err != nil {
			log.Printf("Failed to record a heartbeat: %v", err)
		}
		time.Sleep(*heartFreq)
	}
}

func reconcile() error {
	explen := getHash().Size() * 2
	return filepath.Walk(*root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !info.IsDir() && !strings.HasPrefix(info.Name(), "tmp") &&
			len(info.Name()) == explen {
			// I can do way more efficient stuff than this.
			recordBlobOwnership(info.Name(), info.Size())
			return err
		}
		return nil
	})
}

func reconcileLoop() {
	if *reconcileFreq == 0 {
		return
	}
	for {
		err := reconcile()
		if err != nil {
			log.Printf("Error in reconciliation loop: %v", err)
		}
		time.Sleep(*reconcileFreq)
	}
}

func removeBlobOwnershipRecord(h, node string) {
	log.Printf("Cleaning up %v from %v", h, node)

	k := "/" + h
	err := couchbase.Do(k, func(mc *memcached.Client, vb uint16) error {
		_, err := mc.CAS(vb, k, func(in []byte) []byte {
			ownership := BlobOwnership{}
			err := json.Unmarshal(in, &ownership)
			if err == nil {
				delete(ownership.Nodes, node)
			} else {
				return nil
			}

			rv, err := json.Marshal(&ownership)
			if err != nil {
				log.Fatalf("Error marshaling blob ownership: %v", err)
			}
			return rv
		}, 0)
		return err
	})
	if err != nil {
		log.Printf("Error cleaning %v from %v", node, h)
	}
}

func cleanupNode(node string) {
	log.Printf("Cleaning up node %v", node)
	vres, err := couchbase.View("cbfs", "node_blobs",
		map[string]interface{}{
			"key":    `"` + node + `"`,
			"limit":  *nodeCleanCount,
			"reduce": false,
			"stale":  false,
		})
	if err != nil {
		log.Printf("Error executing node_blobs view: %v", err)
		return
	}
	foundRows := 0
	for _, r := range vres.Rows {
		removeBlobOwnershipRecord(r.ID[1:], node)
		foundRows++
	}
	if foundRows == 0 && len(vres.Errors) == 0 {
		log.Printf("Removing node record: %v", node)
		err = couchbase.Delete("/" + node)
		if err != nil {
			log.Printf("Error deleting %v node record: %v", node, err)
		}
	}
}

func checkStaleNodes() error {
	log.Printf("Checking stale nodes")
	vres, err := couchbase.View("cbfs", "nodes", map[string]interface{}{
		"stale": false})
	if err != nil {
		return err
	}
	for _, r := range vres.Rows {
		ks, ok := r.Key.(string)
		if !ok {
			log.Printf("Wrong key type returned from view: %#v", r)
			continue
		}
		t, err := time.Parse(time.RFC3339Nano, ks)
		if err != nil {
			log.Printf("Error parsing time from %v", r)
			continue
		}
		d := time.Since(t)
		node := r.ID[1:]

		if d > *staleNodeLimit {
			if node == serverId {
				log.Printf("Would've cleaned up myself after %v",
					d)
				continue
			}
			log.Printf("  Node %v missed heartbeat schedule: %v", node, d)
			go cleanupNode(node)
		} else {
			log.Printf("%v is ok at %v", node, d)
		}
	}
	return nil
}

func runPeriodicJob(name string, job *PeriodicJob) {
	for {
		if runNamedGlobalTask(name, job.period, job.f) {
			log.Printf("Attempted job %v", name)
		} else {
			log.Printf("Didn't run job %v", name)
		}
		time.Sleep(job.period + time.Second)
	}
}

func runPeriodicJobs() {
	for n, j := range periodicJobs {
		go runPeriodicJob(n, j)
	}
}
<|endoftext|>


<commit_before>package helm

import (
	_ "fmt"
	_ "os"
	"os/exec"
)

type InfluxdbCommander struct {
	Name, Retention, Pvcsize string
}

type GrafanaCommander struct {
	Name string
}

func (i *InfluxdbCommander) InstallInfluxdb() error {
  releasename := "--name=" + i.Name + "-i"
  opt := "retentionPolicy=" + i.Retention + ",persistence.size=" + i.Pvcsize
  cmdName := "helm"
	cmdArgs := []string{
    "install",
    releasename,
    "influxdb",
    "--set",
    opt,
  }
	return exec.Command(cmdName, cmdArgs...).Run()
}

func (g *GrafanaCommander) InstallGrafana() error {
  releasename := "--name=" + g.Name + "-g"
  cmdName := "helm"
  cmdArgs := []string{
    "install",
    releasename,
    "grafana",
  }
  return exec.Command(cmdName, cmdArgs...).Run()
}

//func main()  {
//  name := "rlxebz"
//  var err error
//
//  influxdb := InfluxdbCommander{
//    Name: name,
//    Retention: "10d",
//    Pvcsize: "10Gi",
//  }
//  err = influxdb.InstallInfluxdb()
//  if err != nil {
//    fmt.Printf("can not create influxdb%+v", err)
//    os.Exit(98)
//  }
//
//  grafana := GrafanaCommander{ Name: name }
//  err = grafana.InstallGrafana()
//  if err != nil {
//    fmt.Printf("can not create grafana%+v", err)
//    os.Exit(99)
//  }
//}
<commit_msg>fix helm repo<commit_after>package helm

import (
	_ "fmt"
	_ "os"
	"os/exec"
)

type InfluxdbCommander struct {
	Name, Retention, Pvcsize string
}

type GrafanaCommander struct {
	Name string
}

func (i *InfluxdbCommander) InstallInfluxdb() error {
  releasename := "--name=" + i.Name + "-i"
  opt := "retentionPolicy=" + i.Retention + ",persistence.size=" + i.Pvcsize
  cmdName := "helm"
	cmdArgs := []string{
    "install",
    releasename,
    "or-charts/influxdb",
    "--set",
    opt,
  }
	return exec.Command(cmdName, cmdArgs...).Run()
}

func (g *GrafanaCommander) InstallGrafana() error {
  releasename := "--name=" + g.Name + "-g"
  cmdName := "helm"
  cmdArgs := []string{
    "install",
    releasename,
    "or-charts/grafana",
  }
  return exec.Command(cmdName, cmdArgs...).Run()
}

//func main()  {
//  name := "rlxebz"
//  var err error
//
//  influxdb := InfluxdbCommander{
//    Name: name,
//    Retention: "10d",
//    Pvcsize: "10Gi",
//  }
//  err = influxdb.InstallInfluxdb()
//  if err != nil {
//    fmt.Printf("can not create influxdb%+v", err)
//    os.Exit(98)
//  }
//
//  grafana := GrafanaCommander{ Name: name }
//  err = grafana.InstallGrafana()
//  if err != nil {
//    fmt.Printf("can not create grafana%+v", err)
//    os.Exit(99)
//  }
//}
<|endoftext|>


<commit_before>package plugin

/*
usage: !ver
usage: !version
*/

import (
	"fmt"
	"github.com/microamp/gerri/cmd"
	"github.com/microamp/gerri/data"
)

const (
	VERSION = "0.3.2"
)

func ReplyVer(pm data.Privmsg, config *data.Config) (string, error) {
	return cmd.Privmsg(pm.Target, fmt.Sprintf("gerri version: %s", VERSION)), nil
}
<commit_msg>v0.3.3<commit_after>package plugin

/*
usage: !ver
usage: !version
*/

import (
	"fmt"
	"github.com/microamp/gerri/cmd"
	"github.com/microamp/gerri/data"
)

const (
	VERSION = "0.3.3"
)

func ReplyVer(pm data.Privmsg, config *data.Config) (string, error) {
	return cmd.Privmsg(pm.Target, fmt.Sprintf("gerri version: %s", VERSION)), nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 The SurgeMQ Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

import (
	"bufio"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"sync"
	"sync/atomic"
	"time"
)

var (
	bufcnt            int64
	DefaultBufferSize int64

	DeviceInBufferSize  int64
	DeviceOutBufferSize int64

	MasterInBufferSize  int64
	MasterOutBufferSize int64
)

type sequence struct {
	// The current position of the producer or consumer
	cursor,

	// The previous known position of the consumer (if producer) or producer (if consumer)
	gate,

	// These are fillers to pad the cache line, which is generally 64 bytes
	p2, p3, p4, p5, p6, p7 int64
}

func newSequence() *sequence {
	return &sequence{}
}

func (this *sequence) get() int64 {
	return atomic.LoadInt64(&this.cursor)
}

func (this *sequence) set(seq int64) {
	atomic.StoreInt64(&this.cursor, seq)
}

type buffer struct {
	id int64

	readIndex  int64 //读序号
	writeIndex int64 //写序号
	buf        []*[]byte

	size int64
	mask int64

	done int64

	pcond *sync.Cond
	ccond *sync.Cond

	b []byte //readfrom中的临时数组，为反复使用不必创建新数组
}

func newBuffer(size int64) (*buffer, error) {
	if size < 0 {
		return nil, bufio.ErrNegativeCount
	}

	if size == 0 {
		size = DefaultBufferSize
	}

	if !powerOfTwo64(size) {
		fmt.Printf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
		return nil, fmt.Errorf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
	}

	return &buffer{
		id:         atomic.AddInt64(&bufcnt, 1),
		readIndex:  int64(0),
		writeIndex: int64(0),
		buf:        make([]*[]byte, size),
		size:       size,
		mask:       size - 1,
		pcond:      sync.NewCond(new(sync.Mutex)),
		ccond:      sync.NewCond(new(sync.Mutex)),
		b:          make([]byte, 5),
	}, nil
}

/**
获取当前读序号
*/
func (this *buffer) GetCurrentReadIndex() int64 {
	return atomic.LoadInt64(&this.readIndex)
}

/**
获取当前写序号
*/
func (this *buffer) GetCurrentWriteIndex() int64 {
	return atomic.LoadInt64(&this.writeIndex)
}

func (this *buffer) ID() int64 {
	return this.id
}

func (this *buffer) Close() error {
	atomic.StoreInt64(&this.done, 1)

	this.pcond.L.Lock()
	this.ccond.Signal()
	//this.ccond.Broadcast()
	this.pcond.L.Unlock()

	this.ccond.L.Lock()
	this.pcond.Signal()
	//this.pcond.Broadcast()
	this.ccond.L.Unlock()

	return nil
}

/**
读取ringbuffer指定的buffer指针，返回该指针并清空ringbuffer该位置存在的指针内容，以及将读序号加1
*/
func (this *buffer) ReadBuffer() (p *[]byte, ok bool) {
	this.ccond.L.Lock()
	defer func() {
		this.pcond.Signal()
		//this.pcond.Broadcast()
		this.ccond.L.Unlock()
	}()
	ok = false
	p = nil
	readIndex := this.GetCurrentReadIndex()
	writeIndex := this.GetCurrentWriteIndex()
	for {
		if this.isDone() {
			return nil, false
		}
		writeIndex = this.GetCurrentWriteIndex()
		if readIndex >= writeIndex {
			this.pcond.Signal()
			//this.pcond.Broadcast()
			this.ccond.Wait()
		} else {
			break
		}
		time.Sleep(500 * time.Microsecond)
	}
	index := readIndex & this.mask //替代求模
	p = this.buf[index]
	this.buf[index] = nil
	atomic.AddInt64(&this.readIndex, int64(1))
	if p != nil {
		ok = true
	}
	return p, ok
}

/**
写入ringbuffer指针，以及将写序号加1
*/
func (this *buffer) WriteBuffer(in *[]byte) (ok bool) {
	this.pcond.L.Lock()
	defer func() {
		this.ccond.Signal()
		//this.ccond.Broadcast()
		this.pcond.L.Unlock()
	}()
	ok = false
	readIndex := this.GetCurrentReadIndex()
	writeIndex := this.GetCurrentWriteIndex()
	for {
		if this.isDone() {
			return false
		}
		readIndex = this.GetCurrentReadIndex()
		if writeIndex >= readIndex && writeIndex-readIndex >= this.size {
			this.ccond.Signal()
			//this.ccond.Broadcast()
			this.pcond.Wait()
			//time.Sleep(1 * time.Millisecond)
		} else {
			break
		}
		time.Sleep(500 * time.Microsecond)
	}
	index := writeIndex & this.mask //替代求模
	this.buf[index] = in
	atomic.AddInt64(&this.writeIndex, int64(1))
	ok = true
	return ok
}

/**
修改 尽量减少数据的创建
*/
func (this *buffer) ReadFrom(r io.Reader) (int64, error) {
	defer this.Close()

	total := int64(0)

	for {
		time.Sleep(5 * time.Millisecond)
		if this.isDone() {
			return total, io.EOF
		}

		var write_bytes []byte

		//b := make([]byte, 5)
		n, err := r.Read(this.b[0:1])
		if err != nil {
			return total, io.EOF
		}
		total += int64(n)
		max_cnt := 1
		for {
			if this.isDone() {
				return total, io.EOF
			}
			// If we have read 5 bytes and still not done, then there's a problem.
			if max_cnt > 4 {
				return 0, fmt.Errorf("sendrecv/peekMessageSize: 4th byte of remaining length has continuation bit set")
			}
			_, err := r.Read(this.b[max_cnt:(max_cnt + 1)])

			//fmt.Println(b)
			if err != nil {
				return total, err
			}
			if this.b[max_cnt] >= 0x80 {
				max_cnt++
			} else {
				break
			}
		}
		remlen, m := binary.Uvarint(this.b[1 : max_cnt+1])
		remlen_tmp := int64(remlen)
		start_ := int64(1) + int64(m)
		total_tmp := remlen_tmp + start_

		write_bytes = make([]byte, total_tmp)
		copy(write_bytes[0:m+1], this.b[0:m+1])
		nlen := int64(0)
		times := 0
		cnt_ := int64(32)
		for nlen < remlen_tmp {
			if this.isDone() {
				return total, io.EOF
			}
			if times > 100 {
				return total, io.EOF
			} else {
				times = 0
			}
			times++
			tmpm := remlen_tmp - nlen

			b_ := write_bytes[(start_ + nlen):]
			if tmpm > cnt_ {
				b_ = write_bytes[(start_ + nlen):(start_ + nlen + cnt_)]
			}

			//b_ := make([]byte, remlen)
			n, err = r.Read(b_[0:])

			if err != nil {
				/*Log.Errorc(func() string {
					return fmt.Sprintf("从conn读取数据失败(%s)(0)", err)
				})
				time.Sleep(5 * time.Millisecond)
				continue*/
				return total, err
			}
			//write_bytes = append(write_bytes, b_[0:]...)
			nlen += int64(n)
			total += int64(n)
		}

		ok := this.WriteBuffer(&write_bytes)

		if !ok {
			return total, errors.New("write ringbuffer failed")
		}
		time.Sleep(5 * time.Millisecond)
	}
}

func (this *buffer) WriteTo(w io.Writer) (int64, error) {
	defer this.Close()

	total := int64(0)

	for {
		if this.isDone() {
			return total, io.EOF
		}

		p, ok := this.ReadBuffer()
		if !ok {
			return total, errors.New("read buffer failed")
		}
		// There's some data, let's process it first
		if len(*p) > 0 {
			n, err := w.Write(*p)
			total += int64(n)
			//Log.Debugc(func() string{ return fmt.Sprintf("Wrote %d bytes, totaling %d bytes", n, total)})

			if err != nil {
				return total, err
			}
		}

	}
}

func (this *buffer) isDone() bool {
	if atomic.LoadInt64(&this.done) == 1 {
		return true
	}

	return false
}

func ringCopy(dst, src []byte, start int64) int {
	n := len(src)

	i, l := 0, 0

	for n > 0 {
		l = copy(dst[start:], src[i:])
		i += l
		n -= l

		if n > 0 {
			start = 0
		}
	}

	return i
}

func powerOfTwo64(n int64) bool {
	return n != 0 && (n&(n-1)) == 0
}

func roundUpPowerOfTwo64(n int64) int64 {
	n--
	n |= n >> 1
	n |= n >> 2
	n |= n >> 4
	n |= n >> 8
	n |= n >> 16
	n |= n >> 32
	n++

	return n
}
<commit_msg>修改 buffer.go process.go sendrecv.go xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx 再次修改readfrom休眠时间（5毫秒）<commit_after>// Copyright (c) 2014 The SurgeMQ Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

import (
	"bufio"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"sync"
	"sync/atomic"
	"time"
)

var (
	bufcnt            int64
	DefaultBufferSize int64

	DeviceInBufferSize  int64
	DeviceOutBufferSize int64

	MasterInBufferSize  int64
	MasterOutBufferSize int64
)

type sequence struct {
	// The current position of the producer or consumer
	cursor,

	// The previous known position of the consumer (if producer) or producer (if consumer)
	gate,

	// These are fillers to pad the cache line, which is generally 64 bytes
	p2, p3, p4, p5, p6, p7 int64
}

func newSequence() *sequence {
	return &sequence{}
}

func (this *sequence) get() int64 {
	return atomic.LoadInt64(&this.cursor)
}

func (this *sequence) set(seq int64) {
	atomic.StoreInt64(&this.cursor, seq)
}

type buffer struct {
	id int64

	readIndex  int64 //读序号
	writeIndex int64 //写序号
	buf        []*[]byte

	size int64
	mask int64

	done int64

	pcond *sync.Cond
	ccond *sync.Cond

	b []byte //readfrom中的临时数组，为反复使用不必创建新数组
}

func newBuffer(size int64) (*buffer, error) {
	if size < 0 {
		return nil, bufio.ErrNegativeCount
	}

	if size == 0 {
		size = DefaultBufferSize
	}

	if !powerOfTwo64(size) {
		fmt.Printf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
		return nil, fmt.Errorf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
	}

	return &buffer{
		id:         atomic.AddInt64(&bufcnt, 1),
		readIndex:  int64(0),
		writeIndex: int64(0),
		buf:        make([]*[]byte, size),
		size:       size,
		mask:       size - 1,
		pcond:      sync.NewCond(new(sync.Mutex)),
		ccond:      sync.NewCond(new(sync.Mutex)),
		b:          make([]byte, 5),
	}, nil
}

/**
获取当前读序号
*/
func (this *buffer) GetCurrentReadIndex() int64 {
	return atomic.LoadInt64(&this.readIndex)
}

/**
获取当前写序号
*/
func (this *buffer) GetCurrentWriteIndex() int64 {
	return atomic.LoadInt64(&this.writeIndex)
}

func (this *buffer) ID() int64 {
	return this.id
}

func (this *buffer) Close() error {
	atomic.StoreInt64(&this.done, 1)

	this.pcond.L.Lock()
	this.ccond.Signal()
	//this.ccond.Broadcast()
	this.pcond.L.Unlock()

	this.ccond.L.Lock()
	this.pcond.Signal()
	//this.pcond.Broadcast()
	this.ccond.L.Unlock()

	return nil
}

/**
读取ringbuffer指定的buffer指针，返回该指针并清空ringbuffer该位置存在的指针内容，以及将读序号加1
*/
func (this *buffer) ReadBuffer() (p *[]byte, ok bool) {
	this.ccond.L.Lock()
	defer func() {
		this.pcond.Signal()
		//this.pcond.Broadcast()
		this.ccond.L.Unlock()
	}()
	ok = false
	p = nil
	readIndex := this.GetCurrentReadIndex()
	writeIndex := this.GetCurrentWriteIndex()
	for {
		if this.isDone() {
			return nil, false
		}
		writeIndex = this.GetCurrentWriteIndex()
		if readIndex >= writeIndex {
			this.pcond.Signal()
			//this.pcond.Broadcast()
			this.ccond.Wait()
		} else {
			break
		}
		time.Sleep(500 * time.Microsecond)
	}
	index := readIndex & this.mask //替代求模
	p = this.buf[index]
	this.buf[index] = nil
	atomic.AddInt64(&this.readIndex, int64(1))
	if p != nil {
		ok = true
	}
	return p, ok
}

/**
写入ringbuffer指针，以及将写序号加1
*/
func (this *buffer) WriteBuffer(in *[]byte) (ok bool) {
	this.pcond.L.Lock()
	defer func() {
		this.ccond.Signal()
		//this.ccond.Broadcast()
		this.pcond.L.Unlock()
	}()
	ok = false
	readIndex := this.GetCurrentReadIndex()
	writeIndex := this.GetCurrentWriteIndex()
	for {
		if this.isDone() {
			return false
		}
		readIndex = this.GetCurrentReadIndex()
		if writeIndex >= readIndex && writeIndex-readIndex >= this.size {
			this.ccond.Signal()
			//this.ccond.Broadcast()
			this.pcond.Wait()
			//time.Sleep(1 * time.Millisecond)
		} else {
			break
		}
		time.Sleep(500 * time.Microsecond)
	}
	index := writeIndex & this.mask //替代求模
	this.buf[index] = in
	atomic.AddInt64(&this.writeIndex, int64(1))
	ok = true
	return ok
}

/**
修改 尽量减少数据的创建
*/
func (this *buffer) ReadFrom(r io.Reader) (int64, error) {
	defer this.Close()

	total := int64(0)

	for {
		time.Sleep(5 * time.Millisecond)
		if this.isDone() {
			return total, io.EOF
		}

		var write_bytes []byte

		//b := make([]byte, 5)
		n, err := r.Read(this.b[0:1])
		if err != nil {
			return total, io.EOF
		}
		total += int64(n)
		max_cnt := 1
		for {
			if this.isDone() {
				return total, io.EOF
			}
			// If we have read 5 bytes and still not done, then there's a problem.
			if max_cnt > 4 {
				return 0, fmt.Errorf("sendrecv/peekMessageSize: 4th byte of remaining length has continuation bit set")
			}
			_, err := r.Read(this.b[max_cnt:(max_cnt + 1)])

			//fmt.Println(b)
			if err != nil {
				return total, err
			}
			if this.b[max_cnt] >= 0x80 {
				max_cnt++
			} else {
				break
			}
		}
		remlen, m := binary.Uvarint(this.b[1 : max_cnt+1])
		remlen_tmp := int64(remlen)
		start_ := int64(1) + int64(m)
		total_tmp := remlen_tmp + start_

		write_bytes = make([]byte, total_tmp)
		copy(write_bytes[0:m+1], this.b[0:m+1])
		nlen := int64(0)
		times := 0
		cnt_ := int64(32)
		for nlen < remlen_tmp {
			if this.isDone() {
				return total, io.EOF
			}
			if times > 100 {
				return total, io.EOF
			} else {
				times = 0
			}
			times++
			tmpm := remlen_tmp - nlen

			b_ := write_bytes[(start_ + nlen):]
			if tmpm > cnt_ {
				b_ = write_bytes[(start_ + nlen):(start_ + nlen + cnt_)]
			}

			//b_ := make([]byte, remlen)
			n, err = r.Read(b_[0:])

			if err != nil {
				/*Log.Errorc(func() string {
					return fmt.Sprintf("从conn读取数据失败(%s)(0)", err)
				})
				time.Sleep(5 * time.Millisecond)
				continue*/
				return total, err
			}
			//write_bytes = append(write_bytes, b_[0:]...)
			nlen += int64(n)
			total += int64(n)
		}

		ok := this.WriteBuffer(&write_bytes)

		if !ok {
			return total, errors.New("write ringbuffer failed")
		}
	}
}

func (this *buffer) WriteTo(w io.Writer) (int64, error) {
	defer this.Close()

	total := int64(0)

	for {
		if this.isDone() {
			return total, io.EOF
		}

		p, ok := this.ReadBuffer()
		if !ok {
			return total, errors.New("read buffer failed")
		}
		// There's some data, let's process it first
		if len(*p) > 0 {
			n, err := w.Write(*p)
			total += int64(n)
			//Log.Debugc(func() string{ return fmt.Sprintf("Wrote %d bytes, totaling %d bytes", n, total)})

			if err != nil {
				return total, err
			}
		}

	}
}

func (this *buffer) isDone() bool {
	if atomic.LoadInt64(&this.done) == 1 {
		return true
	}

	return false
}

func ringCopy(dst, src []byte, start int64) int {
	n := len(src)

	i, l := 0, 0

	for n > 0 {
		l = copy(dst[start:], src[i:])
		i += l
		n -= l

		if n > 0 {
			start = 0
		}
	}

	return i
}

func powerOfTwo64(n int64) bool {
	return n != 0 && (n&(n-1)) == 0
}

func roundUpPowerOfTwo64(n int64) int64 {
	n--
	n |= n >> 1
	n |= n >> 2
	n |= n >> 4
	n |= n >> 8
	n |= n >> 16
	n |= n >> 32
	n++

	return n
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 The SurgeMQ Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

import (
	"bufio"
	"fmt"
	"io"
	"sync/atomic"
	//"github.com/surgemq/message"
	"encoding/binary"
	"errors"
	"math"
	"sync"
)

var (
	bufcnt            int64
	DefaultBufferSize int64

	DeviceInBufferSize  int64
	DeviceOutBufferSize int64

	MasterInBufferSize  int64
	MasterOutBufferSize int64
)

const (
	smallReadBlockSize    = 512
	defaultReadBlockSize  = 8192
	defaultWriteBlockSize = 8192
)

/**
2016.03.03 修改
bingbuffer结构体
*/
type buffer struct {
	readIndex  int64        //读序号
	writeIndex int64        //写序号
	ringBuffer []*ByteArray //环形buffer指针数组
	bufferSize int64        //初始化环形buffer指针数组大小
	mask       int64        //掩码：bufferSize-1
	done       int64        //是否完成
	rcond      *sync.Cond
	wcond      *sync.Cond
}

type ByteArray struct {
	bArray []byte
}

func (this *buffer) ReadCommit(index int64, ok bool) {
	this.rcond.L.Lock()
	defer this.rcond.L.Unlock()
	if ok {
		this.ringBuffer[index] = nil
	}
	this.wcond.Broadcast()
}

func (this *buffer) Len() int {
	cpos := this.GetCurrentReadIndex()
	ppos := this.GetCurrentWriteIndex()
	return int(ppos - cpos)
}

/**
2016.03.03 添加
初始化ringbuffer
参数bufferSize：初始化环形buffer指针数组大小
*/
func newBuffer(size int64) (*buffer, error) {
	if size < 0 {
		return nil, bufio.ErrNegativeCount
	}
	if size == 0 {
		size = DefaultBufferSize
	}
	if !powerOfTwo64(size) {
		fmt.Printf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
		return nil, fmt.Errorf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
	}

	return &buffer{
		readIndex:  int64(0),                 //读序号
		writeIndex: int64(0),                 //写序号
		ringBuffer: make([]*ByteArray, size), //环形buffer指针数组
		bufferSize: size,                     //初始化环形buffer指针数组大小
		mask:       size - 1,
		rcond:      sync.NewCond(new(sync.Mutex)),
		wcond:      sync.NewCond(new(sync.Mutex)),
	}, nil
}

/**
2016.03.03 添加
获取当前读序号
*/
func (this *buffer) GetCurrentReadIndex() int64 {
	return atomic.LoadInt64(&this.readIndex)
}

/**
2016.03.03 添加
获取当前写序号
*/
func (this *buffer) GetCurrentWriteIndex() int64 {
	return atomic.LoadInt64(&this.writeIndex)
}

/**
2016.03.03 添加
读取ringbuffer指定的buffer指针，返回该指针并清空ringbuffer该位置存在的指针内容，以及将读序号加1
*/
func (this *buffer) ReadBuffer() ([]byte, int64, bool) {
	this.rcond.L.Lock()
	defer this.rcond.L.Unlock()
	for {
		readIndex := atomic.LoadInt64(&this.readIndex)
		writeIndex := atomic.LoadInt64(&this.writeIndex)
		switch {
		case readIndex >= writeIndex:
			this.rcond.Wait()
		case writeIndex-readIndex > this.bufferSize:
			this.rcond.Wait()
		default:
			if readIndex == math.MaxInt64 {
				atomic.StoreInt64(&this.readIndex, int64(0))
			} else {
				atomic.AddInt64(&this.readIndex, int64(1))
			}

			index := readIndex & this.mask

			p_ := this.ringBuffer[index]
			//this.ringBuffer[index] = nil
			if p_ == nil {
				return nil, -1, false
			}
			p := p_.bArray

			if p == nil {
				return nil, -1, false
			}

			return p, index, true
		}
	}

}

/**
2016.03.03 添加
写入ringbuffer指针，以及将写序号加1
*/
func (this *buffer) WriteBuffer(in []byte) bool {
	this.wcond.L.Lock()
	defer func() {
		this.rcond.Broadcast()
		this.wcond.L.Unlock()
	}()

	for {
		readIndex := atomic.LoadInt64(&this.readIndex)
		writeIndex := atomic.LoadInt64(&this.writeIndex)
		switch {
		case writeIndex-readIndex < 0:
			this.wcond.Wait()
		default:
			index := writeIndex & this.mask
			if writeIndex == math.MaxInt64 {
				atomic.StoreInt64(&this.writeIndex, int64(0))
			} else {
				atomic.AddInt64(&this.writeIndex, int64(1))
			}
			if this.ringBuffer[index] == nil {
				this.ringBuffer[index] = &ByteArray{bArray: in}
				return true
			} else {
				return false
			}
		}
	}

}

/**
2016.03.03 修改
完成
*/
func (this *buffer) Close() error {
	atomic.StoreInt64(&this.done, 1)

	this.wcond.L.Lock()
	this.rcond.Broadcast()
	this.wcond.L.Unlock()

	this.rcond.L.Lock()
	this.wcond.Broadcast()
	this.rcond.L.Unlock()

	return nil
}

/*

/**
2016.03.03 修改
向ringbuffer中写数据（从connection的中向ringbuffer中写）--生产者
*/
func (this *buffer) ReadFrom(r io.Reader) (int64, error) {
	defer this.Close()
	total := int64(0)
	//for {

	//if this.isDone() {
	//	return total, io.EOF
	//}
	b := make([]byte, int64(5))
	n, err := r.Read(b[0:1])

	if n > 0 {
		total += int64(n)
		if err != nil {
			return total, err
		}
	}

	/**************************/
	cnt := 1

	// Let's read enough bytes to get the message header (msg type, remaining length)
	for {
		// If we have read 5 bytes and still not done, then there's a problem.
		if cnt > 4 {
			return 0, fmt.Errorf("sendrecv/peekMessageSize: 4th byte of remaining length has continuation bit set")
		}

		Log.Infoc(func() string {
			return fmt.Sprintf("sendrecv/peekMessageSize: %d=========", cnt)
		})
		// Peek cnt bytes from the input buffer.

		_, err := r.Read(b[cnt:(cnt + 1)])
		//fmt.Println(b)
		if err != nil {
			return 0, err
		}
		// If we got enough bytes, then check the last byte to see if the continuation
		// bit is set. If so, increment cnt and continue peeking
		if b[cnt] >= 0x80 {
			cnt++
		} else {
			break
		}
	}

	// Get the remaining length of the message
	remlen, m := binary.Uvarint(b[1:])
	//Log.Infoc(func() string {
	//	return fmt.Sprintf("b[cnt:(cnt + 1)]==end")
	//})
	// Total message length is remlen + 1 (msg type) + m (remlen bytes)

	total = int64(remlen) + int64(1) + int64(m)
	//Log.Infoc(func() string {
	//	return fmt.Sprintf("remlen===n===totle: %d===%d===%d", remlen, m, total)
	//})
	//mtype := message.MessageType(b[0] >> 4)
	/****************/
	//var msg message.Message
	//
	//msg, err = mtype.New()
	//if err != nil {
	//	return 0, err
	//}
	b_ := make([]byte, int64(remlen))
	_, err = r.Read(b_[0:])
	if err != nil {
		Log.Errorc(func() string {
			return fmt.Sprintf("从conn读取数据失败")
		})
		return total, err
	}
	b__ := make([]byte, 0, total)
	b__ = append(b__, b[0:1+m]...)
	b__ = append(b__, b_[0:]...)
	fmt.Println(b__)
	//n, err = msg.Decode(b)
	//if err != nil {
	//	return 0, err
	//}

	/*************************/

	if !this.WriteBuffer(b__) {
		return total, err
	}

	return total, nil
	//}
}

/**
2016.03.03 修改
*/
func (this *buffer) WriteTo(w io.Writer) (int64, error) {
	defer this.Close()
	total := int64(0)
	//for {
	//if this.isDone() {
	//	return total, io.EOF
	//}
	p, index, ok := this.ReadBuffer()
	defer this.ReadCommit(index, ok)
	if !ok {
		return total, errors.New("读取过程中出现问题")
	}

	Log.Debugc(func() string {
		return fmt.Sprintf("WriteTo方法读取ringbuffer成功(%d)", index)
	})
	//Log.Debugc(func() string {
	//	return fmt.Sprintf("WriteTo函数》》读取*p：" + p)
	//})

	//
	//Log.Errorc(func() string {
	//	return fmt.Sprintf("msg::" + msg.Name())
	//})
	//
	//p := make([]byte, msg.Len())
	//_, err := msg.Encode(p)
	//if err != nil {
	//	Log.Errorc(func() string {
	//		return fmt.Sprintf("msg.Encode(p)")
	//	})
	//	return total, io.EOF
	//}
	// There's some data, let's process it first
	if len(p) > 0 {
		n, err := w.Write(p)
		total += int64(n)
		Log.Debugc(func() string {
			return fmt.Sprintf("Wrote %d bytes, totaling %d bytes", n, total)
		})

		if err != nil {
			Log.Errorc(func() string {
				return fmt.Sprintf("w.Write(p) error")
			})
			return total, err
		}
	}

	return total, nil
	//}
}

/**
2016.03.03 修改
*/
func (this *buffer) isDone() bool {
	if atomic.LoadInt64(&this.done) == 1 {
		return true
	}

	return false
}

func powerOfTwo64(n int64) bool {
	return n != 0 && (n&(n-1)) == 0
}

func roundUpPowerOfTwo64(n int64) int64 {
	n--
	n |= n >> 1
	n |= n >> 2
	n |= n >> 4
	n |= n >> 8
	n |= n >> 16
	n |= n >> 32
	n++

	return n
}
<commit_msg>添加测试代码<commit_after>// Copyright (c) 2014 The SurgeMQ Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

import (
	"bufio"
	"fmt"
	"io"
	"sync/atomic"
	//"github.com/surgemq/message"
	"encoding/binary"
	"errors"
	"math"
	"sync"
)

var (
	bufcnt            int64
	DefaultBufferSize int64

	DeviceInBufferSize  int64
	DeviceOutBufferSize int64

	MasterInBufferSize  int64
	MasterOutBufferSize int64
)

const (
	smallReadBlockSize    = 512
	defaultReadBlockSize  = 8192
	defaultWriteBlockSize = 8192
)

/**
2016.03.03 修改
bingbuffer结构体
*/
type buffer struct {
	readIndex  int64        //读序号
	writeIndex int64        //写序号
	ringBuffer []*ByteArray //环形buffer指针数组
	bufferSize int64        //初始化环形buffer指针数组大小
	mask       int64        //掩码：bufferSize-1
	done       int64        //是否完成
	rcond      *sync.Cond
	wcond      *sync.Cond
}

type ByteArray struct {
	bArray []byte
}

func (this *buffer) ReadCommit(index int64, ok bool) {
	this.rcond.L.Lock()
	defer this.rcond.L.Unlock()
	if ok {
		this.ringBuffer[index] = nil
	}
	this.wcond.Broadcast()
}

func (this *buffer) Len() int {
	cpos := this.GetCurrentReadIndex()
	ppos := this.GetCurrentWriteIndex()
	return int(ppos - cpos)
}

/**
2016.03.03 添加
初始化ringbuffer
参数bufferSize：初始化环形buffer指针数组大小
*/
func newBuffer(size int64) (*buffer, error) {
	if size < 0 {
		return nil, bufio.ErrNegativeCount
	}
	if size == 0 {
		size = DefaultBufferSize
	}
	if !powerOfTwo64(size) {
		fmt.Printf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
		return nil, fmt.Errorf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
	}

	return &buffer{
		readIndex:  int64(0),                 //读序号
		writeIndex: int64(0),                 //写序号
		ringBuffer: make([]*ByteArray, size), //环形buffer指针数组
		bufferSize: size,                     //初始化环形buffer指针数组大小
		mask:       size - 1,
		rcond:      sync.NewCond(new(sync.Mutex)),
		wcond:      sync.NewCond(new(sync.Mutex)),
	}, nil
}

/**
2016.03.03 添加
获取当前读序号
*/
func (this *buffer) GetCurrentReadIndex() int64 {
	return atomic.LoadInt64(&this.readIndex)
}

/**
2016.03.03 添加
获取当前写序号
*/
func (this *buffer) GetCurrentWriteIndex() int64 {
	return atomic.LoadInt64(&this.writeIndex)
}

/**
2016.03.03 添加
读取ringbuffer指定的buffer指针，返回该指针并清空ringbuffer该位置存在的指针内容，以及将读序号加1
*/
func (this *buffer) ReadBuffer() ([]byte, int64, bool) {
	this.rcond.L.Lock()
	defer this.rcond.L.Unlock()
	for {
		readIndex := atomic.LoadInt64(&this.readIndex)
		writeIndex := atomic.LoadInt64(&this.writeIndex)
		switch {
		case readIndex >= writeIndex:
			this.rcond.Wait()
		case writeIndex-readIndex > this.bufferSize:
			this.rcond.Wait()
		default:
			if readIndex == math.MaxInt64 {
				atomic.StoreInt64(&this.readIndex, int64(0))
			} else {
				atomic.AddInt64(&this.readIndex, int64(1))
			}

			index := readIndex & this.mask

			p_ := this.ringBuffer[index]
			//this.ringBuffer[index] = nil
			if p_ == nil {
				return nil, -1, false
			}
			p := p_.bArray

			if p == nil {
				return nil, -1, false
			}

			return p, index, true
		}
	}

}

/**
2016.03.03 添加
写入ringbuffer指针，以及将写序号加1
*/
func (this *buffer) WriteBuffer(in []byte) bool {
	this.wcond.L.Lock()
	defer func() {
		this.rcond.Broadcast()
		this.wcond.L.Unlock()
	}()

	for {
		readIndex := atomic.LoadInt64(&this.readIndex)
		writeIndex := atomic.LoadInt64(&this.writeIndex)
		switch {
		case writeIndex-readIndex < 0:
			this.wcond.Wait()
		default:
			index := writeIndex & this.mask
			if writeIndex == math.MaxInt64 {
				atomic.StoreInt64(&this.writeIndex, int64(0))
			} else {
				atomic.AddInt64(&this.writeIndex, int64(1))
			}
			if this.ringBuffer[index] == nil {
				this.ringBuffer[index] = &ByteArray{bArray: in}
				return true
			} else {
				return false
			}
		}
	}

}

/**
2016.03.03 修改
完成
*/
func (this *buffer) Close() error {
	atomic.StoreInt64(&this.done, 1)

	this.wcond.L.Lock()
	this.rcond.Broadcast()
	this.wcond.L.Unlock()

	this.rcond.L.Lock()
	this.wcond.Broadcast()
	this.rcond.L.Unlock()

	return nil
}

/*

/**
2016.03.03 修改
向ringbuffer中写数据（从connection的中向ringbuffer中写）--生产者
*/
func (this *buffer) ReadFrom(r io.Reader) (int64, error) {
	defer this.Close()
	total := int64(0)
	//for {

	//if this.isDone() {
	//	return total, io.EOF
	//}
	b := make([]byte, int64(5))
	n, err := r.Read(b[0:1])

	if n > 0 {
		total += int64(n)
		if err != nil {
			return total, err
		}
	}

	/**************************/
	cnt := 1

	// Let's read enough bytes to get the message header (msg type, remaining length)
	for {
		// If we have read 5 bytes and still not done, then there's a problem.
		if cnt > 4 {
			return 0, fmt.Errorf("sendrecv/peekMessageSize: 4th byte of remaining length has continuation bit set")
		}

		Log.Infoc(func() string {
			return fmt.Sprintf("sendrecv/peekMessageSize: %d=========", cnt)
		})
		// Peek cnt bytes from the input buffer.

		_, err := r.Read(b[cnt:(cnt + 1)])
		//fmt.Println(b)
		if err != nil {
			return 0, err
		}
		// If we got enough bytes, then check the last byte to see if the continuation
		// bit is set. If so, increment cnt and continue peeking
		if b[cnt] >= 0x80 {
			cnt++
		} else {
			break
		}
	}

	// Get the remaining length of the message
	remlen, m := binary.Uvarint(b[1:])
	//Log.Infoc(func() string {
	//	return fmt.Sprintf("b[cnt:(cnt + 1)]==end")
	//})
	// Total message length is remlen + 1 (msg type) + m (remlen bytes)

	total = int64(remlen) + int64(1) + int64(m)
	//Log.Infoc(func() string {
	//	return fmt.Sprintf("remlen===n===totle: %d===%d===%d", remlen, m, total)
	//})
	//mtype := message.MessageType(b[0] >> 4)
	/****************/
	//var msg message.Message
	//
	//msg, err = mtype.New()
	//if err != nil {
	//	return 0, err
	//}
	b_ := make([]byte, int64(remlen))
	_, err = r.Read(b_[0:])
	if err != nil {
		Log.Errorc(func() string {
			return fmt.Sprintf("从conn读取数据失败(%s)", err)
		})

		return total, err
	}
	b__ := make([]byte, 0, total)
	b__ = append(b__, b[0:1+m]...)
	b__ = append(b__, b_[0:]...)
	fmt.Println(b__)
	//n, err = msg.Decode(b)
	//if err != nil {
	//	return 0, err
	//}

	/*************************/

	if !this.WriteBuffer(b__) {
		return total, err
	}

	return total, nil
	//}
}

/**
2016.03.03 修改
*/
func (this *buffer) WriteTo(w io.Writer) (int64, error) {
	defer this.Close()
	total := int64(0)
	//for {
	//if this.isDone() {
	//	return total, io.EOF
	//}
	p, index, ok := this.ReadBuffer()
	defer this.ReadCommit(index, ok)
	if !ok {
		return total, errors.New("读取过程中出现问题")
	}

	Log.Debugc(func() string {
		return fmt.Sprintf("WriteTo方法读取ringbuffer成功(%d)", index)
	})
	//Log.Debugc(func() string {
	//	return fmt.Sprintf("WriteTo函数》》读取*p：" + p)
	//})

	//
	//Log.Errorc(func() string {
	//	return fmt.Sprintf("msg::" + msg.Name())
	//})
	//
	//p := make([]byte, msg.Len())
	//_, err := msg.Encode(p)
	//if err != nil {
	//	Log.Errorc(func() string {
	//		return fmt.Sprintf("msg.Encode(p)")
	//	})
	//	return total, io.EOF
	//}
	// There's some data, let's process it first
	if len(p) > 0 {
		n, err := w.Write(p)
		total += int64(n)
		Log.Debugc(func() string {
			return fmt.Sprintf("Wrote %d bytes, totaling %d bytes", n, total)
		})

		if err != nil {
			Log.Errorc(func() string {
				return fmt.Sprintf("w.Write(p) error")
			})
			return total, err
		}
	}

	return total, nil
	//}
}

/**
2016.03.03 修改
*/
func (this *buffer) isDone() bool {
	if atomic.LoadInt64(&this.done) == 1 {
		return true
	}

	return false
}

func powerOfTwo64(n int64) bool {
	return n != 0 && (n&(n-1)) == 0
}

func roundUpPowerOfTwo64(n int64) int64 {
	n--
	n |= n >> 1
	n |= n >> 2
	n |= n >> 4
	n |= n >> 8
	n |= n >> 16
	n |= n >> 32
	n++

	return n
}
<|endoftext|>


<commit_before>package pt

import (
	"fmt"
	"image"
	"math"
	"math/rand"
	"runtime"
	"strings"
	"sync"
	"time"
)

func showProgress(start time.Time, rays uint64, i, h int) {
	pct := int(100 * float64(i) / float64(h))
	elapsed := time.Since(start)
	rps := float64(rays) / elapsed.Seconds()
	fmt.Printf("\r%4d / %d (%3d%%) [", i, h, pct)
	for p := 0; p < 100; p += 3 {
		if pct > p {
			fmt.Print("=")
		} else {
			fmt.Print(" ")
		}
	}
	fmt.Printf("] %s %s ", DurationString(elapsed), NumberString(rps))
}

func render(scene *Scene, camera *Camera, sampler Sampler, w, h, samplesPerPixel int) []Color {
	ncpu := runtime.NumCPU()
	runtime.GOMAXPROCS(ncpu)
	scene.Compile()
	pixels := make([]Color, w*h)
	ch := make(chan int, h)
	absSamples := int(math.Abs(float64(samplesPerPixel)))
	fmt.Printf("%d x %d pixels, %d spp, %d cores\n", w, h, absSamples, ncpu)
	start := time.Now()
	scene.rays = 0
	for i := 0; i < ncpu; i++ {
		go func(i int) {
			rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
			for y := i; y < h; y += ncpu {
				for x := 0; x < w; x++ {
					c := Black
					if samplesPerPixel <= 0 {
						// random subsampling
						for i := 0; i < absSamples; i++ {
							fu := rnd.Float64()
							fv := rnd.Float64()
							ray := camera.CastRay(x, y, w, h, fu, fv, rnd)
							c = c.Add(sampler.Sample(scene, ray, rnd))
						}
						c = c.DivScalar(float64(absSamples))
					} else {
						// stratified subsampling
						n := int(math.Sqrt(float64(samplesPerPixel)))
						for u := 0; u < n; u++ {
							for v := 0; v < n; v++ {
								fu := (float64(u) + 0.5) / float64(n)
								fv := (float64(v) + 0.5) / float64(n)
								ray := camera.CastRay(x, y, w, h, fu, fv, rnd)
								c = c.Add(sampler.Sample(scene, ray, rnd))
							}
						}
						c = c.DivScalar(float64(n * n))
					}
					pixels[y*w+x] = c
				}
				ch <- 1
			}
		}(i)
	}
	showProgress(start, scene.RayCount(), 0, h)
	for i := 0; i < h; i++ {
		<-ch
		showProgress(start, scene.RayCount(), i+1, h)
	}
	fmt.Println()
	return pixels
}

func Render(scene *Scene, camera *Camera, sampler Sampler, w, h, samplesPerPixel int) image.Image {
	pixels := render(scene, camera, sampler, w, h, samplesPerPixel)
	return pixelsToImage(pixels, w, h, 1)
}

func pixelsToImage(pixels []Color, w, h int, scale float64) image.Image {
	result := image.NewRGBA64(image.Rect(0, 0, w, h))
	for y := 0; y < h; y++ {
		for x := 0; x < w; x++ {
			result.SetRGBA64(x, y, pixels[y*w+x].MulScalar(scale).Pow(1/2.2).RGBA64())
		}
	}
	return result
}

func onIteration(pathTemplate string, i, w, h int, pixels []Color, frame []Color, wg *sync.WaitGroup) {
	defer wg.Done()
	for y := 0; y < h; y++ {
		for x := 0; x < w; x++ {
			index := y*w + x
			pixels[index] = pixels[index].Add(frame[index])
		}
	}
	scale := 1 / float64(i)
	result := pixelsToImage(pixels, w, h, scale)
	path := pathTemplate
	if strings.Contains(path, "%") {
		path = fmt.Sprintf(pathTemplate, i)
	}
	if err := SavePNG(path, result); err != nil {
		panic(err)
	}
}

func IterativeRender(pathTemplate string, iterations int, scene *Scene, camera *Camera, sampler Sampler, w, h, samplesPerPixel int) {
	var wg sync.WaitGroup
	scene.Compile()
	pixels := make([]Color, w*h)
	for i := 1; i <= iterations; i++ {
		fmt.Printf("\n[Iteration %d of %d]\n", i, iterations)
		frame := render(scene, camera, sampler, w, h, samplesPerPixel)
		wg.Add(1)
		go onIteration(pathTemplate, i, w, h, pixels, frame, &wg)
	}
	wg.Wait()
}
<commit_msg>CallbackRender returns channel of image.Image<commit_after>package pt

import (
	"fmt"
	"image"
	"math"
	"math/rand"
	"runtime"
	"strings"
	"sync"
	"time"
)

func showProgress(start time.Time, rays uint64, i, h int) {
	pct := int(100 * float64(i) / float64(h))
	elapsed := time.Since(start)
	rps := float64(rays) / elapsed.Seconds()
	fmt.Printf("\r%4d / %d (%3d%%) [", i, h, pct)
	for p := 0; p < 100; p += 3 {
		if pct > p {
			fmt.Print("=")
		} else {
			fmt.Print(" ")
		}
	}
	fmt.Printf("] %s %s ", DurationString(elapsed), NumberString(rps))
}

func render(scene *Scene, camera *Camera, sampler Sampler, w, h, samplesPerPixel int) []Color {
	ncpu := runtime.NumCPU()
	runtime.GOMAXPROCS(ncpu)
	scene.Compile()
	pixels := make([]Color, w*h)
	ch := make(chan int, h)
	absSamples := int(math.Abs(float64(samplesPerPixel)))
	fmt.Printf("%d x %d pixels, %d spp, %d cores\n", w, h, absSamples, ncpu)
	start := time.Now()
	scene.rays = 0
	for i := 0; i < ncpu; i++ {
		go func(i int) {
			rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
			for y := i; y < h; y += ncpu {
				for x := 0; x < w; x++ {
					c := Black
					if samplesPerPixel <= 0 {
						// random subsampling
						for i := 0; i < absSamples; i++ {
							fu := rnd.Float64()
							fv := rnd.Float64()
							ray := camera.CastRay(x, y, w, h, fu, fv, rnd)
							c = c.Add(sampler.Sample(scene, ray, rnd))
						}
						c = c.DivScalar(float64(absSamples))
					} else {
						// stratified subsampling
						n := int(math.Sqrt(float64(samplesPerPixel)))
						for u := 0; u < n; u++ {
							for v := 0; v < n; v++ {
								fu := (float64(u) + 0.5) / float64(n)
								fv := (float64(v) + 0.5) / float64(n)
								ray := camera.CastRay(x, y, w, h, fu, fv, rnd)
								c = c.Add(sampler.Sample(scene, ray, rnd))
							}
						}
						c = c.DivScalar(float64(n * n))
					}
					pixels[y*w+x] = c
				}
				ch <- 1
			}
		}(i)
	}
	showProgress(start, scene.RayCount(), 0, h)
	for i := 0; i < h; i++ {
		<-ch
		showProgress(start, scene.RayCount(), i+1, h)
	}
	fmt.Println()
	return pixels
}

func Render(scene *Scene, camera *Camera, sampler Sampler, w, h, samplesPerPixel int) image.Image {
	pixels := render(scene, camera, sampler, w, h, samplesPerPixel)
	return pixelsToImage(pixels, w, h, 1)
}

func pixelsToImage(pixels []Color, w, h int, scale float64) image.Image {
	result := image.NewRGBA64(image.Rect(0, 0, w, h))
	for y := 0; y < h; y++ {
		for x := 0; x < w; x++ {
			result.SetRGBA64(x, y, pixels[y*w+x].MulScalar(scale).Pow(1/2.2).RGBA64())
		}
	}
	return result
}

func onIteration(pathTemplate string, i, w, h int, pixels []Color, frame []Color, wg *sync.WaitGroup) {
	defer wg.Done()
	for y := 0; y < h; y++ {
		for x := 0; x < w; x++ {
			index := y*w + x
			pixels[index] = pixels[index].Add(frame[index])
		}
	}
	scale := 1 / float64(i)
	result := pixelsToImage(pixels, w, h, scale)
	path := pathTemplate
	if strings.Contains(path, "%") {
		path = fmt.Sprintf(pathTemplate, i)
	}
	if err := SavePNG(path, result); err != nil {
		panic(err)
	}
}

func IterativeRender(pathTemplate string, iterations int, scene *Scene, camera *Camera, sampler Sampler, w, h, samplesPerPixel int) {
	var wg sync.WaitGroup
	scene.Compile()
	pixels := make([]Color, w*h)
	for i := 1; i <= iterations; i++ {
		fmt.Printf("\n[Iteration %d of %d]\n", i, iterations)
		frame := render(scene, camera, sampler, w, h, samplesPerPixel)
		wg.Add(1)
		go onIteration(pathTemplate, i, w, h, pixels, frame, &wg)
	}
	wg.Wait()
}

func CallbackRender(scene *Scene, camera *Camera, sampler Sampler, w, h, samplesPerPixel int) <-chan image.Image {
	ch := make(chan image.Image)
	go func() {
		scene.Compile()
		pixels := make([]Color, w*h)
		for i := 1; ; i++ {
			frame := render(scene, camera, sampler, w, h, samplesPerPixel)
			for y := 0; y < h; y++ {
				for x := 0; x < w; x++ {
					index := y*w + x
					pixels[index] = pixels[index].Add(frame[index])
				}
			}
			scale := 1 / float64(i)
			result := pixelsToImage(pixels, w, h, scale)
			ch <- result
		}
	}()
	return ch
}
<|endoftext|>


<commit_before>package service

import (
	"strconv"

	"github.com/rs/zerolog"

	"github.com/gilcrest/go-api-basic/domain/errs"
	"github.com/gilcrest/go-api-basic/domain/logger"
)

// LoggerRequest is the request struct for the app logger
type LoggerRequest struct {
	GlobalLogLevel string `json:"global_log_level"`
	LogErrorStack  string `json:"log_error_stack"`
}

// LoggerResponse is the response struct for the current
// state of the app logger
type LoggerResponse struct {
	LoggerMinimumLevel string `json:"logger_minimum_level"`
	GlobalLogLevel     string `json:"global_log_level"`
	LogErrorStack      bool   `json:"log_error_stack"`
}

// LoggerService reads and updates the logger state
type LoggerService struct {
	Logger zerolog.Logger
}

// NewLoggerService is an initializer for LoggerService
func NewLoggerService(logger zerolog.Logger) *LoggerService {
	return &LoggerService{Logger: logger}
}

// ReadLogger handles GET requests for the /logger endpoint
func (ls LoggerService) Read() LoggerResponse {

	var logErrorStack bool
	if zerolog.ErrorStackMarshaler != nil {
		logErrorStack = true
	}

	response := LoggerResponse{
		LoggerMinimumLevel: ls.Logger.GetLevel().String(),
		GlobalLogLevel:     zerolog.GlobalLevel().String(),
		LogErrorStack:      logErrorStack,
	}

	return response
}

// Update handles PUT requests for the /logger endpoint
// and updates the logger globals
func (ls LoggerService) Update(r *LoggerRequest) (LoggerResponse, error) {

	if r.GlobalLogLevel != "" {
		// parse input level from request (if present) and set to that
		lvl, err := zerolog.ParseLevel(r.GlobalLogLevel)
		if err != nil {
			return LoggerResponse{}, errs.E(errs.Validation, err)
		}

		clvl := zerolog.GlobalLevel()

		if lvl != clvl {
			// set the global logging level
			zerolog.SetGlobalLevel(lvl)
		}
	}

	if r.LogErrorStack != "" {
		var (
			les bool
			err error
		)
		if les, err = strconv.ParseBool(r.LogErrorStack); err != nil {
			return LoggerResponse{}, errs.E(errs.Validation, "Invalid value sent for log_error_stack")
		}
		// use input LogErrorStack boolean to set whether or not to
		// write error stack
		logger.WriteErrorStackGlobal(les)
	}

	var logErrorStack bool
	if zerolog.ErrorStackMarshaler != nil {
		logErrorStack = true
	}

	response := LoggerResponse{
		LoggerMinimumLevel: ls.Logger.GetLevel().String(),
		GlobalLogLevel:     zerolog.GlobalLevel().String(),
		LogErrorStack:      logErrorStack,
	}

	return response, nil
}
<commit_msg>Remove NewLoggerService initializer<commit_after>package service

import (
	"strconv"

	"github.com/rs/zerolog"

	"github.com/gilcrest/go-api-basic/domain/errs"
	"github.com/gilcrest/go-api-basic/domain/logger"
)

// LoggerRequest is the request struct for the app logger
type LoggerRequest struct {
	GlobalLogLevel string `json:"global_log_level"`
	LogErrorStack  string `json:"log_error_stack"`
}

// LoggerResponse is the response struct for the current
// state of the app logger
type LoggerResponse struct {
	LoggerMinimumLevel string `json:"logger_minimum_level"`
	GlobalLogLevel     string `json:"global_log_level"`
	LogErrorStack      bool   `json:"log_error_stack"`
}

// LoggerService reads and updates the logger state
type LoggerService struct {
	Logger zerolog.Logger
}

// ReadLogger handles GET requests for the /logger endpoint
func (ls LoggerService) Read() LoggerResponse {

	var logErrorStack bool
	if zerolog.ErrorStackMarshaler != nil {
		logErrorStack = true
	}

	response := LoggerResponse{
		LoggerMinimumLevel: ls.Logger.GetLevel().String(),
		GlobalLogLevel:     zerolog.GlobalLevel().String(),
		LogErrorStack:      logErrorStack,
	}

	return response
}

// Update handles PUT requests for the /logger endpoint
// and updates the logger globals
func (ls LoggerService) Update(r *LoggerRequest) (LoggerResponse, error) {

	if r.GlobalLogLevel != "" {
		// parse input level from request (if present) and set to that
		lvl, err := zerolog.ParseLevel(r.GlobalLogLevel)
		if err != nil {
			return LoggerResponse{}, errs.E(errs.Validation, err)
		}

		clvl := zerolog.GlobalLevel()

		if lvl != clvl {
			// set the global logging level
			zerolog.SetGlobalLevel(lvl)
		}
	}

	if r.LogErrorStack != "" {
		var (
			les bool
			err error
		)
		if les, err = strconv.ParseBool(r.LogErrorStack); err != nil {
			return LoggerResponse{}, errs.E(errs.Validation, "Invalid value sent for log_error_stack")
		}
		// use input LogErrorStack boolean to set whether or not to
		// write error stack
		logger.WriteErrorStackGlobal(les)
	}

	var logErrorStack bool
	if zerolog.ErrorStackMarshaler != nil {
		logErrorStack = true
	}

	response := LoggerResponse{
		LoggerMinimumLevel: ls.Logger.GetLevel().String(),
		GlobalLogLevel:     zerolog.GlobalLevel().String(),
		LogErrorStack:      logErrorStack,
	}

	return response, nil
}
<|endoftext|>


<commit_before>package set

import (
	"testing"
)

// BenchmarkAdd checks the performance of the set.Add() method
func BenchmarkAdd(b *testing.B) {
	// Create a new set
	set := New()

	// Run set.Add() b.N times
	for i := 0; i < b.N; i++ {
		set.Add(i)
	}
}

// benchmarkCartesianProduct checks the performance of the set.CartesianProduct() method
func benchmarkCartesianProduct(n int, s *Set, t *Set) {
	// Run set.CartesianProduct() n times
	for i := 0; i < n; i++ {
		s.CartesianProduct(t)
	}
}

// BenchmarkCartesianProductSmall checks the performance of the set.CartesianProduct() method
// over a small data set
func BenchmarkCartesianProductSmall(b *testing.B) {
	benchmarkCartesianProduct(b.N, New(1, 2), New(2, 1))
}

// BenchmarkCartesianProductLarge checks the performance of the set.CartesianProduct() method
// over a large data set
func BenchmarkCartesianProductLarge(b *testing.B) {
	benchmarkCartesianProduct(b.N, New(1, 2, 3, 4, 5, 6, 7, 8, 9), New(9, 8, 7, 6, 5, 4, 3, 2, 1))
}

// benchmarkClone checks the performance of the set.Clone() method
func benchmarkClone(n int, s *Set) {
	// Run set.Clone() n times
	for i := 0; i < n; i++ {
		s.Clone()
	}
}

// BenchmarkCloneSmall checks the performance of the set.Clone() method
// over a small data set
func BenchmarkCloneSmall(b *testing.B) {
	benchmarkClone(b.N, New(1, 2))
}

// BenchmarkCloneLarge checks the performance of the set.Clone() method
// over a large data set
func BenchmarkCloneLarge(b *testing.B) {
	benchmarkClone(b.N, New(1, 2, 3, 4, 5, 6, 7, 8, 9))
}
<commit_msg>Add BenchmarkDifference(), small and large variants<commit_after>package set

import (
	"testing"
)

// BenchmarkAdd checks the performance of the set.Add() method
func BenchmarkAdd(b *testing.B) {
	// Create a new set
	set := New()

	// Run set.Add() b.N times
	for i := 0; i < b.N; i++ {
		set.Add(i)
	}
}

// benchmarkCartesianProduct checks the performance of the set.CartesianProduct() method
func benchmarkCartesianProduct(n int, s *Set, t *Set) {
	// Run set.CartesianProduct() n times
	for i := 0; i < n; i++ {
		s.CartesianProduct(t)
	}
}

// BenchmarkCartesianProductSmall checks the performance of the set.CartesianProduct() method
// over a small data set
func BenchmarkCartesianProductSmall(b *testing.B) {
	benchmarkCartesianProduct(b.N, New(1, 2), New(2, 1))
}

// BenchmarkCartesianProductLarge checks the performance of the set.CartesianProduct() method
// over a large data set
func BenchmarkCartesianProductLarge(b *testing.B) {
	benchmarkCartesianProduct(b.N, New(1, 2, 3, 4, 5, 6, 7, 8, 9), New(9, 8, 7, 6, 5, 4, 3, 2, 1))
}

// benchmarkClone checks the performance of the set.Clone() method
func benchmarkClone(n int, s *Set) {
	// Run set.Clone() n times
	for i := 0; i < n; i++ {
		s.Clone()
	}
}

// BenchmarkCloneSmall checks the performance of the set.Clone() method
// over a small data set
func BenchmarkCloneSmall(b *testing.B) {
	benchmarkClone(b.N, New(1, 2))
}

// BenchmarkCloneLarge checks the performance of the set.Clone() method
// over a large data set
func BenchmarkCloneLarge(b *testing.B) {
	benchmarkClone(b.N, New(1, 2, 3, 4, 5, 6, 7, 8, 9))
}

// benchmarkDifference checks the performance of the set.Difference() method
func benchmarkDifference(n int, s *Set, t *Set) {
	// Run set.Difference() n times
	for i := 0; i < n; i++ {
		s.Difference(t)
	}
}

// BenchmarkDifferenceSmall checks the performance of the set.Difference() method
// over a small data set
func BenchmarkDifferenceSmall(b *testing.B) {
	benchmarkDifference(b.N, New(1, 2), New(2, 1))
}

// BenchmarkDifferenceLarge checks the performance of the set.Difference() method
// over a large data set
func BenchmarkDifferenceLarge(b *testing.B) {
	benchmarkDifference(b.N, New(1, 2, 3, 4, 5, 6, 7, 8, 9), New(9, 8, 7, 6, 5, 4, 3, 2, 1))
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/bmizerany/assert"
	"github.com/yuin/gopher-lua"
	"html/template"
	"io/ioutil"
	"net/http/httptest"
	"testing"
	"time"
)

func TestPongoPage(t *testing.T) {
	fs = NewFileStat(true, time.Minute*1)

	req := httptest.NewRequest("GET", "/", nil)
	w := httptest.NewRecorder()

	filename := "samples/pongo2/index.po2"
	luafilename := "samples/pongo2/data.lua"
	pongodata, err := ioutil.ReadFile(filename)
	assert.Equal(t, err, nil)
	cache := newFileCache(10000000, true, 64*KiB)

	luablock, err := cache.read(luafilename, shouldCache(".po2"))
	assert.Equal(t, err, nil)

	// Make functions from the given Lua data available
	funcs := make(template.FuncMap)
	// luablock can be empty if there was an error or if the file was empty
	assert.Equal(t, luablock.HasData(), true)

	// Lua LState pool
	luapool := &lStatePool{saved: make([]*lua.LState, 0, 4)}
	atShutdown(func() {
		luapool.Shutdown()
	})

	// There was Lua code available. Now make the functions and
	// variables available for the template.
	funcs, err = luaFunctionMap(w, req, luablock.MustData(), luafilename, nil, luapool, cache)
	assert.Equal(t, err, nil)

	// Trigger the error
	for i := 0; i < 10; i++ {
		go pongoPage(w, req, filename, luafilename, pongodata, funcs, cache)
	}
}
<commit_msg>Update test comment<commit_after>package main

import (
	"github.com/bmizerany/assert"
	"github.com/yuin/gopher-lua"
	"html/template"
	"io/ioutil"
	"net/http/httptest"
	"testing"
	"time"
)

func TestPongoPage(t *testing.T) {
	fs = NewFileStat(true, time.Minute*1)

	req := httptest.NewRequest("GET", "/", nil)
	w := httptest.NewRecorder()

	filename := "samples/pongo2/index.po2"
	luafilename := "samples/pongo2/data.lua"
	pongodata, err := ioutil.ReadFile(filename)
	assert.Equal(t, err, nil)
	cache := newFileCache(10000000, true, 64*KiB)

	luablock, err := cache.read(luafilename, shouldCache(".po2"))
	assert.Equal(t, err, nil)

	// Make functions from the given Lua data available
	funcs := make(template.FuncMap)
	// luablock can be empty if there was an error or if the file was empty
	assert.Equal(t, luablock.HasData(), true)

	// Lua LState pool
	luapool := &lStatePool{saved: make([]*lua.LState, 0, 4)}
	atShutdown(func() {
		luapool.Shutdown()
	})

	// There was Lua code available. Now make the functions and
	// variables available for the template.
	funcs, err = luaFunctionMap(w, req, luablock.MustData(), luafilename, nil, luapool, cache)
	assert.Equal(t, err, nil)

	// Trigger the error (now resolved)
	for i := 0; i < 10; i++ {
		go pongoPage(w, req, filename, luafilename, pongodata, funcs, cache)
	}
}
<|endoftext|>