<commit_before>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017 Red Hat, Inc.
 *
 */

package watch

import (
	"net/http"
	"strings"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/ghttp"
	clientv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/util/workqueue"

	kubev1 "k8s.io/api/core/v1"

	v1 "kubevirt.io/kubevirt/pkg/api/v1"
	"kubevirt.io/kubevirt/pkg/kubecli"
	"kubevirt.io/kubevirt/pkg/log"
	"kubevirt.io/kubevirt/pkg/virt-controller/services"
)

var _ = Describe("VM watcher", func() {
	var server *ghttp.Server
	//var vmService services.VMService

	log.Log.SetIOWriter(GinkgoWriter)

	var app VirtControllerApp = VirtControllerApp{}
	app.launcherImage = "kubevirt/virt-launcher"
	BeforeEach(func() {

		server = ghttp.NewServer()
		app.clientSet, _ = kubecli.GetKubevirtClientFromFlags(server.URL(), "")
		app.restClient = app.clientSet.RestClient()
		app.vmCache = cache.NewIndexer(cache.DeletionHandlingMetaNamespaceKeyFunc, nil)
		app.vmQueue = workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())

		app.initCommon()
	})

	Context("Creating a VM ", func() {
		It("should ignore uninitialized VM's", func(done Done) {
			vm := v1.NewMinimalVM("testvm")
			// Register the expected REST call
			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", "/api/v1/namespaces/default/pods"),
					ghttp.RespondWithJSONEncoded(http.StatusOK, nil),
				),
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("POST", "/api/v1/namespaces/default/pods"),
					ghttp.RespondWithJSONEncoded(http.StatusOK, nil),
				),

				ghttp.CombineHandlers(
					ghttp.VerifyRequest("PUT", "/apis/kubevirt.io/v1alpha1/namespaces/default/virtualmachines/testvm"),
					ghttp.RespondWithJSONEncoded(http.StatusOK, vm),
				),
			)

			key, _ := cache.MetaNamespaceKeyFunc(vm)
			app.vmCache.Add(vm)
			app.vmQueue.Add(key)
			app.vmController.Execute()

			// VM's that aren't annotated that presets have been applied
			// should not be acted upon, so 0 requests are expected
			Expect(len(server.ReceivedRequests())).To(Equal(0))
			close(done)
		})

		It("should should schedule a POD.", func(done Done) {

			// Create a VM to be scheduled
			vm := v1.NewMinimalVM("testvm")
			vm.Status.Phase = ""
			vm.ObjectMeta.SetUID(uuid.NewUUID())
			addInitializedAnnotation(vm)

			// Create the expected VM after the update
			expectedVM := vm.DeepCopy()

			// Create a Pod for the VM
			temlateService, err := services.NewTemplateService("whatever", "whatever", "whatever")
			Expect(err).ToNot(HaveOccurred())
			pod, err := temlateService.RenderLaunchManifest(vm)
			Expect(err).ToNot(HaveOccurred())
			pod.Spec.NodeName = "mynode"
			pod.Status.Phase = clientv1.PodSucceeded

			podListInitial := clientv1.PodList{}
			podListInitial.Items = []clientv1.Pod{}

			podListPostCreate := clientv1.PodList{}
			podListPostCreate.Items = []clientv1.Pod{*pod}

			expectedVM.Status.Phase = v1.Scheduling

			// Register the expected REST call
			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", "/api/v1/namespaces/default/pods"),
					ghttp.RespondWithJSONEncoded(http.StatusOK, podListInitial),
				),
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("POST", "/api/v1/namespaces/default/pods"),
					ghttp.RespondWithJSONEncoded(http.StatusOK, pod),
				),

				ghttp.CombineHandlers(
					ghttp.VerifyRequest("PUT", "/apis/kubevirt.io/v1alpha1/namespaces/default/virtualmachines/testvm"),
					ghttp.RespondWithJSONEncoded(http.StatusOK, vm),
				),
			)

			// Tell the controller that there is a new VM
			key, _ := cache.MetaNamespaceKeyFunc(vm)
			app.vmCache.Add(vm)
			app.vmQueue.Add(key)
			app.vmController.Execute()

			Expect(len(server.ReceivedRequests())).To(Equal(3))
			close(done)
		}, 10)

		It("should should schedule a POD with Registry Disk.", func(done Done) {

			// Create a VM to be scheduled
			vm := v1.NewMinimalVM("testvm")
			vm.Status.Phase = ""
			vm.ObjectMeta.SetUID(uuid.NewUUID())
			vm.Spec.Domain.Devices.Disks = append(vm.Spec.Domain.Devices.Disks, v1.Disk{
				Name: "r0",
				DiskDevice: v1.DiskDevice{
					Disk: &v1.DiskTarget{},
				},
			})
			vm.Spec.Volumes = append(vm.Spec.Volumes, v1.Volume{
				Name: "r0",
				VolumeSource: v1.VolumeSource{
					RegistryDisk: &v1.RegistryDiskSource{
						Image: "someimage:v1.2.3.4",
					},
				},
			})
			addInitializedAnnotation(vm)

			// Create a Pod for the VM
			templateService, err := services.NewTemplateService("whatever", "whatever", "whatever")
			Expect(err).ToNot(HaveOccurred())

			// We want to ensure the vm object we initially post
			// doesn't have ports set, so we make a copy in order
			// to render the pod object early for the test.
			vmCopy := vm.DeepCopy()

			pod, err := templateService.RenderLaunchManifest(vmCopy)
			Expect(err).ToNot(HaveOccurred())

			pod.Spec.NodeName = "mynode"
			pod.Status.Phase = clientv1.PodSucceeded

			for idx, _ := range pod.Status.ContainerStatuses {
				if strings.Contains(pod.Status.ContainerStatuses[idx].Name, "disk") == false {
					pod.Status.ContainerStatuses[idx].Ready = true
				}
			}

			podListInitial := clientv1.PodList{}
			podListInitial.Items = []clientv1.Pod{}

			podListPostCreate := clientv1.PodList{}
			podListPostCreate.Items = []clientv1.Pod{*pod}

			// Register the expected REST call
			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", "/api/v1/namespaces/default/pods"),
					ghttp.RespondWithJSONEncoded(http.StatusOK, podListInitial),
				),
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("POST", "/api/v1/namespaces/default/pods"),
					ghttp.RespondWithJSONEncoded(http.StatusOK, pod),
				),
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("PUT", "/apis/kubevirt.io/v1alpha1/namespaces/default/virtualmachines/testvm"),
					ghttp.RespondWithJSONEncoded(http.StatusOK, vm),
				),
			)

			// Tell the controller that there is a new VM
			key, _ := cache.MetaNamespaceKeyFunc(vm)
			app.vmCache.Add(vm)
			app.vmQueue.Add(key)
			app.vmController.Execute()

			Expect(len(server.ReceivedRequests())).To(Equal(3))
			close(done)
		}, 10)
	})

	Context("Running Pod for unscheduled VM given", func() {
		It("should update the VM with the node of the running Pod", func(done Done) {

			// Create a VM which is being scheduled
			vm := v1.NewMinimalVM("testvm")
			vm.Status.Phase = v1.Scheduling
			vm.ObjectMeta.SetUID(uuid.NewUUID())
			addInitializedAnnotation(vm)

			// Create a target Pod for the VM
			temlateService, err := services.NewTemplateService("whatever", "whatever", "whatever")
			Expect(err).ToNot(HaveOccurred())
			var pod *kubev1.Pod
			pod, err = temlateService.RenderLaunchManifest(vm)
			Expect(err).ToNot(HaveOccurred())
			pod.Spec.NodeName = "mynode"
			pod.Status.Phase = kubev1.PodRunning
			pods := clientv1.PodList{
				Items: []kubev1.Pod{*pod},
			}

			// Create the expected VM after the update
			expectedVM := vm.DeepCopy()
			expectedVM.Status.Phase = v1.Scheduled
			expectedVM.Status.NodeName = pod.Spec.NodeName
			expectedVM.Status.Interfaces = []v1.VirtualMachineNetworkInterface{
				v1.VirtualMachineNetworkInterface{IP: pod.Status.PodIP}}
			expectedVM.ObjectMeta.Labels = map[string]string{v1.NodeNameLabel: pod.Spec.NodeName}

			// Register the expected REST call
			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", "/api/v1/namespaces/default/pods"),
					ghttp.RespondWithJSONEncoded(http.StatusOK, pods),
				),
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("PUT", "/apis/kubevirt.io/v1alpha1/namespaces/default/virtualmachines/testvm"),
					ghttp.VerifyJSONRepresenting(expectedVM),
					ghttp.RespondWithJSONEncoded(http.StatusOK, expectedVM),
				),
			)

			// Tell the controller that there is a new running Pod
			key, _ := cache.MetaNamespaceKeyFunc(vm)
			app.vmCache.Add(vm)
			app.vmQueue.Add(key)
			app.vmController.Execute()

			Expect(len(server.ReceivedRequests())).To(Equal(2))
			close(done)
		}, 10)
	})

	AfterEach(func() {
		server.Close()
	})
})
<commit_msg>unit test to verify pod is not deleted immediately after creation<commit_after>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017 Red Hat, Inc.
 *
 */

package watch

import (
	"net/http"
	"strings"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/ghttp"
	clientv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/util/workqueue"

	kubev1 "k8s.io/api/core/v1"

	v1 "kubevirt.io/kubevirt/pkg/api/v1"
	"kubevirt.io/kubevirt/pkg/kubecli"
	"kubevirt.io/kubevirt/pkg/log"
	"kubevirt.io/kubevirt/pkg/virt-controller/services"
)

var _ = Describe("VM watcher", func() {
	var server *ghttp.Server
	//var vmService services.VMService

	log.Log.SetIOWriter(GinkgoWriter)

	var app VirtControllerApp = VirtControllerApp{}
	app.launcherImage = "kubevirt/virt-launcher"
	BeforeEach(func() {

		server = ghttp.NewServer()
		app.clientSet, _ = kubecli.GetKubevirtClientFromFlags(server.URL(), "")
		app.restClient = app.clientSet.RestClient()
		app.vmCache = cache.NewIndexer(cache.DeletionHandlingMetaNamespaceKeyFunc, nil)
		app.vmQueue = workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())

		app.initCommon()
	})

	Context("Creating a VM ", func() {
		It("should ignore uninitialized VM's", func(done Done) {
			vm := v1.NewMinimalVM("testvm")
			// Register the expected REST call
			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", "/api/v1/namespaces/default/pods"),
					ghttp.RespondWithJSONEncoded(http.StatusOK, nil),
				),
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("POST", "/api/v1/namespaces/default/pods"),
					ghttp.RespondWithJSONEncoded(http.StatusOK, nil),
				),

				ghttp.CombineHandlers(
					ghttp.VerifyRequest("PUT", "/apis/kubevirt.io/v1alpha1/namespaces/default/virtualmachines/testvm"),
					ghttp.RespondWithJSONEncoded(http.StatusOK, vm),
				),
			)

			key, _ := cache.MetaNamespaceKeyFunc(vm)
			app.vmCache.Add(vm)
			app.vmQueue.Add(key)
			app.vmController.Execute()

			// VM's that aren't annotated that presets have been applied
			// should not be acted upon, so 0 requests are expected
			Expect(len(server.ReceivedRequests())).To(Equal(0))
			close(done)
		})

		It("should schedule a POD.", func(done Done) {

			// Create a VM to be scheduled
			vm := v1.NewMinimalVM("testvm")
			vm.Status.Phase = ""
			vm.ObjectMeta.SetUID(uuid.NewUUID())
			addInitializedAnnotation(vm)

			// Create the expected VM after the update
			expectedVM := vm.DeepCopy()

			// Create a Pod for the VM
			temlateService, err := services.NewTemplateService("whatever", "whatever", "whatever")
			Expect(err).ToNot(HaveOccurred())
			pod, err := temlateService.RenderLaunchManifest(vm)
			Expect(err).ToNot(HaveOccurred())
			pod.Spec.NodeName = "mynode"
			pod.Status.Phase = clientv1.PodSucceeded

			podListInitial := clientv1.PodList{}
			podListInitial.Items = []clientv1.Pod{}

			expectedVM.Status.Phase = v1.Scheduling

			// Register the expected REST call
			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", "/api/v1/namespaces/default/pods"),
					ghttp.RespondWithJSONEncoded(http.StatusOK, podListInitial),
				),
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("POST", "/api/v1/namespaces/default/pods"),
					ghttp.RespondWithJSONEncoded(http.StatusOK, pod),
				),

				ghttp.CombineHandlers(
					ghttp.VerifyRequest("PUT", "/apis/kubevirt.io/v1alpha1/namespaces/default/virtualmachines/testvm"),
					ghttp.RespondWithJSONEncoded(http.StatusOK, vm),
				),
			)

			// Tell the controller that there is a new VM
			key, _ := cache.MetaNamespaceKeyFunc(vm)
			app.vmCache.Add(vm)
			app.vmQueue.Add(key)
			app.vmController.Execute()

			Expect(len(server.ReceivedRequests())).To(Equal(3))
			close(done)
		}, 10)

		It("should not schedule a POD if pod already exists", func(done Done) {

			// Create a VM to be scheduled
			vm := v1.NewMinimalVM("testvm")
			vm.Status.Phase = ""
			vm.ObjectMeta.SetUID(uuid.NewUUID())
			addInitializedAnnotation(vm)

			// Create the expected VM after the update
			expectedVM := vm.DeepCopy()

			// Create a Pod for the VM
			temlateService, err := services.NewTemplateService("whatever", "whatever", "whatever")
			Expect(err).ToNot(HaveOccurred())
			pod, err := temlateService.RenderLaunchManifest(vm)
			Expect(err).ToNot(HaveOccurred())
			pod.Spec.NodeName = "mynode"
			pod.Status.Phase = clientv1.PodPending

			podListInitial := clientv1.PodList{}
			podListInitial.Items = []clientv1.Pod{*pod}

			expectedVM.Status.Phase = v1.Scheduling

			// Register the expected REST call
			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", "/api/v1/namespaces/default/pods"),
					ghttp.RespondWithJSONEncoded(http.StatusOK, podListInitial),
				),

				ghttp.CombineHandlers(
					ghttp.VerifyRequest("PUT", "/apis/kubevirt.io/v1alpha1/namespaces/default/virtualmachines/testvm"),
					ghttp.RespondWithJSONEncoded(http.StatusOK, vm),
				),
			)

			// Tell the controller that there is a new VM
			key, _ := cache.MetaNamespaceKeyFunc(vm)
			app.vmCache.Add(vm)
			app.vmQueue.Add(key)
			app.vmController.Execute()

			Expect(len(server.ReceivedRequests())).To(Equal(2))
			close(done)
		}, 10)

		It("should schedule a POD with Registry Disk.", func(done Done) {

			// Create a VM to be scheduled
			vm := v1.NewMinimalVM("testvm")
			vm.Status.Phase = ""
			vm.ObjectMeta.SetUID(uuid.NewUUID())
			vm.Spec.Domain.Devices.Disks = append(vm.Spec.Domain.Devices.Disks, v1.Disk{
				Name: "r0",
				DiskDevice: v1.DiskDevice{
					Disk: &v1.DiskTarget{},
				},
			})
			vm.Spec.Volumes = append(vm.Spec.Volumes, v1.Volume{
				Name: "r0",
				VolumeSource: v1.VolumeSource{
					RegistryDisk: &v1.RegistryDiskSource{
						Image: "someimage:v1.2.3.4",
					},
				},
			})
			addInitializedAnnotation(vm)

			// Create a Pod for the VM
			templateService, err := services.NewTemplateService("whatever", "whatever", "whatever")
			Expect(err).ToNot(HaveOccurred())

			// We want to ensure the vm object we initially post
			// doesn't have ports set, so we make a copy in order
			// to render the pod object early for the test.
			vmCopy := vm.DeepCopy()

			pod, err := templateService.RenderLaunchManifest(vmCopy)
			Expect(err).ToNot(HaveOccurred())

			pod.Spec.NodeName = "mynode"
			pod.Status.Phase = clientv1.PodSucceeded

			for idx, _ := range pod.Status.ContainerStatuses {
				if strings.Contains(pod.Status.ContainerStatuses[idx].Name, "disk") == false {
					pod.Status.ContainerStatuses[idx].Ready = true
				}
			}

			podListInitial := clientv1.PodList{}
			podListInitial.Items = []clientv1.Pod{}

			// Register the expected REST call
			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", "/api/v1/namespaces/default/pods"),
					ghttp.RespondWithJSONEncoded(http.StatusOK, podListInitial),
				),
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("POST", "/api/v1/namespaces/default/pods"),
					ghttp.RespondWithJSONEncoded(http.StatusOK, pod),
				),
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("PUT", "/apis/kubevirt.io/v1alpha1/namespaces/default/virtualmachines/testvm"),
					ghttp.RespondWithJSONEncoded(http.StatusOK, vm),
				),
			)

			// Tell the controller that there is a new VM
			key, _ := cache.MetaNamespaceKeyFunc(vm)
			app.vmCache.Add(vm)
			app.vmQueue.Add(key)
			app.vmController.Execute()

			Expect(len(server.ReceivedRequests())).To(Equal(3))
			close(done)
		}, 10)
	})

	Context("Running Pod for unscheduled VM given", func() {
		It("should update the VM with the node of the running Pod", func(done Done) {

			// Create a VM which is being scheduled
			vm := v1.NewMinimalVM("testvm")
			vm.Status.Phase = v1.Scheduling
			vm.ObjectMeta.SetUID(uuid.NewUUID())
			addInitializedAnnotation(vm)

			// Create a target Pod for the VM
			temlateService, err := services.NewTemplateService("whatever", "whatever", "whatever")
			Expect(err).ToNot(HaveOccurred())
			var pod *kubev1.Pod
			pod, err = temlateService.RenderLaunchManifest(vm)
			Expect(err).ToNot(HaveOccurred())
			pod.Spec.NodeName = "mynode"
			pod.Status.Phase = kubev1.PodRunning
			pods := clientv1.PodList{
				Items: []kubev1.Pod{*pod},
			}

			// Create the expected VM after the update
			expectedVM := vm.DeepCopy()
			expectedVM.Status.Phase = v1.Scheduled
			expectedVM.Status.NodeName = pod.Spec.NodeName
			expectedVM.Status.Interfaces = []v1.VirtualMachineNetworkInterface{
				v1.VirtualMachineNetworkInterface{IP: pod.Status.PodIP}}
			expectedVM.ObjectMeta.Labels = map[string]string{v1.NodeNameLabel: pod.Spec.NodeName}

			// Register the expected REST call
			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", "/api/v1/namespaces/default/pods"),
					ghttp.RespondWithJSONEncoded(http.StatusOK, pods),
				),
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("PUT", "/apis/kubevirt.io/v1alpha1/namespaces/default/virtualmachines/testvm"),
					ghttp.VerifyJSONRepresenting(expectedVM),
					ghttp.RespondWithJSONEncoded(http.StatusOK, expectedVM),
				),
			)

			// Tell the controller that there is a new running Pod
			key, _ := cache.MetaNamespaceKeyFunc(vm)
			app.vmCache.Add(vm)
			app.vmQueue.Add(key)
			app.vmController.Execute()

			Expect(len(server.ReceivedRequests())).To(Equal(2))
			close(done)
		}, 10)
	})

	AfterEach(func() {
		server.Close()
	})
})
<|endoftext|>


<commit_before>package builder

import (
	"github.com/modcloth/docker-builder/dclient"
	"github.com/modcloth/docker-builder/log"
	"github.com/modcloth/docker-builder/parser"
)

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/hishboy/gocommons/lang"
	"github.com/modcloth/go-fileutils"
	"github.com/modcloth/queued-command-runner"
	"github.com/onsi/gocleanup"
)

var imageWithTagRegex = regexp.MustCompile("^(.*):(.*)$")

/*
WaitForPush indicates to main() that a `docker push` command has been
started.  Since those are run asynchronously, main() has to wait on the
runner.Done channel.  However, if the build does not require a push, we
don't want to wait or we'll just be stuck forever.
*/
var WaitForPush bool

/*
SkipPush, when set to true, will override any behavior set by a Bobfile and
will cause builders *NOT* to run `docker push` commands.  SkipPush is also set
by the `--skip-push` option when used on the command line.
*/
var SkipPush bool

/*
A Builder is the struct that actually does the work of moving files around and
executing the commands that do the docker build.
*/
type Builder struct {
	dockerClient dclient.DockerClient
	*logrus.Logger
	workdir         string
	isRegular       bool
	nextSubSequence *parser.SubSequence
	Stderr          io.Writer
	Stdout          io.Writer
	Builderfile     string
}

/*
SetNextSubSequence sets the next subsequence within bob to be processed. This
function is exported because it is used explicitly in tests, but in Build(), it
is intended to be used as a helper function.
*/
func (bob *Builder) SetNextSubSequence(subSeq *parser.SubSequence) {
	bob.nextSubSequence = subSeq
}

/*
NewBuilder returns an instance of a Builder struct.  The function exists in
case we want to initialize our Builders with something.
*/
func NewBuilder(logger *logrus.Logger, shouldBeRegular bool) (*Builder, error) {
	if logger == nil {
		logger = logrus.New()
		logger.Level = logrus.Panic
	}

	client, err := dclient.NewDockerClient(logger, shouldBeRegular)

	if err != nil {
		return nil, err
	}

	if logrus.IsTerminal() {
		return &Builder{
			dockerClient: client,
			Logger:       logger,
			isRegular:    shouldBeRegular,
			Stdout:       log.NewOutWriter(logger, "         @{g}%s@{|}"),
			Stderr:       log.NewOutWriter(logger, "         @{r}%s@{|}"),
		}, nil
	}

	return &Builder{
		dockerClient: client,
		Logger:       logger,
		isRegular:    shouldBeRegular,
		Stdout:       log.NewOutWriter(logger, "         %s"),
		Stderr:       log.NewOutWriter(logger, "         %s"),
	}, nil
}

/*
BuilderLogger is an accessor method for bob's internal logger
*/
func (bob *Builder) BuilderLogger() *logrus.Logger {
	return bob.Logger
}

/*
BuildFromFile combines Build() with parser.Parse() to reduce the number of
steps needed to build with bob programatically.
*/
func (bob *Builder) BuildFromFile(file string) error {
	par, err := parser.NewParser(file, bob.BuilderLogger())
	if err != nil {
		return err
	}

	commandSequence, err := par.Parse()
	if err != nil {
		return err
	}

	bob.Builderfile = file

	if err = bob.Build(commandSequence); err != nil {
		return err
	}

	return nil
}

/*
Build does the building!
*/
func (bob *Builder) Build(commandSequence *parser.CommandSequence) error {
	for _, seq := range commandSequence.Commands {
		if err := bob.CleanWorkdir(); err != nil {
			return err
		}
		bob.SetNextSubSequence(seq)
		if err := bob.Setup(); err != nil {
			return err
		}

		workdir := bob.Workdir()

		bob.WithFields(logrus.Fields{
			"container_section": seq.Metadata.Name,
		}).Info("running commands for container section")

		var imageID string
		var err error

		for _, cmd := range seq.SubCommand {
			cmd.Stdout = bob.Stdout
			cmd.Stderr = bob.Stderr
			cmd.Dir = workdir

			if cmd.Path == "docker" {
				path, err := fileutils.Which("docker")
				if err != nil {
					return err
				}

				cmd.Path = path
			}

			switch cmd.Args[1] {
			case "build":
				bob.WithFields(logrus.Fields{
					"command": strings.Join(cmd.Args, " "),
				}).Info("running command")

				if err := cmd.Run(); err != nil {
					return err
				}

				imageID, err = bob.LatestImageTaggedWithUUID(seq.Metadata.UUID)
				if err != nil {
					return err
				}
			case "tag":
				for k, v := range cmd.Args {
					if v == "<IMG>" {
						cmd.Args[k] = imageID
					}
				}
				bob.WithFields(logrus.Fields{
					"command": strings.Join(cmd.Args, " "),
				}).Info("running command")

				if err := cmd.Run(); err != nil {
					return err
				}
			case "push":
				// do final transformation
				runnerCmd := makeRunnerCommandForPush(cmd)

				if !SkipPush {
					// log
					bob.WithFields(logrus.Fields{
						"command": strings.Join(cmd.Args, " "),
					}).Info("running command")

					// set necessary var
					WaitForPush = true

					// run
					runner.Run(runnerCmd)

				} else {
					bob.WithFields(logrus.Fields{
						"key": runnerCmd.Key,
						"cmd": strings.Join(runnerCmd.Cmd.Args, " "),
					}).Warn("not running push command")
				}
			default:
				bob.WithFields(logrus.Fields{
					"command": strings.Join(cmd.Args, " "),
				}).Warn("improperly formatted command")

				return fmt.Errorf("improperly formatted command %q", strings.Join(cmd.Args, " "))
			}
		}
	}

	return nil
}

func makeRunnerCommandForPush(cmd exec.Cmd) *runner.Command {

	imageGiven := cmd.Args[2]
	matches := imageWithTagRegex.FindStringSubmatch(imageGiven)

	// if image includes a tag
	if len(matches) >= 3 {
		return &runner.Command{
			Cmd: &cmd,
			Key: matches[1], // image without tag
		}
	}

	return &runner.Command{
		Cmd: &cmd,
	}
}

/*
Setup moves all of the correct files into place in the temporary directory in
order to perform the docker build.
*/
func (bob *Builder) Setup() error {
	if bob.nextSubSequence == nil {
		return errors.New("no command sub sequence set, cannot perform setup")
	}

	meta := bob.nextSubSequence.Metadata
	fileSet := lang.NewHashSet()

	if len(meta.Included) == 0 {
		files, err := ioutil.ReadDir(bob.Repodir())
		if err != nil {
			return err
		}

		for _, v := range files {
			fileSet.Add(v.Name())
		}
	} else {
		for _, v := range meta.Included {
			fileSet.Add(v)
		}
	}

	// subtract any excludes from fileSet
	for _, exclude := range meta.Excluded {
		if fileSet.Contains(exclude) {
			fileSet.Remove(exclude)
		}
	}

	if fileSet.Contains("Dockerfile") {
		fileSet.Remove("Dockerfile")
	}

	// add the Dockerfile
	fileSet.Add(meta.Dockerfile)

	workdir := bob.Workdir()
	repodir := bob.Repodir()

	// copy the actual files over
	for _, file := range fileSet.ToSlice() {
		src := fmt.Sprintf("%s/%s", repodir, file)
		dest := fmt.Sprintf("%s/%s", workdir, file)

		if file == meta.Dockerfile {
			dest = fmt.Sprintf("%s/%s", workdir, "Dockerfile")
		}

		fileInfo, err := os.Stat(src)
		if err != nil {
			return err
		}

		if fileInfo.IsDir() {
			err = fileutils.CpR(src, dest)
		} else {
			err = fileutils.Cp(src, dest)
		}
		if err != nil {
			return err
		}
	}

	return nil
}

/*
Repodir is the dir from which we are using files for our docker builds.
*/
func (bob *Builder) Repodir() string {
	if !bob.isRegular {
		repoDir := "spec/fixtures/repodir"
		return fmt.Sprintf("%s/%s", os.Getenv("PWD"), repoDir)
	}
	return filepath.Dir(bob.Builderfile)
}

/*
Workdir returns bob's working directory.
*/
func (bob *Builder) Workdir() string {
	return bob.workdir
}

func (bob *Builder) generateWorkDir() string {
	tmp, err := ioutil.TempDir("", "bob")
	if err != nil {
		return ""
	}

	gocleanup.Register(func() {
		fileutils.RmRF(tmp)
	})

	return tmp
}

/*
CleanWorkdir effectively does a rm -rf and mkdir -p on bob's workdir.  Intended
to be used before using the workdir (i.e. before new command groups).
*/
func (bob *Builder) CleanWorkdir() error {
	workdir := bob.generateWorkDir()
	bob.workdir = workdir

	if err := fileutils.RmRF(workdir); err != nil {
		return err
	}

	if err := fileutils.MkdirP(workdir, 0755); err != nil {
		return err
	}

	return nil
}

/*
LatestImageTaggedWithUUID accepts a uuid and invokes the underlying utility
DockerClient to determine the id of the most recently created image tagged with
the provided uuid.
*/
func (bob *Builder) LatestImageTaggedWithUUID(uuid string) (string, error) {
	id, err := bob.dockerClient.LatestImageTaggedWithUUID(uuid)
	if err != nil {
		bob.Println(err)
		return "", err
	}

	return id, nil
}
<commit_msg>Adding a little big more logging<commit_after>package builder

import (
	"github.com/modcloth/docker-builder/dclient"
	"github.com/modcloth/docker-builder/log"
	"github.com/modcloth/docker-builder/parser"
)

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/hishboy/gocommons/lang"
	"github.com/modcloth/go-fileutils"
	"github.com/modcloth/queued-command-runner"
	"github.com/onsi/gocleanup"
)

var imageWithTagRegex = regexp.MustCompile("^(.*):(.*)$")

/*
WaitForPush indicates to main() that a `docker push` command has been
started.  Since those are run asynchronously, main() has to wait on the
runner.Done channel.  However, if the build does not require a push, we
don't want to wait or we'll just be stuck forever.
*/
var WaitForPush bool

/*
SkipPush, when set to true, will override any behavior set by a Bobfile and
will cause builders *NOT* to run `docker push` commands.  SkipPush is also set
by the `--skip-push` option when used on the command line.
*/
var SkipPush bool

/*
A Builder is the struct that actually does the work of moving files around and
executing the commands that do the docker build.
*/
type Builder struct {
	dockerClient dclient.DockerClient
	*logrus.Logger
	workdir         string
	isRegular       bool
	nextSubSequence *parser.SubSequence
	Stderr          io.Writer
	Stdout          io.Writer
	Builderfile     string
}

/*
SetNextSubSequence sets the next subsequence within bob to be processed. This
function is exported because it is used explicitly in tests, but in Build(), it
is intended to be used as a helper function.
*/
func (bob *Builder) SetNextSubSequence(subSeq *parser.SubSequence) {
	bob.nextSubSequence = subSeq
}

/*
NewBuilder returns an instance of a Builder struct.  The function exists in
case we want to initialize our Builders with something.
*/
func NewBuilder(logger *logrus.Logger, shouldBeRegular bool) (*Builder, error) {
	if logger == nil {
		logger = logrus.New()
		logger.Level = logrus.Panic
	}

	client, err := dclient.NewDockerClient(logger, shouldBeRegular)

	if err != nil {
		return nil, err
	}

	if logrus.IsTerminal() {
		return &Builder{
			dockerClient: client,
			Logger:       logger,
			isRegular:    shouldBeRegular,
			Stdout:       log.NewOutWriter(logger, "         @{g}%s@{|}"),
			Stderr:       log.NewOutWriter(logger, "         @{r}%s@{|}"),
		}, nil
	}

	return &Builder{
		dockerClient: client,
		Logger:       logger,
		isRegular:    shouldBeRegular,
		Stdout:       log.NewOutWriter(logger, "         %s"),
		Stderr:       log.NewOutWriter(logger, "         %s"),
	}, nil
}

/*
BuilderLogger is an accessor method for bob's internal logger
*/
func (bob *Builder) BuilderLogger() *logrus.Logger {
	return bob.Logger
}

/*
BuildFromFile combines Build() with parser.Parse() to reduce the number of
steps needed to build with bob programatically.
*/
func (bob *Builder) BuildFromFile(file string) error {
	par, err := parser.NewParser(file, bob.BuilderLogger())
	if err != nil {
		return err
	}

	commandSequence, err := par.Parse()
	if err != nil {
		return err
	}

	bob.Builderfile = file

	if err = bob.Build(commandSequence); err != nil {
		return err
	}

	return nil
}

/*
Build does the building!
*/
func (bob *Builder) Build(commandSequence *parser.CommandSequence) error {
	for _, seq := range commandSequence.Commands {
		if err := bob.CleanWorkdir(); err != nil {
			return err
		}
		bob.SetNextSubSequence(seq)
		if err := bob.Setup(); err != nil {
			return err
		}

		workdir := bob.Workdir()

		bob.WithFields(logrus.Fields{
			"container_section": seq.Metadata.Name,
		}).Info("running commands for container section")

		var imageID string
		var err error

		for _, cmd := range seq.SubCommand {
			cmd.Stdout = bob.Stdout
			cmd.Stderr = bob.Stderr
			cmd.Dir = workdir

			if cmd.Path == "docker" {
				path, err := fileutils.Which("docker")
				if err != nil {
					return err
				}

				cmd.Path = path
			}

			switch cmd.Args[1] {
			case "build":
				bob.WithFields(logrus.Fields{
					"command": strings.Join(cmd.Args, " "),
				}).Info("running build command")

				if err := cmd.Run(); err != nil {
					return err
				}

				imageID, err = bob.LatestImageTaggedWithUUID(seq.Metadata.UUID)
				if err != nil {
					return err
				}
			case "tag":
				for k, v := range cmd.Args {
					if v == "<IMG>" {
						cmd.Args[k] = imageID
					}
				}
				bob.WithFields(logrus.Fields{
					"command": strings.Join(cmd.Args, " "),
				}).Info("running tag command")

				if err := cmd.Run(); err != nil {
					return err
				}
			case "push":
				// do final transformation
				runnerCmd := makeRunnerCommandForPush(cmd)

				if !SkipPush {
					// log
					bob.WithFields(logrus.Fields{
						"command": strings.Join(cmd.Args, " "),
					}).Info("running push command")

					// set necessary var
					WaitForPush = true

					// run
					runner.Run(runnerCmd)

				} else {
					bob.WithFields(logrus.Fields{
						"key": runnerCmd.Key,
						"cmd": strings.Join(runnerCmd.Cmd.Args, " "),
					}).Warn("not running push command")
				}
			default:
				bob.WithFields(logrus.Fields{
					"command": strings.Join(cmd.Args, " "),
				}).Warn("improperly formatted command")

				return fmt.Errorf("improperly formatted command %q", strings.Join(cmd.Args, " "))
			}
		}
	}

	return nil
}

func makeRunnerCommandForPush(cmd exec.Cmd) *runner.Command {

	imageGiven := cmd.Args[2]
	matches := imageWithTagRegex.FindStringSubmatch(imageGiven)

	// if image includes a tag
	if len(matches) >= 3 {
		return &runner.Command{
			Cmd: &cmd,
			Key: matches[1], // image without tag
		}
	}

	return &runner.Command{
		Cmd: &cmd,
	}
}

/*
Setup moves all of the correct files into place in the temporary directory in
order to perform the docker build.
*/
func (bob *Builder) Setup() error {
	if bob.nextSubSequence == nil {
		return errors.New("no command sub sequence set, cannot perform setup")
	}

	meta := bob.nextSubSequence.Metadata
	fileSet := lang.NewHashSet()

	if len(meta.Included) == 0 {
		files, err := ioutil.ReadDir(bob.Repodir())
		if err != nil {
			return err
		}

		for _, v := range files {
			fileSet.Add(v.Name())
		}
	} else {
		for _, v := range meta.Included {
			fileSet.Add(v)
		}
	}

	// subtract any excludes from fileSet
	for _, exclude := range meta.Excluded {
		if fileSet.Contains(exclude) {
			fileSet.Remove(exclude)
		}
	}

	if fileSet.Contains("Dockerfile") {
		fileSet.Remove("Dockerfile")
	}

	// add the Dockerfile
	fileSet.Add(meta.Dockerfile)

	workdir := bob.Workdir()
	repodir := bob.Repodir()

	// copy the actual files over
	for _, file := range fileSet.ToSlice() {
		src := fmt.Sprintf("%s/%s", repodir, file)
		dest := fmt.Sprintf("%s/%s", workdir, file)

		if file == meta.Dockerfile {
			dest = fmt.Sprintf("%s/%s", workdir, "Dockerfile")
		}

		fileInfo, err := os.Stat(src)
		if err != nil {
			return err
		}

		if fileInfo.IsDir() {
			err = fileutils.CpR(src, dest)
		} else {
			err = fileutils.Cp(src, dest)
		}
		if err != nil {
			return err
		}
	}

	return nil
}

/*
Repodir is the dir from which we are using files for our docker builds.
*/
func (bob *Builder) Repodir() string {
	if !bob.isRegular {
		repoDir := "spec/fixtures/repodir"
		return fmt.Sprintf("%s/%s", os.Getenv("PWD"), repoDir)
	}
	return filepath.Dir(bob.Builderfile)
}

/*
Workdir returns bob's working directory.
*/
func (bob *Builder) Workdir() string {
	return bob.workdir
}

func (bob *Builder) generateWorkDir() string {
	tmp, err := ioutil.TempDir("", "bob")
	if err != nil {
		return ""
	}

	gocleanup.Register(func() {
		fileutils.RmRF(tmp)
	})

	return tmp
}

/*
CleanWorkdir effectively does a rm -rf and mkdir -p on bob's workdir.  Intended
to be used before using the workdir (i.e. before new command groups).
*/
func (bob *Builder) CleanWorkdir() error {
	workdir := bob.generateWorkDir()
	bob.workdir = workdir

	if err := fileutils.RmRF(workdir); err != nil {
		return err
	}

	if err := fileutils.MkdirP(workdir, 0755); err != nil {
		return err
	}

	return nil
}

/*
LatestImageTaggedWithUUID accepts a uuid and invokes the underlying utility
DockerClient to determine the id of the most recently created image tagged with
the provided uuid.
*/
func (bob *Builder) LatestImageTaggedWithUUID(uuid string) (string, error) {
	id, err := bob.dockerClient.LatestImageTaggedWithUUID(uuid)
	if err != nil {
		bob.Println(err)
		return "", err
	}

	return id, nil
}
<|endoftext|>


<commit_before>package flagrc

import (
	"bufio"
	"os"
	"strings"
)

func ProcessFlagrc(filePath string) {
	file, err := os.Open(filePath)
	if err == nil {
		defer file.Close()

		scanner := bufio.NewScanner(file)
		for scanner.Scan() {
			argument := strings.Trim(scanner.Text(), " ")
			argumentName := getOnlyArgumentName(argument)
			if !containsArgument(argumentName) {
				appendArgument(argument)
			}
		}
	}
}

func getOnlyArgumentName(line string) string {
	if index := strings.Index(line, "="); index > 0 {
		return line[0:index]
	}
	return line
}

func containsArgument(argumentName string) bool {
	for i, value := range os.Args {
		if i > 0 {
			if strings.HasPrefix(value, argumentName) {
				return true
			}
		}
	}
	return false
}

func appendArgument(argument string) {
	os.Args = append(os.Args, argument)
}
<commit_msg>Add # as comment<commit_after>package flagrc

import (
	"bufio"
	"os"
	"strings"
)

func ProcessFlagrc(filePath string) {
	file, err := os.Open(filePath)
	if err == nil {
		defer file.Close()

		scanner := bufio.NewScanner(file)
		for scanner.Scan() {
			argument := strings.Trim(scanner.Text(), " ")
			if !isComment(argument) {
				argumentName := getOnlyArgumentName(argument)
				if !containsArgument(argumentName) {
					appendArgument(argument)
				}
			}
		}
	}
}

func isComment(line string) bool {
	return strings.HasPrefix(line, "#")
}

func getOnlyArgumentName(line string) string {
	if index := strings.Index(line, "="); index > 0 {
		return line[0:index]
	}
	return line
}

func containsArgument(argumentName string) bool {
	for i, value := range os.Args {
		if i > 0 {
			if strings.HasPrefix(value, argumentName) {
				return true
			}
		}
	}
	return false
}

func appendArgument(argument string) {
	os.Args = append(os.Args, argument)
}
<|endoftext|>


<commit_before>package main

import "fmt"

func main() {
	argMap := map[string]string{
		"user-data": "dummyUserData",
		"meta-data": "dummyMetaData",
	}

	fmt.Println(argMap)
}
<commit_msg>Implement main logic<commit_after>package main

import (
	"flag"
	"fmt"
	"time"

	"github.com/tmrts/flamingo/pkg/datasrc/cloudconfig"
	"github.com/tmrts/flamingo/pkg/datasrc/metadata"
	"github.com/tmrts/flamingo/pkg/sys"
	"github.com/tmrts/flamingo/pkg/sys/identity"
)

var flags struct {
	cloudConfig string
	configDrive string
	metadata    string
}

func init() {
	flag.StringVar(&flags.cloudConfig, "cloud-config", "", "user-data configuration file")
	flag.StringVar(&flags.configDrive, "config-drive", "", "config drive mount path")
	flag.StringVar(&flags.metadata, "meta-data", "", "meta-data file")
}

func StartContextualization() {
	// TODO: Add plug-in hooks
}

func FinalizeContextualization() {
	// TODO: Add plug-in hooks
}

func main() {
	// TODO: Build Meaningful Loggers
	flag.Parse()

	// cloudconfig
	if flags.cloudConfig != "" {
		cloudConfigContext, err := cloudconfig.Parse(flags.cloudConfig)
		if err != nil {
			panic(fmt.Errorf("fatal error config file: %v", err))
		}
	}

	metadataDigest := metadata.Get(10 * time.Second)

	conf := datasrc.Merge(metadataDigest, cloudConfigContext)

	StartContextualization()

	// usergroups
	idm := identity.Manager{Exec: sys.DefaultExecutor}

	for _, grp := range conf.Groups {
		if err := idm.CreateGroup(grp); err != nil {
			panic(err)
		}
	}

	for _, usr := range conf.Users {
		if err := idm.CreateUser(usr); err != nil {
			panic(err)
		}
	}

	for _, grp := range conf.Groups {
		for _, usr := range grp {
			if err := idm.AddUserToGroup(grp, usr); err != nil {
				panic(err)
			}
		}
	}

	// ssh_keys
	if err := ssh.InitializeFor("root"); err != nil {
		panic(err)
	}

	ssh.AuthorizeSSHKey(f, conf.AuthorizedKeys...)

	// write_files
	for _, f := range conf.Files {
		file.New(f.Name, file.Contents(f.Data), file.Uid(0), file.Gid(0), file.Permissions(f.Perms))
	}

	// run_cmd
	for _, cmd := range conf.Commands {
		sys.Execute(cmd)
	}

	if err := FinalizeContextualization(); err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package http2

import (
	"fmt"
	"net"
)

type Stream struct {
	Conn       *net.Conn
	ID         uint32
	WindowSize uint16
	State      STATE
}

// 65535 should be defined in connection
func NewStream(conn *net.Conn, streamID uint32) *Stream {
	return &Stream{conn, streamID, 65535, IDLE}
}

func (self *Stream) ChangeState(state STATE) {
	self.State = state
}

func (self *Stream) DecreaseWindow(size uint16) {
	self.WindowSize -= size
}

func (self *Stream) Send(frame Frame) {
	// do something to self
	fmt.Printf("Send: \n%s\n", frame.String())
	(*self.Conn).Write(frame.GetWire())
}

func (self *Stream) EvaluateFrame(frame Frame) {
	// do something to self, also error handling
}
<commit_msg>added status getter<commit_after>package http2

import (
	"fmt"
	"net"
)

type Stream struct {
	Conn       *net.Conn
	ID         uint32
	WindowSize uint16
	State      STATE
}

// 65535 should be defined in connection
func NewStream(conn *net.Conn, streamID uint32) *Stream {
	return &Stream{conn, streamID, 65535, IDLE}
}

func (self *Stream) ChangeState(state STATE) {
	self.State = state
}

func (self *Stream) GetState() STATE {
	return self.State
}

func (self *Stream) DecreaseWindow(size uint16) {
	self.WindowSize -= size
}

func (self *Stream) Send(frame Frame) {
	// do something to self
	fmt.Printf("Send: \n%s\n", frame.String())
	(*self.Conn).Write(frame.GetWire())
}

func (self *Stream) EvaluateFrame(frame Frame) {
	// do something to self, also error handling
}
<|endoftext|>


<commit_before>// generated by stringer -output=string.go -type=Operation; DO NOT EDIT

package arp

import "fmt"

const _Operation_name = "OperationRequestOperationReply"

var _Operation_index = [...]uint8{0, 16, 30}

func (i Operation) String() string {
	i -= 1
	if i >= Operation(len(_Operation_index)-1) {
		return fmt.Sprintf("Operation(%d)", i+1)
	}
	return _Operation_name[_Operation_index[i]:_Operation_index[i+1]]
}
<commit_msg>arp: use newest stringer and regenerate<commit_after>// Code generated by "stringer -output=string.go -type=Operation"; DO NOT EDIT.

package arp

import "strconv"

const _Operation_name = "OperationRequestOperationReply"

var _Operation_index = [...]uint8{0, 16, 30}

func (i Operation) String() string {
	i -= 1
	if i >= Operation(len(_Operation_index)-1) {
		return "Operation(" + strconv.FormatInt(int64(i+1), 10) + ")"
	}
	return _Operation_name[_Operation_index[i]:_Operation_index[i+1]]
}
<|endoftext|>


<commit_before>package dgamqp

import (
	// standard
	"errors"
	"fmt"
	"sync"

	// open source
	"github.com/streadway/amqp"

	// internal
	"github.com/mission-liao/dingo"
	"github.com/mission-liao/dingo/common"
	"github.com/mission-liao/dingo/transport"
)

type backend struct {
	AmqpConnection

	// store
	stores   *common.HetroRoutines
	rids     map[string]int
	ridsLock sync.Mutex

	// reporter
	reporters *common.HetroRoutines
	cfg       *AmqpConfig
}

func NewBackend(cfg *AmqpConfig) (v *backend, err error) {
	v = &backend{
		reporters: common.NewHetroRoutines(),
		rids:      make(map[string]int),
		cfg:       cfg,
		stores:    common.NewHetroRoutines(),
	}
	err = v.init()
	return
}

func (me *backend) init() (err error) {
	// call parent's Init
	err = me.AmqpConnection.Init(me.cfg.Connection())
	if err != nil {
		return
	}

	// define exchange
	ci, err := me.AmqpConnection.Channel()
	if err != nil {
		return
	}
	defer me.AmqpConnection.ReleaseChannel(ci)

	// init exchange
	err = ci.Channel.ExchangeDeclare(
		"dingo.x.result", // name of exchange
		"direct",         // kind
		true,             // durable
		false,            // auto-delete
		false,            // internal
		false,            // noWait
		nil,              // args
	)
	if err != nil {
		return
	}

	return
}

//
// common.Object interface
//

func (me *backend) Events() ([]<-chan *common.Event, error) {
	return []<-chan *common.Event{
		me.reporters.Events(),
		me.stores.Events(),
	}, nil
}

func (me *backend) Close() (err error) {
	me.reporters.Close()
	me.stores.Close()
	err = me.AmqpConnection.Close()
	return
}

//
// Reporter interface
//

func (me *backend) ReporterHook(eventID int, payload interface{}) (err error) {
	switch eventID {
	case dingo.ReporterEvent.BeforeReport:
		func() {
			ci, err := me.AmqpConnection.Channel()
			if err != nil {
				return
			}
			defer func() {
				if err != nil {
					me.AmqpConnection.ReleaseChannel(ci)
				} else {
					me.AmqpConnection.ReleaseChannel(nil)
				}
			}()

			id := payload.(transport.Meta)
			qName, rKey := getQueueName(id), getRoutingKey(id)

			// TODO: rework here, a new interface API should be added for preparation.
			// declare a queue for this task
			_, err = ci.Channel.QueueDeclare(
				qName, // name of queue
				true,  // durable
				false, // auto-delete
				false, // exclusive
				false, // nowait
				nil,   // args
			)
			if err != nil {
				return
			}

			// bind queue to result-exchange
			err = ci.Channel.QueueBind(
				qName,            // name of queue
				rKey,             // routing key
				"dingo.x.result", // name of exchange
				false,            // nowait
				nil,              // args
			)
			if err != nil {
				return
			}
		}()
	}

	return
}

func (me *backend) Report(reports <-chan *dingo.ReportEnvelope) (id int, err error) {
	quit, done, id := me.reporters.New(0)
	go me._reporter_routine_(quit, done, me.reporters.Events(), reports)
	return
}

//
// Store interface
//

func (me *backend) Poll(id transport.Meta) (reports <-chan []byte, err error) {
	// bind to the queue for this task
	tag, qName, rKey := getConsumerTag(id), getQueueName(id), getRoutingKey(id)
	quit, done, idx := me.stores.New(0)

	me.ridsLock.Lock()
	defer me.ridsLock.Unlock()
	me.rids[id.ID()] = idx

	// acquire a free channel
	ci, err := me.AmqpConnection.Channel()
	if err != nil {
		return
	}
	var (
		dv <-chan amqp.Delivery
		r  chan []byte
	)
	defer func() {
		if err != nil {
			me.AmqpConnection.ReleaseChannel(ci)
		} else {
			go me._store_routine_(quit, done, me.stores.Events(), r, ci, dv, id)
		}
	}()
	// declare a queue for this task
	_, err = ci.Channel.QueueDeclare(
		qName, // name of queue
		true,  // durable
		false, // auto-delete
		false, // exclusive
		false, // nowait
		nil,   // args
	)
	if err != nil {
		return
	}

	// bind queue to result-exchange
	err = ci.Channel.QueueBind(
		qName,            // name of queue
		rKey,             // routing key
		"dingo.x.result", // name of exchange
		false,            // nowait
		nil,              // args
	)
	if err != nil {
		return
	}

	dv, err = ci.Channel.Consume(
		qName, // name of queue
		tag,   // consumer Tag
		false, // autoAck
		false, // exclusive
		false, // noLocal
		false, // noWait
		nil,   // args
	)
	if err != nil {
		return
	}

	r = make(chan []byte, 10)
	reports = r
	return
}

func (me *backend) Done(id transport.Meta) (err error) {
	me.ridsLock.Lock()
	defer me.ridsLock.Unlock()

	v, ok := me.rids[id.ID()]
	if !ok {
		err = errors.New("store id not found")
		return
	}

	return me.stores.Stop(v)
}

//
// routine definition
//

func (me *backend) _reporter_routine_(quit <-chan int, done chan<- int, events chan<- *common.Event, reports <-chan *dingo.ReportEnvelope) {
	defer func() {
		done <- 1
	}()

	out := func(e *dingo.ReportEnvelope) (err error) {
		// report an error event when leaving
		defer func() {
			if err != nil {
				events <- common.NewEventFromError(common.InstT.REPORTER, err)
			}
		}()

		// acquire a channel
		ci, err := me.AmqpConnection.Channel()
		if err != nil {
			return
		}
		defer me.AmqpConnection.ReleaseChannel(ci)

		// QueueDeclare and QueueBind should be done in Poll(...)
		err = ci.Channel.Publish(
			"dingo.x.result",    // name of exchange
			getRoutingKey(e.ID), // routing key
			false,               // madatory
			false,               // immediate
			amqp.Publishing{
				DeliveryMode: amqp.Persistent,
				ContentType:  "text/json",
				Body:         e.Body,
			},
		)
		if err != nil {
			return
		}

		// block until amqp.Channel.NotifyPublish
		// TODO: time out, retry
		cf := <-ci.Confirm
		if !cf.Ack {
			err = errors.New("Unable to publish to server")
			return
		}

		return
	}

finished:
	for {
		select {
		case _, _ = <-quit:
			break finished
		case e, ok := <-reports:
			if !ok {
				// reports channel is closed
				break finished
			}
			out(e)
		}
	}

done:
	// cosume all remaining reports
	for {
		select {
		case e, ok := <-reports:
			if !ok {
				break done
			}
			out(e)
		default:
			break done
		}
	}
}

func (me *backend) _store_routine_(
	quit <-chan int,
	done chan<- int,
	events chan<- *common.Event,
	reports chan<- []byte,
	ci *AmqpChannel,
	dv <-chan amqp.Delivery,
	id transport.Meta) {

	var (
		err            error
		isChannelError bool = false
	)

	defer func() {
		if isChannelError {
			// when error occurs, this channel is
			// automatically closed.
			me.AmqpConnection.ReleaseChannel(nil)
		} else {
			me.AmqpConnection.ReleaseChannel(ci)
		}
		done <- 1
		close(done)
		close(reports)
	}()

finished:
	for {
		select {
		case _, _ = <-quit:
			break finished
		case d, ok := <-dv:
			if !ok {
				break finished
			}

			d.Ack(false)
			reports <- d.Body
		}
	}

	// consuming remaining stuffs in queue
done:
	for {
		select {
		case d, ok := <-dv:
			if !ok {
				break done
			}

			d.Ack(false)
			reports <- d.Body
		default:
			break done
		}
	}

	// cancel consuming
	err = ci.Channel.Cancel(getConsumerTag(id), false)
	if err != nil {
		events <- common.NewEventFromError(common.InstT.STORE, err)
		isChannelError = true
		return
	}

	// unbind queue from exchange
	qName, rKey := getQueueName(id), getRoutingKey(id)
	err = ci.Channel.QueueUnbind(
		qName,            // name of queue
		rKey,             // routing key
		"dingo.x.result", // name of exchange
		nil,              // args
	)
	if err != nil {
		events <- common.NewEventFromError(common.InstT.STORE, err)
		isChannelError = true
		return
	}

	// delete queue
	_, err = ci.Channel.QueueDelete(
		qName, // name of queue
		true,  // ifUnused
		true,  // ifEmpty
		false, // noWait
	)
	if err != nil {
		events <- common.NewEventFromError(common.InstT.STORE, err)
		isChannelError = true
		return
	}
}

//
// private function
//

//
func getQueueName(id transport.Meta) string {
	return fmt.Sprintf("dingo.q.%q", id.ID())
}

//
func getRoutingKey(id transport.Meta) string {
	return fmt.Sprintf("dingo.rkey.%q", id.ID())
}

//
func getConsumerTag(id transport.Meta) string {
	return fmt.Sprintf("dingo.consumer.%q", id.ID())
}
<commit_msg>avoid holding lock when waiting routine closings<commit_after>package dgamqp

import (
	// standard
	"errors"
	"fmt"
	"sync"

	// open source
	"github.com/streadway/amqp"

	// internal
	"github.com/mission-liao/dingo"
	"github.com/mission-liao/dingo/common"
	"github.com/mission-liao/dingo/transport"
)

type backend struct {
	AmqpConnection

	// store
	stores   *common.HetroRoutines
	rids     map[string]int
	ridsLock sync.Mutex

	// reporter
	reporters *common.HetroRoutines
	cfg       *AmqpConfig
}

func NewBackend(cfg *AmqpConfig) (v *backend, err error) {
	v = &backend{
		reporters: common.NewHetroRoutines(),
		rids:      make(map[string]int),
		cfg:       cfg,
		stores:    common.NewHetroRoutines(),
	}
	err = v.init()
	return
}

func (me *backend) init() (err error) {
	// call parent's Init
	err = me.AmqpConnection.Init(me.cfg.Connection())
	if err != nil {
		return
	}

	// define exchange
	ci, err := me.AmqpConnection.Channel()
	if err != nil {
		return
	}
	defer me.AmqpConnection.ReleaseChannel(ci)

	// init exchange
	err = ci.Channel.ExchangeDeclare(
		"dingo.x.result", // name of exchange
		"direct",         // kind
		true,             // durable
		false,            // auto-delete
		false,            // internal
		false,            // noWait
		nil,              // args
	)
	if err != nil {
		return
	}

	return
}

//
// common.Object interface
//

func (me *backend) Events() ([]<-chan *common.Event, error) {
	return []<-chan *common.Event{
		me.reporters.Events(),
		me.stores.Events(),
	}, nil
}

func (me *backend) Close() (err error) {
	me.reporters.Close()
	me.stores.Close()
	err = me.AmqpConnection.Close()
	return
}

//
// Reporter interface
//

func (me *backend) ReporterHook(eventID int, payload interface{}) (err error) {
	switch eventID {
	case dingo.ReporterEvent.BeforeReport:
		func() {
			ci, err := me.AmqpConnection.Channel()
			if err != nil {
				return
			}
			defer func() {
				if err != nil {
					me.AmqpConnection.ReleaseChannel(ci)
				} else {
					me.AmqpConnection.ReleaseChannel(nil)
				}
			}()

			id := payload.(transport.Meta)
			qName, rKey := getQueueName(id), getRoutingKey(id)

			// declare a queue for this task
			_, err = ci.Channel.QueueDeclare(
				qName, // name of queue
				true,  // durable
				false, // auto-delete
				false, // exclusive
				false, // nowait
				nil,   // args
			)
			if err != nil {
				return
			}

			// bind queue to result-exchange
			err = ci.Channel.QueueBind(
				qName,            // name of queue
				rKey,             // routing key
				"dingo.x.result", // name of exchange
				false,            // nowait
				nil,              // args
			)
			if err != nil {
				return
			}
		}()
	}

	return
}

func (me *backend) Report(reports <-chan *dingo.ReportEnvelope) (id int, err error) {
	quit, done, id := me.reporters.New(0)
	go me._reporter_routine_(quit, done, me.reporters.Events(), reports)
	return
}

//
// Store interface
//

func (me *backend) Poll(id transport.Meta) (reports <-chan []byte, err error) {
	// bind to the queue for this task
	tag, qName, rKey := getConsumerTag(id), getQueueName(id), getRoutingKey(id)
	quit, done, idx := me.stores.New(0)

	me.ridsLock.Lock()
	defer me.ridsLock.Unlock()
	me.rids[id.ID()] = idx

	// acquire a free channel
	ci, err := me.AmqpConnection.Channel()
	if err != nil {
		return
	}
	var (
		dv <-chan amqp.Delivery
		r  chan []byte
	)
	defer func() {
		if err != nil {
			me.AmqpConnection.ReleaseChannel(ci)
		} else {
			go me._store_routine_(quit, done, me.stores.Events(), r, ci, dv, id)
		}
	}()
	// declare a queue for this task
	_, err = ci.Channel.QueueDeclare(
		qName, // name of queue
		true,  // durable
		false, // auto-delete
		false, // exclusive
		false, // nowait
		nil,   // args
	)
	if err != nil {
		return
	}

	// bind queue to result-exchange
	err = ci.Channel.QueueBind(
		qName,            // name of queue
		rKey,             // routing key
		"dingo.x.result", // name of exchange
		false,            // nowait
		nil,              // args
	)
	if err != nil {
		return
	}

	dv, err = ci.Channel.Consume(
		qName, // name of queue
		tag,   // consumer Tag
		false, // autoAck
		false, // exclusive
		false, // noLocal
		false, // noWait
		nil,   // args
	)
	if err != nil {
		return
	}

	r = make(chan []byte, 10)
	reports = r
	return
}

func (me *backend) Done(id transport.Meta) (err error) {
	var v int
	err = func() (err error) {
		me.ridsLock.Lock()
		defer me.ridsLock.Unlock()

		var ok bool
		v, ok = me.rids[id.ID()]
		if !ok {
			err = errors.New("store id not found")
			return
		}
		return
	}()
	if err == nil {
		err = me.stores.Stop(v)
	}
	return
}

//
// routine definition
//

func (me *backend) _reporter_routine_(quit <-chan int, done chan<- int, events chan<- *common.Event, reports <-chan *dingo.ReportEnvelope) {
	defer func() {
		done <- 1
	}()

	out := func(e *dingo.ReportEnvelope) (err error) {
		// report an error event when leaving
		defer func() {
			if err != nil {
				events <- common.NewEventFromError(common.InstT.REPORTER, err)
			}
		}()

		// acquire a channel
		ci, err := me.AmqpConnection.Channel()
		if err != nil {
			return
		}
		defer me.AmqpConnection.ReleaseChannel(ci)

		// QueueDeclare and QueueBind should be done in Poll(...)
		err = ci.Channel.Publish(
			"dingo.x.result",    // name of exchange
			getRoutingKey(e.ID), // routing key
			false,               // madatory
			false,               // immediate
			amqp.Publishing{
				DeliveryMode: amqp.Persistent,
				ContentType:  "text/json",
				Body:         e.Body,
			},
		)
		if err != nil {
			return
		}

		// block until amqp.Channel.NotifyPublish
		// TODO: time out, retry
		cf := <-ci.Confirm
		if !cf.Ack {
			err = errors.New("Unable to publish to server")
			return
		}

		return
	}

finished:
	for {
		select {
		case _, _ = <-quit:
			break finished
		case e, ok := <-reports:
			if !ok {
				// reports channel is closed
				break finished
			}
			out(e)
		}
	}

done:
	// cosume all remaining reports
	for {
		select {
		case e, ok := <-reports:
			if !ok {
				break done
			}
			out(e)
		default:
			break done
		}
	}
}

func (me *backend) _store_routine_(
	quit <-chan int,
	done chan<- int,
	events chan<- *common.Event,
	reports chan<- []byte,
	ci *AmqpChannel,
	dv <-chan amqp.Delivery,
	id transport.Meta) {

	var (
		err            error
		isChannelError bool = false
	)

	defer func() {
		if isChannelError {
			// when error occurs, this channel is
			// automatically closed.
			me.AmqpConnection.ReleaseChannel(nil)
		} else {
			me.AmqpConnection.ReleaseChannel(ci)
		}
		done <- 1
		close(done)
		close(reports)
	}()

finished:
	for {
		select {
		case _, _ = <-quit:
			break finished
		case d, ok := <-dv:
			if !ok {
				break finished
			}

			d.Ack(false)
			reports <- d.Body
		}
	}

	// consuming remaining stuffs in queue
done:
	for {
		select {
		case d, ok := <-dv:
			if !ok {
				break done
			}

			d.Ack(false)
			reports <- d.Body
		default:
			break done
		}
	}

	// cancel consuming
	err = ci.Channel.Cancel(getConsumerTag(id), false)
	if err != nil {
		events <- common.NewEventFromError(common.InstT.STORE, err)
		isChannelError = true
		return
	}

	// unbind queue from exchange
	qName, rKey := getQueueName(id), getRoutingKey(id)
	err = ci.Channel.QueueUnbind(
		qName,            // name of queue
		rKey,             // routing key
		"dingo.x.result", // name of exchange
		nil,              // args
	)
	if err != nil {
		events <- common.NewEventFromError(common.InstT.STORE, err)
		isChannelError = true
		return
	}

	// delete queue
	_, err = ci.Channel.QueueDelete(
		qName, // name of queue
		true,  // ifUnused
		true,  // ifEmpty
		false, // noWait
	)
	if err != nil {
		events <- common.NewEventFromError(common.InstT.STORE, err)
		isChannelError = true
		return
	}
}

//
// private function
//

//
func getQueueName(id transport.Meta) string {
	return fmt.Sprintf("dingo.q.%q", id.ID())
}

//
func getRoutingKey(id transport.Meta) string {
	return fmt.Sprintf("dingo.rkey.%q", id.ID())
}

//
func getConsumerTag(id transport.Meta) string {
	return fmt.Sprintf("dingo.consumer.%q", id.ID())
}
<|endoftext|>


<commit_before>package shoauth

import "net/http"

type failureHandler struct {
	clientId string
}

// DefaultFailureHandler returns an HTTP handler that simply redirects to the
// app's callback URL.  This will prompt Shopify to redirect the user back into
// the app with a signed request which can be validated to establish a proper
// session.  This works well for quirks such as the app preferences page, which
// is not signed and thus will fail without a valid session.
func DefaultFailureHandler(clientId string) http.Handler {
	return &failureHandler{clientId: clientId}
}

func (f *failureHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	http.Redirect(w, r, "https://"+r.FormValue("shop")+"/admin/apps/"+f.clientId, http.StatusMovedPermanently)
}
<commit_msg>Simplifying default failure handler.<commit_after>package shoauth

import "net/http"

type failureHandler struct{}

// DefaultFailureHandler returns an HTTP handler that simply redirects to the
// app's callback URL.  This will prompt Shopify to redirect the user back into
// the app with a signed request which can be validated to establish a proper
// session.  This works well for quirks such as the app preferences page, which
// is not signed and thus will fail without a valid session.
func DefaultFailureHandler() http.Handler {
	return &failureHandler{}
}

func (f *failureHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	http.Error(w, "Internal Server Error", http.StatusInternalServerError)
}
<|endoftext|>


<commit_before>package sudoku

import (
	"fmt"
	"math"
	"strconv"
)

// Optimization ideas:
// - keep track of solved/unsolved fields (shorter loops, quicker check for solved state)
// - auto solve on adding n-1th element in deny list
// - dirty state for row/col/block?

// Sudoku is a sudoku puzzle
type Sudoku struct {
	Size        int
	MaxValue    int
	FieldLength int
	Fields      []*Field
	cols        []FieldGroup
	rows        []FieldGroup
	blocks      []FieldGroup
}

// New returns a new sudoku puzzle
func New(size int) Sudoku {
	s := Sudoku{
		Size:     size,
		MaxValue: size * size,
	}
	fieldLength := len(strconv.Itoa(s.MaxValue))
	s.FieldLength = fieldLength
	// init fields
	fieldCount := size * size * size * size

	lineSize := s.MaxValue
	s.rows = make([]FieldGroup, lineSize)
	s.cols = make([]FieldGroup, lineSize)
	s.blocks = make([]FieldGroup, lineSize)

	s.Fields = make([]*Field, fieldCount)
	for row := 0; row < lineSize; row++ {
		for col := 0; col < lineSize; col++ {
			index := row*lineSize + col
			field := NewField(s, index, 0)
			s.Fields[index] = &field
			if col == 0 {
				s.rows[row] = NewFieldGroup(s, lineSize, fmt.Sprintf("row %d", row))
			}
			s.rows[row].Fields[col] = &field
			if row == 0 {
				s.cols[col] = NewFieldGroup(s, lineSize, fmt.Sprintf("col %d", col))
			}
			s.cols[col].Fields[row] = &field

			blockIndex := int(math.Floor(float64(row)/float64(s.Size)))*s.Size + int(math.Floor(float64(col)/float64(s.Size)))
			innerBlockRow := int(math.Mod(float64(row), float64(s.Size)))
			innerBlockCol := int(math.Mod(float64(col), float64(s.Size)))
			isBlockStart := innerBlockRow == 0 && innerBlockCol == 0
			if isBlockStart {
				s.blocks[blockIndex] = NewFieldGroup(s, lineSize, fmt.Sprintf("block %d", blockIndex))
			}
			s.blocks[blockIndex].Fields[innerBlockRow*s.Size+innerBlockCol] = &field
		}
	}
	return s
}

// IsSolved checks if this sudoku is solved
func (s Sudoku) IsSolved() bool {
	for _, f := range s.Fields {
		if !f.IsSolved() {
			return false
		}
	}
	return true
}

// String returns the state of the sudoku
func (s Sudoku) String() string {
	result := ""
	lineSize := s.MaxValue
	for row := 0; row < lineSize; row++ {
		for col := 0; col < lineSize; col++ {
			index := row*lineSize + col
			field := s.Fields[index]
			result += field.String()
		}
		result += "\n"
	}

	/*result += "\nRows:\n"
	for row := 0; row < lineSize; row++ {
		result += s.rows[row].String()
		result += "\n"
	}
	result += "\nCols:\n"
	for col := 0; col < lineSize; col++ {
		result += s.cols[col].String()
		result += "\n"
	}
	result += "\nBlocks:\n"
	for block := 0; block < lineSize; block++ {
		result += s.blocks[block].String()
		result += "\n"
	}

	result += "\nNonValues:\n"
	for _, f := range s.Fields {
		result += fmt.Sprintf("%d: %v", f.Index, f.NonValues)
	}*/

	return result
}

// Init inits
func (s Sudoku) Init(input []int) Sudoku {
	for i, val := range input {
		s.Fields[i].Value = val
	}
	return s
}

// GetRow returns all fields of the same row as given field
func (s Sudoku) GetRow(f *Field) FieldGroup {
	row := int(math.Floor(float64(f.Index) / float64(s.MaxValue)))
	return s.rows[row]
}

// GetCol returns all fields of the same column as given field
func (s Sudoku) GetCol(f *Field) FieldGroup {
	col := int(math.Mod(float64(f.Index), float64(s.MaxValue)))
	return s.cols[col]
}

// GetBlock gets all block Fields for a given field
func (s Sudoku) GetBlock(f *Field) FieldGroup {
	row := int(math.Floor(float64(f.Index) / float64(s.MaxValue)))
	col := int(math.Mod(float64(f.Index), float64(s.MaxValue)))
	blockIndex := int(math.Floor(float64(row)/float64(s.Size)))*s.Size + int(math.Floor(float64(col)/float64(s.Size)))

	return s.blocks[blockIndex]
}

func (s Sudoku) addSolutionByIndex(index, value int) {
	s.addSolution(s.Fields[index], value)
}

func (s Sudoku) addSolution(f *Field, value int) {
	f.Value = value

	rowFields := s.GetRow(f)
	for _, rf := range rowFields.Fields {
		if rf.IsSolved() {
			continue
		}
		//fmt.Printf("Field %d can't be of value %d (row check)\n", rf.Index, f.Value)
		rf.DenyValue(f.Value)
	}

	colFields := s.GetCol(f)
	for _, cf := range colFields.Fields {
		if cf.IsSolved() {
			continue
		}
		//fmt.Printf("Field %d can't be of value %d (col check)\n", cf.Index, f.Value)
		cf.DenyValue(f.Value)
	}

	blockFields := s.GetBlock(f)
	for _, bf := range blockFields.Fields {
		if bf.IsSolved() {
			continue
		}
		//fmt.Printf("Field %d can't be of value %d (block check)\n", bf.Index, f.Value)
		bf.DenyValue(f.Value)
	}
}

// Solve solves
func (s Sudoku) Solve() Sudoku {
	// loop all solved fields at begin
	//   create falsity information along all three dimensions
	for _, f := range s.Fields {
		if f.IsSolved() {
			s.addSolution(f, f.Value)
		}
	}

	for _, f := range s.Fields {
		// not solved
		if !f.IsSolved() {
			if f.Solve() {
				fmt.Printf("Field %d could be deduced because there was only one more possible value which is %d\n", f.Index, f.Value)
			}
		}
	}

	// loop all three dimensions
	//   loop all valid numbers
	//     loop all fields for checking
	//       solve and go back to start
	for _, row := range s.rows {
		row.Solve()
	}
	for _, col := range s.cols {
		col.Solve()
	}
	for _, block := range s.blocks {
		block.Solve()
	}
	return s
}
<commit_msg>Improve String() output with lines<commit_after>package sudoku

import (
	"fmt"
	"math"
	"strconv"
	"strings"
)

// Optimization ideas:
// - keep track of solved/unsolved fields (shorter loops, quicker check for solved state)
// - auto solve on adding n-1th element in deny list
// - dirty state for row/col/block?

// Sudoku is a sudoku puzzle
type Sudoku struct {
	Size        int
	MaxValue    int
	FieldLength int
	Fields      []*Field
	cols        []FieldGroup
	rows        []FieldGroup
	blocks      []FieldGroup
}

// New returns a new sudoku puzzle
func New(size int) Sudoku {
	s := Sudoku{
		Size:     size,
		MaxValue: size * size,
	}
	fieldLength := len(strconv.Itoa(s.MaxValue))
	s.FieldLength = fieldLength
	// init fields
	fieldCount := size * size * size * size

	lineSize := s.MaxValue
	s.rows = make([]FieldGroup, lineSize)
	s.cols = make([]FieldGroup, lineSize)
	s.blocks = make([]FieldGroup, lineSize)

	s.Fields = make([]*Field, fieldCount)
	for row := 0; row < lineSize; row++ {
		for col := 0; col < lineSize; col++ {
			index := row*lineSize + col
			field := NewField(s, index, 0)
			s.Fields[index] = &field
			if col == 0 {
				s.rows[row] = NewFieldGroup(s, lineSize, fmt.Sprintf("row %d", row))
			}
			s.rows[row].Fields[col] = &field
			if row == 0 {
				s.cols[col] = NewFieldGroup(s, lineSize, fmt.Sprintf("col %d", col))
			}
			s.cols[col].Fields[row] = &field

			blockIndex := int(math.Floor(float64(row)/float64(s.Size)))*s.Size + int(math.Floor(float64(col)/float64(s.Size)))
			innerBlockRow := int(math.Mod(float64(row), float64(s.Size)))
			innerBlockCol := int(math.Mod(float64(col), float64(s.Size)))
			isBlockStart := innerBlockRow == 0 && innerBlockCol == 0
			if isBlockStart {
				s.blocks[blockIndex] = NewFieldGroup(s, lineSize, fmt.Sprintf("block %d", blockIndex))
			}
			s.blocks[blockIndex].Fields[innerBlockRow*s.Size+innerBlockCol] = &field
		}
	}
	return s
}

// IsSolved checks if this sudoku is solved
func (s Sudoku) IsSolved() bool {
	for _, f := range s.Fields {
		if !f.IsSolved() {
			return false
		}
	}
	return true
}

// String returns the state of the sudoku
func (s Sudoku) String() string {
	result := ""
	lineSize := s.MaxValue
	drawBorder := true

	// top border
	if drawBorder {
		result += "+" + strings.Repeat(strings.Repeat("-", s.Size)+"+", s.Size) + "\n"
	}
	for row := 0; row < lineSize; row++ {
		if drawBorder {
			result += "|"
		}
		for col := 0; col < lineSize; col++ {
			index := row*lineSize + col
			field := s.Fields[index]
			result += field.String()
			// end of block?
			if drawBorder && col%s.Size == s.Size-1 {
				result += "|"
			}
		}
		result += "\n"
		if drawBorder && row%s.Size == s.Size-1 {
			result += "+" + strings.Repeat(strings.Repeat("-", s.Size)+"+", s.Size) + "\n"
		}
	}

	return result
}

// Init inits
func (s Sudoku) Init(input []int) Sudoku {
	for i, val := range input {
		s.Fields[i].Value = val
	}
	return s
}

// GetRow returns all fields of the same row as given field
func (s Sudoku) GetRow(f *Field) FieldGroup {
	row := int(math.Floor(float64(f.Index) / float64(s.MaxValue)))
	return s.rows[row]
}

// GetCol returns all fields of the same column as given field
func (s Sudoku) GetCol(f *Field) FieldGroup {
	col := int(math.Mod(float64(f.Index), float64(s.MaxValue)))
	return s.cols[col]
}

// GetBlock gets all block Fields for a given field
func (s Sudoku) GetBlock(f *Field) FieldGroup {
	row := int(math.Floor(float64(f.Index) / float64(s.MaxValue)))
	col := int(math.Mod(float64(f.Index), float64(s.MaxValue)))
	blockIndex := int(math.Floor(float64(row)/float64(s.Size)))*s.Size + int(math.Floor(float64(col)/float64(s.Size)))

	return s.blocks[blockIndex]
}

func (s Sudoku) addSolutionByIndex(index, value int) {
	s.addSolution(s.Fields[index], value)
}

func (s Sudoku) addSolution(f *Field, value int) {
	f.Value = value

	rowFields := s.GetRow(f)
	for _, rf := range rowFields.Fields {
		if rf.IsSolved() {
			continue
		}
		//fmt.Printf("Field %d can't be of value %d (row check)\n", rf.Index, f.Value)
		rf.DenyValue(f.Value)
	}

	colFields := s.GetCol(f)
	for _, cf := range colFields.Fields {
		if cf.IsSolved() {
			continue
		}
		//fmt.Printf("Field %d can't be of value %d (col check)\n", cf.Index, f.Value)
		cf.DenyValue(f.Value)
	}

	blockFields := s.GetBlock(f)
	for _, bf := range blockFields.Fields {
		if bf.IsSolved() {
			continue
		}
		//fmt.Printf("Field %d can't be of value %d (block check)\n", bf.Index, f.Value)
		bf.DenyValue(f.Value)
	}
}

// Solve solves
func (s Sudoku) Solve() Sudoku {
	// loop all solved fields at begin
	//   create falsity information along all three dimensions
	for _, f := range s.Fields {
		if f.IsSolved() {
			s.addSolution(f, f.Value)
		}
	}

	for _, f := range s.Fields {
		// not solved
		if !f.IsSolved() {
			if f.Solve() {
				fmt.Printf("Field %d could be deduced because there was only one more possible value which is %d\n", f.Index, f.Value)
			}
		}
	}

	// loop all three dimensions
	//   loop all valid numbers
	//     loop all fields for checking
	//       solve and go back to start
	for _, row := range s.rows {
		row.Solve()
	}
	for _, col := range s.cols {
		col.Solve()
	}
	for _, block := range s.blocks {
		block.Solve()
	}
	return s
}
<|endoftext|>


<commit_before>package jsoniter

import (
	"io"
	"bytes"
)

// Unmarshal adapts to json/encoding APIs
func Unmarshal(data []byte, v interface{}) error {
	data = data[:lastNotSpacePos(data)]
	iter := ParseBytes(data)
	iter.ReadVal(v)
	if iter.head == iter.tail {
		iter.loadMore()
	}
	if iter.Error == io.EOF {
		return nil
	}
	if iter.Error == nil {
		iter.reportError("Unmarshal", "there are bytes left after unmarshal")
	}
	return iter.Error
}

func UnmarshalAny(data []byte) (Any, error) {
	data = data[:lastNotSpacePos(data)]
	iter := ParseBytes(data)
	any := iter.ReadAny()
	if iter.head == iter.tail {
		iter.loadMore()
	}
	if iter.Error == io.EOF {
		return any, nil
	}
	if iter.Error == nil {
		iter.reportError("UnmarshalAny", "there are bytes left after unmarshal")
	}
	return any, iter.Error
}

func lastNotSpacePos(data []byte) int {
	for i := len(data) - 1; i >= 0; i-- {
		if data[i] != ' ' && data[i] != '\t' && data[i] != '\r' && data[i] != '\n' {
			return i + 1
		}
	}
	return 0
}

func UnmarshalFromString(str string, v interface{}) error {
	data := []byte(str)
	data = data[:lastNotSpacePos(data)]
	iter := ParseBytes(data)
	iter.ReadVal(v)
	if iter.head == iter.tail {
		iter.loadMore()
	}
	if iter.Error == io.EOF {
		return nil
	}
	if iter.Error == nil {
		iter.reportError("UnmarshalFromString", "there are bytes left after unmarshal")
	}
	return iter.Error
}

func UnmarshalAnyFromString(str string) (Any, error) {
	data := []byte(str)
	data = data[:lastNotSpacePos(data)]
	iter := ParseBytes(data)
	any := iter.ReadAny()
	if iter.head == iter.tail {
		iter.loadMore()
	}
	if iter.Error == io.EOF {
		return any, nil
	}
	if iter.Error == nil {
		iter.reportError("UnmarshalAnyFromString", "there are bytes left after unmarshal")
	}
	return nil, iter.Error
}

func Marshal(v interface{}) ([]byte, error) {
	buf := &bytes.Buffer{}
	stream := NewStream(buf, 512)
	stream.WriteVal(v)
	stream.Flush()
	if stream.Error != nil {
		return nil, stream.Error
	}
	return buf.Bytes(), nil
}

func MarshalToString(v interface{}) (string, error) {
	buf, err := Marshal(v)
	if err != nil {
		return "", err
	}
	return string(buf), nil
}

func NewDecoder(reader io.Reader) *AdaptedDecoder {
	iter := Parse(reader, 512)
	return &AdaptedDecoder{iter}
}

type AdaptedDecoder struct {
	iter *Iterator
}

func (adapter *AdaptedDecoder) Decode(obj interface{}) error {
	adapter.iter.ReadVal(obj)
	return adapter.iter.Error
}

func (adapter *AdaptedDecoder) More() bool {
	return adapter.iter.head != adapter.iter.tail
}

func (adapter *AdaptedDecoder) Buffered() io.Reader {
	remaining := adapter.iter.buf[adapter.iter.head:adapter.iter.tail]
	return bytes.NewReader(remaining)
}<commit_msg>eof is not error<commit_after>package jsoniter

import (
	"io"
	"bytes"
)

// Unmarshal adapts to json/encoding APIs
func Unmarshal(data []byte, v interface{}) error {
	data = data[:lastNotSpacePos(data)]
	iter := ParseBytes(data)
	iter.ReadVal(v)
	if iter.head == iter.tail {
		iter.loadMore()
	}
	if iter.Error == io.EOF {
		return nil
	}
	if iter.Error == nil {
		iter.reportError("Unmarshal", "there are bytes left after unmarshal")
	}
	return iter.Error
}

func UnmarshalAny(data []byte) (Any, error) {
	data = data[:lastNotSpacePos(data)]
	iter := ParseBytes(data)
	any := iter.ReadAny()
	if iter.head == iter.tail {
		iter.loadMore()
	}
	if iter.Error == io.EOF {
		return any, nil
	}
	if iter.Error == nil {
		iter.reportError("UnmarshalAny", "there are bytes left after unmarshal")
	}
	return any, iter.Error
}

func lastNotSpacePos(data []byte) int {
	for i := len(data) - 1; i >= 0; i-- {
		if data[i] != ' ' && data[i] != '\t' && data[i] != '\r' && data[i] != '\n' {
			return i + 1
		}
	}
	return 0
}

func UnmarshalFromString(str string, v interface{}) error {
	data := []byte(str)
	data = data[:lastNotSpacePos(data)]
	iter := ParseBytes(data)
	iter.ReadVal(v)
	if iter.head == iter.tail {
		iter.loadMore()
	}
	if iter.Error == io.EOF {
		return nil
	}
	if iter.Error == nil {
		iter.reportError("UnmarshalFromString", "there are bytes left after unmarshal")
	}
	return iter.Error
}

func UnmarshalAnyFromString(str string) (Any, error) {
	data := []byte(str)
	data = data[:lastNotSpacePos(data)]
	iter := ParseBytes(data)
	any := iter.ReadAny()
	if iter.head == iter.tail {
		iter.loadMore()
	}
	if iter.Error == io.EOF {
		return any, nil
	}
	if iter.Error == nil {
		iter.reportError("UnmarshalAnyFromString", "there are bytes left after unmarshal")
	}
	return nil, iter.Error
}

func Marshal(v interface{}) ([]byte, error) {
	buf := &bytes.Buffer{}
	stream := NewStream(buf, 512)
	stream.WriteVal(v)
	stream.Flush()
	if stream.Error != nil {
		return nil, stream.Error
	}
	return buf.Bytes(), nil
}

func MarshalToString(v interface{}) (string, error) {
	buf, err := Marshal(v)
	if err != nil {
		return "", err
	}
	return string(buf), nil
}

func NewDecoder(reader io.Reader) *AdaptedDecoder {
	iter := Parse(reader, 512)
	return &AdaptedDecoder{iter}
}

type AdaptedDecoder struct {
	iter *Iterator
}

func (adapter *AdaptedDecoder) Decode(obj interface{}) error {
	adapter.iter.ReadVal(obj)
	err := adapter.iter.Error
	if err == io.EOF {
		return nil
	}
	return adapter.iter.Error
}

func (adapter *AdaptedDecoder) More() bool {
	return adapter.iter.head != adapter.iter.tail
}

func (adapter *AdaptedDecoder) Buffered() io.Reader {
	remaining := adapter.iter.buf[adapter.iter.head:adapter.iter.tail]
	return bytes.NewReader(remaining)
}<|endoftext|>


<commit_before>package copperd

import (
	"log"
	"net"
	"testing"

	"github.com/snaury/copper"
)

func runServer(address string) (string, func()) {
	server := NewServer()
	listener, err := net.Listen("tcp", address)
	if err != nil {
		log.Fatalf("Failed to listen: %s", err)
	}
	err = server.AddListeners(listener)
	if err != nil {
		log.Fatalf("Failed to add listeners: %s", err)
	}
	go server.Serve()
	return listener.Addr().String(), func() {
		server.Shutdown()
	}
}

func connect(address string) Client {
	client, err := dialClient("tcp", address)
	if err != nil {
		log.Fatalf("Failed to connect: %s", err)
	}
	return client
}

func runClientServerConn(clientfunc func(conn copper.Conn), serverfunc func(stream copper.Stream)) {
	target, stopper := runServer("localhost:0")
	defer stopper()

	clientFinished := make(chan int, 1)
	unpublishSeen := make(chan int, 1)

	go func() {
		defer close(clientFinished)
		defer close(unpublishSeen)

		c := connect(target)
		defer c.Close()

		changes, err := c.ServiceChanges()
		if err != nil {
			log.Fatalf("c.ServiceChanges: %s", err)
		}
		defer changes.Stop()

		change, err := changes.Read()
		if err != nil || change.TargetID != 1 || change.Name != "test/myservice" {
			log.Fatalf("changes.Read(1): %#v, %v", change, err)
		}

		clientfunc(c.(*clientConn))

		clientFinished <- 1

		change, err = changes.Read()
		if err != nil || change.TargetID != -1 || change.Name != "test/myservice" {
			log.Fatalf("changes.Read(2): %#v, %v", change, err)
		}

		unpublishSeen <- 1
	}()

	func() {
		c := connect(target)
		defer c.Close()

		pub, err := c.Publish("test/myservice", PublishSettings{
			Distance:    2,
			Concurrency: 3,
		}, copper.StreamHandlerFunc(func(stream copper.Stream) {
			serverfunc(stream)
		}))
		if err != nil {
			log.Fatalf("c.Publish: %v", err)
		}
		defer pub.Stop()

		<-clientFinished
	}()

	<-unpublishSeen
}

func runClientServer(clientfunc func(stream copper.Stream), serverfunc func(stream copper.Stream)) {
	runClientServerConn(
		func(conn copper.Conn) {
			stream, err := conn.Open(1)
			if err != nil {
				log.Fatalf("conn.Open: %v", err)
			}
			defer stream.Close()
			clientfunc(stream)
		},
		serverfunc,
	)
}

func TestClientServer(t *testing.T) {
	runClientServer(
		func(stream copper.Stream) {
			n, err := stream.Write([]byte{1, 2, 3, 4, 5, 6, 7, 8})
			if n != 8 || err != nil {
				t.Fatalf("failed to write: %d, %v", n, err)
			}
			var buf [8]byte
			n, err = stream.Read(buf[:])
			if err == nil {
				_, err = stream.Peek()
			}
			if n != 8 || err != copper.EINTERNAL {
				t.Fatalf("failed to read: %d, %v", n, err)
			}
			t.Logf("client read: % x", buf)
			stream.Close()
		},
		func(stream copper.Stream) {
			var buf [8]byte
			n, err := stream.Read(buf[:])
			if n != 8 || err != nil {
				t.Fatalf("server read: %d, %v", n, err)
			}
			t.Logf("server read: % x", buf[:n])
			stream.Write(buf[:n])
			stream.CloseWithError(copper.EINTERNAL)
		},
	)
}

func TestClientServerCloseRead(t *testing.T) {
	mayCloseRead := make(chan int, 1)
	mayWriteResponse := make(chan int, 1)
	mayReadResponse := make(chan int, 1)
	mayCloseClient := make(chan int, 1)
	mayCloseServer := make(chan int, 1)
	runClientServer(
		func(stream copper.Stream) {
			defer close(mayReadResponse)
			defer close(mayCloseServer)

			if 1 != <-mayCloseRead {
				return
			}
			stream.CloseReadError(copper.EINTERNAL)

			if 1 != <-mayWriteResponse {
				return
			}
			n, err := stream.Write([]byte{1, 2, 3, 4})
			if n != 4 || err != nil {
				t.Fatalf("client: Write: %d, %v", n, err)
			}
			mayReadResponse <- 1

			n, err = stream.WaitAck()
			if n != 0 || err != nil {
				t.Fatalf("client: WaitAck: %d, %v", n, err)
			}
			<-mayCloseClient
			mayCloseServer <- 1
		},
		func(stream copper.Stream) {
			defer close(mayCloseRead)
			defer close(mayWriteResponse)
			defer close(mayCloseClient)

			n, err := stream.Write([]byte{5, 6, 7, 8})
			if n != 4 || err != nil {
				t.Fatalf("server: Write: %d, %v", n, err)
			}
			stream.Flush()
			mayCloseRead <- 1

			err = stream.WaitWriteClosed()
			if err != copper.EINTERNAL {
				t.Fatalf("server: WaitWriteClosed: %v", err)
			}

			mayWriteResponse <- 1
			if 1 != <-mayReadResponse {
				return
			}
			n, err = stream.Read(make([]byte, 16))
			if n != 4 || err != nil {
				t.Fatalf("server: Read: %d, %v", n, err)
			}
			mayCloseClient <- 1
			<-mayCloseServer
		},
	)
}
<commit_msg>Don't fail TestClientServerCloseRead when server side is not reached<commit_after>package copperd

import (
	"log"
	"net"
	"testing"

	"github.com/snaury/copper"
)

func runServer(address string) (string, func()) {
	server := NewServer()
	listener, err := net.Listen("tcp", address)
	if err != nil {
		log.Fatalf("Failed to listen: %s", err)
	}
	err = server.AddListeners(listener)
	if err != nil {
		log.Fatalf("Failed to add listeners: %s", err)
	}
	go server.Serve()
	return listener.Addr().String(), func() {
		server.Shutdown()
	}
}

func connect(address string) Client {
	client, err := dialClient("tcp", address)
	if err != nil {
		log.Fatalf("Failed to connect: %s", err)
	}
	return client
}

func runClientServerConn(clientfunc func(conn copper.Conn), serverfunc func(stream copper.Stream)) {
	target, stopper := runServer("localhost:0")
	defer stopper()

	clientFinished := make(chan int, 1)
	unpublishSeen := make(chan int, 1)

	go func() {
		defer close(clientFinished)
		defer close(unpublishSeen)

		c := connect(target)
		defer c.Close()

		changes, err := c.ServiceChanges()
		if err != nil {
			log.Fatalf("c.ServiceChanges: %s", err)
		}
		defer changes.Stop()

		change, err := changes.Read()
		if err != nil || change.TargetID != 1 || change.Name != "test/myservice" {
			log.Fatalf("changes.Read(1): %#v, %v", change, err)
		}

		clientfunc(c.(*clientConn))

		clientFinished <- 1

		change, err = changes.Read()
		if err != nil || change.TargetID != -1 || change.Name != "test/myservice" {
			log.Fatalf("changes.Read(2): %#v, %v", change, err)
		}

		unpublishSeen <- 1
	}()

	func() {
		c := connect(target)
		defer c.Close()

		pub, err := c.Publish("test/myservice", PublishSettings{
			Distance:    2,
			Concurrency: 3,
		}, copper.StreamHandlerFunc(func(stream copper.Stream) {
			serverfunc(stream)
		}))
		if err != nil {
			log.Fatalf("c.Publish: %v", err)
		}
		defer pub.Stop()

		<-clientFinished
	}()

	<-unpublishSeen
}

func runClientServer(clientfunc func(stream copper.Stream), serverfunc func(stream copper.Stream)) {
	runClientServerConn(
		func(conn copper.Conn) {
			stream, err := conn.Open(1)
			if err != nil {
				log.Fatalf("conn.Open: %v", err)
			}
			defer stream.Close()
			clientfunc(stream)
		},
		serverfunc,
	)
}

func TestClientServer(t *testing.T) {
	runClientServer(
		func(stream copper.Stream) {
			n, err := stream.Write([]byte{1, 2, 3, 4, 5, 6, 7, 8})
			if n != 8 || err != nil {
				t.Fatalf("failed to write: %d, %v", n, err)
			}
			var buf [8]byte
			n, err = stream.Read(buf[:])
			if err == nil {
				_, err = stream.Peek()
			}
			if n != 8 || err != copper.EINTERNAL {
				t.Fatalf("failed to read: %d, %v", n, err)
			}
			t.Logf("client read: % x", buf)
			stream.Close()
		},
		func(stream copper.Stream) {
			var buf [8]byte
			n, err := stream.Read(buf[:])
			if n != 8 || err != nil {
				t.Fatalf("server read: %d, %v", n, err)
			}
			t.Logf("server read: % x", buf[:n])
			stream.Write(buf[:n])
			stream.CloseWithError(copper.EINTERNAL)
		},
	)
}

func TestClientServerCloseRead(t *testing.T) {
	mayCloseRead := make(chan int, 1)
	mayWriteResponse := make(chan int, 1)
	mayReadResponse := make(chan int, 1)
	mayCloseClient := make(chan int, 1)
	mayCloseServer := make(chan int, 1)
	runClientServer(
		func(stream copper.Stream) {
			defer close(mayReadResponse)
			defer close(mayCloseServer)

			_, err := stream.Peek()
			if err != nil {
				t.Fatalf("client: Peek: %v", err)
			}
			if 1 != <-mayCloseRead {
				return
			}
			stream.CloseReadError(copper.EINTERNAL)

			if 1 != <-mayWriteResponse {
				return
			}
			n, err := stream.Write([]byte{1, 2, 3, 4})
			if n != 4 || err != nil {
				t.Fatalf("client: Write: %d, %v", n, err)
			}
			mayReadResponse <- 1

			n, err = stream.WaitAck()
			if n != 0 || err != nil {
				t.Fatalf("client: WaitAck: %d, %v", n, err)
			}
			<-mayCloseClient
			mayCloseServer <- 1
		},
		func(stream copper.Stream) {
			defer close(mayCloseRead)
			defer close(mayWriteResponse)
			defer close(mayCloseClient)

			n, err := stream.Write([]byte{5, 6, 7, 8})
			if n != 4 || err != nil {
				t.Fatalf("server: Write: %d, %v", n, err)
			}
			err = stream.Flush()
			if err != nil {
				t.Fatalf("server: Flush: %v", err)
			}
			mayCloseRead <- 1

			err = stream.WaitWriteClosed()
			if err != copper.EINTERNAL {
				t.Fatalf("server: WaitWriteClosed: %v", err)
			}

			mayWriteResponse <- 1
			if 1 != <-mayReadResponse {
				return
			}
			n, err = stream.Read(make([]byte, 16))
			if n != 4 || err != nil {
				t.Fatalf("server: Read: %d, %v", n, err)
			}
			mayCloseClient <- 1
			<-mayCloseServer
		},
	)
}
<|endoftext|>


<commit_before>package cache

import (
	"github.com/modern-go/reflect2"
	"github.com/rcrowley/go-metrics"
	"sync"
	"sync/atomic"
	"unsafe"
)

var binaryCacheLock sync.Mutex
var binaryCache = NewLRUCache(10 * 1024 * 1024)

var metricMissCount, metricHitCount uint64
var metricReadBinarySize = metrics.NewHistogram(metrics.NewUniformSample(1024 * 8))

func String(str string) string {
	return lookupCache(false, stringToByteSlice(str))
}

func StringForByteSliceCopy(value []byte) string {
	return lookupCache(true, value)
}

func StringForByteSliceNoCopy(value []byte) string {
	return lookupCache(false, value)
}

func lookupCache(copyOnInsert bool, value []byte) string {
	if len(value) == 0 {
		return ""
	}

	key := byteSliceToString(value)

	binaryCacheLock.Lock()
	cachedValue := binaryCache.Get(key)
	binaryCacheLock.Unlock()

	if cachedValue != "" {
		atomic.AddUint64(&metricHitCount, 1)
		return cachedValue
	}

	if copyOnInsert {
		// create a copy of the data that we want to cache
		newValue := make([]byte, 0, len(value))
		key = byteSliceToString(append(newValue, value...))
	}

	atomic.AddUint64(&metricMissCount, 1)

	binaryCacheLock.Lock()
	binaryCache.Set(key)
	binaryCacheLock.Unlock()

	return key
}

func stringToByteSlice(str string) []byte {
	return reflect2.UnsafeCastString(str)
}

func byteSliceToString(bytes []byte) string {
	if bytes == nil {
		return ""
	}

	return *(*string)(unsafe.Pointer(&bytes))
}

func RegisterCacheMetrics(m metrics.Registry) {
	metrics.NewRegisteredFunctionalGaugeFloat64("binary.cache.hit.rate", m, func() float64 {
		hitCount := atomic.SwapUint64(&metricHitCount, 0)
		missCount := atomic.SwapUint64(&metricMissCount, 0)
		return 1000 * float64(hitCount) / float64(hitCount+missCount)
	})

	metrics.NewRegisteredFunctionalGauge("binary.cache.size", m, func() int64 {
		binaryCacheLock.Lock()
		defer binaryCacheLock.Unlock()
		return int64(binaryCache.Size())
	})

	metrics.NewRegisteredFunctionalGauge("binary.cache.count", m, func() int64 {
		binaryCacheLock.Lock()
		binaryCacheLock.Unlock()
		return int64(binaryCache.Count())
	})
}
<commit_msg>Fix cache hit rate metrics.<commit_after>package cache

import (
	"github.com/modern-go/reflect2"
	"github.com/rcrowley/go-metrics"
	"sync"
	"sync/atomic"
	"unsafe"
)

var binaryCacheLock sync.Mutex
var binaryCache = NewLRUCache(10 * 1024 * 1024)

var metricMissCount, metricHitCount uint64
var metricReadBinarySize = metrics.NewHistogram(metrics.NewUniformSample(1024 * 8))

func String(str string) string {
	return lookupCache(false, stringToByteSlice(str))
}

func StringForByteSliceCopy(value []byte) string {
	return lookupCache(true, value)
}

func StringForByteSliceNoCopy(value []byte) string {
	return lookupCache(false, value)
}

func lookupCache(copyOnInsert bool, value []byte) string {
	if len(value) == 0 {
		return ""
	}

	key := byteSliceToString(value)

	binaryCacheLock.Lock()
	cachedValue := binaryCache.Get(key)
	binaryCacheLock.Unlock()

	if cachedValue != "" {
		atomic.AddUint64(&metricHitCount, 1)
		return cachedValue
	}

	if copyOnInsert {
		// create a copy of the data that we want to cache
		newValue := make([]byte, 0, len(value))
		key = byteSliceToString(append(newValue, value...))
	}

	atomic.AddUint64(&metricMissCount, 1)

	binaryCacheLock.Lock()
	binaryCache.Set(key)
	binaryCacheLock.Unlock()

	return key
}

func stringToByteSlice(str string) []byte {
	return reflect2.UnsafeCastString(str)
}

func byteSliceToString(bytes []byte) string {
	if bytes == nil {
		return ""
	}

	return *(*string)(unsafe.Pointer(&bytes))
}

func RegisterCacheMetrics(m metrics.Registry) {
	metrics.NewRegisteredFunctionalGaugeFloat64("binary.cache.hit.rate", m, func() float64 {
		hitCount := atomic.SwapUint64(&metricHitCount, 0)
		missCount := atomic.SwapUint64(&metricMissCount, 0)
		return float64(hitCount) / float64(hitCount+missCount)
	})

	metrics.NewRegisteredFunctionalGauge("binary.cache.size", m, func() int64 {
		binaryCacheLock.Lock()
		defer binaryCacheLock.Unlock()
		return int64(binaryCache.Size())
	})

	metrics.NewRegisteredFunctionalGauge("binary.cache.count", m, func() int64 {
		binaryCacheLock.Lock()
		binaryCacheLock.Unlock()
		return int64(binaryCache.Count())
	})
}
<|endoftext|>


<commit_before>package cache

import (
	"errors"
	"html/template"
	"io/ioutil"
	"log"
	"os"
	"strings"
)

type DocCache struct {
	docs  map[string]*template.Template //map of documents in 'path', indexed by filename
	size  int64                         //Size of the document in bytes
	count int                           //Amount of documents in the cache
	Path  string                        //Path to the document directory
}

var Docs *DocCache // Represents the global document cache

//Allocated and builds a new cache from the directory 'path'
func InitCache(path string) error {
	if Docs = NewDocCache(path); Docs == nil {
		return errors.New("Failed to get document cache")
	} else {
		return Docs.BuildCache()
	}
}

//Allocates a new DocCache with path 'path'
func NewDocCache(path string) *DocCache {
	return &DocCache{make(map[string]*template.Template), 0, 0, path}
}

//Builds a cache of documents from the files in the 'docDir' path
func (cache *DocCache) BuildCache() error {
	log.Println("Building document cache from", cache.Path)

	list, err := ioutil.ReadDir(cache.Path)
	if err != nil {
		return err
	}

	for _, file := range list {
		if strings.HasSuffix(file.Name(), ".html") == false {
			log.Println("\t!! Ignore:", file.Name())
			continue
		}
		log.Println("\t++ Cache:", file.Name())
		cache.CacheDoc(file.Name())
	}

	log.Println("\t!! Cached", cache.count, "file(s) (", cache.size, " bytes) in", cache.Path)
	return nil
}

func (cache *DocCache) RefreshDoc(name string) error {
	log.Println("\t!! Refreshing modified document:", name)
	if cache.IsCached(name) == false {
		return errors.New("Document does not exist in cache")
	} else {
		delete(cache.docs, name)
		data, err := ioutil.ReadFile(cache.Path + name)
		if err != nil {
			return err
		}

		cache.docs[name], err = template.New(name).Parse(string(data))
		if err != nil {
			return err
		}
	}
	return nil
}

//Adds the document 'name' to the cache
func (cache *DocCache) CacheDoc(name string) error {
	data, err := ioutil.ReadFile(cache.Path + name)
	if err != nil {
		return err
	}

	cache.docs[name], err = template.New(name).Parse(string(data))
	if err != nil {
		return err
	}

	stat, _ := os.Stat(cache.Path + name)
	cache.size += stat.Size()
	cache.count++
	return nil
}

//Returns true if 'name' is in cache, false otherwise
func (cache *DocCache) IsCached(name string) bool {
	_, ok := cache.docs[name]
	return ok
}

//Returns true if doc 'name' is on disk, false otherwise
func (cache *DocCache) IsOnDisk(name string) bool {
	_, err := os.Stat(name)
	return err == nil
}

//Looks for the document 'name' in the cache, then on the disk, then gives up and returns 404
// If the document is found in the cache it returns the document immediately
// If the document isn't in the cache, but on the disk, the document is read and
//	added to the cache and returned
// If the document isn't in the cache or on the disk, return 404
func (cache *DocCache) GetDoc(name string) *template.Template {

	log.Println("\t", name, "?? Querying cache")
	//File is in cache, simply return it
	if cache.IsCached(name) == true {
		log.Println("\t", name, "++ Cached/Found")
		return cache.docs[name]
		//Doc isn't in cache, see if it's on disk
		//cache and return it if it is
	} else if cache.IsOnDisk(name) == true {
		log.Println("\t", name, "!! Not in cache")
		log.Println("\t", name, "++ On disk/Found")
		log.Println("\t", name, "!! Caching")

		cache.CacheDoc(name)
		return cache.docs[name]
	} else {
		//No luck in cache or on disk, return 404
		log.Println("\t", name, "-- Not cached/Not on disk")
		return nil
	}
	return nil
}
<commit_msg>Use filepath.Join() in cache.go:CacheDoc()<commit_after>package cache

import (
	"errors"
	"html/template"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
)

type DocCache struct {
	docs  map[string]*template.Template //map of documents in 'path', indexed by filename
	size  int64                         //Size of the document in bytes
	count int                           //Amount of documents in the cache
	Path  string                        //Path to the document directory
}

var Docs *DocCache // Represents the global document cache

//Allocated and builds a new cache from the directory 'path'
func InitCache(path string) error {
	if Docs = NewDocCache(path); Docs == nil {
		return errors.New("Failed to get document cache")
	} else {
		return Docs.BuildCache()
	}
}

//Allocates a new DocCache with path 'path'
func NewDocCache(path string) *DocCache {
	return &DocCache{make(map[string]*template.Template), 0, 0, path}
}

//Builds a cache of documents from the files in the 'docDir' path
func (cache *DocCache) BuildCache() error {
	log.Println("Building document cache from", cache.Path)

	list, err := ioutil.ReadDir(cache.Path)
	if err != nil {
		return err
	}

	for _, file := range list {
		if strings.HasSuffix(file.Name(), ".html") == false {
			log.Println("\t!! Ignore:", file.Name())
			continue
		}
		log.Println("\t++ Cache:", file.Name())
		cache.CacheDoc(file.Name())
	}

	log.Println("\t!! Cached", cache.count, "file(s) (", cache.size, " bytes) in", cache.Path)
	return nil
}

func (cache *DocCache) RefreshDoc(name string) error {
	log.Println("\t!! Refreshing modified document:", name)
	if cache.IsCached(name) == false {
		return errors.New("Document does not exist in cache")
	} else {
		delete(cache.docs, name)
		data, err := ioutil.ReadFile(cache.Path + name)
		if err != nil {
			return err
		}

		cache.docs[name], err = template.New(name).Parse(string(data))
		if err != nil {
			return err
		}
	}
	return nil
}

//Adds the document 'name' to the cache
func (cache *DocCache) CacheDoc(name string) error {
	data, err := ioutil.ReadFile(cache.Path + name)
	if err != nil {
		return err
	}

	cache.docs[name], err = template.New(name).Parse(string(data))
	if err != nil {
		return err
	}

	stat, _ := os.Stat(filepath.Join(cache.Path, name))
	cache.size += stat.Size()
	cache.count++
	return nil
}

//Returns true if 'name' is in cache, false otherwise
func (cache *DocCache) IsCached(name string) bool {
	_, ok := cache.docs[name]
	return ok
}

//Returns true if doc 'name' is on disk, false otherwise
func (cache *DocCache) IsOnDisk(name string) bool {
	_, err := os.Stat(name)
	return err == nil
}

//Looks for the document 'name' in the cache, then on the disk, then gives up and returns 404
// If the document is found in the cache it returns the document immediately
// If the document isn't in the cache, but on the disk, the document is read and
//	added to the cache and returned
// If the document isn't in the cache or on the disk, return 404
func (cache *DocCache) GetDoc(name string) *template.Template {

	log.Println("\t", name, "?? Querying cache")
	//File is in cache, simply return it
	if cache.IsCached(name) == true {
		log.Println("\t", name, "++ Cached/Found")
		return cache.docs[name]
		//Doc isn't in cache, see if it's on disk
		//cache and return it if it is
	} else if cache.IsOnDisk(name) == true {
		log.Println("\t", name, "!! Not in cache")
		log.Println("\t", name, "++ On disk/Found")
		log.Println("\t", name, "!! Caching")

		cache.CacheDoc(name)
		return cache.docs[name]
	} else {
		//No luck in cache or on disk, return 404
		log.Println("\t", name, "-- Not cached/Not on disk")
		return nil
	}
	return nil
}
<|endoftext|>