<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"encoding/json"
	"fmt"
	"time"

	cadvisorapi "github.com/google/cadvisor/info/v1"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/resource"
	client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/util/wait"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	mb = 1024 * 1024
	gb = 1024 * mb

	// TODO(madhusudancs): find a way to query kubelet's disk space manager to obtain this value. 256MB
	// is the default that is set today. This test might break if the default value changes. This value
	// can be configured by setting the "low-diskspace-threshold-mb" flag while starting a kubelet.
	// However, kubelets are started as part of the cluster start up, once, before any e2e test is run,
	// and remain unchanged until all the tests are run and the cluster is brought down. Changing the
	// flag value affects all the e2e tests. So we are hard-coding this value for now.
	lowDiskSpaceThreshold uint64 = 256 * mb

	nodeOODTimeOut = 5 * time.Minute

	numNodeOODPods = 3
)

// Plan:
// 1. Fill disk space on all nodes except one. One node is left out so that we can schedule pods
//    on that node. Arbitrarily choose that node to be node with index 0.
// 2. Get the CPU capacity on unfilled node.
// 3. Divide the available CPU into one less than the number of pods we want to schedule. We want
//    to schedule 3 pods, so divide CPU capacity by 2.
// 4. Request the divided CPU for each pod.
// 5. Observe that 2 of the pods schedule onto the node whose disk is not full, and the remaining
//    pod stays pending and does not schedule onto the nodes whose disks are full nor the node
//    with the other two pods, since there is not enough free CPU capacity there.
// 6. Recover disk space from one of the nodes whose disk space was previously filled. Arbritrarily
//    choose that node to be node with index 1.
// 7. Observe that the pod in pending status schedules on that node.
//
// Flaky issue #20015.  We have no clear path for how to test this functionality in a non-flaky way.
var _ = Describe("NodeOutOfDisk [Serial] [Flaky]", func() {
	var c *client.Client
	var unfilledNodeName, recoveredNodeName string
	framework := Framework{BaseName: "node-outofdisk"}

	BeforeEach(func() {
		framework.beforeEach()
		c = framework.Client

		nodelist := ListSchedulableNodesOrDie(c)
		Expect(len(nodelist.Items)).To(BeNumerically(">", 1))

		unfilledNodeName = nodelist.Items[0].Name
		for _, node := range nodelist.Items[1:] {
			fillDiskSpace(c, &node)
		}
	})

	AfterEach(func() {
		defer framework.afterEach()

		nodelist := ListSchedulableNodesOrDie(c)
		Expect(len(nodelist.Items)).ToNot(BeZero())
		for _, node := range nodelist.Items {
			if unfilledNodeName == node.Name || recoveredNodeName == node.Name {
				continue
			}
			recoverDiskSpace(c, &node)
		}
	})

	It("runs out of disk space", func() {
		unfilledNode, err := c.Nodes().Get(unfilledNodeName)
		expectNoError(err)

		By(fmt.Sprintf("Calculating CPU availability on node %s", unfilledNode.Name))
		milliCpu, err := availCpu(c, unfilledNode)
		expectNoError(err)

		// Per pod CPU should be just enough to fit only (numNodeOODPods - 1) pods on the given
		// node. We compute this value by dividing the available CPU capacity on the node by
		// (numNodeOODPods - 1) and subtracting ϵ from it. We arbitrarily choose ϵ to be 1%
		// of the available CPU per pod, i.e. 0.01 * milliCpu/(numNodeOODPods-1). Instead of
		// subtracting 1% from the value, we directly use 0.99 as the multiplier.
		podCPU := int64(float64(milliCpu/(numNodeOODPods-1)) * 0.99)

		ns := framework.Namespace.Name
		podClient := c.Pods(ns)

		By("Creating pods and waiting for all but one pods to be scheduled")

		for i := 0; i < numNodeOODPods-1; i++ {
			name := fmt.Sprintf("pod-node-outofdisk-%d", i)
			createOutOfDiskPod(c, ns, name, podCPU)

			expectNoError(framework.WaitForPodRunning(name))
			pod, err := podClient.Get(name)
			expectNoError(err)
			Expect(pod.Spec.NodeName).To(Equal(unfilledNodeName))
		}

		pendingPodName := fmt.Sprintf("pod-node-outofdisk-%d", numNodeOODPods-1)
		createOutOfDiskPod(c, ns, pendingPodName, podCPU)

		By(fmt.Sprintf("Finding a failed scheduler event for pod %s", pendingPodName))
		wait.Poll(2*time.Second, 5*time.Minute, func() (bool, error) {
			selector := fields.Set{
				"involvedObject.kind":      "Pod",
				"involvedObject.name":      pendingPodName,
				"involvedObject.namespace": ns,
				"source":                   api.DefaultSchedulerName,
				"reason":                   "FailedScheduling",
			}.AsSelector()
			options := api.ListOptions{FieldSelector: selector}
			schedEvents, err := c.Events(ns).List(options)
			expectNoError(err)

			if len(schedEvents.Items) > 0 {
				return true, nil
			} else {
				return false, nil
			}
		})

		nodelist := ListSchedulableNodesOrDie(c)
		Expect(len(nodelist.Items)).To(BeNumerically(">", 1))

		nodeToRecover := nodelist.Items[1]
		Expect(nodeToRecover.Name).ToNot(Equal(unfilledNodeName))

		recoverDiskSpace(c, &nodeToRecover)
		recoveredNodeName = nodeToRecover.Name

		By(fmt.Sprintf("Verifying that pod %s schedules on node %s", pendingPodName, recoveredNodeName))
		expectNoError(framework.WaitForPodRunning(pendingPodName))
		pendingPod, err := podClient.Get(pendingPodName)
		expectNoError(err)
		Expect(pendingPod.Spec.NodeName).To(Equal(recoveredNodeName))
	})
})

// createOutOfDiskPod creates a pod in the given namespace with the requested amount of CPU.
func createOutOfDiskPod(c *client.Client, ns, name string, milliCPU int64) {
	podClient := c.Pods(ns)

	pod := &api.Pod{
		ObjectMeta: api.ObjectMeta{
			Name: name,
		},
		Spec: api.PodSpec{
			Containers: []api.Container{
				{
					Name:  "pause",
					Image: "beta.gcr.io/google_containers/pause:2.0",
					Resources: api.ResourceRequirements{
						Requests: api.ResourceList{
							// Request enough CPU to fit only two pods on a given node.
							api.ResourceCPU: *resource.NewMilliQuantity(milliCPU, resource.DecimalSI),
						},
					},
				},
			},
		},
	}

	_, err := podClient.Create(pod)
	expectNoError(err)
}

// availCpu calculates the available CPU on a given node by subtracting the CPU requested by
// all the pods from the total available CPU capacity on the node.
func availCpu(c *client.Client, node *api.Node) (int64, error) {
	podClient := c.Pods(api.NamespaceAll)

	selector := fields.Set{"spec.nodeName": node.Name}.AsSelector()
	options := api.ListOptions{FieldSelector: selector}
	pods, err := podClient.List(options)
	if err != nil {
		return 0, fmt.Errorf("failed to retrieve all the pods on node %s: %v", node.Name, err)
	}
	avail := node.Status.Capacity.Cpu().MilliValue()
	for _, pod := range pods.Items {
		for _, cont := range pod.Spec.Containers {
			avail -= cont.Resources.Requests.Cpu().MilliValue()
		}
	}
	return avail, nil
}

// availSize returns the available disk space on a given node by querying node stats which
// is in turn obtained internally from cadvisor.
func availSize(c *client.Client, node *api.Node) (uint64, error) {
	statsResource := fmt.Sprintf("api/v1/proxy/nodes/%s/stats/", node.Name)
	Logf("Querying stats for node %s using url %s", node.Name, statsResource)
	res, err := c.Get().AbsPath(statsResource).Timeout(time.Minute).Do().Raw()
	if err != nil {
		return 0, fmt.Errorf("error querying cAdvisor API: %v", err)
	}
	ci := cadvisorapi.ContainerInfo{}
	err = json.Unmarshal(res, &ci)
	if err != nil {
		return 0, fmt.Errorf("couldn't unmarshal container info: %v", err)
	}
	return ci.Stats[len(ci.Stats)-1].Filesystem[0].Available, nil
}

// fillDiskSpace fills the available disk space on a given node by creating a large file. The disk
// space on the node is filled in such a way that the available space after filling the disk is just
// below the lowDiskSpaceThreshold mark.
func fillDiskSpace(c *client.Client, node *api.Node) {
	avail, err := availSize(c, node)
	expectNoError(err, "Node %s: couldn't obtain available disk size %v", node.Name, err)

	fillSize := (avail - lowDiskSpaceThreshold + (100 * mb))

	Logf("Node %s: disk space available %d bytes", node.Name, avail)
	By(fmt.Sprintf("Node %s: creating a file of size %d bytes to fill the available disk space", node.Name, fillSize))

	cmd := fmt.Sprintf("fallocate -l %d test.img", fillSize)
	expectNoError(issueSSHCommand(cmd, testContext.Provider, node))

	ood := waitForNodeToBe(c, node.Name, api.NodeOutOfDisk, true, nodeOODTimeOut)
	Expect(ood).To(BeTrue(), "Node %s did not run out of disk within %v", node.Name, nodeOODTimeOut)

	avail, err = availSize(c, node)
	Logf("Node %s: disk space available %d bytes", node.Name, avail)
	Expect(avail < lowDiskSpaceThreshold).To(BeTrue())
}

// recoverDiskSpace recovers disk space, filled by creating a large file, on a given node.
func recoverDiskSpace(c *client.Client, node *api.Node) {
	By(fmt.Sprintf("Recovering disk space on node %s", node.Name))
	cmd := "rm -f test.img"
	expectNoError(issueSSHCommand(cmd, testContext.Provider, node))

	ood := waitForNodeToBe(c, node.Name, api.NodeOutOfDisk, false, nodeOODTimeOut)
	Expect(ood).To(BeTrue(), "Node %s's out of disk condition status did not change to false within %v", node.Name, nodeOODTimeOut)
}
<commit_msg>annotate NodeOutOfDisk as disruptive<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"encoding/json"
	"fmt"
	"time"

	cadvisorapi "github.com/google/cadvisor/info/v1"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/resource"
	client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/util/wait"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	mb = 1024 * 1024
	gb = 1024 * mb

	// TODO(madhusudancs): find a way to query kubelet's disk space manager to obtain this value. 256MB
	// is the default that is set today. This test might break if the default value changes. This value
	// can be configured by setting the "low-diskspace-threshold-mb" flag while starting a kubelet.
	// However, kubelets are started as part of the cluster start up, once, before any e2e test is run,
	// and remain unchanged until all the tests are run and the cluster is brought down. Changing the
	// flag value affects all the e2e tests. So we are hard-coding this value for now.
	lowDiskSpaceThreshold uint64 = 256 * mb

	nodeOODTimeOut = 5 * time.Minute

	numNodeOODPods = 3
)

// Plan:
// 1. Fill disk space on all nodes except one. One node is left out so that we can schedule pods
//    on that node. Arbitrarily choose that node to be node with index 0.  This makes this a disruptive test.
// 2. Get the CPU capacity on unfilled node.
// 3. Divide the available CPU into one less than the number of pods we want to schedule. We want
//    to schedule 3 pods, so divide CPU capacity by 2.
// 4. Request the divided CPU for each pod.
// 5. Observe that 2 of the pods schedule onto the node whose disk is not full, and the remaining
//    pod stays pending and does not schedule onto the nodes whose disks are full nor the node
//    with the other two pods, since there is not enough free CPU capacity there.
// 6. Recover disk space from one of the nodes whose disk space was previously filled. Arbritrarily
//    choose that node to be node with index 1.
// 7. Observe that the pod in pending status schedules on that node.
//
// Flaky issue #20015.  We have no clear path for how to test this functionality in a non-flaky way.
var _ = Describe("NodeOutOfDisk [Serial] [Flaky] [Disruptive]", func() {
	var c *client.Client
	var unfilledNodeName, recoveredNodeName string
	framework := Framework{BaseName: "node-outofdisk"}

	BeforeEach(func() {
		framework.beforeEach()
		c = framework.Client

		nodelist := ListSchedulableNodesOrDie(c)
		Expect(len(nodelist.Items)).To(BeNumerically(">", 1))

		unfilledNodeName = nodelist.Items[0].Name
		for _, node := range nodelist.Items[1:] {
			fillDiskSpace(c, &node)
		}
	})

	AfterEach(func() {
		defer framework.afterEach()

		nodelist := ListSchedulableNodesOrDie(c)
		Expect(len(nodelist.Items)).ToNot(BeZero())
		for _, node := range nodelist.Items {
			if unfilledNodeName == node.Name || recoveredNodeName == node.Name {
				continue
			}
			recoverDiskSpace(c, &node)
		}
	})

	It("runs out of disk space", func() {
		unfilledNode, err := c.Nodes().Get(unfilledNodeName)
		expectNoError(err)

		By(fmt.Sprintf("Calculating CPU availability on node %s", unfilledNode.Name))
		milliCpu, err := availCpu(c, unfilledNode)
		expectNoError(err)

		// Per pod CPU should be just enough to fit only (numNodeOODPods - 1) pods on the given
		// node. We compute this value by dividing the available CPU capacity on the node by
		// (numNodeOODPods - 1) and subtracting ϵ from it. We arbitrarily choose ϵ to be 1%
		// of the available CPU per pod, i.e. 0.01 * milliCpu/(numNodeOODPods-1). Instead of
		// subtracting 1% from the value, we directly use 0.99 as the multiplier.
		podCPU := int64(float64(milliCpu/(numNodeOODPods-1)) * 0.99)

		ns := framework.Namespace.Name
		podClient := c.Pods(ns)

		By("Creating pods and waiting for all but one pods to be scheduled")

		for i := 0; i < numNodeOODPods-1; i++ {
			name := fmt.Sprintf("pod-node-outofdisk-%d", i)
			createOutOfDiskPod(c, ns, name, podCPU)

			expectNoError(framework.WaitForPodRunning(name))
			pod, err := podClient.Get(name)
			expectNoError(err)
			Expect(pod.Spec.NodeName).To(Equal(unfilledNodeName))
		}

		pendingPodName := fmt.Sprintf("pod-node-outofdisk-%d", numNodeOODPods-1)
		createOutOfDiskPod(c, ns, pendingPodName, podCPU)

		By(fmt.Sprintf("Finding a failed scheduler event for pod %s", pendingPodName))
		wait.Poll(2*time.Second, 5*time.Minute, func() (bool, error) {
			selector := fields.Set{
				"involvedObject.kind":      "Pod",
				"involvedObject.name":      pendingPodName,
				"involvedObject.namespace": ns,
				"source":                   api.DefaultSchedulerName,
				"reason":                   "FailedScheduling",
			}.AsSelector()
			options := api.ListOptions{FieldSelector: selector}
			schedEvents, err := c.Events(ns).List(options)
			expectNoError(err)

			if len(schedEvents.Items) > 0 {
				return true, nil
			} else {
				return false, nil
			}
		})

		nodelist := ListSchedulableNodesOrDie(c)
		Expect(len(nodelist.Items)).To(BeNumerically(">", 1))

		nodeToRecover := nodelist.Items[1]
		Expect(nodeToRecover.Name).ToNot(Equal(unfilledNodeName))

		recoverDiskSpace(c, &nodeToRecover)
		recoveredNodeName = nodeToRecover.Name

		By(fmt.Sprintf("Verifying that pod %s schedules on node %s", pendingPodName, recoveredNodeName))
		expectNoError(framework.WaitForPodRunning(pendingPodName))
		pendingPod, err := podClient.Get(pendingPodName)
		expectNoError(err)
		Expect(pendingPod.Spec.NodeName).To(Equal(recoveredNodeName))
	})
})

// createOutOfDiskPod creates a pod in the given namespace with the requested amount of CPU.
func createOutOfDiskPod(c *client.Client, ns, name string, milliCPU int64) {
	podClient := c.Pods(ns)

	pod := &api.Pod{
		ObjectMeta: api.ObjectMeta{
			Name: name,
		},
		Spec: api.PodSpec{
			Containers: []api.Container{
				{
					Name:  "pause",
					Image: "beta.gcr.io/google_containers/pause:2.0",
					Resources: api.ResourceRequirements{
						Requests: api.ResourceList{
							// Request enough CPU to fit only two pods on a given node.
							api.ResourceCPU: *resource.NewMilliQuantity(milliCPU, resource.DecimalSI),
						},
					},
				},
			},
		},
	}

	_, err := podClient.Create(pod)
	expectNoError(err)
}

// availCpu calculates the available CPU on a given node by subtracting the CPU requested by
// all the pods from the total available CPU capacity on the node.
func availCpu(c *client.Client, node *api.Node) (int64, error) {
	podClient := c.Pods(api.NamespaceAll)

	selector := fields.Set{"spec.nodeName": node.Name}.AsSelector()
	options := api.ListOptions{FieldSelector: selector}
	pods, err := podClient.List(options)
	if err != nil {
		return 0, fmt.Errorf("failed to retrieve all the pods on node %s: %v", node.Name, err)
	}
	avail := node.Status.Capacity.Cpu().MilliValue()
	for _, pod := range pods.Items {
		for _, cont := range pod.Spec.Containers {
			avail -= cont.Resources.Requests.Cpu().MilliValue()
		}
	}
	return avail, nil
}

// availSize returns the available disk space on a given node by querying node stats which
// is in turn obtained internally from cadvisor.
func availSize(c *client.Client, node *api.Node) (uint64, error) {
	statsResource := fmt.Sprintf("api/v1/proxy/nodes/%s/stats/", node.Name)
	Logf("Querying stats for node %s using url %s", node.Name, statsResource)
	res, err := c.Get().AbsPath(statsResource).Timeout(time.Minute).Do().Raw()
	if err != nil {
		return 0, fmt.Errorf("error querying cAdvisor API: %v", err)
	}
	ci := cadvisorapi.ContainerInfo{}
	err = json.Unmarshal(res, &ci)
	if err != nil {
		return 0, fmt.Errorf("couldn't unmarshal container info: %v", err)
	}
	return ci.Stats[len(ci.Stats)-1].Filesystem[0].Available, nil
}

// fillDiskSpace fills the available disk space on a given node by creating a large file. The disk
// space on the node is filled in such a way that the available space after filling the disk is just
// below the lowDiskSpaceThreshold mark.
func fillDiskSpace(c *client.Client, node *api.Node) {
	avail, err := availSize(c, node)
	expectNoError(err, "Node %s: couldn't obtain available disk size %v", node.Name, err)

	fillSize := (avail - lowDiskSpaceThreshold + (100 * mb))

	Logf("Node %s: disk space available %d bytes", node.Name, avail)
	By(fmt.Sprintf("Node %s: creating a file of size %d bytes to fill the available disk space", node.Name, fillSize))

	cmd := fmt.Sprintf("fallocate -l %d test.img", fillSize)
	expectNoError(issueSSHCommand(cmd, testContext.Provider, node))

	ood := waitForNodeToBe(c, node.Name, api.NodeOutOfDisk, true, nodeOODTimeOut)
	Expect(ood).To(BeTrue(), "Node %s did not run out of disk within %v", node.Name, nodeOODTimeOut)

	avail, err = availSize(c, node)
	Logf("Node %s: disk space available %d bytes", node.Name, avail)
	Expect(avail < lowDiskSpaceThreshold).To(BeTrue())
}

// recoverDiskSpace recovers disk space, filled by creating a large file, on a given node.
func recoverDiskSpace(c *client.Client, node *api.Node) {
	By(fmt.Sprintf("Recovering disk space on node %s", node.Name))
	cmd := "rm -f test.img"
	expectNoError(issueSSHCommand(cmd, testContext.Provider, node))

	ood := waitForNodeToBe(c, node.Name, api.NodeOutOfDisk, false, nodeOODTimeOut)
	Expect(ood).To(BeTrue(), "Node %s's out of disk condition status did not change to false within %v", node.Name, nodeOODTimeOut)
}
<|endoftext|>


<commit_before>package pure

import (
	"bytes"
	"regexp"
)

const eof = byte(0)

type pureError struct {
	error
}

type scanner struct {
	buf   *bytes.Buffer
	index int

	line, col int
}

func newScanner(b []byte) *scanner {
	return &scanner{
		buf:   bytes.NewBuffer(b),
		index: -1,
		line:  0,
		col:   0,
	}
}

func (s *scanner) scan() (b byte) {
	if s.index >= len(s.buf.Bytes()) {
		s.buf.UnreadByte()
		return eof
	}
	b, _ = s.buf.ReadByte()

	if b == '\n' {
		s.line++
		s.col = 0
	}
	s.col++
	return
}

func (s *scanner) Peek() byte {
	b, _ := s.buf.ReadByte()
	s.buf.UnreadByte()
	return b
}

func (s *scanner) unread() {
	s.buf.UnreadByte()
	s.col--
}

func IsWhitespace(b byte) bool {
	return b == '\n' || b == '\r' || b == '\t' || b == ' '
}

func IsNumber(b byte) bool {
	return b >= '0' && b <= '9'
}

func IsAlpha(b byte) bool {
	return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z')
}

func IsAlphaNum(b byte) bool {
	return IsNumber(b) || IsAlpha(b)
}

func SpecialCharacter(b byte) bool {
	return regexp.MustCompile("[<|>,;.:-_'*¨^~!§½\"@#£¤$%€&/{(\\[\\])}=+?´`]?").MatchString(string(b))
}

func (s *scanner) ScanIdentifier() (tok Token, lit string) {
	var buf bytes.Buffer
	buf.WriteByte(s.scan())

	for {
		c := s.scan()

		if c == eof {
			return EOF, buf.String()
		}

		if !IsAlphaNum(c) {
			if c == '.' || (IsWhitespace(c) && IsWhitespace(s.Peek())) {
				s.unread()
				return GROUP, buf.String()
			}

			s.unread()
			return IDENTIFIER, buf.String()
		}

		buf.WriteByte(c)
	}
}

func (s *scanner) ScanNumber() (tok Token, lit string) {
	var buf bytes.Buffer
	buf.WriteByte(s.scan())
	tok = INT
	for {
		c := s.scan()

		if c == eof {
			return EOF, buf.String()
		}

		if !IsNumber(c) {
			if c == '.' {
				tok = DOUBLE
				buf.WriteByte(c)
				continue
			}
			if IsAlpha(c) || SpecialCharacter(c) && (c != '\r' && c != '\n') {
				tok = QUANTITY
				buf.WriteByte(c)
				continue
			}
			s.unread()
			lit = buf.String()
			return
		}

		buf.WriteByte(c)
	}
}

func (s *scanner) ScanString() (tok Token, lit string) {
	var buf bytes.Buffer

	for c := s.scan(); c != '"'; c = s.scan() {
		if c == eof {
			return EOF, buf.String()
		}

		if c == '\\' {
			if p := s.Peek(); p == '\n' || p == '\r' {
				for {
					c = s.scan()

					if IsWhitespace(c) {
						continue
					}
					s.unread()
					break
				}
			}
			buf.WriteByte(s.scan())
			continue
		}

		buf.WriteByte(c)
	}
	s.scan()
	return STRING, buf.String()
}

func (s *scanner) ScanPath() (tok Token, lit string) {
	var buf bytes.Buffer
	c := s.scan()
	buf.WriteByte(c) // consume the '.' or '/'

	for {
		c = s.scan()
		if c == eof {
			return EOF, buf.String()
		}

		if !IsAlphaNum(c) {
			if c == '/' || c == '\\' || c == '.' || c == '-' || c == '_' || c == ' ' {
				buf.WriteByte(c)
				continue
			}
			s.unread()
			return PATH, buf.String()
		}
		buf.WriteByte(c)
	}
}

func (s *scanner) ScanEnv() (tok Token, lit string) {
	var buf bytes.Buffer
	buf.WriteByte(s.scan()) // consume the '$'
	for {
		c := s.scan()

		if c == eof {
			return EOF, buf.String()
		}

		if !IsAlpha(c) {
			if c == '{' {
				buf.WriteByte(c)
				continue
			}
			if c == '}' {
				buf.WriteByte(c)
				return ENV, buf.String()
			}
			s.unread()
			return ENV, buf.String()
		}

		buf.WriteByte(c)
	}
}

func (s *scanner) ScanInclude() (tok Token, lit string) {
	var buf bytes.Buffer

	for {
		c := s.scan()

		if c == eof {
			return EOF, buf.String()
		}

		if !IsAlphaNum(c) {
			if buf.String() == "include" {
				_, lit := s.ScanPath()
				buf.Reset()
				buf.WriteString(lit)
			}
			return INCLUDE, buf.String()
		}

		buf.WriteByte(c)
	}
}

func (s *scanner) Scan() (tok Token, lit string) {
	var buf bytes.Buffer
	c := s.scan()
	buf.WriteByte(c)

	if IsWhitespace(c) {
		return WHITESPACE, buf.String()
	}

	if IsAlpha(c) {
		s.unread()
		return s.ScanIdentifier()
	}

	if IsNumber(c) {
		s.unread()
		return s.ScanNumber()
	}

	switch c {
	case eof:
		return EOF, buf.String()
	case '"':
		return s.ScanString()
	case '.':
		if c = s.Peek(); c == '/' {
			s.unread()
			s.unread()
			return s.ScanPath()
		}
		s.unread()
		return DOT, "."
	case '$':
		s.unread()
		return s.ScanEnv()
	case '%':
		return s.ScanInclude()
	case '[':
		return ARRAY, "["
	case '=':
		if s.Peek() == '>' {
			s.scan()
			return REF, "=>"
		}
		return EQUALS, "="
	case ':':
		return COLON, ":"
	case '/':
		return s.ScanPath()
	}
	return Illegal, buf.String()
}
<commit_msg>fixed a scanner bug<commit_after>package pure

import (
	"bytes"
	"regexp"
)

const eof = byte(0)

type pureError struct {
	error
}

type scanner struct {
	buf   *bytes.Buffer
	index int

	line, col int
}

func newScanner(b []byte) *scanner {
	return &scanner{
		buf:   bytes.NewBuffer(b),
		index: -1,
		line:  0,
		col:   0,
	}
}

func (s *scanner) scan() (b byte) {
	if s.index >= len(s.buf.Bytes()) {
		s.buf.UnreadByte()
		return eof
	}
	b, _ = s.buf.ReadByte()

	if b == '\n' {
		s.line++
		s.col = 0
	}
	s.col++
	return
}

func (s *scanner) Peek() byte {
	b, _ := s.buf.ReadByte()
	s.buf.UnreadByte()
	return b
}

func (s *scanner) unread() {
	s.buf.UnreadByte()
	s.col--
}

func IsWhitespace(b byte) bool {
	return b == '\n' || b == '\r' || b == '\t' || b == ' '
}

func IsNumber(b byte) bool {
	return b >= '0' && b <= '9'
}

func IsAlpha(b byte) bool {
	return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z')
}

func IsAlphaNum(b byte) bool {
	return IsNumber(b) || IsAlpha(b)
}

func SpecialCharacter(b byte) bool {
	return regexp.MustCompile("[<|>,;.:-_'*¨^~!§½\"@#£¤$%€&/{(\\[\\])}=+?´`]?").MatchString(string(b))
}

func (s *scanner) ScanIdentifier() (tok Token, lit string) {
	var buf bytes.Buffer
	buf.WriteByte(s.scan())

	for {
		c := s.scan()

		if c == eof {
			return tok, buf.String()
		}

		if !IsAlphaNum(c) {
			if c == '.' || (IsWhitespace(c) && IsWhitespace(s.Peek())) {
				s.unread()
				return GROUP, buf.String()
			}

			s.unread()
			return IDENTIFIER, buf.String()
		}

		buf.WriteByte(c)
	}
}

func (s *scanner) ScanNumber() (tok Token, lit string) {
	var buf bytes.Buffer
	buf.WriteByte(s.scan())
	tok = INT
	for {
		c := s.scan()

		if c == eof {
			return tok, buf.String()
		}

		if !IsNumber(c) {
			if c == '.' {
				tok = DOUBLE
				buf.WriteByte(c)
				continue
			}
			if IsAlpha(c) || SpecialCharacter(c) && (c != '\r' && c != '\n') {
				tok = QUANTITY
				buf.WriteByte(c)
				continue
			}
			s.unread()
			lit = buf.String()
			return
		}

		buf.WriteByte(c)
	}
}

func (s *scanner) ScanString() (tok Token, lit string) {
	var buf bytes.Buffer

	for c := s.scan(); c != '"'; c = s.scan() {
		if c == eof {
			return tok, buf.String()
		}

		if c == '\\' {
			if p := s.Peek(); p == '\n' || p == '\r' {
				for {
					c = s.scan()

					if IsWhitespace(c) {
						continue
					}
					s.unread()
					break
				}
			}
			buf.WriteByte(s.scan())
			continue
		}

		buf.WriteByte(c)
	}
	s.scan()
	return STRING, buf.String()
}

func (s *scanner) ScanPath() (tok Token, lit string) {
	var buf bytes.Buffer
	c := s.scan()
	buf.WriteByte(c) // consume the '.' or '/'

	for {
		c = s.scan()
		if c == eof {
			return tok, buf.String()
		}

		if !IsAlphaNum(c) {
			if c == '/' || c == '\\' || c == '.' || c == '-' || c == '_' || c == ' ' {
				buf.WriteByte(c)
				continue
			}
			s.unread()
			return PATH, buf.String()
		}
		buf.WriteByte(c)
	}
}

func (s *scanner) ScanEnv() (tok Token, lit string) {
	var buf bytes.Buffer
	buf.WriteByte(s.scan()) // consume the '$'
	for {
		c := s.scan()

		if c == eof {
			return tok, buf.String()
		}

		if !IsAlpha(c) {
			if c == '{' {
				buf.WriteByte(c)
				continue
			}
			if c == '}' {
				buf.WriteByte(c)
				return ENV, buf.String()
			}
			s.unread()
			return ENV, buf.String()
		}

		buf.WriteByte(c)
	}
}

func (s *scanner) ScanInclude() (tok Token, lit string) {
	var buf bytes.Buffer

	for {
		c := s.scan()

		if c == eof {
			return tok, buf.String()
		}

		if !IsAlphaNum(c) {
			if buf.String() == "include" {
				_, lit := s.ScanPath()
				buf.Reset()
				buf.WriteString(lit)
			}
			return INCLUDE, buf.String()
		}

		buf.WriteByte(c)
	}
}

func (s *scanner) Scan() (tok Token, lit string) {
	var buf bytes.Buffer
	c := s.scan()
	buf.WriteByte(c)

	if IsWhitespace(c) {
		return WHITESPACE, buf.String()
	}

	if IsAlpha(c) {
		s.unread()
		return s.ScanIdentifier()
	}

	if IsNumber(c) {
		s.unread()
		return s.ScanNumber()
	}

	switch c {
	case eof:
		return EOF, buf.String()
	case '"':
		return s.ScanString()
	case '.':
		if c = s.Peek(); c == '/' {
			s.unread()
			s.unread()
			return s.ScanPath()
		}
		s.unread()
		return DOT, "."
	case '$':
		s.unread()
		return s.ScanEnv()
	case '%':
		return s.ScanInclude()
	case '[':
		return ARRAY, "["
	case '=':
		if s.Peek() == '>' {
			s.scan()
			return REF, "=>"
		}
		return EQUALS, "="
	case ':':
		return COLON, ":"
	case '/':
		return s.ScanPath()
	}
	return Illegal, buf.String()
}
<|endoftext|>


<commit_before>// Package diary provides a simple JSON logger.
package diary

import (
	"encoding/json"
	"io"
	"os"
	"time"
)

// Default keys for log output
const (
	DefaultTimeKey    = "ts"
	DefaultLevelKey   = "lvl"
	DefaultMessageKey = "message"
)

// Level is the level of the log entry
type Level int

// Log Levels
const (
	LevelFatal = iota
	LevelError
	LevelInfo
	LevelDebug
)

type (
	// Context is a map of key/value pairs. These are Marshalled and included in the log output.
	Context map[string]interface{}

	// Logger is the actual logger. The default log level is info and the default writer is STDOUT.
	Logger struct {
		level      Level
		context    Context
		writer     io.Writer
		timeKey    string
		levelKey   string
		messageKey string
	}
)

// SetLevel creates a function that sets the log level. Generally, used when create a new logger.
func SetLevel(lvl Level) func(*Logger) {
	return func(l *Logger) {
		l.level = lvl
		return
	}
}

// SetContext creates a function that sets the context. Generally, used when create a new logger.
func SetContext(ctx Context) func(*Logger) {
	return func(l *Logger) {
		l.context = ctx
		return
	}
}

// SetWriter creates a function that will set the writer. Generally, used when create a new logger.
func SetWriter(w io.Writer) func(*Logger) {
	return func(l *Logger) {
		l.writer = w
		return
	}
}

// SetTimeKey creates a funtion that sets the time key. Generally, used when create a new logger.
func SetTimeKey(key string) func(*Logger) {
	return func(l *Logger) {
		l.timeKey = key
		return
	}
}

// SetLevelKey creates a funtion that sets the level key. Generally, used when create a new logger.
func SetLevelKey(key string) func(*Logger) {
	return func(l *Logger) {
		l.levelKey = key
		return
	}
}

// SetMessageKey creates a funtion that sets the message key. Generally, used when create a new logger.
func SetMessageKey(key string) func(*Logger) {
	return func(l *Logger) {
		l.messageKey = key
		return
	}
}

func (l *Logger) doOptions(options []func(*Logger)) {
	for _, f := range options {
		f(l)
	}
	return
}

// New creates a logger.
func New(context Context, options ...func(*Logger)) *Logger {
	l := &Logger{
		level:      LevelInfo,
		context:    context,
		writer:     os.Stdout,
		timeKey:    DefaultTimeKey,
		levelKey:   DefaultLevelKey,
		messageKey: DefaultMessageKey,
	}

	l.doOptions(options)

	return l
}

// New creates a child logger.  Initial options are inherited from the parent.
func (l *Logger) New(context Context, options ...func(*Logger)) *Logger {
	n := &Logger{
		level:      l.level,
		writer:     l.writer,
		timeKey:    l.timeKey,
		levelKey:   l.levelKey,
		messageKey: l.messageKey,
	}

	ctx := make(Context)

	for k, v := range l.context {
		ctx[k] = v
	}

	for k, v := range context {
		ctx[k] = v
	}

	n.context = ctx

	n.doOptions(options)

	return n
}

// Fatal logs a message at the "fatal" log level. It then calls os.Exit
func (l *Logger) Fatal(msg string, context ...Context) {
	l.write(LevelFatal, msg, context)
	os.Exit(-1)
}

// Error logs a message at the "error" log level.
func (l *Logger) Error(msg string, context ...Context) {
	l.write(LevelError, msg, context)
}

// Info logs a message at the "info" log level.
func (l *Logger) Info(msg string, context ...Context) {
	l.write(LevelInfo, msg, context)
}

// Debug logs a message at the "debug" log level.
func (l *Logger) Debug(msg string, context ...Context) {
	l.write(LevelDebug, msg, context)
}

func (l *Logger) write(level Level, msg string, context []Context) {
	if level > l.level {
		return
	}

	record := make(map[string]interface{}, 8)

	for k, v := range l.context {
		record[k] = v
	}

	for _, ctx := range context {
		for k, v := range ctx {
			record[k] = v
		}
	}

	record[l.timeKey] = time.Now()
	record[l.messageKey] = msg
	record[l.levelKey] = l.level.String()

	if data, err := json.Marshal(record); err == nil {
		data = append(data, '\n')
		l.writer.Write(data)
	}
}

// String returns the name of a Level.
func (l Level) String() string {
	switch l {
	case LevelDebug:
		return "dbug"
	case LevelInfo:
		return "info"
	case LevelError:
		return "eror"
	case LevelFatal:
		return "fatal"
	default:
		return "unknown"
	}
}

// LevelFromString returns the appropriate Level from a string name.
// Useful for parsing command line args and configuration files.
func LevelFromString(levelString string) (Level, bool) {
	switch levelString {
	case "debug":
		return LevelDebug, true
	case "info":
		return LevelInfo, true
	case "error", "eror", "err":
		return LevelError, true
	case "fatal":
		return LevelFatal, true
	default:
		return LevelDebug, false
	}
}
<commit_msg>spelling<commit_after>// Package diary provides a simple JSON logger.
package diary

import (
	"encoding/json"
	"io"
	"os"
	"time"
)

// Default keys for log output
const (
	DefaultTimeKey    = "ts"
	DefaultLevelKey   = "lvl"
	DefaultMessageKey = "message"
)

// Level is the level of the log entry
type Level int

// Log Levels
const (
	LevelFatal = iota
	LevelError
	LevelInfo
	LevelDebug
)

type (
	// Context is a map of key/value pairs. These are Marshalled and included in the log output.
	Context map[string]interface{}

	// Logger is the actual logger. The default log level is info and the default writer is STDOUT.
	Logger struct {
		level      Level
		context    Context
		writer     io.Writer
		timeKey    string
		levelKey   string
		messageKey string
	}
)

// SetLevel creates a function that sets the log level. Generally, used when create a new logger.
func SetLevel(lvl Level) func(*Logger) {
	return func(l *Logger) {
		l.level = lvl
		return
	}
}

// SetContext creates a function that sets the context. Generally, used when create a new logger.
func SetContext(ctx Context) func(*Logger) {
	return func(l *Logger) {
		l.context = ctx
		return
	}
}

// SetWriter creates a function that will set the writer. Generally, used when create a new logger.
func SetWriter(w io.Writer) func(*Logger) {
	return func(l *Logger) {
		l.writer = w
		return
	}
}

// SetTimeKey creates a funtion that sets the time key. Generally, used when create a new logger.
func SetTimeKey(key string) func(*Logger) {
	return func(l *Logger) {
		l.timeKey = key
		return
	}
}

// SetLevelKey creates a funtion that sets the level key. Generally, used when create a new logger.
func SetLevelKey(key string) func(*Logger) {
	return func(l *Logger) {
		l.levelKey = key
		return
	}
}

// SetMessageKey creates a funtion that sets the message key. Generally, used when create a new logger.
func SetMessageKey(key string) func(*Logger) {
	return func(l *Logger) {
		l.messageKey = key
		return
	}
}

func (l *Logger) doOptions(options []func(*Logger)) {
	for _, f := range options {
		f(l)
	}
	return
}

// New creates a logger.
func New(context Context, options ...func(*Logger)) *Logger {
	l := &Logger{
		level:      LevelInfo,
		context:    context,
		writer:     os.Stdout,
		timeKey:    DefaultTimeKey,
		levelKey:   DefaultLevelKey,
		messageKey: DefaultMessageKey,
	}

	l.doOptions(options)

	return l
}

// New creates a child logger.  Initial options are inherited from the parent.
func (l *Logger) New(context Context, options ...func(*Logger)) *Logger {
	n := &Logger{
		level:      l.level,
		writer:     l.writer,
		timeKey:    l.timeKey,
		levelKey:   l.levelKey,
		messageKey: l.messageKey,
	}

	ctx := make(Context)

	for k, v := range l.context {
		ctx[k] = v
	}

	for k, v := range context {
		ctx[k] = v
	}

	n.context = ctx

	n.doOptions(options)

	return n
}

// Fatal logs a message at the "fatal" log level. It then calls os.Exit
func (l *Logger) Fatal(msg string, context ...Context) {
	l.write(LevelFatal, msg, context)
	os.Exit(-1)
}

// Error logs a message at the "error" log level.
func (l *Logger) Error(msg string, context ...Context) {
	l.write(LevelError, msg, context)
}

// Info logs a message at the "info" log level.
func (l *Logger) Info(msg string, context ...Context) {
	l.write(LevelInfo, msg, context)
}

// Debug logs a message at the "debug" log level.
func (l *Logger) Debug(msg string, context ...Context) {
	l.write(LevelDebug, msg, context)
}

func (l *Logger) write(level Level, msg string, context []Context) {
	if level > l.level {
		return
	}

	record := make(map[string]interface{}, 8)

	for k, v := range l.context {
		record[k] = v
	}

	for _, ctx := range context {
		for k, v := range ctx {
			record[k] = v
		}
	}

	record[l.timeKey] = time.Now()
	record[l.messageKey] = msg
	record[l.levelKey] = l.level.String()

	if data, err := json.Marshal(record); err == nil {
		data = append(data, '\n')
		l.writer.Write(data)
	}
}

// String returns the name of a Level.
func (l Level) String() string {
	switch l {
	case LevelDebug:
		return "debug"
	case LevelInfo:
		return "info"
	case LevelError:
		return "error"
	case LevelFatal:
		return "fatal"
	default:
		return "unknown"
	}
}

// LevelFromString returns the appropriate Level from a string name.
// Useful for parsing command line args and configuration files.
func LevelFromString(levelString string) (Level, bool) {
	switch levelString {
	case "debug":
		return LevelDebug, true
	case "info":
		return LevelInfo, true
	case "error", "eror", "err":
		return LevelError, true
	case "fatal":
		return LevelFatal, true
	default:
		return LevelDebug, false
	}
}
<|endoftext|>


<commit_before>package forge

import (
	"bufio"
	"io"
	"strings"

	"github.com/brettlangdon/forge/token"
)

var eof = rune(0)

func isLetter(ch rune) bool {
	return ('a' <= ch && ch <= 'z') || ('A' <= ch && ch <= 'Z')
}

func isDigit(ch rune) bool {
	return ('0' <= ch && ch <= '9')
}

func isWhitespace(ch rune) bool {
	return (ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r')
}

func isBoolean(str string) bool {
	lower := strings.ToLower(str)
	return lower == "true" || lower == "false"
}

func isNull(str string) bool {
	return strings.ToLower(str) == "null"
}

func isInclude(str string) bool {
	return strings.ToLower(str) == "include"
}

type Scanner struct {
	cur_line int
	cur_col  int
	cur_tok  token.Token
	cur_ch   rune
	newline  bool
	reader   *bufio.Reader
}

func NewScanner(reader io.Reader) *Scanner {
	scanner := &Scanner{
		reader:   bufio.NewReader(reader),
		cur_line: 0,
		cur_col:  0,
		newline:  false,
	}
	scanner.readRune()
	return scanner
}

func (this *Scanner) readRune() {
	if this.newline {
		this.cur_line += 1
		this.cur_col = 0
		this.newline = false
	} else {
		this.cur_col += 1
	}

	next_ch, _, err := this.reader.ReadRune()
	if err != nil {
		this.cur_ch = eof
		return
	}

	this.cur_ch = next_ch

	if this.cur_ch == '\n' {
		this.newline = true
	}
}

func (this *Scanner) parseIdentifier() {
	this.cur_tok.ID = token.IDENTIFIER
	this.cur_tok.Literal = string(this.cur_ch)
	for {
		this.readRune()
		if !isLetter(this.cur_ch) && this.cur_ch != '_' {
			break
		}
		this.cur_tok.Literal += string(this.cur_ch)
	}

	if isBoolean(this.cur_tok.Literal) {
		this.cur_tok.ID = token.BOOLEAN
	} else if isNull(this.cur_tok.Literal) {
		this.cur_tok.ID = token.NULL
	} else if isInclude(this.cur_tok.Literal) {
		this.cur_tok.ID = token.INCLUDE
	}
}

func (this *Scanner) parseNumber() {
	this.cur_tok.ID = token.INTEGER
	this.cur_tok.Literal = string(this.cur_ch)
	digit := false
	for {
		this.readRune()
		if this.cur_ch == '.' && digit == false {
			this.cur_tok.ID = token.FLOAT
			digit = true
		} else if !isDigit(this.cur_ch) {
			break
		}
		this.cur_tok.Literal += string(this.cur_ch)
	}
}

func (this *Scanner) parseString() {
	this.cur_tok.ID = token.STRING
	this.cur_tok.Literal = string(this.cur_ch)
	for {
		this.readRune()
		if this.cur_ch == '"' {
			break
		}
		this.cur_tok.Literal += string(this.cur_ch)
	}
	this.readRune()
}

func (this *Scanner) parseComment() {
	this.cur_tok.ID = token.COMMENT
	this.cur_tok.Literal = ""
	for {
		this.readRune()
		if this.cur_ch == '\n' {
			break
		}
		this.cur_tok.Literal += string(this.cur_ch)
	}
	this.readRune()
}

func (this *Scanner) skipWhitespace() {
	for {
		this.readRune()
		if !isWhitespace(this.cur_ch) {
			break
		}
	}
}

func (this *Scanner) NextToken() token.Token {
	if isWhitespace(this.cur_ch) {
		this.skipWhitespace()
	}

	this.cur_tok = token.Token{
		ID:      token.ILLEGAL,
		Literal: string(this.cur_ch),
		Line:    this.cur_line,
		Column:  this.cur_col,
	}

	switch ch := this.cur_ch; {
	case isLetter(ch) || ch == '_':
		this.parseIdentifier()
	case isDigit(ch):
		this.parseNumber()
	case ch == '#':
		this.parseComment()
	case ch == eof:
		this.cur_tok.ID = token.EOF
		this.cur_tok.Literal = "EOF"
	default:
		this.readRune()
		this.cur_tok.Literal = string(ch)
		switch ch {
		case '=':
			this.cur_tok.ID = token.EQUAL
		case '"':
			this.parseString()
		case '{':
			this.cur_tok.ID = token.LBRACKET
		case '}':
			this.cur_tok.ID = token.RBRACKET
		case ';':
			this.cur_tok.ID = token.SEMICOLON
		case '.':
			this.cur_tok.ID = token.PERIOD
		}
	}

	return this.cur_tok
}
<commit_msg>remove underscore names in scanner<commit_after>package forge

import (
	"bufio"
	"io"
	"strings"

	"github.com/brettlangdon/forge/token"
)

var eof = rune(0)

func isLetter(ch rune) bool {
	return ('a' <= ch && ch <= 'z') || ('A' <= ch && ch <= 'Z')
}

func isDigit(ch rune) bool {
	return ('0' <= ch && ch <= '9')
}

func isWhitespace(ch rune) bool {
	return (ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r')
}

func isBoolean(str string) bool {
	lower := strings.ToLower(str)
	return lower == "true" || lower == "false"
}

func isNull(str string) bool {
	return strings.ToLower(str) == "null"
}

func isInclude(str string) bool {
	return strings.ToLower(str) == "include"
}

type Scanner struct {
	curLine int
	curCol  int
	curTok  token.Token
	curCh   rune
	newline bool
	reader  *bufio.Reader
}

func NewScanner(reader io.Reader) *Scanner {
	scanner := &Scanner{
		reader:  bufio.NewReader(reader),
		curLine: 0,
		curCol:  0,
		newline: false,
	}
	scanner.readRune()
	return scanner
}

func (this *Scanner) readRune() {
	if this.newline {
		this.curLine += 1
		this.curCol = 0
		this.newline = false
	} else {
		this.curCol += 1
	}

	nextCh, _, err := this.reader.ReadRune()
	if err != nil {
		this.curCh = eof
		return
	}

	this.curCh = nextCh

	if this.curCh == '\n' {
		this.newline = true
	}
}

func (this *Scanner) parseIdentifier() {
	this.curTok.ID = token.IDENTIFIER
	this.curTok.Literal = string(this.curCh)
	for {
		this.readRune()
		if !isLetter(this.curCh) && this.curCh != '_' {
			break
		}
		this.curTok.Literal += string(this.curCh)
	}

	if isBoolean(this.curTok.Literal) {
		this.curTok.ID = token.BOOLEAN
	} else if isNull(this.curTok.Literal) {
		this.curTok.ID = token.NULL
	} else if isInclude(this.curTok.Literal) {
		this.curTok.ID = token.INCLUDE
	}
}

func (this *Scanner) parseNumber() {
	this.curTok.ID = token.INTEGER
	this.curTok.Literal = string(this.curCh)
	digit := false
	for {
		this.readRune()
		if this.curCh == '.' && digit == false {
			this.curTok.ID = token.FLOAT
			digit = true
		} else if !isDigit(this.curCh) {
			break
		}
		this.curTok.Literal += string(this.curCh)
	}
}

func (this *Scanner) parseString() {
	this.curTok.ID = token.STRING
	this.curTok.Literal = string(this.curCh)
	for {
		this.readRune()
		if this.curCh == '"' {
			break
		}
		this.curTok.Literal += string(this.curCh)
	}
	this.readRune()
}

func (this *Scanner) parseComment() {
	this.curTok.ID = token.COMMENT
	this.curTok.Literal = ""
	for {
		this.readRune()
		if this.curCh == '\n' {
			break
		}
		this.curTok.Literal += string(this.curCh)
	}
	this.readRune()
}

func (this *Scanner) skipWhitespace() {
	for {
		this.readRune()
		if !isWhitespace(this.curCh) {
			break
		}
	}
}

func (this *Scanner) NextToken() token.Token {
	if isWhitespace(this.curCh) {
		this.skipWhitespace()
	}

	this.curTok = token.Token{
		ID:      token.ILLEGAL,
		Literal: string(this.curCh),
		Line:    this.curLine,
		Column:  this.curCol,
	}

	switch ch := this.curCh; {
	case isLetter(ch) || ch == '_':
		this.parseIdentifier()
	case isDigit(ch):
		this.parseNumber()
	case ch == '#':
		this.parseComment()
	case ch == eof:
		this.curTok.ID = token.EOF
		this.curTok.Literal = "EOF"
	default:
		this.readRune()
		this.curTok.Literal = string(ch)
		switch ch {
		case '=':
			this.curTok.ID = token.EQUAL
		case '"':
			this.parseString()
		case '{':
			this.curTok.ID = token.LBRACKET
		case '}':
			this.curTok.ID = token.RBRACKET
		case ';':
			this.curTok.ID = token.SEMICOLON
		case '.':
			this.curTok.ID = token.PERIOD
		}
	}

	return this.curTok
}
<|endoftext|>


<commit_before>package harness

import (
	"fmt"
	"github.com/robfig/revel"
	"go/build"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"text/template"
)

var importErrorPattern = regexp.MustCompile("import \"([^\"]+)\": cannot find package")

// Build the app:
// 1. Generate the the main.go file.
// 2. Run the appropriate "go build" command.
// Requires that revel.Init has been called previously.
// Returns the path to the built binary, and an error if there was a problem building it.
func Build() (app *App, compileError *revel.Error) {
	sourceInfo, compileError := ProcessSource(revel.CodePaths)
	if compileError != nil {
		return nil, compileError
	}

	// Add the db.import to the import paths.
	if dbImportPath, found := revel.Config.String("db.import"); found {
		sourceInfo.InitImportPaths = append(sourceInfo.InitImportPaths, dbImportPath)
	}

	// Generate two source files.
	templateArgs := map[string]interface{}{
		"Controllers":    sourceInfo.ControllerSpecs(),
		"ValidationKeys": sourceInfo.ValidationKeys,
		"ImportPaths":    calcImportAliases(sourceInfo),
		"TestSuites":     sourceInfo.TestSuites(),
	}
	genSource("tmp", "main.go", MAIN, templateArgs)
	genSource("routes", "routes.go", ROUTES, templateArgs)

	// Read build config.
	buildTags := revel.Config.StringDefault("build.tags", "")

	// Build the user program (all code under app).
	// It relies on the user having "go" installed.
	goPath, err := exec.LookPath("go")
	if err != nil {
		revel.ERROR.Fatalf("Go executable not found in PATH.")
	}

	pkg, err := build.Default.Import(revel.ImportPath, "", build.FindOnly)
	if err != nil {
		revel.ERROR.Fatalln("Failure importing", revel.ImportPath)
	}
	binName := path.Join(pkg.BinDir, path.Base(revel.BasePath))
	if runtime.GOOS == "windows" {
		binName += ".exe"
	}

	gotten := make(map[string]struct{})
	for {
		buildCmd := exec.Command(goPath, "build",
			"-tags", buildTags,
			"-o", binName, path.Join(revel.ImportPath, "app", "tmp"))
		revel.TRACE.Println("Exec:", buildCmd.Args)
		output, err := buildCmd.CombinedOutput()

		// If the build succeeded, we're done.
		if err == nil {
			return NewApp(binName), nil
		}
		revel.ERROR.Println(string(output))

		// See if it was an import error that we can go get.
		matches := importErrorPattern.FindStringSubmatch(string(output))
		if matches == nil {
			return nil, newCompileError(output)
		}

		// Ensure we haven't already tried to go get it.
		pkgName := matches[1]
		if _, alreadyTried := gotten[pkgName]; alreadyTried {
			return nil, newCompileError(output)
		}
		gotten[pkgName] = struct{}{}

		// Execute "go get <pkg>"
		getCmd := exec.Command(goPath, "get", pkgName)
		revel.TRACE.Println("Exec:", getCmd.Args)
		getOutput, err := getCmd.CombinedOutput()
		if err != nil {
			revel.ERROR.Println(string(getOutput))
			return nil, newCompileError(output)
		}

		// Success getting the import, attempt to build again.
	}
	revel.ERROR.Fatalf("Not reachable")
	return nil, nil
}

// getSource renders the given template to produce source code, which it writes
// to the given directory and file.
func genSource(dir, filename, templateSource string, args map[string]interface{}) {
	sourceCode := revel.ExecuteTemplate(
		template.Must(template.New("").Parse(templateSource)),
		args)

	// Create a fresh dir.
	tmpPath := path.Join(revel.AppPath, dir)
	err := os.RemoveAll(tmpPath)
	if err != nil {
		revel.ERROR.Println("Failed to remove dir:", err)
	}
	err = os.Mkdir(tmpPath, 0777)
	if err != nil {
		revel.ERROR.Fatalf("Failed to make tmp directory: %v", err)
	}

	// Create the file
	file, err := os.Create(path.Join(tmpPath, filename))
	defer file.Close()
	if err != nil {
		revel.ERROR.Fatalf("Failed to create file: %v", err)
	}
	_, err = file.WriteString(sourceCode)
	if err != nil {
		revel.ERROR.Fatalf("Failed to write to file: %v", err)
	}
}

// Looks through all the method args and returns a set of unique import paths
// that cover all the method arg types.
// Additionally, assign package aliases when necessary to resolve ambiguity.
func calcImportAliases(src *SourceInfo) map[string]string {
	aliases := make(map[string]string)
	typeArrays := [][]*TypeInfo{src.ControllerSpecs(), src.TestSuites()}
	for _, specs := range typeArrays {
		for _, spec := range specs {
			addAlias(aliases, spec.ImportPath, spec.PackageName)

			for _, methSpec := range spec.MethodSpecs {
				for _, methArg := range methSpec.Args {
					if methArg.ImportPath == "" {
						continue
					}

					addAlias(aliases, methArg.ImportPath, methArg.TypeExpr.PkgName)
				}
			}
		}
	}

	// Add the "InitImportPaths", with alias "_"
	for _, importPath := range src.InitImportPaths {
		if _, ok := aliases[importPath]; !ok {
			aliases[importPath] = "_"
		}
	}

	return aliases
}

func addAlias(aliases map[string]string, importPath, pkgName string) {
	alias, ok := aliases[importPath]
	if ok {
		return
	}
	alias = makePackageAlias(aliases, pkgName)
	aliases[importPath] = alias
}

func makePackageAlias(aliases map[string]string, pkgName string) string {
	i := 0
	alias := pkgName
	for containsValue(aliases, alias) {
		alias = fmt.Sprintf("%s%d", pkgName, i)
		i++
	}
	return alias
}

func containsValue(m map[string]string, val string) bool {
	for _, v := range m {
		if v == val {
			return true
		}
	}
	return false
}

// Parse the output of the "go build" command.
// Return a detailed Error.
func newCompileError(output []byte) *revel.Error {
	errorMatch := regexp.MustCompile(`(?m)^([^:#]+):(\d+):(\d+:)? (.*)$`).
		FindSubmatch(output)
	if errorMatch == nil {
		revel.ERROR.Println("Failed to parse build errors:\n", string(output))
		return &revel.Error{
			SourceType:  "Go code",
			Title:       "Go Compilation Error",
			Description: "See console for build error.",
		}
	}

	// Read the source for the offending file.
	var (
		relFilename    = string(errorMatch[1]) // e.g. "src/revel/sample/app/controllers/app.go"
		absFilename, _ = filepath.Abs(relFilename)
		line, _        = strconv.Atoi(string(errorMatch[2]))
		description    = string(errorMatch[4])
		compileError   = &revel.Error{
			SourceType:  "Go code",
			Title:       "Go Compilation Error",
			Path:        relFilename,
			Description: description,
			Line:        line,
		}
	)

	fileStr, err := revel.ReadLines(absFilename)
	if err != nil {
		compileError.MetaError = absFilename + ": " + err.Error()
		revel.ERROR.Println(compileError.MetaError)
		return compileError
	}

	compileError.SourceLines = fileStr
	return compileError
}

const MAIN = `// GENERATED CODE - DO NOT EDIT
package main

import (
	"flag"
	"reflect"
	"github.com/robfig/revel"{{range $k, $v := $.ImportPaths}}
	{{$v}} "{{$k}}"{{end}}
)

var (
	runMode    *string = flag.String("runMode", "", "Run mode.")
	port       *int    = flag.Int("port", 0, "By default, read from app.conf")
	importPath *string = flag.String("importPath", "", "Go Import Path for the app.")
	srcPath    *string = flag.String("srcPath", "", "Path to the source root.")

	// So compiler won't complain if the generated code doesn't reference reflect package...
	_ = reflect.Invalid
)

func main() {
	flag.Parse()
	revel.Init(*runMode, *importPath, *srcPath)
	revel.INFO.Println("Running revel server")
	{{range $i, $c := .Controllers}}
	revel.RegisterController((*{{index $.ImportPaths .ImportPath}}.{{.StructName}})(nil),
		[]*revel.MethodType{
			{{range .MethodSpecs}}&revel.MethodType{
				Name: "{{.Name}}",
				Args: []*revel.MethodArg{ {{range .Args}}
					&revel.MethodArg{Name: "{{.Name}}", Type: reflect.TypeOf((*{{index $.ImportPaths .ImportPath | .TypeExpr.TypeName}})(nil)) },{{end}}
				},
				RenderArgNames: map[int][]string{ {{range .RenderCalls}}
					{{.Line}}: []string{ {{range .Names}}
						"{{.}}",{{end}}
					},{{end}}
				},
			},
			{{end}}
		})
	{{end}}
	revel.DefaultValidationKeys = map[string]map[int]string{ {{range $path, $lines := .ValidationKeys}}
		"{{$path}}": { {{range $line, $key := $lines}}
			{{$line}}: "{{$key}}",{{end}}
		},{{end}}
	}
	revel.TestSuites = []interface{}{ {{range .TestSuites}}
		(*{{index $.ImportPaths .ImportPath}}.{{.StructName}})(nil),{{end}}
	}

	revel.Run(*port)
}
`
const ROUTES = `// GENERATED CODE - DO NOT EDIT
package routes

import "github.com/robfig/revel"

{{range $i, $c := .Controllers}}
type t{{.StructName}} struct {}
var {{.StructName}} t{{.StructName}}

{{range .MethodSpecs}}
func (p t{{$c.StructName}}) {{.Name}}({{range .Args}}
		{{.Name}} {{if .ImportPath}}interface{}{{else}}{{.TypeExpr.TypeName ""}}{{end}},{{end}}
		) string {
	args := make(map[string]string)
	{{range .Args}}
	revel.Unbind(args, "{{.Name}}", {{.Name}}){{end}}
	return revel.MainRouter.Reverse("{{$c.StructName}}.{{.Name}}", args).Url
}
{{end}}
{{end}}
`
<commit_msg>Update the auto-go-getter for Go1.1<commit_after>package harness

import (
	"fmt"
	"github.com/robfig/revel"
	"go/build"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"text/template"
)

var importErrorPattern = regexp.MustCompile("cannot find package \"([^\"]+)\"")

// Build the app:
// 1. Generate the the main.go file.
// 2. Run the appropriate "go build" command.
// Requires that revel.Init has been called previously.
// Returns the path to the built binary, and an error if there was a problem building it.
func Build() (app *App, compileError *revel.Error) {
	sourceInfo, compileError := ProcessSource(revel.CodePaths)
	if compileError != nil {
		return nil, compileError
	}

	// Add the db.import to the import paths.
	if dbImportPath, found := revel.Config.String("db.import"); found {
		sourceInfo.InitImportPaths = append(sourceInfo.InitImportPaths, dbImportPath)
	}

	// Generate two source files.
	templateArgs := map[string]interface{}{
		"Controllers":    sourceInfo.ControllerSpecs(),
		"ValidationKeys": sourceInfo.ValidationKeys,
		"ImportPaths":    calcImportAliases(sourceInfo),
		"TestSuites":     sourceInfo.TestSuites(),
	}
	genSource("tmp", "main.go", MAIN, templateArgs)
	genSource("routes", "routes.go", ROUTES, templateArgs)

	// Read build config.
	buildTags := revel.Config.StringDefault("build.tags", "")

	// Build the user program (all code under app).
	// It relies on the user having "go" installed.
	goPath, err := exec.LookPath("go")
	if err != nil {
		revel.ERROR.Fatalf("Go executable not found in PATH.")
	}

	pkg, err := build.Default.Import(revel.ImportPath, "", build.FindOnly)
	if err != nil {
		revel.ERROR.Fatalln("Failure importing", revel.ImportPath)
	}
	binName := path.Join(pkg.BinDir, path.Base(revel.BasePath))
	if runtime.GOOS == "windows" {
		binName += ".exe"
	}

	gotten := make(map[string]struct{})
	for {
		buildCmd := exec.Command(goPath, "build",
			"-tags", buildTags,
			"-o", binName, path.Join(revel.ImportPath, "app", "tmp"))
		revel.TRACE.Println("Exec:", buildCmd.Args)
		output, err := buildCmd.CombinedOutput()

		// If the build succeeded, we're done.
		if err == nil {
			return NewApp(binName), nil
		}
		revel.ERROR.Println(string(output))

		// See if it was an import error that we can go get.
		matches := importErrorPattern.FindStringSubmatch(string(output))
		if matches == nil {
			return nil, newCompileError(output)
		}

		// Ensure we haven't already tried to go get it.
		pkgName := matches[1]
		if _, alreadyTried := gotten[pkgName]; alreadyTried {
			return nil, newCompileError(output)
		}
		gotten[pkgName] = struct{}{}

		// Execute "go get <pkg>"
		getCmd := exec.Command(goPath, "get", pkgName)
		revel.TRACE.Println("Exec:", getCmd.Args)
		getOutput, err := getCmd.CombinedOutput()
		if err != nil {
			revel.ERROR.Println(string(getOutput))
			return nil, newCompileError(output)
		}

		// Success getting the import, attempt to build again.
	}
	revel.ERROR.Fatalf("Not reachable")
	return nil, nil
}

// getSource renders the given template to produce source code, which it writes
// to the given directory and file.
func genSource(dir, filename, templateSource string, args map[string]interface{}) {
	sourceCode := revel.ExecuteTemplate(
		template.Must(template.New("").Parse(templateSource)),
		args)

	// Create a fresh dir.
	tmpPath := path.Join(revel.AppPath, dir)
	err := os.RemoveAll(tmpPath)
	if err != nil {
		revel.ERROR.Println("Failed to remove dir:", err)
	}
	err = os.Mkdir(tmpPath, 0777)
	if err != nil {
		revel.ERROR.Fatalf("Failed to make tmp directory: %v", err)
	}

	// Create the file
	file, err := os.Create(path.Join(tmpPath, filename))
	defer file.Close()
	if err != nil {
		revel.ERROR.Fatalf("Failed to create file: %v", err)
	}
	_, err = file.WriteString(sourceCode)
	if err != nil {
		revel.ERROR.Fatalf("Failed to write to file: %v", err)
	}
}

// Looks through all the method args and returns a set of unique import paths
// that cover all the method arg types.
// Additionally, assign package aliases when necessary to resolve ambiguity.
func calcImportAliases(src *SourceInfo) map[string]string {
	aliases := make(map[string]string)
	typeArrays := [][]*TypeInfo{src.ControllerSpecs(), src.TestSuites()}
	for _, specs := range typeArrays {
		for _, spec := range specs {
			addAlias(aliases, spec.ImportPath, spec.PackageName)

			for _, methSpec := range spec.MethodSpecs {
				for _, methArg := range methSpec.Args {
					if methArg.ImportPath == "" {
						continue
					}

					addAlias(aliases, methArg.ImportPath, methArg.TypeExpr.PkgName)
				}
			}
		}
	}

	// Add the "InitImportPaths", with alias "_"
	for _, importPath := range src.InitImportPaths {
		if _, ok := aliases[importPath]; !ok {
			aliases[importPath] = "_"
		}
	}

	return aliases
}

func addAlias(aliases map[string]string, importPath, pkgName string) {
	alias, ok := aliases[importPath]
	if ok {
		return
	}
	alias = makePackageAlias(aliases, pkgName)
	aliases[importPath] = alias
}

func makePackageAlias(aliases map[string]string, pkgName string) string {
	i := 0
	alias := pkgName
	for containsValue(aliases, alias) {
		alias = fmt.Sprintf("%s%d", pkgName, i)
		i++
	}
	return alias
}

func containsValue(m map[string]string, val string) bool {
	for _, v := range m {
		if v == val {
			return true
		}
	}
	return false
}

// Parse the output of the "go build" command.
// Return a detailed Error.
func newCompileError(output []byte) *revel.Error {
	errorMatch := regexp.MustCompile(`(?m)^([^:#]+):(\d+):(\d+:)? (.*)$`).
		FindSubmatch(output)
	if errorMatch == nil {
		revel.ERROR.Println("Failed to parse build errors:\n", string(output))
		return &revel.Error{
			SourceType:  "Go code",
			Title:       "Go Compilation Error",
			Description: "See console for build error.",
		}
	}

	// Read the source for the offending file.
	var (
		relFilename    = string(errorMatch[1]) // e.g. "src/revel/sample/app/controllers/app.go"
		absFilename, _ = filepath.Abs(relFilename)
		line, _        = strconv.Atoi(string(errorMatch[2]))
		description    = string(errorMatch[4])
		compileError   = &revel.Error{
			SourceType:  "Go code",
			Title:       "Go Compilation Error",
			Path:        relFilename,
			Description: description,
			Line:        line,
		}
	)

	fileStr, err := revel.ReadLines(absFilename)
	if err != nil {
		compileError.MetaError = absFilename + ": " + err.Error()
		revel.ERROR.Println(compileError.MetaError)
		return compileError
	}

	compileError.SourceLines = fileStr
	return compileError
}

const MAIN = `// GENERATED CODE - DO NOT EDIT
package main

import (
	"flag"
	"reflect"
	"github.com/robfig/revel"{{range $k, $v := $.ImportPaths}}
	{{$v}} "{{$k}}"{{end}}
)

var (
	runMode    *string = flag.String("runMode", "", "Run mode.")
	port       *int    = flag.Int("port", 0, "By default, read from app.conf")
	importPath *string = flag.String("importPath", "", "Go Import Path for the app.")
	srcPath    *string = flag.String("srcPath", "", "Path to the source root.")

	// So compiler won't complain if the generated code doesn't reference reflect package...
	_ = reflect.Invalid
)

func main() {
	flag.Parse()
	revel.Init(*runMode, *importPath, *srcPath)
	revel.INFO.Println("Running revel server")
	{{range $i, $c := .Controllers}}
	revel.RegisterController((*{{index $.ImportPaths .ImportPath}}.{{.StructName}})(nil),
		[]*revel.MethodType{
			{{range .MethodSpecs}}&revel.MethodType{
				Name: "{{.Name}}",
				Args: []*revel.MethodArg{ {{range .Args}}
					&revel.MethodArg{Name: "{{.Name}}", Type: reflect.TypeOf((*{{index $.ImportPaths .ImportPath | .TypeExpr.TypeName}})(nil)) },{{end}}
				},
				RenderArgNames: map[int][]string{ {{range .RenderCalls}}
					{{.Line}}: []string{ {{range .Names}}
						"{{.}}",{{end}}
					},{{end}}
				},
			},
			{{end}}
		})
	{{end}}
	revel.DefaultValidationKeys = map[string]map[int]string{ {{range $path, $lines := .ValidationKeys}}
		"{{$path}}": { {{range $line, $key := $lines}}
			{{$line}}: "{{$key}}",{{end}}
		},{{end}}
	}
	revel.TestSuites = []interface{}{ {{range .TestSuites}}
		(*{{index $.ImportPaths .ImportPath}}.{{.StructName}})(nil),{{end}}
	}

	revel.Run(*port)
}
`
const ROUTES = `// GENERATED CODE - DO NOT EDIT
package routes

import "github.com/robfig/revel"

{{range $i, $c := .Controllers}}
type t{{.StructName}} struct {}
var {{.StructName}} t{{.StructName}}

{{range .MethodSpecs}}
func (p t{{$c.StructName}}) {{.Name}}({{range .Args}}
		{{.Name}} {{if .ImportPath}}interface{}{{else}}{{.TypeExpr.TypeName ""}}{{end}},{{end}}
		) string {
	args := make(map[string]string)
	{{range .Args}}
	revel.Unbind(args, "{{.Name}}", {{.Name}}){{end}}
	return revel.MainRouter.Reverse("{{$c.StructName}}.{{.Name}}", args).Url
}
{{end}}
{{end}}
`
<|endoftext|>


<commit_before>package shared

import (
	"time"
)

type GatewayStatus struct {
	Eui               string    `json:"eui,omitempty"`
	Time              time.Time `json:"time,omitempty"`
	Latitude          *float64  `json:"latitude,omitempty"`
	Longitude         *float64  `json:"longitude,omitempty"`
	Altitude          *float64  `json:"altitude,omitempty"`
	RxCount           *int      `json:"rxCount,omitempty"`
	RxOk              *int      `json:"rxOk,omitempty"`
	RxForwarded       *int      `json:"rxForwarded,omitempty"`
	AckRatio          *float64  `json:"ackRatio,omitempty"`
	DatagramsReceived *int      `json:"datagramsReceived,omitempty"`
	DatagramsSent     *int      `json:"datagramsSent,omitempty"`
}

type RxPacket struct {
	GatewayEui string    `json:"gatewayEui,omitempty"`
	NodeEui    string    `json:"nodeEui,omitempty"`
	Time       time.Time `json:"time,omitempty"`
	Data       string    `json:"data,omitempty"`
}
<commit_msg>Added RxPacket.RawData and updated JSON serialization hints<commit_after>package shared

import (
	"time"
)

type GatewayStatus struct {
	Eui               string    `json:"eui"`
	Time              time.Time `json:"time"`
	Latitude          *float64  `json:"latitude,omitempty"`
	Longitude         *float64  `json:"longitude,omitempty"`
	Altitude          *float64  `json:"altitude,omitempty"`
	RxCount           *int      `json:"rxCount,omitempty"`
	RxOk              *int      `json:"rxOk,omitempty"`
	RxForwarded       *int      `json:"rxForwarded,omitempty"`
	AckRatio          *float64  `json:"ackRatio,omitempty"`
	DatagramsReceived *int      `json:"datagramsReceived,omitempty"`
	DatagramsSent     *int      `json:"datagramsSent,omitempty"`
}

type RxPacket struct {
	GatewayEui string    `json:"gatewayEui"`
	NodeEui    string    `json:"nodeEui"`
	Time       time.Time `json:"time"`
	RawData    string    `json:"rawData"`
	Data       string    `json:"data,omitempty"`
}
<|endoftext|>


<commit_before>package script

const (
	DefaultDockerNetworkMode = "bridge"
)

// Docker stores the configuration details for
// configuring docker container.
type Docker struct {
	// NetworkMode (also known as `--net` option)
	// Could be set only if Docker is running in privileged mode
	NetworkMode *string `yaml:"net,omitempty"`

	// Hostname (also known as `--hostname` option)
	// Could be set only if Docker is running in privileged mode
	Hostname *string `yaml:"hostname,omitempty"`
}

// DockerNetworkMode returns DefaultNetworkMode
// when Docker.NetworkMode is empty.
// DockerNetworkMode returns Docker.NetworkMode
// when it is not empty.
func DockerNetworkMode(d *Docker) string {
	if d == nil || d.NetworkMode == nil {
		return DefaultDockerNetworkMode
	}
	return *d.NetworkMode
}

// DockerNetworkMode returns empty string
// when Docker.NetworkMode is empty.
// DockerNetworkMode returns Docker.NetworkMode
// when it is not empty.
func DockerHostname(d *Docker) string {
	if d == nil || d.Hostname == nil {
		return ""
	}
	return *d.Hostname
}
<commit_msg>Enable pseudo tty setting<commit_after>package script

const (
	DefaultDockerNetworkMode = "bridge"
)

// Docker stores the configuration details for
// configuring docker container.
type Docker struct {
	// NetworkMode (also known as `--net` option)
	// Could be set only if Docker is running in privileged mode
	NetworkMode *string `yaml:"net,omitempty"`

	// Hostname (also known as `--hostname` option)
	// Could be set only if Docker is running in privileged mode
	Hostname *string `yaml:"hostname,omitempty"`

	// Allocate a pseudo-TTY (also known as `--tty` option)
	TTY bool `yaml:"tty,omitempty"`
}

// DockerNetworkMode returns DefaultNetworkMode
// when Docker.NetworkMode is empty.
// DockerNetworkMode returns Docker.NetworkMode
// when it is not empty.
func DockerNetworkMode(d *Docker) string {
	if d == nil || d.NetworkMode == nil {
		return DefaultDockerNetworkMode
	}
	return *d.NetworkMode
}

// DockerNetworkMode returns empty string
// when Docker.NetworkMode is empty.
// DockerNetworkMode returns Docker.NetworkMode
// when it is not empty.
func DockerHostname(d *Docker) string {
	if d == nil || d.Hostname == nil {
		return ""
	}
	return *d.Hostname
}

// DockerTty returns true if the build
// should allocate a pseudo tty
func DockerTty(d *Docker) string {
	if d == nil {
		return false
	}
	return d.TTY
}
<|endoftext|>


<commit_before>package cmap

import (
	"fmt"
	"unsafe"
)

const (
	max_exponent = 32
	default_load_factor = 0.75
)

type ConcurrentLongHashMap struct {
	exponent uint32
	buckets []unsafe.Pointer
	size	int64
	loadFactor	float64
}

func NewConcurrentLongHashMap() *ConcurrentLongHashMap {
	hashMap := &ConcurrentLongHashMap{0, make([]unsafe.Pointer, max_exponent), 0, default_load_factor}
	b := make([]unsafe.Pointer, 1)
	hashMap.buckets[0] = unsafe.Pointer(&b)
	return hashMap
}<commit_msg>ConcurrentLongHashMap add more implementation....<commit_after>package cmap

import (
	"fmt"
	"unsafe"
)

const (
	max_exponent = 32
	default_load_factor = 0.75
)

type Any interface{}

type Equable interface {
	Equals(Any) bool
}

type WrapperKey interface {
	Equable
	HashCode() int
}

type Node struct {
	hashCode int
	hashKey int
	key WrapperKey
	value unsafe.Pointer
}

func newRealNodeWithHashCode(key WrapperKey, value Any, hashCode int) *Node {
	return &Node{hashCode, , key, unsafe.Pointer(&value)}
}

func newRealNode(key WrapperKey, value Any) *Node {
	return newRealNodeWithHashCode(key, value, key.HashCode())
}

type LongKey int64

func (key LongKey) HashCode() int {
	h := int(key ^ (key >> 32))
	h ^= (h >> 20) ^ (h >> 12)
	return h ^ (h >> 7) ^ (h >> 4)
}

func (key LongKey) Equals(any Any) bool {
	if lk, ok := any.(LongKey); ok {
		return int64(key) == int64(lk)
	}

	return false
}

type ConcurrentLongHashMap struct {
	exponent uint32
	buckets []unsafe.Pointer
	size	int64
	loadFactor	float64
}

func NewConcurrentLongHashMap() *ConcurrentLongHashMap {
	hashMap := &ConcurrentLongHashMap{0, make([]unsafe.Pointer, max_exponent), 0, default_load_factor}
	b := make([]unsafe.Pointer, 1)
	hashMap.buckets[0] = unsafe.Pointer(&b)
	return hashMap
}

func (m *ConcurrentLongHashMap) GetByHashCode(hashCode int, key WrapperKey) (any Any, ok bool) {

}

func (m *ConcurrentLongHashMap) Get(key WrapperKey) (Any, bool) {

}<|endoftext|>


<commit_before>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"strconv"
	"sync"
	"time"

	// Injection related imports.
	kubeclient "knative.dev/pkg/client/injection/kube/client"
	"knative.dev/pkg/injection"
	revisioninformer "knative.dev/serving/pkg/client/injection/informers/serving/v1alpha1/revision"

	"github.com/kelseyhightower/envconfig"
	"go.opencensus.io/stats/view"
	"go.uber.org/zap"
	"k8s.io/apimachinery/pkg/util/wait"
	"knative.dev/pkg/configmap"
	"knative.dev/pkg/controller"
	"knative.dev/pkg/injection/sharedmain"
	pkglogging "knative.dev/pkg/logging"
	"knative.dev/pkg/logging/logkey"
	"knative.dev/pkg/metrics"
	pkgnet "knative.dev/pkg/network"
	"knative.dev/pkg/profiling"
	"knative.dev/pkg/signals"
	"knative.dev/pkg/system"
	"knative.dev/pkg/tracing"
	tracingconfig "knative.dev/pkg/tracing/config"
	"knative.dev/pkg/version"
	"knative.dev/pkg/websocket"
	"knative.dev/serving/pkg/activator"
	activatorconfig "knative.dev/serving/pkg/activator/config"
	activatorhandler "knative.dev/serving/pkg/activator/handler"
	activatornet "knative.dev/serving/pkg/activator/net"
	"knative.dev/serving/pkg/apis/networking"
	"knative.dev/serving/pkg/autoscaler"
	"knative.dev/serving/pkg/goversion"
	pkghttp "knative.dev/serving/pkg/http"
	"knative.dev/serving/pkg/logging"
	"knative.dev/serving/pkg/network"
	"knative.dev/serving/pkg/queue"
)

// Fail if using unsupported go version.
var _ = goversion.IsSupported()

const (
	component = "activator"

	// Add enough buffer to not block request serving on stats collection
	requestCountingQueueLength = 100

	// The number of requests that are queued on the breaker before the 503s are sent.
	// The value must be adjusted depending on the actual production requirements.
	breakerQueueDepth = 10000

	// The upper bound for concurrent requests sent to the revision.
	// As new endpoints show up, the Breakers concurrency increases up to this value.
	breakerMaxConcurrency = 1000

	// The port on which autoscaler WebSocket server listens.
	autoscalerPort = ":8080"
)

var (
	masterURL = flag.String("master", "", "The address of the Kubernetes API server. "+
		"Overrides any value in kubeconfig. Only required if out-of-cluster.")
	kubeconfig = flag.String("kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.")
)

func statReporter(statSink *websocket.ManagedConnection, stopCh <-chan struct{},
	statChan <-chan []autoscaler.StatMessage, logger *zap.SugaredLogger) {
	for {
		select {
		case sm := <-statChan:
			go func() {
				for _, msg := range sm {
					if err := statSink.Send(msg); err != nil {
						logger.Errorw("Error while sending stat", zap.Error(err))
					}
				}
			}()
		case <-stopCh:
			// It's a sending connection, so no drainage required.
			statSink.Shutdown()
			return
		}
	}
}

type config struct {
	PodName string `split_words:"true" required:"true"`
	PodIP   string `split_words:"true" required:"true"`
}

func main() {
	flag.Parse()

	// Set up a context that we can cancel to tell informers and other subprocesses to stop.
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	// Report stats on Go memory usage every 30 seconds.
	msp := metrics.NewMemStatsAll()
	msp.Start(ctx, 30*time.Second)
	if err := view.Register(msp.DefaultViews()...); err != nil {
		log.Fatalf("Error exporting go memstats view: %v", err)
	}

	cfg, err := sharedmain.GetConfig(*masterURL, *kubeconfig)
	if err != nil {
		log.Fatal("Error building kubeconfig:", err)
	}

	log.Printf("Registering %d clients", len(injection.Default.GetClients()))
	log.Printf("Registering %d informer factories", len(injection.Default.GetInformerFactories()))
	log.Printf("Registering %d informers", len(injection.Default.GetInformers()))

	ctx, informers := injection.Default.SetupInformers(ctx, cfg)

	// Set up our logger.
	loggingConfig, err := sharedmain.GetLoggingConfig(ctx)
	if err != nil {
		log.Fatal("Error loading/parsing logging configuration:", err)
	}
	logger, atomicLevel := pkglogging.NewLoggerFromConfig(loggingConfig, component)
	logger = logger.With(zap.String(logkey.ControllerType, component))
	ctx = pkglogging.WithLogger(ctx, logger)
	defer flush(logger)

	kubeClient := kubeclient.Get(ctx)

	// Run informers instead of starting them from the factory to prevent the sync hanging because of empty handler.
	if err := controller.StartInformers(ctx.Done(), informers...); err != nil {
		logger.Fatalw("Failed to start informers", zap.Error(err))
	}

	logger.Info("Starting the knative activator")

	var env config
	if err := envconfig.Process("", &env); err != nil {
		logger.Fatalw("Failed to process env", zap.Error(err))
	}

	// We sometimes startup faster than we can reach kube-api. Poll on failure to prevent us terminating
	if perr := wait.PollImmediate(time.Second, 60*time.Second, func() (bool, error) {
		if err = version.CheckMinimumVersion(kubeClient.Discovery()); err != nil {
			logger.Errorw("Failed to get k8s version", zap.Error(err))
		}
		return err == nil, nil
	}); perr != nil {
		logger.Fatalw("Timed out attempting to get k8s version", zap.Error(err))
	}

	reporter, err := activator.NewStatsReporter()
	if err != nil {
		logger.Fatalw("Failed to create stats reporter", zap.Error(err))
	}

	statCh := make(chan []autoscaler.StatMessage)
	defer close(statCh)

	reqCh := make(chan activatorhandler.ReqEvent, requestCountingQueueLength)
	defer close(reqCh)

	params := queue.BreakerParams{QueueDepth: breakerQueueDepth, MaxConcurrency: breakerMaxConcurrency, InitialCapacity: 0}

	// Start throttler.
	throttler := activatornet.NewThrottler(ctx, params,
		// We want to join host port since that will be our search space in the Throttler.
		net.JoinHostPort(env.PodIP, strconv.Itoa(networking.BackendHTTPPort)))
	go throttler.Run(ctx)

	oct := tracing.NewOpenCensusTracer(tracing.WithExporter(networking.ActivatorServiceName, logger))

	tracerUpdater := configmap.TypeFilter(&tracingconfig.Config{})(func(name string, value interface{}) {
		cfg := value.(*tracingconfig.Config)
		if err := oct.ApplyConfig(cfg); err != nil {
			logger.Errorw("Unable to apply open census tracer config", zap.Error(err))
			return
		}
	})

	// Set up our config store
	configMapWatcher := configmap.NewInformedWatcher(kubeClient, system.Namespace())
	configStore := activatorconfig.NewStore(logger, tracerUpdater)
	configStore.WatchConfigs(configMapWatcher)

	// Open a WebSocket connection to the autoscaler.
	autoscalerEndpoint := fmt.Sprintf("ws://%s.%s.svc.%s%s", "autoscaler", system.Namespace(), pkgnet.GetClusterDomainName(), autoscalerPort)
	logger.Info("Connecting to Autoscaler at ", autoscalerEndpoint)
	statSink := websocket.NewDurableSendingConnection(autoscalerEndpoint, logger)
	go statReporter(statSink, ctx.Done(), statCh, logger)

	// Create and run our concurrency reporter
	reportTicker := time.NewTicker(time.Second)
	defer reportTicker.Stop()
	cr := activatorhandler.NewConcurrencyReporter(ctx, env.PodName, reqCh,
		reportTicker.C, statCh, reporter)
	go cr.Run(ctx.Done())

	// Create activation handler chain
	// Note: innermost handlers are specified first, ie. the last handler in the chain will be executed first
	var ah http.Handler = activatorhandler.New(
		ctx,
		throttler,
		reporter)
	ah = activatorhandler.NewRequestEventHandler(reqCh, ah)
	ah = tracing.HTTPSpanMiddleware(ah)
	ah = configStore.HTTPMiddleware(ah)
	reqLogHandler, err := pkghttp.NewRequestLogHandler(ah, logging.NewSyncFileWriter(os.Stdout), "",
		requestLogTemplateInputGetter(revisioninformer.Get(ctx).Lister()), false /*enableProbeRequestLog*/)
	if err != nil {
		logger.Fatalw("Unable to create request log handler", zap.Error(err))
	}
	ah = reqLogHandler
	ah = &activatorhandler.ProbeHandler{NextHandler: ah}

	// Set up our health check based on the health of stat sink and environmental factors.
	// When drainCh is closed, we should start to drain connections.
	hc, drainCh := newHealthCheck(logger, statSink)
	ah = &activatorhandler.HealthHandler{HealthCheck: hc, NextHandler: ah}

	ah = network.NewProbeHandler(ah)

	// NOTE: MetricHandler is being used as the outermost handler for the purpose of measuring the request latency.
	ah = activatorhandler.NewMetricHandler(ctx, reporter, ah)

	profilingHandler := profiling.NewHandler(logger, false)
	// Watch the logging config map and dynamically update logging levels.
	configMapWatcher.Watch(pkglogging.ConfigMapName(), pkglogging.UpdateLevelFromConfigMap(logger, atomicLevel, component))

	// Watch the observability config map
	configMapWatcher.Watch(metrics.ConfigMapName(),
		metrics.UpdateExporterFromConfigMap(component, logger),
		updateRequestLogFromConfigMap(logger, reqLogHandler),
		profilingHandler.UpdateFromConfigMap)

	if err = configMapWatcher.Start(ctx.Done()); err != nil {
		logger.Fatalw("Failed to start configuration manager", zap.Error(err))
	}

	servers := map[string]*http.Server{
		"http1":   network.NewServer(":"+strconv.Itoa(networking.BackendHTTPPort), ah),
		"h2c":     network.NewServer(":"+strconv.Itoa(networking.BackendHTTP2Port), ah),
		"profile": profiling.NewServer(profilingHandler),
	}

	errCh := make(chan error, len(servers))
	for name, server := range servers {
		go func(name string, s *http.Server) {
			// Don't forward ErrServerClosed as that indicates we're already shutting down.
			if err := s.ListenAndServe(); err != nil && err != http.ErrServerClosed {
				errCh <- fmt.Errorf("%s server failed: %w", name, err)
			}
		}(name, server)
	}

	// Wait for the signal to drain.
	select {
	case <-drainCh:
		logger.Info("Received the drain signal.")
	case err := <-errCh:
		logger.Errorw("Failed to run HTTP server", zap.Error(err))
	}

	// The drain has started (we are now failing readiness probes).  Let the effects of this
	// propagate so that new requests are no longer routed our way.
	time.Sleep(30 * time.Second)
	logger.Info("Done waiting, shutting down servers.")

	// Drain outstanding requests, and stop accepting new ones.
	for _, server := range servers {
		server.Shutdown(context.Background())
	}
	logger.Info("Servers shutdown.")
}

func newHealthCheck(logger *zap.SugaredLogger, statSink *websocket.ManagedConnection) (func() error, <-chan struct{}) {
	// When we get SIGTERM (sigCh closes), start failing readiness probes.
	sigCh := signals.SetupSignalHandler()

	// Some duration after our first readiness probe failure (to allow time
	// for the network to reprogram) send the signal to drain connections.
	drainCh := make(chan struct{})
	once := sync.Once{}

	return func() error {
		select {
		case <-sigCh:
			// Signal to start the process of draining.
			once.Do(func() {
				logger.Info("Received SIGTERM")
				close(drainCh)
			})
			return errors.New("received SIGTERM from kubelet")
		default:
			logger.Debug("No signal yet.")
			return statSink.Status()
		}
	}, drainCh
}

func flush(logger *zap.SugaredLogger) {
	logger.Sync()
	os.Stdout.Sync()
	os.Stderr.Sync()
	metrics.FlushExporter()
}
<commit_msg>Log activator pod name as key. (#5835)<commit_after>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"strconv"
	"sync"
	"time"

	// Injection related imports.
	kubeclient "knative.dev/pkg/client/injection/kube/client"
	"knative.dev/pkg/injection"
	revisioninformer "knative.dev/serving/pkg/client/injection/informers/serving/v1alpha1/revision"

	"github.com/kelseyhightower/envconfig"
	"go.opencensus.io/stats/view"
	"go.uber.org/zap"
	"k8s.io/apimachinery/pkg/util/wait"
	"knative.dev/pkg/configmap"
	"knative.dev/pkg/controller"
	"knative.dev/pkg/injection/sharedmain"
	pkglogging "knative.dev/pkg/logging"
	"knative.dev/pkg/logging/logkey"
	"knative.dev/pkg/metrics"
	pkgnet "knative.dev/pkg/network"
	"knative.dev/pkg/profiling"
	"knative.dev/pkg/signals"
	"knative.dev/pkg/system"
	"knative.dev/pkg/tracing"
	tracingconfig "knative.dev/pkg/tracing/config"
	"knative.dev/pkg/version"
	"knative.dev/pkg/websocket"
	"knative.dev/serving/pkg/activator"
	activatorconfig "knative.dev/serving/pkg/activator/config"
	activatorhandler "knative.dev/serving/pkg/activator/handler"
	activatornet "knative.dev/serving/pkg/activator/net"
	"knative.dev/serving/pkg/apis/networking"
	"knative.dev/serving/pkg/autoscaler"
	"knative.dev/serving/pkg/goversion"
	pkghttp "knative.dev/serving/pkg/http"
	"knative.dev/serving/pkg/logging"
	"knative.dev/serving/pkg/network"
	"knative.dev/serving/pkg/queue"
)

// Fail if using unsupported go version.
var _ = goversion.IsSupported()

const (
	component = "activator"

	// Add enough buffer to not block request serving on stats collection
	requestCountingQueueLength = 100

	// The number of requests that are queued on the breaker before the 503s are sent.
	// The value must be adjusted depending on the actual production requirements.
	breakerQueueDepth = 10000

	// The upper bound for concurrent requests sent to the revision.
	// As new endpoints show up, the Breakers concurrency increases up to this value.
	breakerMaxConcurrency = 1000

	// The port on which autoscaler WebSocket server listens.
	autoscalerPort = ":8080"
)

var (
	masterURL = flag.String("master", "", "The address of the Kubernetes API server. "+
		"Overrides any value in kubeconfig. Only required if out-of-cluster.")
	kubeconfig = flag.String("kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.")
)

func statReporter(statSink *websocket.ManagedConnection, stopCh <-chan struct{},
	statChan <-chan []autoscaler.StatMessage, logger *zap.SugaredLogger) {
	for {
		select {
		case sm := <-statChan:
			go func() {
				for _, msg := range sm {
					if err := statSink.Send(msg); err != nil {
						logger.Errorw("Error while sending stat", zap.Error(err))
					}
				}
			}()
		case <-stopCh:
			// It's a sending connection, so no drainage required.
			statSink.Shutdown()
			return
		}
	}
}

type config struct {
	PodName string `split_words:"true" required:"true"`
	PodIP   string `split_words:"true" required:"true"`
}

func main() {
	flag.Parse()

	// Set up a context that we can cancel to tell informers and other subprocesses to stop.
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	// Report stats on Go memory usage every 30 seconds.
	msp := metrics.NewMemStatsAll()
	msp.Start(ctx, 30*time.Second)
	if err := view.Register(msp.DefaultViews()...); err != nil {
		log.Fatalf("Error exporting go memstats view: %v", err)
	}

	cfg, err := sharedmain.GetConfig(*masterURL, *kubeconfig)
	if err != nil {
		log.Fatal("Error building kubeconfig:", err)
	}

	log.Printf("Registering %d clients", len(injection.Default.GetClients()))
	log.Printf("Registering %d informer factories", len(injection.Default.GetInformerFactories()))
	log.Printf("Registering %d informers", len(injection.Default.GetInformers()))

	ctx, informers := injection.Default.SetupInformers(ctx, cfg)

	// Set up our logger.
	loggingConfig, err := sharedmain.GetLoggingConfig(ctx)
	if err != nil {
		log.Fatal("Error loading/parsing logging configuration:", err)
	}
	var env config
	if err := envconfig.Process("", &env); err != nil {
		log.Fatalf("Failed to process env: %v", err)
	}

	logger, atomicLevel := pkglogging.NewLoggerFromConfig(loggingConfig, component)
	logger = logger.With(zap.String(logkey.ControllerType, component),
		zap.String(logkey.Pod, env.PodName))
	ctx = pkglogging.WithLogger(ctx, logger)
	defer flush(logger)

	kubeClient := kubeclient.Get(ctx)

	// Run informers instead of starting them from the factory to prevent the sync hanging because of empty handler.
	if err := controller.StartInformers(ctx.Done(), informers...); err != nil {
		logger.Fatalw("Failed to start informers", zap.Error(err))
	}

	logger.Info("Starting the knative activator")

	// We sometimes startup faster than we can reach kube-api. Poll on failure to prevent us terminating
	if perr := wait.PollImmediate(time.Second, 60*time.Second, func() (bool, error) {
		if err = version.CheckMinimumVersion(kubeClient.Discovery()); err != nil {
			logger.Errorw("Failed to get k8s version", zap.Error(err))
		}
		return err == nil, nil
	}); perr != nil {
		logger.Fatalw("Timed out attempting to get k8s version", zap.Error(err))
	}

	reporter, err := activator.NewStatsReporter()
	if err != nil {
		logger.Fatalw("Failed to create stats reporter", zap.Error(err))
	}

	statCh := make(chan []autoscaler.StatMessage)
	defer close(statCh)

	reqCh := make(chan activatorhandler.ReqEvent, requestCountingQueueLength)
	defer close(reqCh)

	params := queue.BreakerParams{QueueDepth: breakerQueueDepth, MaxConcurrency: breakerMaxConcurrency, InitialCapacity: 0}

	// Start throttler.
	throttler := activatornet.NewThrottler(ctx, params,
		// We want to join host port since that will be our search space in the Throttler.
		net.JoinHostPort(env.PodIP, strconv.Itoa(networking.BackendHTTPPort)))
	go throttler.Run(ctx)

	oct := tracing.NewOpenCensusTracer(tracing.WithExporter(networking.ActivatorServiceName, logger))

	tracerUpdater := configmap.TypeFilter(&tracingconfig.Config{})(func(name string, value interface{}) {
		cfg := value.(*tracingconfig.Config)
		if err := oct.ApplyConfig(cfg); err != nil {
			logger.Errorw("Unable to apply open census tracer config", zap.Error(err))
			return
		}
	})

	// Set up our config store
	configMapWatcher := configmap.NewInformedWatcher(kubeClient, system.Namespace())
	configStore := activatorconfig.NewStore(logger, tracerUpdater)
	configStore.WatchConfigs(configMapWatcher)

	// Open a WebSocket connection to the autoscaler.
	autoscalerEndpoint := fmt.Sprintf("ws://%s.%s.svc.%s%s", "autoscaler", system.Namespace(), pkgnet.GetClusterDomainName(), autoscalerPort)
	logger.Info("Connecting to Autoscaler at ", autoscalerEndpoint)
	statSink := websocket.NewDurableSendingConnection(autoscalerEndpoint, logger)
	go statReporter(statSink, ctx.Done(), statCh, logger)

	// Create and run our concurrency reporter
	reportTicker := time.NewTicker(time.Second)
	defer reportTicker.Stop()
	cr := activatorhandler.NewConcurrencyReporter(ctx, env.PodName, reqCh,
		reportTicker.C, statCh, reporter)
	go cr.Run(ctx.Done())

	// Create activation handler chain
	// Note: innermost handlers are specified first, ie. the last handler in the chain will be executed first
	var ah http.Handler = activatorhandler.New(
		ctx,
		throttler,
		reporter)
	ah = activatorhandler.NewRequestEventHandler(reqCh, ah)
	ah = tracing.HTTPSpanMiddleware(ah)
	ah = configStore.HTTPMiddleware(ah)
	reqLogHandler, err := pkghttp.NewRequestLogHandler(ah, logging.NewSyncFileWriter(os.Stdout), "",
		requestLogTemplateInputGetter(revisioninformer.Get(ctx).Lister()), false /*enableProbeRequestLog*/)
	if err != nil {
		logger.Fatalw("Unable to create request log handler", zap.Error(err))
	}
	ah = reqLogHandler
	ah = &activatorhandler.ProbeHandler{NextHandler: ah}

	// Set up our health check based on the health of stat sink and environmental factors.
	// When drainCh is closed, we should start to drain connections.
	hc, drainCh := newHealthCheck(logger, statSink)
	ah = &activatorhandler.HealthHandler{HealthCheck: hc, NextHandler: ah}

	ah = network.NewProbeHandler(ah)

	// NOTE: MetricHandler is being used as the outermost handler for the purpose of measuring the request latency.
	ah = activatorhandler.NewMetricHandler(ctx, reporter, ah)

	profilingHandler := profiling.NewHandler(logger, false)
	// Watch the logging config map and dynamically update logging levels.
	configMapWatcher.Watch(pkglogging.ConfigMapName(), pkglogging.UpdateLevelFromConfigMap(logger, atomicLevel, component))

	// Watch the observability config map
	configMapWatcher.Watch(metrics.ConfigMapName(),
		metrics.UpdateExporterFromConfigMap(component, logger),
		updateRequestLogFromConfigMap(logger, reqLogHandler),
		profilingHandler.UpdateFromConfigMap)

	if err = configMapWatcher.Start(ctx.Done()); err != nil {
		logger.Fatalw("Failed to start configuration manager", zap.Error(err))
	}

	servers := map[string]*http.Server{
		"http1":   network.NewServer(":"+strconv.Itoa(networking.BackendHTTPPort), ah),
		"h2c":     network.NewServer(":"+strconv.Itoa(networking.BackendHTTP2Port), ah),
		"profile": profiling.NewServer(profilingHandler),
	}

	errCh := make(chan error, len(servers))
	for name, server := range servers {
		go func(name string, s *http.Server) {
			// Don't forward ErrServerClosed as that indicates we're already shutting down.
			if err := s.ListenAndServe(); err != nil && err != http.ErrServerClosed {
				errCh <- fmt.Errorf("%s server failed: %w", name, err)
			}
		}(name, server)
	}

	// Wait for the signal to drain.
	select {
	case <-drainCh:
		logger.Info("Received the drain signal.")
	case err := <-errCh:
		logger.Errorw("Failed to run HTTP server", zap.Error(err))
	}

	// The drain has started (we are now failing readiness probes).  Let the effects of this
	// propagate so that new requests are no longer routed our way.
	time.Sleep(30 * time.Second)
	logger.Info("Done waiting, shutting down servers.")

	// Drain outstanding requests, and stop accepting new ones.
	for _, server := range servers {
		server.Shutdown(context.Background())
	}
	logger.Info("Servers shutdown.")
}

func newHealthCheck(logger *zap.SugaredLogger, statSink *websocket.ManagedConnection) (func() error, <-chan struct{}) {
	// When we get SIGTERM (sigCh closes), start failing readiness probes.
	sigCh := signals.SetupSignalHandler()

	// Some duration after our first readiness probe failure (to allow time
	// for the network to reprogram) send the signal to drain connections.
	drainCh := make(chan struct{})
	once := sync.Once{}

	return func() error {
		select {
		case <-sigCh:
			// Signal to start the process of draining.
			once.Do(func() {
				logger.Info("Received SIGTERM")
				close(drainCh)
			})
			return errors.New("received SIGTERM from kubelet")
		default:
			logger.Debug("No signal yet.")
			return statSink.Status()
		}
	}, drainCh
}

func flush(logger *zap.SugaredLogger) {
	logger.Sync()
	os.Stdout.Sync()
	os.Stderr.Sync()
	metrics.FlushExporter()
}
<|endoftext|>


<commit_before>/*
stwmmensa fetches the canteen menu from the Studentenwerk Muenchen website

stwmmensa accepts the following arguments:
	-h --help: Print this usage info and quit
	-o --output: set the path to the output file (mandatory)
	-l --location: set mensa location id (default: 421)
	-f --format: select format (default: xml)

	** Location
	Here is a list of valid location codes with the name of the corresponding mensa:
	411 : Mensa Leopoldstraße
	412 : Mensa Martinsried
	421 : Mensa Arcisstraße
	422 : Mensa Garching
	423 : Mensa Weihenstephan
	431 : Mensa Lothstraße
	432 : Mensa Pasing

	** Format
	xml : a generic xml file is created
	lis : generate html snippet for LIS-infoscreen

	** Examples:
	stwmmensa -l 411 -o /my/path/leopold.xml -f xml
	stwmmensa --location=423 --output=weihenstephan.html --format=lis
*/
package main

import (
	"fmt"
	"log"
	"os"
	"time"

	"github.com/folixg/stwmmensa/mensa"
)

const help = `stwmmensa accepts the following arguments:
-h --help: Print this usage info and quit
-o --output: set the path to the output file (mandatory)
-l --location: set mensa location id (default: 421)
-f --format: select format (default: xml)

** Location
Here is a list of valid location codes with the name of the corresponding mensa:
411 : Mensa Leopoldstraße
412 : Mensa Martinsried
421 : Mensa Arcisstraße
422 : Mensa Garching
423 : Mensa Weihenstephan
431 : Mensa Lothstraße
432 : Mensa Pasing

** Format
xml : a generic xml file is created
lis : generate html snippet for LIS-infoscreen

** Examples:
stwmmensa -l 411 -o /my/path/leopold.xml -f xml
stwmmensa --location=423 --output=weihenstephan.html --format=lis

`

func main() {
	// parse command line arguments
	args, err := mensa.ParseArgs(os.Args)
	if err != nil {
		if err.Error() == "help" {
			fmt.Print(help)
			os.Exit(0)
		}
		log.Fatal(err)
	}
	// get the date (today/tomorrow/monday)
	date := mensa.GetDate(time.Now())
	// fetch the menu
	menu := mensa.FetchMenu(date, args.Location)
	// create output file
	mensa.WriteOutput(menu, args.Format, args.Outfile)
}
<commit_msg>fixed too long godoc description<commit_after>//stwmmensa fetches the canteen menu from the Studentenwerk Muenchen website
package main

/*stwmmensa accepts the following arguments:
-h --help: Print this usage info and quit
-o --output: set the path to the output file (mandatory)
-l --location: set mensa location id (default: 421)
-f --format: select format (default: xml)

** Location
Here is a list of valid location codes with the name of the corresponding mensa:
411 : Mensa Leopoldstraße
412 : Mensa Martinsried
421 : Mensa Arcisstraße
422 : Mensa Garching
423 : Mensa Weihenstephan
431 : Mensa Lothstraße
432 : Mensa Pasing

** Format
xml : a generic xml file is created
lis : generate html snippet for LIS-infoscreen

** Examples:
stwmmensa -l 411 -o /my/path/leopold.xml -f xml
stwmmensa --location=423 --output=weihenstephan.html --format=lis
*/

import (
	"fmt"
	"log"
	"os"
	"time"

	"github.com/folixg/stwmmensa/mensa"
)

const help = `stwmmensa accepts the following arguments:
-h --help: Print this usage info and quit
-o --output: set the path to the output file (mandatory)
-l --location: set mensa location id (default: 421)
-f --format: select format (default: xml)

** Location
Here is a list of valid location codes with the name of the corresponding mensa:
411 : Mensa Leopoldstraße
412 : Mensa Martinsried
421 : Mensa Arcisstraße
422 : Mensa Garching
423 : Mensa Weihenstephan
431 : Mensa Lothstraße
432 : Mensa Pasing

** Format
xml : a generic xml file is created
lis : generate html snippet for LIS-infoscreen

** Examples:
stwmmensa -l 411 -o /my/path/leopold.xml -f xml
stwmmensa --location=423 --output=weihenstephan.html --format=lis

`

func main() {
	// parse command line arguments
	args, err := mensa.ParseArgs(os.Args)
	if err != nil {
		if err.Error() == "help" {
			fmt.Print(help)
			os.Exit(0)
		}
		log.Fatal(err)
	}
	// get the date (today/tomorrow/monday)
	date := mensa.GetDate(time.Now())
	// fetch the menu
	menu := mensa.FetchMenu(date, args.Location)
	// create output file
	mensa.WriteOutput(menu, args.Format, args.Outfile)
}
<|endoftext|>