<commit_before>// Copyright 2020-2021 Buf Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package appprotoexec

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/bufbuild/buf/internal/pkg/app"
	"github.com/bufbuild/buf/internal/pkg/app/appproto"
	"github.com/bufbuild/buf/internal/pkg/ioextended"
	"github.com/bufbuild/buf/internal/pkg/protoencoding"
	"github.com/bufbuild/buf/internal/pkg/storage"
	"github.com/bufbuild/buf/internal/pkg/storage/storageos"
	"github.com/bufbuild/buf/internal/pkg/tmp"
	"go.opencensus.io/trace"
	"go.uber.org/multierr"
	"go.uber.org/zap"
	"google.golang.org/protobuf/proto"
	"google.golang.org/protobuf/types/descriptorpb"
	"google.golang.org/protobuf/types/pluginpb"
)

type protocProxyHandler struct {
	logger            *zap.Logger
	storageosProvider storageos.Provider
	protocPath        string
	pluginName        string
}

func newProtocProxyHandler(
	logger *zap.Logger,
	storageosProvider storageos.Provider,
	protocPath string,
	pluginName string,
) *protocProxyHandler {
	return &protocProxyHandler{
		logger:            logger.Named("appprotoexec"),
		storageosProvider: storageosProvider,
		protocPath:        protocPath,
		pluginName:        pluginName,
	}
}

func (h *protocProxyHandler) Handle(
	ctx context.Context,
	container app.EnvStderrContainer,
	responseWriter appproto.ResponseWriter,
	request *pluginpb.CodeGeneratorRequest,
) (retErr error) {
	ctx, span := trace.StartSpan(ctx, "protoc_proxy")
	span.AddAttributes(trace.StringAttribute("plugin", filepath.Base(h.pluginName)))
	defer span.End()
	if app.DevStdinFilePath == "" {
		return errors.New("app.DevStdinFilePath is empty for this platform")
	}
	protocVersion, err := h.getProtocVersion(ctx, container)
	if err != nil {
		return err
	}
	if h.pluginName == "kotlin" && !getKotlinSupported(protocVersion) {
		return fmt.Errorf("kotlin is not supported for protoc version %s", versionString(protocVersion))
	}
	fileDescriptorSet := &descriptorpb.FileDescriptorSet{
		File: request.ProtoFile,
	}
	fileDescriptorSetData, err := protoencoding.NewWireMarshaler().Marshal(fileDescriptorSet)
	if err != nil {
		return err
	}
	tmpDir, err := tmp.NewDir()
	if err != nil {
		return err
	}
	defer func() {
		retErr = multierr.Append(retErr, tmpDir.Close())
	}()
	args := []string{
		fmt.Sprintf("--descriptor_set_in=%s", app.DevStdinFilePath),
		fmt.Sprintf("--%s_out=%s", h.pluginName, tmpDir.AbsPath()),
	}
	featureProto3Optional := getFeatureProto3Optional(protocVersion)
	if featureProto3Optional {
		args = append(
			args,
			// this flag may be deleted someday
			// in that case, we will have to check that minor < certain_value
			"--experimental_allow_proto3_optional",
		)
	}
	if parameter := request.GetParameter(); parameter != "" {
		args = append(
			args,
			fmt.Sprintf("--%s_opt=%s", h.pluginName, parameter),
		)
	}
	args = append(
		args,
		request.FileToGenerate...,
	)
	cmd := exec.CommandContext(ctx, h.protocPath, args...)
	cmd.Env = app.Environ(container)
	cmd.Stdin = bytes.NewReader(fileDescriptorSetData)
	cmd.Stdout = io.Discard
	cmd.Stderr = container.Stderr()
	if err := cmd.Run(); err != nil {
		// TODO: strip binary path as well?
		// We don't know if this is a system error or plugin error, so we assume system error
		return err
	}
	if featureProto3Optional {
		responseWriter.SetFeatureProto3Optional()
	}
	// no need for symlinks here, and don't want to support
	readWriteBucket, err := h.storageosProvider.NewReadWriteBucket(tmpDir.AbsPath())
	if err != nil {
		return err
	}
	return storage.WalkReadObjects(
		ctx,
		readWriteBucket,
		"",
		func(readObject storage.ReadObject) error {
			data, err := io.ReadAll(readObject)
			if err != nil {
				return err
			}
			return responseWriter.AddFile(
				&pluginpb.CodeGeneratorResponse_File{
					Name:    proto.String(readObject.Path()),
					Content: proto.String(string(data)),
				},
			)
		},
	)
}

func (h *protocProxyHandler) getProtocVersion(
	ctx context.Context,
	container app.EnvContainer,
) (*pluginpb.Version, error) {
	stdoutBuffer := bytes.NewBuffer(nil)
	stderrBuffer := bytes.NewBuffer(nil)
	cmd := exec.CommandContext(ctx, h.protocPath, "--version")
	cmd.Env = app.Environ(container)
	cmd.Stdin = ioextended.DiscardReader
	// do we want to do this?
	cmd.Stdout = stdoutBuffer
	cmd.Stderr = stderrBuffer
	if err := cmd.Run(); err != nil {
		// TODO: strip binary path as well?
		return nil, fmt.Errorf("%v\n%v", err, stderrBuffer.String())
	}
	return parseVersionForCLIVersion(strings.TrimSpace(stdoutBuffer.String()))
}
<commit_msg>Fix windows generation of protoc default plugins (#360)<commit_after>// Copyright 2020-2021 Buf Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package appprotoexec

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/bufbuild/buf/internal/pkg/app"
	"github.com/bufbuild/buf/internal/pkg/app/appproto"
	"github.com/bufbuild/buf/internal/pkg/ioextended"
	"github.com/bufbuild/buf/internal/pkg/protoencoding"
	"github.com/bufbuild/buf/internal/pkg/storage"
	"github.com/bufbuild/buf/internal/pkg/storage/storageos"
	"github.com/bufbuild/buf/internal/pkg/tmp"
	"go.opencensus.io/trace"
	"go.uber.org/multierr"
	"go.uber.org/zap"
	"google.golang.org/protobuf/proto"
	"google.golang.org/protobuf/types/descriptorpb"
	"google.golang.org/protobuf/types/pluginpb"
)

type protocProxyHandler struct {
	logger            *zap.Logger
	storageosProvider storageos.Provider
	protocPath        string
	pluginName        string
}

func newProtocProxyHandler(
	logger *zap.Logger,
	storageosProvider storageos.Provider,
	protocPath string,
	pluginName string,
) *protocProxyHandler {
	return &protocProxyHandler{
		logger:            logger.Named("appprotoexec"),
		storageosProvider: storageosProvider,
		protocPath:        protocPath,
		pluginName:        pluginName,
	}
}

func (h *protocProxyHandler) Handle(
	ctx context.Context,
	container app.EnvStderrContainer,
	responseWriter appproto.ResponseWriter,
	request *pluginpb.CodeGeneratorRequest,
) (retErr error) {
	ctx, span := trace.StartSpan(ctx, "protoc_proxy")
	span.AddAttributes(trace.StringAttribute("plugin", filepath.Base(h.pluginName)))
	defer span.End()
	protocVersion, err := h.getProtocVersion(ctx, container)
	if err != nil {
		return err
	}
	if h.pluginName == "kotlin" && !getKotlinSupported(protocVersion) {
		return fmt.Errorf("kotlin is not supported for protoc version %s", versionString(protocVersion))
	}
	fileDescriptorSet := &descriptorpb.FileDescriptorSet{
		File: request.ProtoFile,
	}
	fileDescriptorSetData, err := protoencoding.NewWireMarshaler().Marshal(fileDescriptorSet)
	if err != nil {
		return err
	}
	descriptorFilePath := app.DevStdinFilePath
	var tmpFile tmp.File
	if descriptorFilePath == "" {
		// since we have no stdin file (i.e. Windows), we're going to have to use a temporary file
		if tmpFile, err = tmp.NewFileWithData(fileDescriptorSetData); err != nil {
			return err
		} else {
			descriptorFilePath = tmpFile.AbsPath()
		}
	}
	tmpDir, err := tmp.NewDir()
	if err != nil {
		return err
	}
	defer func() {
		retErr = multierr.Append(retErr, tmpDir.Close())
	}()
	args := []string{
		fmt.Sprintf("--descriptor_set_in=%s", descriptorFilePath),
		fmt.Sprintf("--%s_out=%s", h.pluginName, tmpDir.AbsPath()),
	}
	featureProto3Optional := getFeatureProto3Optional(protocVersion)
	if featureProto3Optional {
		args = append(
			args,
			// this flag may be deleted someday
			// in that case, we will have to check that minor < certain_value
			"--experimental_allow_proto3_optional",
		)
	}
	if parameter := request.GetParameter(); parameter != "" {
		args = append(
			args,
			fmt.Sprintf("--%s_opt=%s", h.pluginName, parameter),
		)
	}
	args = append(
		args,
		request.FileToGenerate...,
	)
	cmd := exec.CommandContext(ctx, h.protocPath, args...)
	cmd.Env = app.Environ(container)
	if app.DevStdinFilePath != "" {
		cmd.Stdin = bytes.NewReader(fileDescriptorSetData)
	}
	cmd.Stdout = io.Discard
	cmd.Stderr = container.Stderr()
	if err := cmd.Run(); err != nil {
		// TODO: strip binary path as well?
		// We don't know if this is a system error or plugin error, so we assume system error
		return err
	}
	if featureProto3Optional {
		responseWriter.SetFeatureProto3Optional()
	}
	if tmpFile != nil {
		_ = tmpFile.Close()
	}
	// no need for symlinks here, and don't want to support
	readWriteBucket, err := h.storageosProvider.NewReadWriteBucket(tmpDir.AbsPath())
	if err != nil {
		return err
	}
	return storage.WalkReadObjects(
		ctx,
		readWriteBucket,
		"",
		func(readObject storage.ReadObject) error {
			data, err := io.ReadAll(readObject)
			if err != nil {
				return err
			}
			return responseWriter.AddFile(
				&pluginpb.CodeGeneratorResponse_File{
					Name:    proto.String(readObject.Path()),
					Content: proto.String(string(data)),
				},
			)
		},
	)
}

func (h *protocProxyHandler) getProtocVersion(
	ctx context.Context,
	container app.EnvContainer,
) (*pluginpb.Version, error) {
	stdoutBuffer := bytes.NewBuffer(nil)
	stderrBuffer := bytes.NewBuffer(nil)
	cmd := exec.CommandContext(ctx, h.protocPath, "--version")
	cmd.Env = app.Environ(container)
	cmd.Stdin = ioextended.DiscardReader
	// do we want to do this?
	cmd.Stdout = stdoutBuffer
	cmd.Stderr = stderrBuffer
	if err := cmd.Run(); err != nil {
		// TODO: strip binary path as well?
		return nil, fmt.Errorf("%v\n%v", err, stderrBuffer.String())
	}
	return parseVersionForCLIVersion(strings.TrimSpace(stdoutBuffer.String()))
}
<|endoftext|>


<commit_before>package main

const (
	sname           = "test.net.local"
	software        = "echochat"
	softwarev       = "v0.1"
	CONN_HOST       = "0.0.0.0"
	CONN_PORT       = "6667"
	ping_time       = 45 //something
	ping_check_time = 20 // time between the user's ping checks
	resolvehosts    = true  //Note: make forward confirmed reverse dns optional.
	isupport        = "NAMESX CHANTYPES=#& PREFIX=(o)@"
)

var (
	valid_chan_prefix = []string{"#", "&"}
	global_bad_chars  = []string{":", "!", "@", "*", "(", ")", "<", ">", ",", "~", "/", "\\"}
)
<commit_msg>go fmt<commit_after>package main

const (
	sname           = "test.net.local"
	software        = "echochat"
	softwarev       = "v0.1"
	CONN_HOST       = "0.0.0.0"
	CONN_PORT       = "6667"
	ping_time       = 45   //something
	ping_check_time = 20   // time between the user's ping checks
	resolvehosts    = true //Note: make forward confirmed reverse dns optional.
	isupport        = "NAMESX CHANTYPES=#& PREFIX=(o)@"
)

var (
	valid_chan_prefix = []string{"#", "&"}
	global_bad_chars  = []string{":", "!", "@", "*", "(", ")", "<", ">", ",", "~", "/", "\\"}
)
<|endoftext|>


<commit_before>package ssh

import (
	"errors"
	"fmt"
	"strings"

	"gopkg.in/src-d/go-git.v4/plumbing/transport"
	"gopkg.in/src-d/go-git.v4/plumbing/transport/internal/common"

	"golang.org/x/crypto/ssh"
)

var (
	errAlreadyConnected = errors.New("ssh session already created")
)

// DefaultClient is the default SSH client.
var DefaultClient = common.NewClient(&runner{})

type runner struct{}

func (r *runner) Command(cmd string, ep transport.Endpoint) (common.Command, error) {
	c := &command{command: cmd, endpoint: ep}
	if err := c.connect(); err != nil {
		return nil, err
	}

	return c, nil
}

type command struct {
	*ssh.Session
	connected bool
	command   string
	endpoint  transport.Endpoint
	client    *ssh.Client
	auth      AuthMethod
}

func (c *command) SetAuth(auth transport.AuthMethod) error {
	a, ok := auth.(AuthMethod)
	if !ok {
		return transport.ErrInvalidAuthMethod
	}

	c.auth = a
	return nil
}

func (c *command) Start() error {
	return c.Session.Start(endpointToCommand(c.command, c.endpoint))
}

// Close closes the SSH session and connection.
func (c *command) Close() error {
	if !c.connected {
		return nil
	}

	c.connected = false

	//XXX: If did read the full packfile, then the session might be already
	//     closed.
	_ = c.Session.Close()

	return c.client.Close()
}

// connect connects to the SSH server, unless a AuthMethod was set with
// SetAuth method, by default uses an auth method based on PublicKeysCallback,
// it connects to a SSH agent, using the address stored in the SSH_AUTH_SOCK
// environment var.
func (c *command) connect() error {
	if c.connected {
		return errAlreadyConnected
	}

	if err := c.setAuthFromEndpoint(); err != nil {
		return err
	}

	var err error
	c.client, err = ssh.Dial("tcp", c.getHostWithPort(), c.auth.clientConfig())
	if err != nil {
		return err
	}

	c.Session, err = c.client.NewSession()
	if err != nil {
		_ = c.client.Close()
		return err
	}

	c.connected = true
	return nil
}

func (c *command) getHostWithPort() string {
	host := c.endpoint.Host
	if strings.Index(c.endpoint.Host, ":") == -1 {
		host += ":22"
	}

	return host
}

func (c *command) setAuthFromEndpoint() error {
	var u string
	if info := c.endpoint.User; info != nil {
		u = info.Username()
	}

	var err error
	c.auth, err = NewSSHAgentAuth(u)
	return err
}

func endpointToCommand(cmd string, ep transport.Endpoint) string {
	directory := ep.Path
	directory = directory[1:]

	return fmt.Sprintf("%s '%s'", cmd, directory)
}
<commit_msg>transport: ssh, keeping the original path (#189)<commit_after>package ssh

import (
	"errors"
	"fmt"
	"strings"

	"gopkg.in/src-d/go-git.v4/plumbing/transport"
	"gopkg.in/src-d/go-git.v4/plumbing/transport/internal/common"

	"golang.org/x/crypto/ssh"
)

var (
	errAlreadyConnected = errors.New("ssh session already created")
)

// DefaultClient is the default SSH client.
var DefaultClient = common.NewClient(&runner{})

type runner struct{}

func (r *runner) Command(cmd string, ep transport.Endpoint) (common.Command, error) {
	c := &command{command: cmd, endpoint: ep}
	if err := c.connect(); err != nil {
		return nil, err
	}

	return c, nil
}

type command struct {
	*ssh.Session
	connected bool
	command   string
	endpoint  transport.Endpoint
	client    *ssh.Client
	auth      AuthMethod
}

func (c *command) SetAuth(auth transport.AuthMethod) error {
	a, ok := auth.(AuthMethod)
	if !ok {
		return transport.ErrInvalidAuthMethod
	}

	c.auth = a
	return nil
}

func (c *command) Start() error {
	return c.Session.Start(endpointToCommand(c.command, c.endpoint))
}

// Close closes the SSH session and connection.
func (c *command) Close() error {
	if !c.connected {
		return nil
	}

	c.connected = false

	//XXX: If did read the full packfile, then the session might be already
	//     closed.
	_ = c.Session.Close()

	return c.client.Close()
}

// connect connects to the SSH server, unless a AuthMethod was set with
// SetAuth method, by default uses an auth method based on PublicKeysCallback,
// it connects to a SSH agent, using the address stored in the SSH_AUTH_SOCK
// environment var.
func (c *command) connect() error {
	if c.connected {
		return errAlreadyConnected
	}

	if err := c.setAuthFromEndpoint(); err != nil {
		return err
	}

	var err error
	c.client, err = ssh.Dial("tcp", c.getHostWithPort(), c.auth.clientConfig())
	if err != nil {
		return err
	}

	c.Session, err = c.client.NewSession()
	if err != nil {
		_ = c.client.Close()
		return err
	}

	c.connected = true
	return nil
}

func (c *command) getHostWithPort() string {
	host := c.endpoint.Host
	if strings.Index(c.endpoint.Host, ":") == -1 {
		host += ":22"
	}

	return host
}

func (c *command) setAuthFromEndpoint() error {
	var u string
	if info := c.endpoint.User; info != nil {
		u = info.Username()
	}

	var err error
	c.auth, err = NewSSHAgentAuth(u)
	return err
}

func endpointToCommand(cmd string, ep transport.Endpoint) string {
	return fmt.Sprintf("%s '%s'", cmd, ep.Path)
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package flags holds command line options common to all GXUI samples.
package flags

import (
	"flag"
	"github.com/google/gxui"
	"github.com/google/gxui/themes/dark"
	"github.com/google/gxui/themes/light"
)

var DefaultScaleFactor float32
var FlagTheme string

func init() {
	flagTheme := flag.String("theme", "dark", "Theme to use {dark|light}.")
	defaultScaleFactor := flag.Float64("scaling", 1.0, "Adjusts the scaling of UI rendering")
	flag.Parse()

	DefaultScaleFactor = float32(*defaultScaleFactor)
	FlagTheme = *flagTheme
}

// CreateTheme creates and returns the theme specified on the command line.
// The default theme is dark.
func CreateTheme(driver gxui.Driver) gxui.Theme {
	if FlagTheme == "light" {
		return light.CreateTheme(driver)
	}
	return dark.CreateTheme(driver)
}
<commit_msg>Remove gxui flags<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package flags holds command line options common to all GXUI samples.
package flags

import (
	"github.com/google/gxui"
	"github.com/google/gxui/themes/dark"
)

var DefaultScaleFactor float32
var FlagTheme string

func init() {
	DefaultScaleFactor = 1.0
	FlagTheme = "dark"
}

// CreateTheme creates and returns the theme specified on the command line.
// The default theme is dark.
func CreateTheme(driver gxui.Driver) gxui.Theme {
	return dark.CreateTheme(driver)
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package logic

import (
	"context"
	"fmt"
	"time"

	"golang.org/x/time/rate"

	apiv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/labels"
	vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1"
	vpa_clientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned"
	vpa_lister "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/listers/autoscaling.k8s.io/v1"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/updater/eviction"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/updater/priority"
	metrics_updater "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/updater"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/status"
	vpa_api_util "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/vpa"
	kube_client "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/fake"
	"k8s.io/client-go/kubernetes/scheme"
	clientv1 "k8s.io/client-go/kubernetes/typed/core/v1"
	v1lister "k8s.io/client-go/listers/core/v1"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/record"
	"k8s.io/klog/v2"
)

// Updater performs updates on pods if recommended by Vertical Pod Autoscaler
type Updater interface {
	// RunOnce represents single iteration in the main-loop of Updater
	RunOnce(context.Context)
}

type updater struct {
	vpaLister                    vpa_lister.VerticalPodAutoscalerLister
	podLister                    v1lister.PodLister
	eventRecorder                record.EventRecorder
	evictionFactory              eviction.PodsEvictionRestrictionFactory
	recommendationProcessor      vpa_api_util.RecommendationProcessor
	evictionAdmission            priority.PodEvictionAdmission
	priorityProcessor            priority.PriorityProcessor
	evictionRateLimiter          *rate.Limiter
	selectorFetcher              target.VpaTargetSelectorFetcher
	useAdmissionControllerStatus bool
	statusValidator              status.Validator
}

// NewUpdater creates Updater with given configuration
func NewUpdater(
	kubeClient kube_client.Interface,
	vpaClient *vpa_clientset.Clientset,
	minReplicasForEvicition int,
	evictionRateLimit float64,
	evictionRateBurst int,
	evictionToleranceFraction float64,
	useAdmissionControllerStatus bool,
	statusNamespace string,
	recommendationProcessor vpa_api_util.RecommendationProcessor,
	evictionAdmission priority.PodEvictionAdmission,
	selectorFetcher target.VpaTargetSelectorFetcher,
	priorityProcessor priority.PriorityProcessor,
	namespace string,
) (Updater, error) {
	evictionRateLimiter := getRateLimiter(evictionRateLimit, evictionRateBurst)
	factory, err := eviction.NewPodsEvictionRestrictionFactory(kubeClient, minReplicasForEvicition, evictionToleranceFraction)
	if err != nil {
		return nil, fmt.Errorf("Failed to create eviction restriction factory: %v", err)
	}
	return &updater{
		vpaLister:                    vpa_api_util.NewVpasLister(vpaClient, make(chan struct{}), namespace),
		podLister:                    newPodLister(kubeClient, namespace),
		eventRecorder:                newEventRecorder(kubeClient),
		evictionFactory:              factory,
		recommendationProcessor:      recommendationProcessor,
		evictionRateLimiter:          evictionRateLimiter,
		evictionAdmission:            evictionAdmission,
		priorityProcessor:            priorityProcessor,
		selectorFetcher:              selectorFetcher,
		useAdmissionControllerStatus: useAdmissionControllerStatus,
		statusValidator: status.NewValidator(
			kubeClient,
			status.AdmissionControllerStatusName,
			statusNamespace,
		),
	}, nil
}

// RunOnce represents single iteration in the main-loop of Updater
func (u *updater) RunOnce(ctx context.Context) {
	timer := metrics_updater.NewExecutionTimer()
	defer timer.ObserveTotal()

	if u.useAdmissionControllerStatus {
		isValid, err := u.statusValidator.IsStatusValid(status.AdmissionControllerStatusTimeout)
		if err != nil {
			klog.Errorf("Error getting Admission Controller status: %v. Skipping eviction loop", err)
			return
		}
		if !isValid {
			klog.Warningf("Admission Controller status has been refreshed more than %v ago. Skipping eviction loop",
				status.AdmissionControllerStatusTimeout)
			return
		}
	}

	vpaList, err := u.vpaLister.List(labels.Everything())
	if err != nil {
		klog.Fatalf("failed get VPA list: %v", err)
	}
	timer.ObserveStep("ListVPAs")

	vpas := make([]*vpa_api_util.VpaWithSelector, 0)

	for _, vpa := range vpaList {
		if vpa_api_util.GetUpdateMode(vpa) != vpa_types.UpdateModeRecreate &&
			vpa_api_util.GetUpdateMode(vpa) != vpa_types.UpdateModeAuto {
			klog.V(3).Infof("skipping VPA object %v because its mode is not \"Recreate\" or \"Auto\"", vpa.Name)
			continue
		}
		selector, err := u.selectorFetcher.Fetch(vpa)
		if err != nil {
			klog.V(3).Infof("skipping VPA object %v because we cannot fetch selector", vpa.Name)
			continue
		}

		vpas = append(vpas, &vpa_api_util.VpaWithSelector{
			Vpa:      vpa,
			Selector: selector,
		})
	}

	if len(vpas) == 0 {
		klog.Warningf("no VPA objects to process")
		if u.evictionAdmission != nil {
			u.evictionAdmission.CleanUp()
		}
		return
	}

	podsList, err := u.podLister.List(labels.Everything())
	if err != nil {
		klog.Errorf("failed to get pods list: %v", err)
		return
	}
	timer.ObserveStep("ListPods")
	allLivePods := filterDeletedPods(podsList)

	controlledPods := make(map[*vpa_types.VerticalPodAutoscaler][]*apiv1.Pod)
	for _, pod := range allLivePods {
		controllingVPA := vpa_api_util.GetControllingVPAForPod(pod, vpas)
		if controllingVPA != nil {
			controlledPods[controllingVPA.Vpa] = append(controlledPods[controllingVPA.Vpa], pod)
		}
	}
	timer.ObserveStep("FilterPods")

	if u.evictionAdmission != nil {
		u.evictionAdmission.LoopInit(allLivePods, controlledPods)
	}
	timer.ObserveStep("AdmissionInit")

	// wrappers for metrics which are computed every loop run
	controlledPodsCounter := metrics_updater.NewControlledPodsCounter()
	evictablePodsCounter := metrics_updater.NewEvictablePodsCounter()
	vpasWithEvictablePodsCounter := metrics_updater.NewVpasWithEvictablePodsCounter()
	vpasWithEvictedPodsCounter := metrics_updater.NewVpasWithEvictedPodsCounter()

	// using defer to protect against 'return' after evictionRateLimiter.Wait
	defer controlledPodsCounter.Observe()
	defer evictablePodsCounter.Observe()
	defer vpasWithEvictablePodsCounter.Observe()
	defer vpasWithEvictedPodsCounter.Observe()

	// NOTE: this loop assumes that controlledPods are filtered
	// to contain only Pods controlled by a VPA in auto or recreate mode
	for vpa, livePods := range controlledPods {
		vpaSize := len(livePods)
		controlledPodsCounter.Add(vpaSize, vpaSize)
		evictionLimiter := u.evictionFactory.NewPodsEvictionRestriction(livePods, vpa)
		podsForUpdate := u.getPodsUpdateOrder(filterNonEvictablePods(livePods, evictionLimiter), vpa)
		evictablePodsCounter.Add(vpaSize, len(podsForUpdate))

		withEvictable := false
		withEvicted := false
		for _, pod := range podsForUpdate {
			withEvictable = true
			if !evictionLimiter.CanEvict(pod) {
				continue
			}
			err := u.evictionRateLimiter.Wait(ctx)
			if err != nil {
				klog.Warningf("evicting pod %v failed: %v", pod.Name, err)
				return
			}
			klog.V(2).Infof("evicting pod %v", pod.Name)
			evictErr := evictionLimiter.Evict(pod, u.eventRecorder)
			if evictErr != nil {
				klog.Warningf("evicting pod %v failed: %v", pod.Name, evictErr)
			} else {
				withEvicted = true
				metrics_updater.AddEvictedPod(vpaSize)
			}
		}

		if withEvictable {
			vpasWithEvictablePodsCounter.Add(vpaSize, 1)
		}
		if withEvicted {
			vpasWithEvictedPodsCounter.Add(vpaSize, 1)
		}
	}
	timer.ObserveStep("EvictPods")
}

func getRateLimiter(evictionRateLimit float64, evictionRateLimitBurst int) *rate.Limiter {
	var evictionRateLimiter *rate.Limiter
	if evictionRateLimit <= 0 {
		// As a special case if the rate is set to rate.Inf, the burst rate is ignored
		// see https://github.com/golang/time/blob/master/rate/rate.go#L37
		evictionRateLimiter = rate.NewLimiter(rate.Inf, 0)
		klog.V(1).Info("Rate limit disabled")
	} else {
		evictionRateLimiter = rate.NewLimiter(rate.Limit(evictionRateLimit), evictionRateLimitBurst)
	}
	return evictionRateLimiter
}

// getPodsUpdateOrder returns list of pods that should be updated ordered by update priority
func (u *updater) getPodsUpdateOrder(pods []*apiv1.Pod, vpa *vpa_types.VerticalPodAutoscaler) []*apiv1.Pod {
	priorityCalculator := priority.NewUpdatePriorityCalculator(
		vpa,
		nil,
		u.recommendationProcessor,
		u.priorityProcessor)

	for _, pod := range pods {
		priorityCalculator.AddPod(pod, time.Now())
	}

	return priorityCalculator.GetSortedPods(u.evictionAdmission)
}

func filterNonEvictablePods(pods []*apiv1.Pod, evictionRestriciton eviction.PodsEvictionRestriction) []*apiv1.Pod {
	result := make([]*apiv1.Pod, 0)
	for _, pod := range pods {
		if evictionRestriciton.CanEvict(pod) {
			result = append(result, pod)
		}
	}
	return result
}

func filterDeletedPods(pods []*apiv1.Pod) []*apiv1.Pod {
	result := make([]*apiv1.Pod, 0)
	for _, pod := range pods {
		if pod.DeletionTimestamp == nil {
			result = append(result, pod)
		}
	}
	return result
}

func newPodLister(kubeClient kube_client.Interface, namespace string) v1lister.PodLister {
	selector := fields.ParseSelectorOrDie("spec.nodeName!=" + "" + ",status.phase!=" +
		string(apiv1.PodSucceeded) + ",status.phase!=" + string(apiv1.PodFailed))
	podListWatch := cache.NewListWatchFromClient(kubeClient.CoreV1().RESTClient(), "pods", apiv1.NamespaceAll, selector)
	store := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
	podLister := v1lister.NewPodLister(store)
	podReflector := cache.NewReflector(podListWatch, &apiv1.Pod{}, store, time.Hour)
	stopCh := make(chan struct{})
	go podReflector.Run(stopCh)

	return podLister
}

func newEventRecorder(kubeClient kube_client.Interface) record.EventRecorder {
	eventBroadcaster := record.NewBroadcaster()
	eventBroadcaster.StartLogging(klog.V(4).Infof)
	if _, isFake := kubeClient.(*fake.Clientset); !isFake {
		eventBroadcaster.StartRecordingToSink(&clientv1.EventSinkImpl{Interface: clientv1.New(kubeClient.CoreV1().RESTClient()).Events("")})
	}
	return eventBroadcaster.NewRecorder(scheme.Scheme, apiv1.EventSource{Component: "vpa-updater"})
}
<commit_msg>Restrict Updater PodLister to namespace<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package logic

import (
	"context"
	"fmt"
	"time"

	"golang.org/x/time/rate"

	apiv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/labels"
	vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1"
	vpa_clientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned"
	vpa_lister "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/listers/autoscaling.k8s.io/v1"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/updater/eviction"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/updater/priority"
	metrics_updater "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/updater"
	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/status"
	vpa_api_util "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/vpa"
	kube_client "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/fake"
	"k8s.io/client-go/kubernetes/scheme"
	clientv1 "k8s.io/client-go/kubernetes/typed/core/v1"
	v1lister "k8s.io/client-go/listers/core/v1"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/record"
	"k8s.io/klog/v2"
)

// Updater performs updates on pods if recommended by Vertical Pod Autoscaler
type Updater interface {
	// RunOnce represents single iteration in the main-loop of Updater
	RunOnce(context.Context)
}

type updater struct {
	vpaLister                    vpa_lister.VerticalPodAutoscalerLister
	podLister                    v1lister.PodLister
	eventRecorder                record.EventRecorder
	evictionFactory              eviction.PodsEvictionRestrictionFactory
	recommendationProcessor      vpa_api_util.RecommendationProcessor
	evictionAdmission            priority.PodEvictionAdmission
	priorityProcessor            priority.PriorityProcessor
	evictionRateLimiter          *rate.Limiter
	selectorFetcher              target.VpaTargetSelectorFetcher
	useAdmissionControllerStatus bool
	statusValidator              status.Validator
}

// NewUpdater creates Updater with given configuration
func NewUpdater(
	kubeClient kube_client.Interface,
	vpaClient *vpa_clientset.Clientset,
	minReplicasForEvicition int,
	evictionRateLimit float64,
	evictionRateBurst int,
	evictionToleranceFraction float64,
	useAdmissionControllerStatus bool,
	statusNamespace string,
	recommendationProcessor vpa_api_util.RecommendationProcessor,
	evictionAdmission priority.PodEvictionAdmission,
	selectorFetcher target.VpaTargetSelectorFetcher,
	priorityProcessor priority.PriorityProcessor,
	namespace string,
) (Updater, error) {
	evictionRateLimiter := getRateLimiter(evictionRateLimit, evictionRateBurst)
	factory, err := eviction.NewPodsEvictionRestrictionFactory(kubeClient, minReplicasForEvicition, evictionToleranceFraction)
	if err != nil {
		return nil, fmt.Errorf("Failed to create eviction restriction factory: %v", err)
	}
	return &updater{
		vpaLister:                    vpa_api_util.NewVpasLister(vpaClient, make(chan struct{}), namespace),
		podLister:                    newPodLister(kubeClient, namespace),
		eventRecorder:                newEventRecorder(kubeClient),
		evictionFactory:              factory,
		recommendationProcessor:      recommendationProcessor,
		evictionRateLimiter:          evictionRateLimiter,
		evictionAdmission:            evictionAdmission,
		priorityProcessor:            priorityProcessor,
		selectorFetcher:              selectorFetcher,
		useAdmissionControllerStatus: useAdmissionControllerStatus,
		statusValidator: status.NewValidator(
			kubeClient,
			status.AdmissionControllerStatusName,
			statusNamespace,
		),
	}, nil
}

// RunOnce represents single iteration in the main-loop of Updater
func (u *updater) RunOnce(ctx context.Context) {
	timer := metrics_updater.NewExecutionTimer()
	defer timer.ObserveTotal()

	if u.useAdmissionControllerStatus {
		isValid, err := u.statusValidator.IsStatusValid(status.AdmissionControllerStatusTimeout)
		if err != nil {
			klog.Errorf("Error getting Admission Controller status: %v. Skipping eviction loop", err)
			return
		}
		if !isValid {
			klog.Warningf("Admission Controller status has been refreshed more than %v ago. Skipping eviction loop",
				status.AdmissionControllerStatusTimeout)
			return
		}
	}

	vpaList, err := u.vpaLister.List(labels.Everything())
	if err != nil {
		klog.Fatalf("failed get VPA list: %v", err)
	}
	timer.ObserveStep("ListVPAs")

	vpas := make([]*vpa_api_util.VpaWithSelector, 0)

	for _, vpa := range vpaList {
		if vpa_api_util.GetUpdateMode(vpa) != vpa_types.UpdateModeRecreate &&
			vpa_api_util.GetUpdateMode(vpa) != vpa_types.UpdateModeAuto {
			klog.V(3).Infof("skipping VPA object %v because its mode is not \"Recreate\" or \"Auto\"", vpa.Name)
			continue
		}
		selector, err := u.selectorFetcher.Fetch(vpa)
		if err != nil {
			klog.V(3).Infof("skipping VPA object %v because we cannot fetch selector", vpa.Name)
			continue
		}

		vpas = append(vpas, &vpa_api_util.VpaWithSelector{
			Vpa:      vpa,
			Selector: selector,
		})
	}

	if len(vpas) == 0 {
		klog.Warningf("no VPA objects to process")
		if u.evictionAdmission != nil {
			u.evictionAdmission.CleanUp()
		}
		return
	}

	podsList, err := u.podLister.List(labels.Everything())
	if err != nil {
		klog.Errorf("failed to get pods list: %v", err)
		return
	}
	timer.ObserveStep("ListPods")
	allLivePods := filterDeletedPods(podsList)

	controlledPods := make(map[*vpa_types.VerticalPodAutoscaler][]*apiv1.Pod)
	for _, pod := range allLivePods {
		controllingVPA := vpa_api_util.GetControllingVPAForPod(pod, vpas)
		if controllingVPA != nil {
			controlledPods[controllingVPA.Vpa] = append(controlledPods[controllingVPA.Vpa], pod)
		}
	}
	timer.ObserveStep("FilterPods")

	if u.evictionAdmission != nil {
		u.evictionAdmission.LoopInit(allLivePods, controlledPods)
	}
	timer.ObserveStep("AdmissionInit")

	// wrappers for metrics which are computed every loop run
	controlledPodsCounter := metrics_updater.NewControlledPodsCounter()
	evictablePodsCounter := metrics_updater.NewEvictablePodsCounter()
	vpasWithEvictablePodsCounter := metrics_updater.NewVpasWithEvictablePodsCounter()
	vpasWithEvictedPodsCounter := metrics_updater.NewVpasWithEvictedPodsCounter()

	// using defer to protect against 'return' after evictionRateLimiter.Wait
	defer controlledPodsCounter.Observe()
	defer evictablePodsCounter.Observe()
	defer vpasWithEvictablePodsCounter.Observe()
	defer vpasWithEvictedPodsCounter.Observe()

	// NOTE: this loop assumes that controlledPods are filtered
	// to contain only Pods controlled by a VPA in auto or recreate mode
	for vpa, livePods := range controlledPods {
		vpaSize := len(livePods)
		controlledPodsCounter.Add(vpaSize, vpaSize)
		evictionLimiter := u.evictionFactory.NewPodsEvictionRestriction(livePods, vpa)
		podsForUpdate := u.getPodsUpdateOrder(filterNonEvictablePods(livePods, evictionLimiter), vpa)
		evictablePodsCounter.Add(vpaSize, len(podsForUpdate))

		withEvictable := false
		withEvicted := false
		for _, pod := range podsForUpdate {
			withEvictable = true
			if !evictionLimiter.CanEvict(pod) {
				continue
			}
			err := u.evictionRateLimiter.Wait(ctx)
			if err != nil {
				klog.Warningf("evicting pod %v failed: %v", pod.Name, err)
				return
			}
			klog.V(2).Infof("evicting pod %v", pod.Name)
			evictErr := evictionLimiter.Evict(pod, u.eventRecorder)
			if evictErr != nil {
				klog.Warningf("evicting pod %v failed: %v", pod.Name, evictErr)
			} else {
				withEvicted = true
				metrics_updater.AddEvictedPod(vpaSize)
			}
		}

		if withEvictable {
			vpasWithEvictablePodsCounter.Add(vpaSize, 1)
		}
		if withEvicted {
			vpasWithEvictedPodsCounter.Add(vpaSize, 1)
		}
	}
	timer.ObserveStep("EvictPods")
}

func getRateLimiter(evictionRateLimit float64, evictionRateLimitBurst int) *rate.Limiter {
	var evictionRateLimiter *rate.Limiter
	if evictionRateLimit <= 0 {
		// As a special case if the rate is set to rate.Inf, the burst rate is ignored
		// see https://github.com/golang/time/blob/master/rate/rate.go#L37
		evictionRateLimiter = rate.NewLimiter(rate.Inf, 0)
		klog.V(1).Info("Rate limit disabled")
	} else {
		evictionRateLimiter = rate.NewLimiter(rate.Limit(evictionRateLimit), evictionRateLimitBurst)
	}
	return evictionRateLimiter
}

// getPodsUpdateOrder returns list of pods that should be updated ordered by update priority
func (u *updater) getPodsUpdateOrder(pods []*apiv1.Pod, vpa *vpa_types.VerticalPodAutoscaler) []*apiv1.Pod {
	priorityCalculator := priority.NewUpdatePriorityCalculator(
		vpa,
		nil,
		u.recommendationProcessor,
		u.priorityProcessor)

	for _, pod := range pods {
		priorityCalculator.AddPod(pod, time.Now())
	}

	return priorityCalculator.GetSortedPods(u.evictionAdmission)
}

func filterNonEvictablePods(pods []*apiv1.Pod, evictionRestriciton eviction.PodsEvictionRestriction) []*apiv1.Pod {
	result := make([]*apiv1.Pod, 0)
	for _, pod := range pods {
		if evictionRestriciton.CanEvict(pod) {
			result = append(result, pod)
		}
	}
	return result
}

func filterDeletedPods(pods []*apiv1.Pod) []*apiv1.Pod {
	result := make([]*apiv1.Pod, 0)
	for _, pod := range pods {
		if pod.DeletionTimestamp == nil {
			result = append(result, pod)
		}
	}
	return result
}

func newPodLister(kubeClient kube_client.Interface, namespace string) v1lister.PodLister {
	selector := fields.ParseSelectorOrDie("spec.nodeName!=" + "" + ",status.phase!=" +
		string(apiv1.PodSucceeded) + ",status.phase!=" + string(apiv1.PodFailed))
	podListWatch := cache.NewListWatchFromClient(kubeClient.CoreV1().RESTClient(), "pods", namespace, selector)
	store := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
	podLister := v1lister.NewPodLister(store)
	podReflector := cache.NewReflector(podListWatch, &apiv1.Pod{}, store, time.Hour)
	stopCh := make(chan struct{})
	go podReflector.Run(stopCh)

	return podLister
}

func newEventRecorder(kubeClient kube_client.Interface) record.EventRecorder {
	eventBroadcaster := record.NewBroadcaster()
	eventBroadcaster.StartLogging(klog.V(4).Infof)
	if _, isFake := kubeClient.(*fake.Clientset); !isFake {
		eventBroadcaster.StartRecordingToSink(&clientv1.EventSinkImpl{Interface: clientv1.New(kubeClient.CoreV1().RESTClient()).Events("")})
	}
	return eventBroadcaster.NewRecorder(scheme.Scheme, apiv1.EventSource{Component: "vpa-updater"})
}
<|endoftext|>


<commit_before>package performanceboard

import (
	"appengine"
	"appengine/datastore"
	"appengine/user"
	"github.com/gorilla/mux"
	"net/http"
)

const BoardKind = "Board"

type Board struct {
	Key    *datastore.Key `datastore:"-"`
	UserID string
}

func (board *Board) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
	api, _ := router.Get("board").URL("board", board.Key.Encode())
	JsonResponse{
		"board": board.Key.Encode(),
		"api":   AbsURL(*api, request),
	}.Write(writer)
}

// HTTP handlers

func createBoard(writer http.ResponseWriter, request *http.Request) {
	context := appengine.NewContext(request)
	board := Board{}
	if u := user.Current(context); u != nil {
		board.UserID = u.ID
	}
	board.Key, _ = datastore.Put(context, datastore.NewIncompleteKey(context, BoardKind, nil), &board)
	board.ServeHTTP(writer, request)
}

func clearBoard(w http.ResponseWriter, r *http.Request) {
	keyString := mux.Vars(r)["board"]
	key, err := datastore.DecodeKey(keyString)
	if err != nil || key.Kind() != BoardKind {
		http.Error(w, "Invalid board key: "+keyString, http.StatusBadRequest)
		return
	}
	c := appengine.NewContext(r)
	board := Board{Key: key}
	if err = datastore.Get(c, key, &board); err != nil {
		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
		return
	}
	keys := []*datastore.Key{}
	t := datastore.NewQuery(MetricKind).Ancestor(key).KeysOnly().Run(c)
	keyChan := make(chan *datastore.Key, 50)
	go func() {
		for {
			key, err := t.Next(nil)
			if err == datastore.Done {
				keyChan <- key
				break
			} else if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				break
			}
		}
		close(keyChan)
	}()
	errChan := make(chan error, 50)
	go func() {
		for key := range keyChan {
			go func(key *datastore.Key) {
				errChan <- datastore.Delete(c, key)
			}(key)
		}
		close(errChan)
	}()
	for err := range errChan {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	board.ServeHTTP(w, r)
}
<commit_msg>Need to figure out err chan better<commit_after>package performanceboard

import (
	"appengine"
	"appengine/datastore"
	"appengine/user"
	"github.com/gorilla/mux"
	"net/http"
)

const BoardKind = "Board"

type Board struct {
	Key    *datastore.Key `datastore:"-"`
	UserID string
}

func (board *Board) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
	api, _ := router.Get("board").URL("board", board.Key.Encode())
	JsonResponse{
		"board": board.Key.Encode(),
		"api":   AbsURL(*api, request),
	}.Write(writer)
}

// HTTP handlers

func createBoard(writer http.ResponseWriter, request *http.Request) {
	context := appengine.NewContext(request)
	board := Board{}
	if u := user.Current(context); u != nil {
		board.UserID = u.ID
	}
	board.Key, _ = datastore.Put(context, datastore.NewIncompleteKey(context, BoardKind, nil), &board)
	board.ServeHTTP(writer, request)
}

func clearBoard(w http.ResponseWriter, r *http.Request) {
	keyString := mux.Vars(r)["board"]
	key, err := datastore.DecodeKey(keyString)
	if err != nil || key.Kind() != BoardKind {
		http.Error(w, "Invalid board key: "+keyString, http.StatusBadRequest)
		return
	}
	c := appengine.NewContext(r)
	board := Board{Key: key}
	if err = datastore.Get(c, key, &board); err != nil {
		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
		return
	}
	t := datastore.NewQuery(MetricKind).Ancestor(key).KeysOnly().Run(c)

	// Deletes metrics concurrently with 20 goroutines
	keyChan := make(chan *datastore.Key)
	errChan := make(chan error, 20)
	go func() {
		for {
			key, err := t.Next(nil)
			if err == datastore.Done {
				break
			} else if err != nil {
				errChan <- err
				break
			}
			keyChan <- key
		}
		close(keyChan)
	}()
	n := 0
	for key := range keyChan {
		go func(key *datastore.Key) {
			errChan <- datastore.Delete(c, key)
		}(key)
		n++
	}
	for i := 0; i < n; i++ {
		if err := <- errChan; err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	}
	board.ServeHTTP(w, r)
}
<|endoftext|>


<commit_before>package elkrem

import (
	"fmt"

	"github.com/btcsuite/btcd/wire"
)

/* elkrem is a simpler alternative to the 64 dimensional sha-chain.
it's basically a reverse merkle tree.  If we want to provide 2**64 possible
hashes, this requires a worst case computation of 63 hashes for the
sender, and worst-case storage of 64 hashes for the receiver.

The operations are left hash L() and right hash R(), which are
hash(parent)  and  hash(parent, 1)  respectively.  (concatenate one byte)

Here is a shorter example of a tree with 8 leaves and 15 total nodes.

The sender first computes the bottom left leaf 0b0000.  This is
L(L(L(L(root)))).  The receiver stores leaf 0.

Next the sender computes 0b0001.  R(L(L(L(root)))).  Receiver stores.
Next sender computes 0b1000 (8).  L(L(L(root))).  Receiver stores this, and
discards leaves 0b0000 and 0b0001, as they have the parent node 8.

For total hashes (2**h)-1 requires a tree of height h.

Sender:
as state, must store 1 hash (root) and current index (h bits)
to move to the next index, compute at most h hashes.

Receiver:
as state, must store at most h+1 hashes and the index of each hash (h*(h+1)) bits
to compute a previous index, compute at most h hashes.
*/

// You can calculate h from i but I can't figure out how without taking
// O(i) ops.  Feels like there should be a clever O(h) way.  1 byte, whatever.
type ElkremNode struct {
	h   uint8         // height of this node
	i   uint64        // index (ith node)
	sha *wire.ShaHash // hash
}
type ElkremSender struct {
	treeHeight uint8         // height of tree (size is 2**height -1 )
	current    uint64        // last sent hash index
	maxIndex   uint64        // top of the tree
	root       *wire.ShaHash // root hash of the tree
}
type ElkremReceiver struct {
	current    uint64       // last received index (actually don't need it?)
	treeHeight uint8        // height of tree (size is 2**height -1 )
	s          []ElkremNode // store of received hashes, max size = height
}

func LeftSha(in wire.ShaHash) wire.ShaHash {
	return wire.DoubleSha256SH(in.Bytes()) // left is sha(sha(in))
}
func RightSha(in wire.ShaHash) wire.ShaHash {
	return wire.DoubleSha256SH(append(in.Bytes(), 0x01)) // sha(sha(in, 1))
}

// iterative descent of sub-tree. w = hash number you want. i = input index
// h = height of input index. sha = input hash
func descend(w, i uint64, h uint8, sha wire.ShaHash) (wire.ShaHash, error) {
	for w < i {
		if w <= i-(1<<h) { // left
			sha = LeftSha(sha)
			i = i - (1 << h) // left descent reduces index by 2**h
		} else { // right
			sha = RightSha(sha)
			i-- // right descent reduces index by 1
		}
		if h == 0 { // avoid underflowing h
			break
		}
		h-- // either descent reduces height by 1
	}
	if w != i { // somehow couldn't / didn't end up where we wanted to go
		return sha, fmt.Errorf("can't generate index %d from %d", w, i)
	}
	return sha, nil
}

// Creates an Elkrem Sender from a root hash and tree height
func NewElkremSender(th uint8, r wire.ShaHash) ElkremSender {
	var e ElkremSender
	e.root = &r
	e.treeHeight = th
	// set max index based on tree height
	for j := uint8(0); j <= e.treeHeight; j++ {
		e.maxIndex = e.maxIndex<<1 | 1
	}
	e.maxIndex-- // 1 less than 2**h
	return e
}

// Creates an Elkrem Receiver from a tree height
func NewElkremReceiver(th uint8) ElkremReceiver {
	var e ElkremReceiver
	e.treeHeight = th
	return e
}

// Next() increments the index to the next hash and outputs it
func (e *ElkremSender) Next() (*wire.ShaHash, error) {
	sha, err := e.AtIndex(e.current)
	if err != nil {
		return nil, err
	}
	// increment index for next time
	e.current++
	return sha, nil
}

// w is the wanted index, i is the root index
func (e *ElkremSender) AtIndex(w uint64) (*wire.ShaHash, error) {
	out, err := descend(w, e.maxIndex, e.treeHeight, *e.root)
	return &out, err
}

func (e *ElkremReceiver) AddNext(sha *wire.ShaHash) error {
	// note: careful about atomicity / disk writes here
	var n ElkremNode
	n.sha = sha
	t := len(e.s) - 1                    // top of stack
	if t > 0 && e.s[t-1].h == e.s[t].h { // top 2 elements are equal height
		// next node must be parent; verify and remove children
		n.h = e.s[t].h + 1             // assign height
		l := LeftSha(*sha)             // calc l child
		r := RightSha(*sha)            // calc r child
		if !e.s[t-1].sha.IsEqual(&l) { // test l child
			return fmt.Errorf("left child doesn't match, expect %s got %s",
				e.s[t-1].sha.String(), l.String())
		}
		if !e.s[t].sha.IsEqual(&r) { // test r child
			return fmt.Errorf("right child doesn't match, expect %s got %s",
				e.s[t].sha.String(), r.String())
		}
		e.s = e.s[:len(e.s)-2] // l and r children OK, remove them
	} // if that didn't happen, height defaults to 0
	e.current++          // increment current index
	n.i = e.current      // set new node to that incremented index
	e.s = append(e.s, n) // append new node to stack
	return nil
}
func (e *ElkremReceiver) AtIndex(w uint64) (*wire.ShaHash, error) {
	var out ElkremNode      // node we will eventually return
	for _, n := range e.s { // go through stack
		if w <= n.i { // found one bigger than or equal to what we want
			out = n
			break
		}
	}
	if out.sha == nil { // didn't find anything
		return nil, fmt.Errorf("receiver has max %d, less than requested %d",
			e.s[len(e.s)-1].i, w)
	}
	sha, err := descend(w, out.i, out.h, *out.sha)
	return &sha, err
}
<commit_msg>actually remove current from sender and calc on the fly for addnext<commit_after>package elkrem

import (
	"fmt"

	"github.com/btcsuite/btcd/wire"
)

/* elkrem is a simpler alternative to the 64 dimensional sha-chain.
it's basically a reverse merkle tree.  If we want to provide 2**64 possible
hashes, this requires a worst case computation of 63 hashes for the
sender, and worst-case storage of 64 hashes for the receiver.

The operations are left hash L() and right hash R(), which are
hash(parent)  and  hash(parent, 1)  respectively.  (concatenate one byte)

Here is a shorter example of a tree with 8 leaves and 15 total nodes.

The sender first computes the bottom left leaf 0b0000.  This is
L(L(L(L(root)))).  The receiver stores leaf 0.

Next the sender computes 0b0001.  R(L(L(L(root)))).  Receiver stores.
Next sender computes 0b1000 (8).  L(L(L(root))).  Receiver stores this, and
discards leaves 0b0000 and 0b0001, as they have the parent node 8.

For total hashes (2**h)-1 requires a tree of height h.

Sender:
as state, must store 1 hash (root) and current index (h bits)
to move to the next index, compute at most h hashes.

Receiver:
as state, must store at most h+1 hashes and the index of each hash (h*(h+1)) bits
to compute a previous index, compute at most h hashes.
*/

// You can calculate h from i but I can't figure out how without taking
// O(i) ops.  Feels like there should be a clever O(h) way.  1 byte, whatever.
type ElkremNode struct {
	h   uint8         // height of this node
	i   uint64        // index (ith node)
	sha *wire.ShaHash // hash
}
type ElkremSender struct {
	treeHeight uint8         // height of tree (size is 2**height -1 )
	current    uint64        // last sent hash index
	maxIndex   uint64        // top of the tree
	root       *wire.ShaHash // root hash of the tree
}
type ElkremReceiver struct {
	treeHeight uint8        // height of tree (size is 2**height -1 )
	s          []ElkremNode // store of received hashes, max size = height
}

func LeftSha(in wire.ShaHash) wire.ShaHash {
	return wire.DoubleSha256SH(in.Bytes()) // left is sha(sha(in))
}
func RightSha(in wire.ShaHash) wire.ShaHash {
	return wire.DoubleSha256SH(append(in.Bytes(), 0x01)) // sha(sha(in, 1))
}

// iterative descent of sub-tree. w = hash number you want. i = input index
// h = height of input index. sha = input hash
func descend(w, i uint64, h uint8, sha wire.ShaHash) (wire.ShaHash, error) {
	for w < i {
		if w <= i-(1<<h) { // left
			sha = LeftSha(sha)
			i = i - (1 << h) // left descent reduces index by 2**h
		} else { // right
			sha = RightSha(sha)
			i-- // right descent reduces index by 1
		}
		if h == 0 { // avoid underflowing h
			break
		}
		h-- // either descent reduces height by 1
	}
	if w != i { // somehow couldn't / didn't end up where we wanted to go
		return sha, fmt.Errorf("can't generate index %d from %d", w, i)
	}
	return sha, nil
}

// Creates an Elkrem Sender from a root hash and tree height
func NewElkremSender(th uint8, r wire.ShaHash) ElkremSender {
	var e ElkremSender
	e.root = &r
	e.treeHeight = th
	// set max index based on tree height
	for j := uint8(0); j <= e.treeHeight; j++ {
		e.maxIndex = e.maxIndex<<1 | 1
	}
	e.maxIndex-- // 1 less than 2**h
	return e
}

// Creates an Elkrem Receiver from a tree height
func NewElkremReceiver(th uint8) ElkremReceiver {
	var e ElkremReceiver
	e.treeHeight = th
	return e
}

// Next() increments the index to the next hash and outputs it
func (e *ElkremSender) Next() (*wire.ShaHash, error) {
	sha, err := e.AtIndex(e.current)
	if err != nil {
		return nil, err
	}
	// increment index for next time
	e.current++
	return sha, nil
}

// w is the wanted index, i is the root index
func (e *ElkremSender) AtIndex(w uint64) (*wire.ShaHash, error) {
	out, err := descend(w, e.maxIndex, e.treeHeight, *e.root)
	return &out, err
}

func (e *ElkremReceiver) AddNext(sha *wire.ShaHash) error {
	// note: careful about atomicity / disk writes here
	var n ElkremNode
	n.sha = sha
	t := len(e.s) - 1 // top of stack
	if t > 0 {        // if this is not the first hash
		n.i = e.s[t].i + 1 // incoming index is tip of stack index + 1
	}
	if t > 0 && e.s[t-1].h == e.s[t].h { // top 2 elements are equal height
		// next node must be parent; verify and remove children
		n.h = e.s[t].h + 1             // assign height
		l := LeftSha(*sha)             // calc l child
		r := RightSha(*sha)            // calc r child
		if !e.s[t-1].sha.IsEqual(&l) { // test l child
			return fmt.Errorf("left child doesn't match, expect %s got %s",
				e.s[t-1].sha.String(), l.String())
		}
		if !e.s[t].sha.IsEqual(&r) { // test r child
			return fmt.Errorf("right child doesn't match, expect %s got %s",
				e.s[t].sha.String(), r.String())
		}
		e.s = e.s[:len(e.s)-2] // l and r children OK, remove them
	} // if that didn't happen, height defaults to 0
	e.s = append(e.s, n) // append new node to stack
	return nil
}
func (e *ElkremReceiver) AtIndex(w uint64) (*wire.ShaHash, error) {
	var out ElkremNode      // node we will eventually return
	for _, n := range e.s { // go through stack
		if w <= n.i { // found one bigger than or equal to what we want
			out = n
			break
		}
	}
	if out.sha == nil { // didn't find anything
		return nil, fmt.Errorf("receiver has max %d, less than requested %d",
			e.s[len(e.s)-1].i, w)
	}
	sha, err := descend(w, out.i, out.h, *out.sha)
	return &sha, err
}
<|endoftext|>


<commit_before>package parser

import (
	"fmt"

	"github.com/adnanh/hookman/lexer"
	"github.com/adnanh/webhook/hook"
)

// Parser is a struct that contains Lexer and the GeneratedRule
type Parser struct {
	Lexer         *lexer.Lexer
	GeneratedRule *hook.Rules
}

// New returns a new instance of Parser for given input string
func New(input string) *Parser {
	return &Parser{Lexer: lexer.New(input), GeneratedRule: nil}
}

// Parse performs lexical analysis of the input string and generates rules based on the lexer output
func (parser *Parser) Parse() error {
	if errors := parser.Lexer.Lex(); len(errors) > 0 {
		return fmt.Errorf("Error while parsing input string: %s", errors[0])
	}

	return nil
}
<commit_msg>parser<commit_after>package parser

import (
	"fmt"

	"github.com/adnanh/hookman/lexer"
	"github.com/adnanh/webhook/hook"
)

// Parser is a struct that contains Lexer and the GeneratedRule
type Parser struct {
	Lexer         *lexer.Lexer
	GeneratedRule *hook.Rules
}

// New returns a new instance of Parser for given input string
func New(input string) *Parser {
	return &Parser{Lexer: lexer.New(input), GeneratedRule: &hook.Rules{}}
}

// Parse performs lexical analysis of the input string and generates rules based on the lexer output
func (parser *Parser) Parse() error {
	if errors := parser.Lexer.Lex(); len(errors) > 0 {
		return fmt.Errorf("Error while parsing input string: %s", errors[0])
	}

	//var nextRule *hook.Rules

	tokens := parser.Lexer.Tokens

	for i := 0; i < len(tokens); i++ {
		fmt.Printf("%+v\n", tokens[i])
	}

	return nil
}
<|endoftext|>


<commit_before>// Package parser implements all the parser functionality for Makefiles. This
// is supposed to be a parser with a very small feature set that just supports
// what is needed to do linting and checking and not actual full Makefile
// parsing. And it's handrolled because apparently GNU Make doesn't have a
// grammar (see http://www.mail-archive.com/help-make@gnu.org/msg02778.html)
package parser

import (
	"errors"
	"regexp"
	"strings"
)

// Makefile provides a data structure to describe a parsed Makefile
type Makefile struct {
	Rules     RuleList
	Variables VariableList
}

// Rule represents a Make rule
type Rule struct {
	Target       string
	Dependencies []string
	Body         []string
	LineNumber   int
}

// RuleList represents a list of rules
type RuleList []Rule

// Variable represents a Make variable
type Variable struct {
	Name            string
	SimplyExpanded  bool
	Assignment      string
	SpecialVariable bool
	LineNumber      int
}

// VariableList represents a list of variables
type VariableList []Variable

var (
	reFindRule             = regexp.MustCompile("^([a-zA-Z]+):(.*)")
	reFindRuleBody         = regexp.MustCompile("^\t+(.*)")
	reFindSimpleVariable   = regexp.MustCompile("^([a-zA-Z]+) ?:=(.*)")
	reFindExpandedVariable = regexp.MustCompile("^([a-zA-Z]+) ?=(.*)")
	reFindSpecialVariable  = regexp.MustCompile("^\\.([a-zA-Z_]+):(.*)")
)

// Parse is the main function to parse a Makefile from a file path string to a
// Makefile struct. This function should be kept fairly small and ideally most
// of the heavy lifting will live in the specific parsing functions below that
// know how to deal with individual lines.
func Parse(filepath string) (ret Makefile, err error) {

	var scanner *MakefileScanner
	scanner, err = NewMakefileScanner(filepath)
	if err != nil {
		return ret, err
	}

	for {
		switch {
		case strings.HasPrefix(scanner.Text(), "#"):
			// parse comments here, ignoring them for now
			scanner.Scan()
		case strings.HasPrefix(scanner.Text(), "."):
			if matches := reFindSpecialVariable.FindStringSubmatch(scanner.Text()); matches != nil {
				specialVar := Variable{
					Name:            strings.TrimSpace(matches[1]),
					Assignment:      strings.TrimSpace(matches[2]),
					SpecialVariable: true,
					LineNumber:      scanner.LineNumber}
				ret.Variables = append(ret.Variables, specialVar)
			}
			scanner.Scan()
		default:
			// parse target or variable here, the function advances the scanner
			// itself to be able to detect rule bodies
			ruleOrVariable, parseError := parseRuleOrVariable(scanner)
			if parseError != nil {
				return ret, parseError
			}
			switch ruleOrVariable.(type) {
			case Rule:
				rule, found := ruleOrVariable.(Rule)
				if found != true {
					return ret, errors.New("Parse error")
				}
				ret.Rules = append(ret.Rules, rule)
			case Variable:
				variable, found := ruleOrVariable.(Variable)
				if found != true {
					return ret, errors.New("Parse error")
				}
				ret.Variables = append(ret.Variables, variable)
			}
		}

		if scanner.Finished == true {
			return
		}
	}
}

// parseRuleOrVariable gets the parsing scanner in a state where it resides on
// a line that could be a variable or a rule. The function parses the line and
// subsequent lines if there is a rule body to parse and returns an interface
// that is either a Variable or a Rule struct and leaves the scanner in a
// state where it resides on the first line after the content parsed into the
// returned struct. The parsing of line details is done via regexing for now
// since it seems ok as a first pass but will likely have to change later into
// a proper lexer/parser setup.
func parseRuleOrVariable(scanner *MakefileScanner) (ret interface{}, err error) {
	line := scanner.Text()

	if matches := reFindRule.FindStringSubmatch(line); matches != nil {
		// we found a rule so we need to advance the scanner to figure out if
		// there is a body
		scanner.Scan()
		bodyMatches := reFindRuleBody.FindStringSubmatch(scanner.Text())
		ruleBody := make([]string, 0, 20)
		for bodyMatches != nil {

			ruleBody = append(ruleBody, strings.TrimSpace(bodyMatches[1]))

			// done parsing the rule body line, advance the scanner and potentially
			// go into the next loop iteration
			scanner.Scan()
			bodyMatches = reFindRuleBody.FindStringSubmatch(scanner.Text())
		}
		// trim whitespace from all dependencies
		deps := strings.Split(matches[2], " ")
		filteredDeps := make([]string, 0, cap(deps))

		for idx := range deps {
			item := strings.TrimSpace(deps[idx])
			if item != "" {
				filteredDeps = append(filteredDeps, item)
			}
		}
		ret = Rule{
			Target:       strings.TrimSpace(matches[1]),
			Dependencies: filteredDeps,
			Body:         ruleBody,
			LineNumber:   scanner.LineNumber}
	} else if matches := reFindSimpleVariable.FindStringSubmatch(line); matches != nil {
		ret = Variable{
			Name:           strings.TrimSpace(matches[1]),
			Assignment:     strings.TrimSpace(matches[2]),
			SimplyExpanded: true,
			LineNumber:     scanner.LineNumber}
		scanner.Scan()
	} else if matches := reFindExpandedVariable.FindStringSubmatch(line); matches != nil {
		ret = Variable{
			Name:           strings.TrimSpace(matches[1]),
			Assignment:     strings.TrimSpace(matches[2]),
			SimplyExpanded: false,
			LineNumber:     scanner.LineNumber}
		scanner.Scan()
	} else {
		scanner.Scan()
	}

	return
}
<commit_msg>add debug logging to parser<commit_after>// Package parser implements all the parser functionality for Makefiles. This
// is supposed to be a parser with a very small feature set that just supports
// what is needed to do linting and checking and not actual full Makefile
// parsing. And it's handrolled because apparently GNU Make doesn't have a
// grammar (see http://www.mail-archive.com/help-make@gnu.org/msg02778.html)
package parser

import (
	"errors"
	"fmt"
	"github.com/mrtazz/checkmake/logger"
	"regexp"
	"strings"
)

// Makefile provides a data structure to describe a parsed Makefile
type Makefile struct {
	Rules     RuleList
	Variables VariableList
}

// Rule represents a Make rule
type Rule struct {
	Target       string
	Dependencies []string
	Body         []string
	LineNumber   int
}

// RuleList represents a list of rules
type RuleList []Rule

// Variable represents a Make variable
type Variable struct {
	Name            string
	SimplyExpanded  bool
	Assignment      string
	SpecialVariable bool
	LineNumber      int
}

// VariableList represents a list of variables
type VariableList []Variable

var (
	reFindRule             = regexp.MustCompile("^([a-zA-Z]+):(.*)")
	reFindRuleBody         = regexp.MustCompile("^\t+(.*)")
	reFindSimpleVariable   = regexp.MustCompile("^([a-zA-Z]+) ?:=(.*)")
	reFindExpandedVariable = regexp.MustCompile("^([a-zA-Z]+) ?=(.*)")
	reFindSpecialVariable  = regexp.MustCompile("^\\.([a-zA-Z_]+):(.*)")
)

// Parse is the main function to parse a Makefile from a file path string to a
// Makefile struct. This function should be kept fairly small and ideally most
// of the heavy lifting will live in the specific parsing functions below that
// know how to deal with individual lines.
func Parse(filepath string) (ret Makefile, err error) {

	var scanner *MakefileScanner
	scanner, err = NewMakefileScanner(filepath)
	if err != nil {
		return ret, err
	}

	for {
		switch {
		case strings.HasPrefix(scanner.Text(), "#"):
			// parse comments here, ignoring them for now
			scanner.Scan()
		case strings.HasPrefix(scanner.Text(), "."):
			if matches := reFindSpecialVariable.FindStringSubmatch(scanner.Text()); matches != nil {
				specialVar := Variable{
					Name:            strings.TrimSpace(matches[1]),
					Assignment:      strings.TrimSpace(matches[2]),
					SpecialVariable: true,
					LineNumber:      scanner.LineNumber}
				ret.Variables = append(ret.Variables, specialVar)
			}
			scanner.Scan()
		default:
			// parse target or variable here, the function advances the scanner
			// itself to be able to detect rule bodies
			ruleOrVariable, parseError := parseRuleOrVariable(scanner)
			if parseError != nil {
				return ret, parseError
			}
			switch ruleOrVariable.(type) {
			case Rule:
				rule, found := ruleOrVariable.(Rule)
				if found != true {
					return ret, errors.New("Parse error")
				}
				ret.Rules = append(ret.Rules, rule)
			case Variable:
				variable, found := ruleOrVariable.(Variable)
				if found != true {
					return ret, errors.New("Parse error")
				}
				ret.Variables = append(ret.Variables, variable)
			}
		}

		if scanner.Finished == true {
			return
		}
	}
}

// parseRuleOrVariable gets the parsing scanner in a state where it resides on
// a line that could be a variable or a rule. The function parses the line and
// subsequent lines if there is a rule body to parse and returns an interface
// that is either a Variable or a Rule struct and leaves the scanner in a
// state where it resides on the first line after the content parsed into the
// returned struct. The parsing of line details is done via regexing for now
// since it seems ok as a first pass but will likely have to change later into
// a proper lexer/parser setup.
func parseRuleOrVariable(scanner *MakefileScanner) (ret interface{}, err error) {
	line := scanner.Text()

	if matches := reFindRule.FindStringSubmatch(line); matches != nil {
		// we found a rule so we need to advance the scanner to figure out if
		// there is a body
		scanner.Scan()
		bodyMatches := reFindRuleBody.FindStringSubmatch(scanner.Text())
		ruleBody := make([]string, 0, 20)
		for bodyMatches != nil {

			ruleBody = append(ruleBody, strings.TrimSpace(bodyMatches[1]))

			// done parsing the rule body line, advance the scanner and potentially
			// go into the next loop iteration
			scanner.Scan()
			bodyMatches = reFindRuleBody.FindStringSubmatch(scanner.Text())
		}
		// trim whitespace from all dependencies
		deps := strings.Split(matches[2], " ")
		filteredDeps := make([]string, 0, cap(deps))

		for idx := range deps {
			item := strings.TrimSpace(deps[idx])
			if item != "" {
				filteredDeps = append(filteredDeps, item)
			}
		}
		ret = Rule{
			Target:       strings.TrimSpace(matches[1]),
			Dependencies: filteredDeps,
			Body:         ruleBody,
			LineNumber:   scanner.LineNumber}
	} else if matches := reFindSimpleVariable.FindStringSubmatch(line); matches != nil {
		ret = Variable{
			Name:           strings.TrimSpace(matches[1]),
			Assignment:     strings.TrimSpace(matches[2]),
			SimplyExpanded: true,
			LineNumber:     scanner.LineNumber}
		scanner.Scan()
	} else if matches := reFindExpandedVariable.FindStringSubmatch(line); matches != nil {
		ret = Variable{
			Name:           strings.TrimSpace(matches[1]),
			Assignment:     strings.TrimSpace(matches[2]),
			SimplyExpanded: false,
			LineNumber:     scanner.LineNumber}
		scanner.Scan()
	} else {
		logger.Debug(fmt.Sprintf("Unable to match line '%s' to a Rule or Variable", line))
		scanner.Scan()
	}

	return
}
<|endoftext|>


<commit_before>// Copyright 2016 The clang-server Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package parser

import (
	"encoding/binary"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/go-clang/v3.9/clang"
	"github.com/pkg/errors"
	"github.com/pkgutil/osutil"
	"github.com/pkgutil/stringsutil"
	"github.com/zchee/clang-server/compilationdatabase"
	"github.com/zchee/clang-server/indexdb"
	"github.com/zchee/clang-server/internal/log"
	"github.com/zchee/clang-server/internal/pathutil"
	"github.com/zchee/clang-server/parser/builtinheader"
	"github.com/zchee/clang-server/symbol"
)

// defaultClangOption defalut global clang options.
// clang.TranslationUnit_DetailedPreprocessingRecord = 0x01
// clang.TranslationUnit_Incomplete = 0x02
// clang.TranslationUnit_PrecompiledPreamble = 0x04
// clang.TranslationUnit_CacheCompletionResults = 0x08
// clang.TranslationUnit_ForSerialization = 0x10
// clang.TranslationUnit_CXXChainedPCH = 0x20
// clang.TranslationUnit_SkipFunctionBodies = 0x40
// clang.TranslationUnit_IncludeBriefCommentsInCodeCompletion = 0x80
// clang.TranslationUnit_CreatePreambleOnFirstParse = 0x100
// clang.TranslationUnit_KeepGoing = 0x200
// const defaultClangOption uint32 = 0x445 // Use all flags for now
var defaultClangOption = clang.DefaultEditingTranslationUnitOptions() | uint32(clang.TranslationUnit_KeepGoing)

// Parser represents a C/C++ AST parser.
type Parser struct {
	root        string
	clangOption uint32

	idx    clang.Index
	cd     *compilationdatabase.CompilationDatabase
	db     *indexdb.IndexDB
	server *symbol.Server

	dispatcher *dispatcher

	debugUncatched bool                     // for debug
	uncachedKind   map[clang.CursorKind]int // for debug
}

// Config represents a parser config.
type Config struct {
	Root        string
	JSONName    string
	PathRange   []string
	ClangOption uint32

	Debug bool
}

// NewParser return the new Parser.
func NewParser(path string, config *Config) *Parser {
	root := config.Root
	if root == "" {
		proot, err := pathutil.FindProjectRoot(path)
		if err != nil {
			log.Fatal(err)
		}
		root = proot
	}

	cd := compilationdatabase.NewCompilationDatabase(root)
	if err := cd.Parse(config.JSONName, config.PathRange); err != nil {
		log.Fatal(err)
	}

	db, err := indexdb.NewIndexDB(root)
	if err != nil {
		log.Fatal(err)
	}

	clangOption := config.ClangOption
	if clangOption == 0 {
		clangOption = defaultClangOption
	}

	p := &Parser{
		root:        root,
		clangOption: clangOption,
		idx:         clang.NewIndex(0, 0), // disable excludeDeclarationsFromPCH, enable displayDiagnostics
		cd:          cd,
		db:          db,
		server:      symbol.NewServer(),
	}

	if config.Debug {
		p.debugUncatched = true
		p.uncachedKind = make(map[clang.CursorKind]int)
	}

	if err := CreateBulitinHeaders(); err != nil {
		log.Fatal(err)
	}

	return p
}

// CreateBulitinHeaders creates(dumps) a clang builtin header to cache directory.
func CreateBulitinHeaders() error {
	builtinHdrDir := filepath.Join(pathutil.CacheDir(), "clang", "include")
	if !osutil.IsExist(builtinHdrDir) {
		if err := os.MkdirAll(builtinHdrDir, 0700); err != nil {
			return errors.WithStack(err)
		}
	}

	for _, fname := range builtinheader.AssetNames() {
		data, err := builtinheader.AssetInfo(fname)
		if err != nil {
			return errors.WithStack(err)
		}

		if strings.Contains(data.Name(), string(filepath.Separator)) {
			dir, _ := filepath.Split(data.Name())
			if err := os.MkdirAll(filepath.Join(builtinHdrDir, dir), 0700); err != nil {
				return errors.WithStack(err)
			}
		}

		buf, err := builtinheader.Asset(data.Name())
		if err != nil {
			return errors.WithStack(err)
		}

		if err := ioutil.WriteFile(filepath.Join(builtinHdrDir, data.Name()), buf, 0600); err != nil {
			return errors.WithStack(err)
		}
	}

	return nil
}

// Parse parses the project directories.
func (p *Parser) Parse() {
	defer func() {
		p.db.Close()
		p.server.Serve()
	}()
	defer profile(time.Now(), "Parse")

	ccs := p.cd.CompileCommands()
	if len(ccs) == 0 {
		log.Fatal("not walk")
	}

	compilerConfig := p.cd.CompilerConfig
	flags := append(compilerConfig.SystemCIncludeDir, compilerConfig.SystemFrameworkDir...)

	// TODO(zchee): needs include stdint.h?
	if i := stringsutil.IndexContainsSlice(ccs[0].Arguments, "-std="); i > 0 {
		std := ccs[0].Arguments[i][5:]
		switch {
		case strings.HasPrefix(std, "c"), strings.HasPrefix(std, "gnu"):
			if std[len(std)-2] == '8' || std[len(std)-2] == '9' || std[len(std)-2] == '1' {
				flags = append(flags, "-include", "/usr/include/stdint.h")
			}
		}
	} else {
		flags = append(flags, "-include", "/usr/include/stdint.h")
	}
	if !(filepath.Ext(ccs[0].File) == ".c") {
		flags = append(flags, compilerConfig.SystemCXXIncludeDir...)
	}

	builtinHdrDir := filepath.Join(pathutil.CacheDir(), "clang", "include")
	flags = append(flags, "-I"+builtinHdrDir)

	p.dispatcher = newDispatcher(p.ParseFile)
	p.dispatcher.Start()
	for i := 0; i < len(ccs); i++ {
		args := ccs[i].Arguments
		args = append(flags, args...)
		p.dispatcher.Add(parseArg{ccs[i].File, args})
	}
	p.dispatcher.Wait()
}

type parseArg struct {
	filename string
	flag     []string
}

// ParseFile parses the C/C++ file.
func (p *Parser) ParseFile(arg parseArg) error {
	var tu clang.TranslationUnit

	if p.db.Has(arg.filename) {
		buf, err := p.db.Get(arg.filename)
		if err != nil {
			return err
		}

		data := symbol.GetRootAsFile(buf, 0)
		tu, err = deserializeTranslationUnit(p.idx, data.TranslationUnit())
		if err != nil {
			return err
		}
		defer tu.Dispose()

		log.Debugf("tu.Spelling(): %T => %+v\n", tu.Spelling(), tu.Spelling()) // for debug

		return nil
	}

	if cErr := p.idx.ParseTranslationUnit2(arg.filename, arg.flag, nil, p.clangOption, &tu); clang.ErrorCode(cErr) != clang.Error_Success {
		return errors.New(clang.ErrorCode(cErr).Spelling())
	}
	defer tu.Dispose()

	tuch := make(chan []byte, 1)
	go func() {
		tuch <- p.SerializeTranslationUnit(arg.filename, tu)
	}()

	// printDiagnostics(tu.Diagnostics())

	rootCursor := tu.TranslationUnitCursor()
	file := symbol.NewFile(arg.filename, arg.flag)
	visitNode := func(cursor, parent clang.Cursor) clang.ChildVisitResult {
		if cursor.IsNull() {
			log.Debug("cursor: <none>")
			return clang.ChildVisit_Continue
		}

		cursorLoc := symbol.FromCursor(cursor)
		if cursorLoc.FileName() == "" || cursorLoc.FileName() == "." {
			// TODO(zchee): Ignore system header(?)
			return clang.ChildVisit_Continue
		}

		kind := cursor.Kind()
		switch kind {
		case clang.Cursor_FunctionDecl, clang.Cursor_StructDecl, clang.Cursor_FieldDecl, clang.Cursor_TypedefDecl, clang.Cursor_EnumDecl, clang.Cursor_EnumConstantDecl:
			defCursor := cursor.Definition()
			if defCursor.IsNull() {
				file.AddDecl(cursorLoc)
			} else {
				defLoc := symbol.FromCursor(defCursor)
				file.AddDefinition(cursorLoc, defLoc)
			}
		case clang.Cursor_MacroDefinition:
			file.AddDefinition(cursorLoc, cursorLoc)
		case clang.Cursor_VarDecl:
			file.AddDecl(cursorLoc)
		case clang.Cursor_ParmDecl:
			if cursor.Spelling() != "" {
				file.AddDecl(cursorLoc)
			}
		case clang.Cursor_CallExpr:
			refCursor := cursor.Referenced()
			refLoc := symbol.FromCursor(refCursor)
			file.AddCaller(cursorLoc, refLoc, true)
		case clang.Cursor_DeclRefExpr, clang.Cursor_TypeRef, clang.Cursor_MemberRefExpr, clang.Cursor_MacroExpansion:
			refCursor := cursor.Referenced()
			refLoc := symbol.FromCursor(refCursor)
			file.AddCaller(cursorLoc, refLoc, false)
		case clang.Cursor_InclusionDirective:
			incFile := cursor.IncludedFile()
			file.AddHeader(cursor.Spelling(), incFile)
		default:
			if p.debugUncatched {
				p.uncachedKind[kind]++
			}
		}

		return clang.ChildVisit_Recurse
	}

	rootCursor.Visit(visitNode)
	file.AddTranslationUnit(<-tuch)
	buf := file.Serialize()

	out := symbol.GetRootAsFile(buf.FinishedBytes(), 0)
	printFile(out) // for debug

	log.Debugf("Goroutine:%d", runtime.NumGoroutine())
	log.Debugf("================== DONE: filename: %+v ==================\n\n\n", arg.filename)

	return p.db.Put(arg.filename, buf.FinishedBytes())
}

// SerializeTranslationUnit serialize the TranslationUnit to Clang serialized representation.
// TODO(zchee): Avoid ioutil.TempFile if possible.
func (p *Parser) SerializeTranslationUnit(filename string, tu clang.TranslationUnit) []byte {
	tmpFile, err := ioutil.TempFile(os.TempDir(), filepath.Base(filename))
	if err != nil {
		log.Fatal(err)
	}

	saveOptions := uint32(clang.TranslationUnit_KeepGoing)
	if cErr := tu.SaveTranslationUnit(tmpFile.Name(), saveOptions); clang.SaveError(cErr) != clang.SaveError_None {
		log.Fatal(clang.SaveError(cErr))
	}

	buf, err := ioutil.ReadFile(tmpFile.Name())
	if err != nil {
		log.Fatal(err)
	}
	os.Remove(tmpFile.Name())

	return buf
}

func deserializeTranslationUnit(idx clang.Index, buf []byte) (clang.TranslationUnit, error) {
	var tu clang.TranslationUnit

	tmpfile, err := ioutil.TempFile(os.TempDir(), "clang-server")
	if err != nil {
		return tu, err
	}
	binary.Write(tmpfile, binary.LittleEndian, buf)

	if err := idx.TranslationUnit2(tmpfile.Name(), &tu); clang.ErrorCode(err) != clang.Error_Success {
		return tu, errors.New(err.Spelling())
	}
	// finished create a translation unit from an AST file, remove tmpfile
	os.Remove(tmpfile.Name())

	return tu, nil
}

// ClangVersion return the current clang version.
func ClangVersion() string {
	return clang.GetClangVersion()
}
<commit_msg>parser: move deserializeTranslationUnit to Parser method and export<commit_after>// Copyright 2016 The clang-server Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package parser

import (
	"encoding/binary"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/go-clang/v3.9/clang"
	"github.com/pkg/errors"
	"github.com/pkgutil/osutil"
	"github.com/pkgutil/stringsutil"
	"github.com/zchee/clang-server/compilationdatabase"
	"github.com/zchee/clang-server/indexdb"
	"github.com/zchee/clang-server/internal/log"
	"github.com/zchee/clang-server/internal/pathutil"
	"github.com/zchee/clang-server/parser/builtinheader"
	"github.com/zchee/clang-server/symbol"
)

// defaultClangOption defalut global clang options.
// clang.TranslationUnit_DetailedPreprocessingRecord = 0x01
// clang.TranslationUnit_Incomplete = 0x02
// clang.TranslationUnit_PrecompiledPreamble = 0x04
// clang.TranslationUnit_CacheCompletionResults = 0x08
// clang.TranslationUnit_ForSerialization = 0x10
// clang.TranslationUnit_CXXChainedPCH = 0x20
// clang.TranslationUnit_SkipFunctionBodies = 0x40
// clang.TranslationUnit_IncludeBriefCommentsInCodeCompletion = 0x80
// clang.TranslationUnit_CreatePreambleOnFirstParse = 0x100
// clang.TranslationUnit_KeepGoing = 0x200
// const defaultClangOption uint32 = 0x445 // Use all flags for now
var defaultClangOption = clang.DefaultEditingTranslationUnitOptions() | uint32(clang.TranslationUnit_KeepGoing)

// Parser represents a C/C++ AST parser.
type Parser struct {
	root        string
	clangOption uint32

	idx    clang.Index
	cd     *compilationdatabase.CompilationDatabase
	db     *indexdb.IndexDB
	server *symbol.Server

	dispatcher *dispatcher

	debugUncatched bool                     // for debug
	uncachedKind   map[clang.CursorKind]int // for debug
}

// Config represents a parser config.
type Config struct {
	Root        string
	JSONName    string
	PathRange   []string
	ClangOption uint32

	Debug bool
}

// NewParser return the new Parser.
func NewParser(path string, config *Config) *Parser {
	root := config.Root
	if root == "" {
		proot, err := pathutil.FindProjectRoot(path)
		if err != nil {
			log.Fatal(err)
		}
		root = proot
	}

	cd := compilationdatabase.NewCompilationDatabase(root)
	if err := cd.Parse(config.JSONName, config.PathRange); err != nil {
		log.Fatal(err)
	}

	db, err := indexdb.NewIndexDB(root)
	if err != nil {
		log.Fatal(err)
	}

	clangOption := config.ClangOption
	if clangOption == 0 {
		clangOption = defaultClangOption
	}

	p := &Parser{
		root:        root,
		clangOption: clangOption,
		idx:         clang.NewIndex(0, 0), // disable excludeDeclarationsFromPCH, enable displayDiagnostics
		cd:          cd,
		db:          db,
		server:      symbol.NewServer(),
	}

	if config.Debug {
		p.debugUncatched = true
		p.uncachedKind = make(map[clang.CursorKind]int)
	}

	if err := CreateBulitinHeaders(); err != nil {
		log.Fatal(err)
	}

	return p
}

// CreateBulitinHeaders creates(dumps) a clang builtin header to cache directory.
func CreateBulitinHeaders() error {
	builtinHdrDir := filepath.Join(pathutil.CacheDir(), "clang", "include")
	if !osutil.IsExist(builtinHdrDir) {
		if err := os.MkdirAll(builtinHdrDir, 0700); err != nil {
			return errors.WithStack(err)
		}
	}

	for _, fname := range builtinheader.AssetNames() {
		data, err := builtinheader.AssetInfo(fname)
		if err != nil {
			return errors.WithStack(err)
		}

		if strings.Contains(data.Name(), string(filepath.Separator)) {
			dir, _ := filepath.Split(data.Name())
			if err := os.MkdirAll(filepath.Join(builtinHdrDir, dir), 0700); err != nil {
				return errors.WithStack(err)
			}
		}

		buf, err := builtinheader.Asset(data.Name())
		if err != nil {
			return errors.WithStack(err)
		}

		if err := ioutil.WriteFile(filepath.Join(builtinHdrDir, data.Name()), buf, 0600); err != nil {
			return errors.WithStack(err)
		}
	}

	return nil
}

// Parse parses the project directories.
func (p *Parser) Parse() {
	defer func() {
		p.db.Close()
		p.server.Serve()
	}()
	defer profile(time.Now(), "Parse")

	ccs := p.cd.CompileCommands()
	if len(ccs) == 0 {
		log.Fatal("not walk")
	}

	compilerConfig := p.cd.CompilerConfig
	flags := append(compilerConfig.SystemCIncludeDir, compilerConfig.SystemFrameworkDir...)

	// TODO(zchee): needs include stdint.h?
	if i := stringsutil.IndexContainsSlice(ccs[0].Arguments, "-std="); i > 0 {
		std := ccs[0].Arguments[i][5:]
		switch {
		case strings.HasPrefix(std, "c"), strings.HasPrefix(std, "gnu"):
			if std[len(std)-2] == '8' || std[len(std)-2] == '9' || std[len(std)-2] == '1' {
				flags = append(flags, "-include", "/usr/include/stdint.h")
			}
		}
	} else {
		flags = append(flags, "-include", "/usr/include/stdint.h")
	}
	if !(filepath.Ext(ccs[0].File) == ".c") {
		flags = append(flags, compilerConfig.SystemCXXIncludeDir...)
	}

	builtinHdrDir := filepath.Join(pathutil.CacheDir(), "clang", "include")
	flags = append(flags, "-I"+builtinHdrDir)

	p.dispatcher = newDispatcher(p.ParseFile)
	p.dispatcher.Start()
	for i := 0; i < len(ccs); i++ {
		args := ccs[i].Arguments
		args = append(flags, args...)
		p.dispatcher.Add(parseArg{ccs[i].File, args})
	}
	p.dispatcher.Wait()
}

type parseArg struct {
	filename string
	flag     []string
}

// ParseFile parses the C/C++ file.
func (p *Parser) ParseFile(arg parseArg) error {
	var tu clang.TranslationUnit

	if p.db.Has(arg.filename) {
		buf, err := p.db.Get(arg.filename)
		if err != nil {
			return err
		}

		data := symbol.GetRootAsFile(buf, 0)
		tu, err = p.DeserializeTranslationUnit(p.idx, data.TranslationUnit())
		if err != nil {
			return err
		}
		defer tu.Dispose()

		log.Debugf("tu.Spelling(): %T => %+v\n", tu.Spelling(), tu.Spelling()) // for debug

		return nil
	}

	if cErr := p.idx.ParseTranslationUnit2(arg.filename, arg.flag, nil, p.clangOption, &tu); clang.ErrorCode(cErr) != clang.Error_Success {
		return errors.New(clang.ErrorCode(cErr).Spelling())
	}
	defer tu.Dispose()

	tuch := make(chan []byte, 1)
	go func() {
		tuch <- p.SerializeTranslationUnit(arg.filename, tu)
	}()

	// printDiagnostics(tu.Diagnostics())

	rootCursor := tu.TranslationUnitCursor()
	file := symbol.NewFile(arg.filename, arg.flag)
	visitNode := func(cursor, parent clang.Cursor) clang.ChildVisitResult {
		if cursor.IsNull() {
			log.Debug("cursor: <none>")
			return clang.ChildVisit_Continue
		}

		cursorLoc := symbol.FromCursor(cursor)
		if cursorLoc.FileName() == "" || cursorLoc.FileName() == "." {
			// TODO(zchee): Ignore system header(?)
			return clang.ChildVisit_Continue
		}

		kind := cursor.Kind()
		switch kind {
		case clang.Cursor_FunctionDecl, clang.Cursor_StructDecl, clang.Cursor_FieldDecl, clang.Cursor_TypedefDecl, clang.Cursor_EnumDecl, clang.Cursor_EnumConstantDecl:
			defCursor := cursor.Definition()
			if defCursor.IsNull() {
				file.AddDecl(cursorLoc)
			} else {
				defLoc := symbol.FromCursor(defCursor)
				file.AddDefinition(cursorLoc, defLoc)
			}
		case clang.Cursor_MacroDefinition:
			file.AddDefinition(cursorLoc, cursorLoc)
		case clang.Cursor_VarDecl:
			file.AddDecl(cursorLoc)
		case clang.Cursor_ParmDecl:
			if cursor.Spelling() != "" {
				file.AddDecl(cursorLoc)
			}
		case clang.Cursor_CallExpr:
			refCursor := cursor.Referenced()
			refLoc := symbol.FromCursor(refCursor)
			file.AddCaller(cursorLoc, refLoc, true)
		case clang.Cursor_DeclRefExpr, clang.Cursor_TypeRef, clang.Cursor_MemberRefExpr, clang.Cursor_MacroExpansion:
			refCursor := cursor.Referenced()
			refLoc := symbol.FromCursor(refCursor)
			file.AddCaller(cursorLoc, refLoc, false)
		case clang.Cursor_InclusionDirective:
			incFile := cursor.IncludedFile()
			file.AddHeader(cursor.Spelling(), incFile)
		default:
			if p.debugUncatched {
				p.uncachedKind[kind]++
			}
		}

		return clang.ChildVisit_Recurse
	}

	rootCursor.Visit(visitNode)
	file.AddTranslationUnit(<-tuch)
	buf := file.Serialize()

	out := symbol.GetRootAsFile(buf.FinishedBytes(), 0)
	printFile(out) // for debug

	log.Debugf("Goroutine:%d", runtime.NumGoroutine())
	log.Debugf("================== DONE: filename: %+v ==================\n\n\n", arg.filename)

	return p.db.Put(arg.filename, buf.FinishedBytes())
}

// SerializeTranslationUnit serialize the TranslationUnit to Clang serialized representation.
// TODO(zchee): Avoid ioutil.TempFile if possible.
func (p *Parser) SerializeTranslationUnit(filename string, tu clang.TranslationUnit) []byte {
	tmpFile, err := ioutil.TempFile(os.TempDir(), filepath.Base(filename))
	if err != nil {
		log.Fatal(err)
	}

	saveOptions := uint32(clang.TranslationUnit_KeepGoing)
	if cErr := tu.SaveTranslationUnit(tmpFile.Name(), saveOptions); clang.SaveError(cErr) != clang.SaveError_None {
		log.Fatal(clang.SaveError(cErr))
	}

	buf, err := ioutil.ReadFile(tmpFile.Name())
	if err != nil {
		log.Fatal(err)
	}
	os.Remove(tmpFile.Name())

	return buf
}

// DeserializeTranslationUnit deserialize the TranslationUnit from buf Clang serialized representation.
func (p *Parser) DeserializeTranslationUnit(idx clang.Index, buf []byte) (clang.TranslationUnit, error) {
	var tu clang.TranslationUnit

	tmpfile, err := ioutil.TempFile(os.TempDir(), "clang-server")
	if err != nil {
		return tu, err
	}
	binary.Write(tmpfile, binary.LittleEndian, buf)

	if err := idx.TranslationUnit2(tmpfile.Name(), &tu); clang.ErrorCode(err) != clang.Error_Success {
		return tu, errors.New(err.Spelling())
	}
	// finished create a translation unit from an AST file, remove tmpfile
	os.Remove(tmpfile.Name())

	return tu, nil
}

// ClangVersion return the current clang version.
func ClangVersion() string {
	return clang.GetClangVersion()
}
<|endoftext|>


<commit_before>package parser

import ()

type callbackFn func(key, val string)

type parser struct {
	lexer    *lexer
	item     item
	err      error
	callback callbackFn
}

func NewParser(input string) *parser {
	return &parser{lexer: lex(input)}
}

func (p *parser) next() {
	if p.item.token == itemEOF {
		return
	}

	for {
		p.item = p.lexer.nextItem()
		if p.item.token != itemComment {
			break
		}
	}
}

func (p *parser) accept(token rune) bool {
	if p.item.token == token {
		p.next()
		return true
	}

	return false
}

func (p *parser) expect(token rune) bool {
	if p.accept(token) {
		return true
	}

	p.err = &expectError{token}
	return false
}

func (p *parser) eof() bool {
	if p.item.token == itemEOF {
		return true
	}

	return false
}

func (p *parser) Parse(callback callbackFn) error {
	p.callback = callback
	p.next()

	for p.statement() || p.ignore() {
	}

	err := p.err
	if err == nil && !p.eof() {
		err = &parseError{p.item.val}
	}

	return err
}

func (p *parser) ignore() bool {
	ok := false

	for p.accept('\n') {
		ok = true
	}

	return ok
}

func (p *parser) statement() bool {
	p.accept(itemExport)

	k := p.item
	if p.key() {
		if p.accept('=') || p.accept(':') {
			s := ""

			v := p.item
			if p.value() {
				s = v.value()
			}

			if p.callback != nil {
				p.callback(k.value(), s)
			}
		}

		if p.accept('\n') || p.accept(itemEOF) {
			return true
		}
	}

	return false
}

func (p *parser) key() bool {
	return p.accept(itemIdentifier) || p.accept(itemString)
}

func (p *parser) value() bool {
	return p.accept(itemText) || p.accept(itemString)
}
<commit_msg>parser を修正<commit_after>package parser

import ()

type callbackFn func(key, val string)

type parser struct {
	lexer    *lexer
	item     item
	err      error
	callback callbackFn
}

func NewParser(input string) *parser {
	return &parser{lexer: lex(input)}
}

func (p *parser) next() {
	if p.item.token == itemEOF {
		return
	}

	for {
		p.item = p.lexer.nextItem()
		if p.item.token != itemComment {
			break
		}
	}
}

func (p *parser) accept(token rune) bool {
	if p.item.token == token {
		p.next()
		return true
	}

	return false
}

func (p *parser) expect(token rune) bool {
	if p.accept(token) {
		return true
	}

	p.err = &expectError{token}
	return false
}

func (p *parser) eof() bool {
	if p.item.token == itemEOF {
		return true
	}

	return false
}

func (p *parser) Parse(callback callbackFn) error {
	p.callback = callback
	p.next()

	for p.statement() || p.ignore() {
	}

	err := p.err
	if err == nil && !p.eof() {
		err = &parseError{p.item.val}
	}

	return err
}

func (p *parser) ignore() bool {
	ok := false

	for p.accept('\n') {
		ok = true
	}

	return ok
}

func (p *parser) statement() bool {
	switch {
	case p.accept(itemExport):
		p.expression()
		return true
	case p.expression():
		return true
	}

	return false
}

func (p *parser) expression() bool {
	k := p.item
	if p.key() {
		if p.accept('=') || p.accept(':') {
			s := ""

			v := p.item
			if p.value() {
				s = v.value()
			}

			if p.callback != nil {
				p.callback(k.value(), s)
			}
		}

		if p.accept('\n') || p.accept(itemEOF) {
			return true
		}
	}

	return false
}

func (p *parser) key() bool {
	return p.accept(itemIdentifier) || p.accept(itemString)
}

func (p *parser) value() bool {
	return p.accept(itemText) || p.accept(itemString)
}
<|endoftext|>


<commit_before>package arbreader

import (
	"encoding/json"
	"io"
	"strings"

	"github.com/juju/errgo"
)

type Message struct {
  Key string
  Type string
  Extended map[string]string
  Description string
  Value string
}

func Read(reader io.Reader) ([]*Message, error) {
	data := map[string]interface{}{}
	if err := json.NewDecoder(reader).Decode(&data); err != nil {
		return nil, errgo.Mask(err)
	}

	messages := []*Message{}
	for key, value := range data {
		if key[0] == '@' {
			continue
		}

		vv := data["@" + key].(map[string]interface{})

		extended := map[string]string{}
		for ekey, evalue := range vv {
			if !strings.HasPrefix(ekey, "x-") {
				continue
			}

			extended[ekey] = evalue.(string)
		}

		messages = append(messages, &Message{
			Key: key,
			Type: vv["type"].(string),
			Description: vv["description"].(string),
			Extended: extended,
			Value: value,
		})
	}

	return messages, nil
}
<commit_msg>Cast to string to save the value.<commit_after>package arbreader

import (
	"encoding/json"
	"io"
	"strings"

	"github.com/juju/errgo"
)

type Message struct {
  Key string
  Type string
  Extended map[string]string
  Description string
  Value string
}

func Read(reader io.Reader) ([]*Message, error) {
	data := map[string]interface{}{}
	if err := json.NewDecoder(reader).Decode(&data); err != nil {
		return nil, errgo.Mask(err)
	}

	messages := []*Message{}
	for key, value := range data {
		if key[0] == '@' {
			continue
		}

		vv := data["@" + key].(map[string]interface{})

		extended := map[string]string{}
		for ekey, evalue := range vv {
			if !strings.HasPrefix(ekey, "x-") {
				continue
			}

			extended[ekey] = evalue.(string)
		}

		messages = append(messages, &Message{
			Key: key,
			Type: vv["type"].(string),
			Description: vv["description"].(string),
			Extended: extended,
			Value: value.(string),
		})
	}

	return messages, nil
}
<|endoftext|>


<commit_before>// Go test

/*
  This is at test of the go syntax parser
*/

/*

Here is a function in a comment:
func main() int {
  fmt.Println("hello")
}
*/

int main(int argc, char *argv)
{
  printf("Hello");
}

package main
import (
  "fmt"
  "github.com/user/newmath"
)

// Main method
func main() {
  fmt.Println("Hello world\"")
  fmt.Println(`Hello world!
  that spans more than one line`)
  fmt.Println('l')
  fmt.Println('\'')
}

// A function
func Sqrt(x float64) float64 {
  z := 1.0
  for i := 0; i < 1000; i++ {
    z -= (z*z - x) / (2 * z)
  }
  return z
}

func foo() int {
  return 5;
}

func bar() {
  return 6;
}

func (p *parser) init(fset *token.Fileset, filename string, src []byte) (ret int) {
  p.file = fset.Addfile(filename, -1, len(src))
}
<commit_msg>Run gofmt on test.go<commit_after>// Go test

/*
This is at test of the go syntax parser
*/

/*

Here is a function in a comment:
func main() int {
fmt.Println("hello")
}
*/

package main

import (
	"fmt"
	"github.com/user/newmath"
)

// Main method
func main() {
	fmt.Println("Hello world\"")
	fmt.Println(`Hello world!
that spans more than one line`)
	fmt.Println('l')
	fmt.Println('\'')

}

// A function
func Sqrt(x float64) float64 {
	z := 1.0
	for i := 0; i < 1000; i++ {
		z -= (z*z - x) / (2 * z)
	}
	return z
}

func foo() int {
	return 5
}

func bar() {
	return 6
}

func (p *parser) init(fset *token.Fileset, filename string, src []byte) (ret int) {
	p.file = fset.Addfile(filename, -1, len(src))
}
<|endoftext|>


<commit_before>package rpc

import (
	"io"

	"golang.org/x/net/context"
)

type dispatcher interface {
	Call(ctx context.Context, name string, arg interface{}, res interface{}, u ErrorUnwrapper) error
	Notify(ctx context.Context, name string, arg interface{}) error
	Close()
}

type dispatch struct {
	writer encoder
	calls  *callContainer

	// Stops all loops when closed
	stopCh chan struct{}
	// Closed once all loops are finished
	closedCh chan struct{}

	log LogInterface
}

func newDispatch(enc encoder, calls *callContainer, l LogInterface) *dispatch {
	d := &dispatch{
		writer:   enc,
		calls:    calls,
		stopCh:   make(chan struct{}),
		closedCh: make(chan struct{}),

		log: l,
	}
	return d
}

func (d *dispatch) Call(ctx context.Context, name string, arg interface{}, res interface{}, u ErrorUnwrapper) error {
	profiler := d.log.StartProfiler("call %s", name)
	defer profiler.Stop()

	c := d.calls.NewCall(ctx, name, arg, res, u)

	// Have to add call before encoding otherwise we'll race the response
	d.calls.AddCall(c)
	defer d.calls.RemoveCall(c.seqid)
	errCh := d.writer.Encode([]interface{}{MethodCall, c.seqid, c.method, c.arg})

	select {
	case err := <-errCh:
		if err != nil {
			return err
		}
	case <-c.ctx.Done():
		return d.handleCancel(c)
	case <-d.stopCh:
		return io.EOF
	}

	d.log.ClientCall(c.seqid, c.method, c.arg)

	select {
	case res := <-c.resultCh:
		d.log.ClientReply(c.seqid, c.method, res.Err(), res.Res())
		return res.Err()
	case <-c.ctx.Done():
		return d.handleCancel(c)
	case <-d.stopCh:
		return io.EOF
	}
}

func (d *dispatch) Notify(ctx context.Context, name string, arg interface{}) error {
	errCh := d.writer.Encode([]interface{}{MethodNotify, name, arg})
	select {
	case err := <-errCh:
		if err == nil {
			d.log.ClientNotify(name, arg)
		}
		return err
	case <-d.stopCh:
		return io.EOF
	case <-ctx.Done():
		d.log.ClientCancel(-1, name, nil)
		return newCanceledError(name, -1)
	}
}

func (d *dispatch) Close() {
	close(d.stopCh)
}

func (d *dispatch) handleCancel(c *call) error {
	d.log.ClientCancel(c.seqid, c.method, nil)
	errCh := d.writer.Encode([]interface{}{MethodCancel, c.seqid, c.method})
	select {
	case err := <-errCh:
		if err != nil {
			d.log.Info("error while dispatching cancellation: %+v", err.Error())
		}
	default:
		// Don't block on receiving the error from the Encode
	}
	return newCanceledError(c.method, c.seqid)
}
<commit_msg>Don't dispatch call if the dispatcher has been closed<commit_after>package rpc

import (
	"io"

	"golang.org/x/net/context"
)

type dispatcher interface {
	Call(ctx context.Context, name string, arg interface{}, res interface{}, u ErrorUnwrapper) error
	Notify(ctx context.Context, name string, arg interface{}) error
	Close()
}

type dispatch struct {
	writer encoder
	calls  *callContainer

	// Stops all loops when closed
	stopCh chan struct{}
	// Closed once all loops are finished
	closedCh chan struct{}

	log LogInterface
}

func newDispatch(enc encoder, calls *callContainer, l LogInterface) *dispatch {
	d := &dispatch{
		writer:   enc,
		calls:    calls,
		stopCh:   make(chan struct{}),
		closedCh: make(chan struct{}),

		log: l,
	}
	return d
}

func (d *dispatch) Call(ctx context.Context, name string, arg interface{}, res interface{}, u ErrorUnwrapper) error {
	profiler := d.log.StartProfiler("call %s", name)
	defer profiler.Stop()

	select {
	case <-d.stopCh:
		return io.EOF
	default:
	}

	c := d.calls.NewCall(ctx, name, arg, res, u)

	// Have to add call before encoding otherwise we'll race the response
	d.calls.AddCall(c)
	defer d.calls.RemoveCall(c.seqid)
	errCh := d.writer.Encode([]interface{}{MethodCall, c.seqid, c.method, c.arg})

	select {
	case err := <-errCh:
		if err != nil {
			return err
		}
	case <-c.ctx.Done():
		return d.handleCancel(c)
	case <-d.stopCh:
		return io.EOF
	}

	d.log.ClientCall(c.seqid, c.method, c.arg)

	select {
	case res := <-c.resultCh:
		d.log.ClientReply(c.seqid, c.method, res.Err(), res.Res())
		return res.Err()
	case <-c.ctx.Done():
		return d.handleCancel(c)
	case <-d.stopCh:
		return io.EOF
	}
}

func (d *dispatch) Notify(ctx context.Context, name string, arg interface{}) error {
	errCh := d.writer.Encode([]interface{}{MethodNotify, name, arg})
	select {
	case err := <-errCh:
		if err == nil {
			d.log.ClientNotify(name, arg)
		}
		return err
	case <-d.stopCh:
		return io.EOF
	case <-ctx.Done():
		d.log.ClientCancel(-1, name, nil)
		return newCanceledError(name, -1)
	}
}

func (d *dispatch) Close() {
	close(d.stopCh)
}

func (d *dispatch) handleCancel(c *call) error {
	d.log.ClientCancel(c.seqid, c.method, nil)
	errCh := d.writer.Encode([]interface{}{MethodCancel, c.seqid, c.method})
	select {
	case err := <-errCh:
		if err != nil {
			d.log.Info("error while dispatching cancellation: %+v", err.Error())
		}
	default:
		// Don't block on receiving the error from the Encode
	}
	return newCanceledError(c.method, c.seqid)
}
<|endoftext|>