<commit_before>// +build linux

package cgroups

import (
	"fmt"
	systemd1 "github.com/coreos/go-systemd/dbus"
	"github.com/dotcloud/docker/pkg/systemd"
	"github.com/godbus/dbus"
	"path/filepath"
	"strings"
	"sync"
)

type systemdCgroup struct {
}

var (
	connLock              sync.Mutex
	theConn               *systemd1.Conn
	hasStartTransientUnit bool
)

func useSystemd() bool {
	if !systemd.SdBooted() {
		return false
	}

	connLock.Lock()
	defer connLock.Unlock()

	if theConn == nil {
		var err error
		theConn, err = systemd1.New()
		if err != nil {
			return false
		}

		// Assume we have StartTransientUnit
		hasStartTransientUnit = true

		// But if we get UnknownMethod error we don't
		if _, err := theConn.StartTransientUnit("test.scope", "invalid"); err != nil {
			if dbusError, ok := err.(dbus.Error); ok {
				if dbusError.Name == "org.freedesktop.DBus.Error.UnknownMethod" {
					hasStartTransientUnit = false
				}
			}
		}
	}

	return hasStartTransientUnit
}

type DeviceAllow struct {
	Node        string
	Permissions string
}

func getIfaceForUnit(unitName string) string {
	if strings.HasSuffix(unitName, ".scope") {
		return "Scope"
	}
	if strings.HasSuffix(unitName, ".service") {
		return "Service"
	}
	return "Unit"
}

func systemdApply(c *Cgroup, pid int) (ActiveCgroup, error) {
	unitName := c.Parent + "-" + c.Name + ".scope"
	slice := "system.slice"

	var properties []systemd1.Property

	for _, v := range c.UnitProperties {
		switch v[0] {
		case "Slice":
			slice = v[1]
		default:
			return nil, fmt.Errorf("Unknown unit propery %s", v[0])
		}
	}

	properties = append(properties,
		systemd1.Property{"Slice", dbus.MakeVariant(slice)},
		systemd1.Property{"Description", dbus.MakeVariant("docker container " + c.Name)},
		systemd1.Property{"PIDs", dbus.MakeVariant([]uint32{uint32(pid)})})

	if !c.DeviceAccess {
		properties = append(properties,
			systemd1.Property{"DevicePolicy", dbus.MakeVariant("strict")},
			systemd1.Property{"DeviceAllow", dbus.MakeVariant([]DeviceAllow{
				{"/dev/null", "rwm"},
				{"/dev/zero", "rwm"},
				{"/dev/full", "rwm"},
				{"/dev/random", "rwm"},
				{"/dev/urandom", "rwm"},
				{"/dev/tty", "rwm"},
				{"/dev/console", "rwm"},
				{"/dev/tty0", "rwm"},
				{"/dev/tty1", "rwm"},
				{"/dev/pts/ptmx", "rwm"},
				// There is no way to add /dev/pts/* here atm, so we hack this manually below
				// /dev/pts/* (how to add this?)
				// Same with tuntap, which doesn't exist as a node most of the time
			})})
	}

	if c.Memory != 0 {
		properties = append(properties,
			systemd1.Property{"MemoryLimit", dbus.MakeVariant(uint64(c.Memory))})
	}
	// TODO: MemorySwap not available in systemd

	if c.CpuShares != 0 {
		properties = append(properties,
			systemd1.Property{"CPUShares", dbus.MakeVariant(uint64(c.CpuShares))})
	}

	if _, err := theConn.StartTransientUnit(unitName, "replace", properties...); err != nil {
		return nil, err
	}

	// To work around the lack of /dev/pts/* support above we need to manually add these
	// so, ask systemd for the cgroup used
	props, err := theConn.GetUnitTypeProperties(unitName, getIfaceForUnit(unitName))
	if err != nil {
		return nil, err
	}

	cgroup := props["ControlGroup"].(string)

	if !c.DeviceAccess {
		mountpoint, err := FindCgroupMountpoint("devices")
		if err != nil {
			return nil, err
		}

		path := filepath.Join(mountpoint, cgroup)

		// /dev/pts/*
		if err := writeFile(path, "devices.allow", "c 136:* rwm"); err != nil {
			return nil, err
		}
		// tuntap
		if err := writeFile(path, "devices.allow", "c 10:200 rwm"); err != nil {
			return nil, err
		}
	}

	return &systemdCgroup{}, nil
}

func (c *systemdCgroup) Cleanup() error {
	// systemd cleans up, we don't need to do anything
	return nil
}
<commit_msg>Join memory and cpu cgroup in systemd too<commit_after>// +build linux

package cgroups

import (
	"fmt"
	systemd1 "github.com/coreos/go-systemd/dbus"
	"github.com/dotcloud/docker/pkg/systemd"
	"github.com/godbus/dbus"
	"path/filepath"
	"strings"
	"sync"
)

type systemdCgroup struct {
}

var (
	connLock              sync.Mutex
	theConn               *systemd1.Conn
	hasStartTransientUnit bool
)

func useSystemd() bool {
	if !systemd.SdBooted() {
		return false
	}

	connLock.Lock()
	defer connLock.Unlock()

	if theConn == nil {
		var err error
		theConn, err = systemd1.New()
		if err != nil {
			return false
		}

		// Assume we have StartTransientUnit
		hasStartTransientUnit = true

		// But if we get UnknownMethod error we don't
		if _, err := theConn.StartTransientUnit("test.scope", "invalid"); err != nil {
			if dbusError, ok := err.(dbus.Error); ok {
				if dbusError.Name == "org.freedesktop.DBus.Error.UnknownMethod" {
					hasStartTransientUnit = false
				}
			}
		}
	}

	return hasStartTransientUnit
}

type DeviceAllow struct {
	Node        string
	Permissions string
}

func getIfaceForUnit(unitName string) string {
	if strings.HasSuffix(unitName, ".scope") {
		return "Scope"
	}
	if strings.HasSuffix(unitName, ".service") {
		return "Service"
	}
	return "Unit"
}

func systemdApply(c *Cgroup, pid int) (ActiveCgroup, error) {
	unitName := c.Parent + "-" + c.Name + ".scope"
	slice := "system.slice"

	var properties []systemd1.Property

	for _, v := range c.UnitProperties {
		switch v[0] {
		case "Slice":
			slice = v[1]
		default:
			return nil, fmt.Errorf("Unknown unit propery %s", v[0])
		}
	}

	properties = append(properties,
		systemd1.Property{"Slice", dbus.MakeVariant(slice)},
		systemd1.Property{"Description", dbus.MakeVariant("docker container " + c.Name)},
		systemd1.Property{"PIDs", dbus.MakeVariant([]uint32{uint32(pid)})})

	if !c.DeviceAccess {
		properties = append(properties,
			systemd1.Property{"DevicePolicy", dbus.MakeVariant("strict")},
			systemd1.Property{"DeviceAllow", dbus.MakeVariant([]DeviceAllow{
				{"/dev/null", "rwm"},
				{"/dev/zero", "rwm"},
				{"/dev/full", "rwm"},
				{"/dev/random", "rwm"},
				{"/dev/urandom", "rwm"},
				{"/dev/tty", "rwm"},
				{"/dev/console", "rwm"},
				{"/dev/tty0", "rwm"},
				{"/dev/tty1", "rwm"},
				{"/dev/pts/ptmx", "rwm"},
				// There is no way to add /dev/pts/* here atm, so we hack this manually below
				// /dev/pts/* (how to add this?)
				// Same with tuntap, which doesn't exist as a node most of the time
			})})
	}

	// Always enable accounting, this gets us the same behaviour as the raw implementation,
	// plus the kernel has some problems with joining the memory cgroup at a later time.
	properties = append(properties,
		systemd1.Property{"MemoryAccounting", dbus.MakeVariant(true)},
		systemd1.Property{"CPUAccounting", dbus.MakeVariant(true)})

	if c.Memory != 0 {
		properties = append(properties,
			systemd1.Property{"MemoryLimit", dbus.MakeVariant(uint64(c.Memory))})
	}
	// TODO: MemorySwap not available in systemd

	if c.CpuShares != 0 {
		properties = append(properties,
			systemd1.Property{"CPUShares", dbus.MakeVariant(uint64(c.CpuShares))})
	}

	if _, err := theConn.StartTransientUnit(unitName, "replace", properties...); err != nil {
		return nil, err
	}

	// To work around the lack of /dev/pts/* support above we need to manually add these
	// so, ask systemd for the cgroup used
	props, err := theConn.GetUnitTypeProperties(unitName, getIfaceForUnit(unitName))
	if err != nil {
		return nil, err
	}

	cgroup := props["ControlGroup"].(string)

	if !c.DeviceAccess {
		mountpoint, err := FindCgroupMountpoint("devices")
		if err != nil {
			return nil, err
		}

		path := filepath.Join(mountpoint, cgroup)

		// /dev/pts/*
		if err := writeFile(path, "devices.allow", "c 136:* rwm"); err != nil {
			return nil, err
		}
		// tuntap
		if err := writeFile(path, "devices.allow", "c 10:200 rwm"); err != nil {
			return nil, err
		}
	}

	return &systemdCgroup{}, nil
}

func (c *systemdCgroup) Cleanup() error {
	// systemd cleans up, we don't need to do anything
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2018 FairwindsOps Inc
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package reconciler

import (
	"reflect"

	"github.com/fairwindsops/rbac-manager/pkg/kube"

	rbacmanagerv1beta1 "github.com/fairwindsops/rbac-manager/pkg/apis/rbacmanager/v1beta1"
	logrus "github.com/sirupsen/logrus"
	v1 "k8s.io/api/core/v1"
	rbacv1 "k8s.io/api/rbac/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/client-go/kubernetes"
)

// Reconciler creates and deletes Kubernetes resources to achieve the desired state of an RBAC Definition
type Reconciler struct {
	Clientset kubernetes.Interface
	ownerRefs []metav1.OwnerReference
}

// ReconcileNamespaceChange reconciles relevant portions of RBAC Definitions
//   after changes to namespaces within the cluster
func (r *Reconciler) ReconcileNamespaceChange(rbacDef *rbacmanagerv1beta1.RBACDefinition, namespace *v1.Namespace) error {
	r.ownerRefs = rbacDefOwnerRefs(rbacDef)

	p := Parser{
		Clientset: r.Clientset,
		ownerRefs: r.ownerRefs,
	}

	if p.hasNamespaceSelectors(rbacDef) {
		logrus.Infof("Reconciling %v namespace for %v", namespace.Name, rbacDef.Name)
		p.parseRoleBindings(rbacDef)
		err := r.reconcileRoleBindings(&p.parsedRoleBindings)
		if err != nil {
			return err
		}
	}

	return nil
}

// ReconcileOwners reconciles any RBACDefinitions found in owner references
func (r *Reconciler) ReconcileOwners(ownerRefs []metav1.OwnerReference, kind string) error {
	for _, ownerRef := range ownerRefs {
		if ownerRef.Kind == "RBACDefinition" {
			rbacDef, err := kube.GetRbacDefinition(ownerRef.Name)
			if err != nil {
				return err
			}

			r.ownerRefs = rbacDefOwnerRefs(&rbacDef)

			p := Parser{
				Clientset: r.Clientset,
				ownerRefs: r.ownerRefs,
			}

			if kind == "RoleBinding" {
				p.parseRoleBindings(&rbacDef)
				return r.reconcileRoleBindings(&p.parsedRoleBindings)
			} else if kind == "ClusterRoleBinding" {
				p.parseClusterRoleBindings(&rbacDef)
				return r.reconcileClusterRoleBindings(&p.parsedClusterRoleBindings)
			} else if kind == "ServiceAccount" {
				err := p.Parse(rbacDef)
				if err != nil {
					return err
				}
				return r.reconcileServiceAccounts(&p.parsedServiceAccounts)
			}
		}
	}
	return nil
}

// Reconcile creates, updates, or deletes Kubernetes resources to match
//   the desired state defined in an RBAC Definition
func (r *Reconciler) Reconcile(rbacDef *rbacmanagerv1beta1.RBACDefinition) error {
	logrus.Infof("Reconciling RBACDefinition %v", rbacDef.Name)

	r.ownerRefs = rbacDefOwnerRefs(rbacDef)

	p := Parser{
		Clientset: r.Clientset,
		ownerRefs: r.ownerRefs,
	}

	var err error

	err = p.Parse(*rbacDef)
	if err != nil {
		return err
	}

	err = r.reconcileServiceAccounts(&p.parsedServiceAccounts)
	if err != nil {
		return err
	}

	err = r.reconcileClusterRoleBindings(&p.parsedClusterRoleBindings)
	if err != nil {
		return err
	}

	err = r.reconcileRoleBindings(&p.parsedRoleBindings)
	if err != nil {
		return err
	}

	return nil
}

func (r *Reconciler) reconcileServiceAccounts(requested *[]v1.ServiceAccount) error {
	existing, err := r.Clientset.CoreV1().ServiceAccounts("").List(kube.ListOptions)
	if err != nil {
		return err
	}

	matchingServiceAccounts := []v1.ServiceAccount{}
	serviceAccountsToCreate := []v1.ServiceAccount{}

	for _, requestedSA := range *requested {
		alreadyExists := false
		for _, existingSA := range existing.Items {
			if saMatches(&existingSA, &requestedSA) {
				alreadyExists = true
				matchingServiceAccounts = append(matchingServiceAccounts, existingSA)
				break
			}
		}

		if !alreadyExists {
			serviceAccountsToCreate = append(serviceAccountsToCreate, requestedSA)
		} else {
			logrus.Debugf("Service Account already exists %v", requestedSA.Name)
		}
	}

	for _, existingSA := range existing.Items {
		if reflect.DeepEqual(existingSA.OwnerReferences, r.ownerRefs) {
			matchingRequest := false
			for _, matchingSA := range matchingServiceAccounts {
				if saMatches(&existingSA, &matchingSA) {
					matchingRequest = true
					break
				}
			}

			if !matchingRequest {
				logrus.Infof("Deleting Service Account %v", existingSA.Name)
				err := r.Clientset.CoreV1().ServiceAccounts(existingSA.Namespace).Delete(existingSA.Name, &metav1.DeleteOptions{})
				if err != nil {
					logrus.Infof("Error deleting Service Account: %v", err)
				}
			} else {
				logrus.Debugf("Matches requested Service Account %v", existingSA.Name)
			}
		}
	}

	for _, serviceAccountToCreate := range serviceAccountsToCreate {
		logrus.Infof("Creating Service Account: %v", serviceAccountToCreate.Name)
		_, err := r.Clientset.CoreV1().ServiceAccounts(serviceAccountToCreate.ObjectMeta.Namespace).Create(&serviceAccountToCreate)
		if err != nil {
			logrus.Errorf("Error creating Service Account: %v", err)
		}
	}

	return nil
}

func (r *Reconciler) reconcileClusterRoleBindings(requested *[]rbacv1.ClusterRoleBinding) error {
	existing, err := r.Clientset.RbacV1().ClusterRoleBindings().List(kube.ListOptions)
	if err != nil {
		return err
	}

	matchingClusterRoleBindings := []rbacv1.ClusterRoleBinding{}
	clusterRoleBindingsToCreate := []rbacv1.ClusterRoleBinding{}

	for _, requestedCRB := range *requested {
		alreadyExists := false
		for _, existingCRB := range existing.Items {
			if crbMatches(&existingCRB, &requestedCRB) {
				alreadyExists = true
				matchingClusterRoleBindings = append(matchingClusterRoleBindings, existingCRB)
				break
			}
		}

		if !alreadyExists {
			clusterRoleBindingsToCreate = append(clusterRoleBindingsToCreate, requestedCRB)
		} else {
			logrus.Debugf("Cluster Role Binding already exists %v", requestedCRB.Name)
		}
	}

	for _, existingCRB := range existing.Items {
		if reflect.DeepEqual(existingCRB.OwnerReferences, r.ownerRefs) {
			matchingRequest := false
			for _, requestedCRB := range matchingClusterRoleBindings {
				if crbMatches(&existingCRB, &requestedCRB) {
					matchingRequest = true
					break
				}
			}

			if !matchingRequest {
				logrus.Infof("Deleting Cluster Role Binding: %v", existingCRB.Name)
				err := r.Clientset.RbacV1().ClusterRoleBindings().Delete(existingCRB.Name, &metav1.DeleteOptions{})
				if err != nil {
					logrus.Errorf("Error deleting Cluster Role Binding: %v", err)
				}
			} else {
				logrus.Debugf("Matches requested Cluster Role Binding: %v", existingCRB.Name)
			}
		}
	}

	for _, clusterRoleBindingToCreate := range clusterRoleBindingsToCreate {
		logrus.Infof("Creating Cluster Role Binding: %v", clusterRoleBindingToCreate.Name)
		_, err := r.Clientset.RbacV1().ClusterRoleBindings().Create(&clusterRoleBindingToCreate)
		if err != nil {
			logrus.Errorf("Error creating Cluster Role Binding: %v", err)
		}
	}

	return nil
}

func (r *Reconciler) reconcileRoleBindings(requested *[]rbacv1.RoleBinding) error {
	existing, err := r.Clientset.RbacV1().RoleBindings("").List(kube.ListOptions)
	if err != nil {
		return err
	}

	matchingRoleBindings := []rbacv1.RoleBinding{}
	roleBindingsToCreate := []rbacv1.RoleBinding{}

	for _, requestedRB := range *requested {
		alreadyExists := false
		for _, existingRB := range existing.Items {
			if rbMatches(&existingRB, &requestedRB) {
				alreadyExists = true
				matchingRoleBindings = append(matchingRoleBindings, existingRB)
				break
			}
		}

		if !alreadyExists {
			roleBindingsToCreate = append(roleBindingsToCreate, requestedRB)
		} else {
			logrus.Debugf("Role Binding already exists %v", requestedRB.Name)
		}
	}

	for _, existingRB := range existing.Items {
		if reflect.DeepEqual(existingRB.OwnerReferences, r.ownerRefs) {
			matchingRequest := false
			for _, requestedRB := range matchingRoleBindings {
				if rbMatches(&existingRB, &requestedRB) {
					matchingRequest = true
					break
				}
			}

			if !matchingRequest {
				logrus.Infof("Deleting Role Binding %v", existingRB.Name)
				err := r.Clientset.RbacV1().RoleBindings(existingRB.Namespace).Delete(existingRB.Name, &metav1.DeleteOptions{})
				if err != nil {
					logrus.Infof("Error deleting Role Binding: %v", err)
				}
			} else {
				logrus.Debugf("Matches requested Role Binding %v", existingRB.Name)
			}
		}
	}

	for _, roleBindingToCreate := range roleBindingsToCreate {
		logrus.Infof("Creating Role Binding: %v", roleBindingToCreate.Name)
		_, err := r.Clientset.RbacV1().RoleBindings(roleBindingToCreate.ObjectMeta.Namespace).Create(&roleBindingToCreate)
		if err != nil {
			logrus.Errorf("Error creating Role Binding: %v", err)
		}
	}

	return nil
}

func rbacDefOwnerRefs(rbacDef *rbacmanagerv1beta1.RBACDefinition) []metav1.OwnerReference {
	return []metav1.OwnerReference{
		*metav1.NewControllerRef(rbacDef, schema.GroupVersionKind{
			Group:   rbacmanagerv1beta1.SchemeGroupVersion.Group,
			Version: rbacmanagerv1beta1.SchemeGroupVersion.Version,
			Kind:    "RBACDefinition",
		}),
	}
}
<commit_msg>add a mutex to the reconciler to avoid race conditions between runs<commit_after>// Copyright 2018 FairwindsOps Inc
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package reconciler

import (
	"reflect"
	"sync"

	"github.com/fairwindsops/rbac-manager/pkg/kube"

	rbacmanagerv1beta1 "github.com/fairwindsops/rbac-manager/pkg/apis/rbacmanager/v1beta1"
	logrus "github.com/sirupsen/logrus"
	v1 "k8s.io/api/core/v1"
	rbacv1 "k8s.io/api/rbac/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/client-go/kubernetes"
)

// Reconciler creates and deletes Kubernetes resources to achieve the desired state of an RBAC Definition
type Reconciler struct {
	Clientset kubernetes.Interface
	ownerRefs []metav1.OwnerReference
}

var mux = sync.Mutex{}

// ReconcileNamespaceChange reconciles relevant portions of RBAC Definitions
//   after changes to namespaces within the cluster
func (r *Reconciler) ReconcileNamespaceChange(rbacDef *rbacmanagerv1beta1.RBACDefinition, namespace *v1.Namespace) error {
	mux.Lock()
	defer mux.Unlock()

	r.ownerRefs = rbacDefOwnerRefs(rbacDef)

	p := Parser{
		Clientset: r.Clientset,
		ownerRefs: r.ownerRefs,
	}

	if p.hasNamespaceSelectors(rbacDef) {
		logrus.Infof("Reconciling %v namespace for %v", namespace.Name, rbacDef.Name)
		p.parseRoleBindings(rbacDef)
		err := r.reconcileRoleBindings(&p.parsedRoleBindings)
		if err != nil {
			return err
		}
	}

	return nil
}

// ReconcileOwners reconciles any RBACDefinitions found in owner references
func (r *Reconciler) ReconcileOwners(ownerRefs []metav1.OwnerReference, kind string) error {
	mux.Lock()
	defer mux.Unlock()

	for _, ownerRef := range ownerRefs {
		if ownerRef.Kind == "RBACDefinition" {
			rbacDef, err := kube.GetRbacDefinition(ownerRef.Name)
			if err != nil {
				return err
			}

			r.ownerRefs = rbacDefOwnerRefs(&rbacDef)

			p := Parser{
				Clientset: r.Clientset,
				ownerRefs: r.ownerRefs,
			}

			if kind == "RoleBinding" {
				p.parseRoleBindings(&rbacDef)
				return r.reconcileRoleBindings(&p.parsedRoleBindings)
			} else if kind == "ClusterRoleBinding" {
				p.parseClusterRoleBindings(&rbacDef)
				return r.reconcileClusterRoleBindings(&p.parsedClusterRoleBindings)
			} else if kind == "ServiceAccount" {
				err := p.Parse(rbacDef)
				if err != nil {
					return err
				}
				return r.reconcileServiceAccounts(&p.parsedServiceAccounts)
			}
		}
	}
	return nil
}

// Reconcile creates, updates, or deletes Kubernetes resources to match
//   the desired state defined in an RBAC Definition
func (r *Reconciler) Reconcile(rbacDef *rbacmanagerv1beta1.RBACDefinition) error {
	mux.Lock()
	defer mux.Unlock()

	logrus.Infof("Reconciling RBACDefinition %v", rbacDef.Name)

	r.ownerRefs = rbacDefOwnerRefs(rbacDef)

	p := Parser{
		Clientset: r.Clientset,
		ownerRefs: r.ownerRefs,
	}

	var err error

	err = p.Parse(*rbacDef)
	if err != nil {
		return err
	}

	err = r.reconcileServiceAccounts(&p.parsedServiceAccounts)
	if err != nil {
		return err
	}

	err = r.reconcileClusterRoleBindings(&p.parsedClusterRoleBindings)
	if err != nil {
		return err
	}

	err = r.reconcileRoleBindings(&p.parsedRoleBindings)
	if err != nil {
		return err
	}

	return nil
}

func (r *Reconciler) reconcileServiceAccounts(requested *[]v1.ServiceAccount) error {
	existing, err := r.Clientset.CoreV1().ServiceAccounts("").List(kube.ListOptions)
	if err != nil {
		return err
	}

	matchingServiceAccounts := []v1.ServiceAccount{}
	serviceAccountsToCreate := []v1.ServiceAccount{}

	for _, requestedSA := range *requested {
		alreadyExists := false
		for _, existingSA := range existing.Items {
			if saMatches(&existingSA, &requestedSA) {
				alreadyExists = true
				matchingServiceAccounts = append(matchingServiceAccounts, existingSA)
				break
			}
		}

		if !alreadyExists {
			serviceAccountsToCreate = append(serviceAccountsToCreate, requestedSA)
		} else {
			logrus.Debugf("Service Account already exists %v", requestedSA.Name)
		}
	}

	for _, existingSA := range existing.Items {
		if reflect.DeepEqual(existingSA.OwnerReferences, r.ownerRefs) {
			matchingRequest := false
			for _, matchingSA := range matchingServiceAccounts {
				if saMatches(&existingSA, &matchingSA) {
					matchingRequest = true
					break
				}
			}

			if !matchingRequest {
				logrus.Infof("Deleting Service Account %v", existingSA.Name)
				err := r.Clientset.CoreV1().ServiceAccounts(existingSA.Namespace).Delete(existingSA.Name, &metav1.DeleteOptions{})
				if err != nil {
					logrus.Infof("Error deleting Service Account: %v", err)
				}
			} else {
				logrus.Debugf("Matches requested Service Account %v", existingSA.Name)
			}
		}
	}

	for _, serviceAccountToCreate := range serviceAccountsToCreate {
		logrus.Infof("Creating Service Account: %v", serviceAccountToCreate.Name)
		_, err := r.Clientset.CoreV1().ServiceAccounts(serviceAccountToCreate.ObjectMeta.Namespace).Create(&serviceAccountToCreate)
		if err != nil {
			logrus.Errorf("Error creating Service Account: %v", err)
		}
	}

	return nil
}

func (r *Reconciler) reconcileClusterRoleBindings(requested *[]rbacv1.ClusterRoleBinding) error {
	existing, err := r.Clientset.RbacV1().ClusterRoleBindings().List(kube.ListOptions)
	if err != nil {
		return err
	}

	matchingClusterRoleBindings := []rbacv1.ClusterRoleBinding{}
	clusterRoleBindingsToCreate := []rbacv1.ClusterRoleBinding{}

	for _, requestedCRB := range *requested {
		alreadyExists := false
		for _, existingCRB := range existing.Items {
			if crbMatches(&existingCRB, &requestedCRB) {
				alreadyExists = true
				matchingClusterRoleBindings = append(matchingClusterRoleBindings, existingCRB)
				break
			}
		}

		if !alreadyExists {
			clusterRoleBindingsToCreate = append(clusterRoleBindingsToCreate, requestedCRB)
		} else {
			logrus.Debugf("Cluster Role Binding already exists %v", requestedCRB.Name)
		}
	}

	for _, existingCRB := range existing.Items {
		if reflect.DeepEqual(existingCRB.OwnerReferences, r.ownerRefs) {
			matchingRequest := false
			for _, requestedCRB := range matchingClusterRoleBindings {
				if crbMatches(&existingCRB, &requestedCRB) {
					matchingRequest = true
					break
				}
			}

			if !matchingRequest {
				logrus.Infof("Deleting Cluster Role Binding: %v", existingCRB.Name)
				err := r.Clientset.RbacV1().ClusterRoleBindings().Delete(existingCRB.Name, &metav1.DeleteOptions{})
				if err != nil {
					logrus.Errorf("Error deleting Cluster Role Binding: %v", err)
				}
			} else {
				logrus.Debugf("Matches requested Cluster Role Binding: %v", existingCRB.Name)
			}
		}
	}

	for _, clusterRoleBindingToCreate := range clusterRoleBindingsToCreate {
		logrus.Infof("Creating Cluster Role Binding: %v", clusterRoleBindingToCreate.Name)
		_, err := r.Clientset.RbacV1().ClusterRoleBindings().Create(&clusterRoleBindingToCreate)
		if err != nil {
			logrus.Errorf("Error creating Cluster Role Binding: %v", err)
		}
	}

	return nil
}

func (r *Reconciler) reconcileRoleBindings(requested *[]rbacv1.RoleBinding) error {
	existing, err := r.Clientset.RbacV1().RoleBindings("").List(kube.ListOptions)
	if err != nil {
		return err
	}

	matchingRoleBindings := []rbacv1.RoleBinding{}
	roleBindingsToCreate := []rbacv1.RoleBinding{}

	for _, requestedRB := range *requested {
		alreadyExists := false
		for _, existingRB := range existing.Items {
			if rbMatches(&existingRB, &requestedRB) {
				alreadyExists = true
				matchingRoleBindings = append(matchingRoleBindings, existingRB)
				break
			}
		}

		if !alreadyExists {
			roleBindingsToCreate = append(roleBindingsToCreate, requestedRB)
		} else {
			logrus.Debugf("Role Binding already exists %v", requestedRB.Name)
		}
	}

	for _, existingRB := range existing.Items {
		if reflect.DeepEqual(existingRB.OwnerReferences, r.ownerRefs) {
			matchingRequest := false
			for _, requestedRB := range matchingRoleBindings {
				if rbMatches(&existingRB, &requestedRB) {
					matchingRequest = true
					break
				}
			}

			if !matchingRequest {
				logrus.Infof("Deleting Role Binding %v", existingRB.Name)
				err := r.Clientset.RbacV1().RoleBindings(existingRB.Namespace).Delete(existingRB.Name, &metav1.DeleteOptions{})
				if err != nil {
					logrus.Infof("Error deleting Role Binding: %v", err)
				}
			} else {
				logrus.Debugf("Matches requested Role Binding %v", existingRB.Name)
			}
		}
	}

	for _, roleBindingToCreate := range roleBindingsToCreate {
		logrus.Infof("Creating Role Binding: %v", roleBindingToCreate.Name)
		_, err := r.Clientset.RbacV1().RoleBindings(roleBindingToCreate.ObjectMeta.Namespace).Create(&roleBindingToCreate)
		if err != nil {
			logrus.Errorf("Error creating Role Binding: %v", err)
		}
	}

	return nil
}

func rbacDefOwnerRefs(rbacDef *rbacmanagerv1beta1.RBACDefinition) []metav1.OwnerReference {
	return []metav1.OwnerReference{
		*metav1.NewControllerRef(rbacDef, schema.GroupVersionKind{
			Group:   rbacmanagerv1beta1.SchemeGroupVersion.Group,
			Version: rbacmanagerv1beta1.SchemeGroupVersion.Version,
			Kind:    "RBACDefinition",
		}),
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"strconv"
	"time"

	"github.com/ae6rt/retry"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/awsutil"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/aws/aws-sdk-go/service/s3"
)

type AWSStorageService struct {
	Config *aws.Config
}

func NewAWSStorageService(key, secret, region string) StorageService {
	return AWSStorageService{aws.NewConfig().WithCredentials(credentials.NewStaticCredentials(key, secret, "")).WithRegion(region).WithMaxRetries(3)}
}

func (c AWSStorageService) GetBuildsByProject(project Project, since uint64, limit uint64) ([]Build, error) {

	var resp *dynamodb.QueryOutput

	work := func() error {
		svc := dynamodb.New(c.Config)
		params := &dynamodb.QueryInput{
			TableName:              aws.String("decap-build-metadata"),
			IndexName:              aws.String("projectKey-buildTime-index"),
			KeyConditionExpression: aws.String("projectKey = :pkey and buildTime > :since"),
			ExpressionAttributeValues: map[string]*dynamodb.AttributeValue{
				":pkey": {
					S: aws.String(projectKey(project.Team, project.Library)),
				},
				":since": {
					N: aws.String(fmt.Sprintf("%d", since)),
				},
			},
			ScanIndexForward: aws.Bool(false),
			Limit:            aws.Int64(int64(limit)),
		}

		var err error
		resp, err = svc.Query(params)

		if err != nil {
			if awsErr, ok := err.(awserr.Error); ok {
				Log.Println(awsErr.Code(), awsErr.Message(), awsErr.OrigErr())
				if reqErr, ok := err.(awserr.RequestFailure); ok {
					Log.Println(reqErr.Code(), reqErr.Message(), reqErr.StatusCode(), reqErr.RequestID())
				}
			} else {
				Log.Println(err.Error())
			}
			return err
		}
		return nil
	}
	err := retry.New(5*time.Second, 3, retry.DefaultBackoffFunc).Try(work)
	if err != nil {
		return nil, err
	}

	builds := make([]Build, 0)
	for _, v := range resp.Items {
		buildElapsedTime, err := strconv.ParseUint(*v["buildElapsedTime"].N, 10, 64)
		if err != nil {
			Log.Printf("Error converting buildElapsedTime to ordinal value: %v\n", err)
		}
		buildResult, err := strconv.ParseInt(*v["buildResult"].N, 10, 32)
		if err != nil {
			Log.Printf("Error converting buildResult to ordinal value: %v\n", err)
		}
		buildTime, err := strconv.ParseUint(*v["buildTime"].N, 10, 64)
		if err != nil {
			Log.Printf("Error converting buildTime to ordinal value: %v\n", err)
		}

		build := Build{
			ID:       *v["buildID"].S,
			Branch:   *v["branch"].S,
			Duration: buildElapsedTime,
			Result:   int(buildResult),
			UnixTime: buildTime,
		}
		builds = append(builds, build)
	}
	return builds, nil
}

func (c AWSStorageService) GetBuildsBuildling() ([]Build, error) {

	var resp *dynamodb.QueryOutput

	work := func() error {
		svc := dynamodb.New(c.Config)
		params := &dynamodb.QueryInput{
			TableName:              aws.String("decap-build-metadata"),
			IndexName:              aws.String("isBuilding-index"),
			KeyConditionExpression: aws.String("isBuilding = :val"),
			ExpressionAttributeValues: map[string]*dynamodb.AttributeValue{
				":val": {
					N: aws.String("1"),
				},
			},
			ScanIndexForward: aws.Bool(false),
		}

		var err error
		resp, err = svc.Query(params)

		if err != nil {
			if awsErr, ok := err.(awserr.Error); ok {
				Log.Println(awsErr.Code(), awsErr.Message(), awsErr.OrigErr())
				if reqErr, ok := err.(awserr.RequestFailure); ok {
					Log.Println(reqErr.Code(), reqErr.Message(), reqErr.StatusCode(), reqErr.RequestID())
				}
			} else {
				Log.Println(err.Error())
			}
			return err
		}
		return nil
	}
	err := retry.New(5*time.Second, 3, retry.DefaultBackoffFunc).Try(work)
	if err != nil {
		return nil, err
	}

	builds := make([]Build, 0)
	for _, v := range resp.Items {
		buildTime, err := strconv.ParseUint(*v["buildTime"].N, 10, 64)
		if err != nil {
			Log.Printf("Error converting buildTime to ordinal value: %v\n", err)
		}

		build := Build{
			ProjectKey: *v["projectKey"].S,
			ID:         *v["buildID"].S,
			Branch:     *v["branch"].S,
			UnixTime:   buildTime,
		}
		builds = append(builds, build)
	}
	return builds, nil
}

func (c AWSStorageService) GetArtifacts(buildID string) ([]byte, error) {
	return c.bytesFromBucket("decap-build-artifacts", buildID)
}

func (c AWSStorageService) GetConsoleLog(buildID string) ([]byte, error) {
	return c.bytesFromBucket("decap-console-logs", buildID)
}

func (c AWSStorageService) bytesFromBucket(bucketName, objectKey string) ([]byte, error) {

	var resp *s3.GetObjectOutput

	work := func() error {
		svc := s3.New(c.Config)

		params := &s3.GetObjectInput{
			Bucket: aws.String(bucketName),
			Key:    aws.String(objectKey),
		}

		var err error
		if resp, err = svc.GetObject(params); err != nil {
			Log.Println(err.Error())
			return err
		}
		return nil
	}

	if err := retry.New(5*time.Second, 3, retry.DefaultBackoffFunc).Try(work); err != nil {
		return nil, err
	}

	return ioutil.ReadAll(resp.Body)
}
<commit_msg>remove unused log import<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"strconv"
	"time"

	"github.com/ae6rt/retry"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/aws/aws-sdk-go/service/s3"
)

type AWSStorageService struct {
	Config *aws.Config
}

func NewAWSStorageService(key, secret, region string) StorageService {
	return AWSStorageService{aws.NewConfig().WithCredentials(credentials.NewStaticCredentials(key, secret, "")).WithRegion(region).WithMaxRetries(3)}
}

func (c AWSStorageService) GetBuildsByProject(project Project, since uint64, limit uint64) ([]Build, error) {

	var resp *dynamodb.QueryOutput

	work := func() error {
		svc := dynamodb.New(c.Config)
		params := &dynamodb.QueryInput{
			TableName:              aws.String("decap-build-metadata"),
			IndexName:              aws.String("projectKey-buildTime-index"),
			KeyConditionExpression: aws.String("projectKey = :pkey and buildTime > :since"),
			ExpressionAttributeValues: map[string]*dynamodb.AttributeValue{
				":pkey": {
					S: aws.String(projectKey(project.Team, project.Library)),
				},
				":since": {
					N: aws.String(fmt.Sprintf("%d", since)),
				},
			},
			ScanIndexForward: aws.Bool(false),
			Limit:            aws.Int64(int64(limit)),
		}

		var err error
		resp, err = svc.Query(params)

		if err != nil {
			if awsErr, ok := err.(awserr.Error); ok {
				Log.Println(awsErr.Code(), awsErr.Message(), awsErr.OrigErr())
				if reqErr, ok := err.(awserr.RequestFailure); ok {
					Log.Println(reqErr.Code(), reqErr.Message(), reqErr.StatusCode(), reqErr.RequestID())
				}
			} else {
				Log.Println(err.Error())
			}
			return err
		}
		return nil
	}
	err := retry.New(5*time.Second, 3, retry.DefaultBackoffFunc).Try(work)
	if err != nil {
		return nil, err
	}

	builds := make([]Build, 0)
	for _, v := range resp.Items {
		buildElapsedTime, err := strconv.ParseUint(*v["buildElapsedTime"].N, 10, 64)
		if err != nil {
			Log.Printf("Error converting buildElapsedTime to ordinal value: %v\n", err)
		}
		buildResult, err := strconv.ParseInt(*v["buildResult"].N, 10, 32)
		if err != nil {
			Log.Printf("Error converting buildResult to ordinal value: %v\n", err)
		}
		buildTime, err := strconv.ParseUint(*v["buildTime"].N, 10, 64)
		if err != nil {
			Log.Printf("Error converting buildTime to ordinal value: %v\n", err)
		}

		build := Build{
			ID:       *v["buildID"].S,
			Branch:   *v["branch"].S,
			Duration: buildElapsedTime,
			Result:   int(buildResult),
			UnixTime: buildTime,
		}
		builds = append(builds, build)
	}
	return builds, nil
}

func (c AWSStorageService) GetBuildsBuildling() ([]Build, error) {

	var resp *dynamodb.QueryOutput

	work := func() error {
		svc := dynamodb.New(c.Config)
		params := &dynamodb.QueryInput{
			TableName:              aws.String("decap-build-metadata"),
			IndexName:              aws.String("isBuilding-index"),
			KeyConditionExpression: aws.String("isBuilding = :val"),
			ExpressionAttributeValues: map[string]*dynamodb.AttributeValue{
				":val": {
					N: aws.String("1"),
				},
			},
			ScanIndexForward: aws.Bool(false),
		}

		var err error
		resp, err = svc.Query(params)

		if err != nil {
			if awsErr, ok := err.(awserr.Error); ok {
				Log.Println(awsErr.Code(), awsErr.Message(), awsErr.OrigErr())
				if reqErr, ok := err.(awserr.RequestFailure); ok {
					Log.Println(reqErr.Code(), reqErr.Message(), reqErr.StatusCode(), reqErr.RequestID())
				}
			} else {
				Log.Println(err.Error())
			}
			return err
		}
		return nil
	}
	err := retry.New(5*time.Second, 3, retry.DefaultBackoffFunc).Try(work)
	if err != nil {
		return nil, err
	}

	builds := make([]Build, 0)
	for _, v := range resp.Items {
		buildTime, err := strconv.ParseUint(*v["buildTime"].N, 10, 64)
		if err != nil {
			Log.Printf("Error converting buildTime to ordinal value: %v\n", err)
		}

		build := Build{
			ProjectKey: *v["projectKey"].S,
			ID:         *v["buildID"].S,
			Branch:     *v["branch"].S,
			UnixTime:   buildTime,
		}
		builds = append(builds, build)
	}
	return builds, nil
}

func (c AWSStorageService) GetArtifacts(buildID string) ([]byte, error) {
	return c.bytesFromBucket("decap-build-artifacts", buildID)
}

func (c AWSStorageService) GetConsoleLog(buildID string) ([]byte, error) {
	return c.bytesFromBucket("decap-console-logs", buildID)
}

func (c AWSStorageService) bytesFromBucket(bucketName, objectKey string) ([]byte, error) {

	var resp *s3.GetObjectOutput

	work := func() error {
		svc := s3.New(c.Config)

		params := &s3.GetObjectInput{
			Bucket: aws.String(bucketName),
			Key:    aws.String(objectKey),
		}

		var err error
		if resp, err = svc.GetObject(params); err != nil {
			Log.Println(err.Error())
			return err
		}
		return nil
	}

	if err := retry.New(5*time.Second, 3, retry.DefaultBackoffFunc).Try(work); err != nil {
		return nil, err
	}

	return ioutil.ReadAll(resp.Body)
}
<|endoftext|>


<commit_before>// Package web provides a thin layer of support for writing web services. It
// integrates with the ardanlabs kit repo to provide support for routing and
// application ctx. The base things you need to write a web service is
// provided.
package web

import (
	"context"
	"encoding/json"
	"io"
	"net/http"
	"os"
	"os/signal"
	"time"

	"github.com/braintree/manners"
	"github.com/dimfeld/httptreemux"
	"github.com/pborman/uuid"
	"gopkg.in/go-playground/validator.v8"
)

// TraceIDHeader is the header added to outgoing requests which adds the
// traceID to it.
const TraceIDHeader = "X-Trace-ID"

//==============================================================================

// validate provides a validator for checking models.
var validate = validator.New(&validator.Config{
	TagName:      "validate",
	FieldNameTag: "json",
})

// Unmarshal decodes the input to the struct type and checks the
// fields to verify the value is in a proper state.
func Unmarshal(r io.Reader, v interface{}) error {
	if err := json.NewDecoder(r).Decode(v); err != nil {
		return err
	}

	var inv InvalidError
	if fve := validate.Struct(v); fve != nil {
		for _, fe := range fve.(validator.ValidationErrors) {
			inv = append(inv, Invalid{Fld: fe.Field, Err: fe.Tag})
		}
		return inv
	}

	return nil
}

//==============================================================================

// Key represents the type of value for the context key.
type ctxKey int

// KeyValues is how request values or stored/retrieved.
const KeyValues ctxKey = 1

// Values represent state for each request.
type Values struct {
	TraceID    string
	Now        time.Time
	StatusCode int
}

//==============================================================================

// A Handler is a type that handles an http request within our own little mini
// framework.
type Handler func(ctx context.Context, w http.ResponseWriter, r *http.Request, params map[string]string) error

// A Middleware is a type that wraps a handler to remove boilerplate or other
// concerns not direct to any given Handler.
type Middleware func(Handler) Handler

//==============================================================================

// App is the entrypoint into our application and what configures our context
// object for each of our http handlers. Feel free to add any configuration
// data/logic on this App struct
type App struct {
	*httptreemux.TreeMux
	Values map[string]interface{}

	mw []Middleware
}

// New create an App value that handle a set of routes for the application.
// You can provide any number of middleware and they'll be used to wrap every
// request handler.
func New(mw ...Middleware) *App {
	return &App{
		TreeMux: httptreemux.New(),
		Values:  make(map[string]interface{}),
		mw:      mw,
	}
}

// Group creates a new App Group based on the current App and provided
// middleware.
func (a *App) Group(mw ...Middleware) *Group {
	return &Group{
		app: a,
		mw:  mw,
	}
}

// Use adds the set of provided middleware onto the Application middleware
// chain. Any route running off of this App will use all the middleware provided
// this way always regardless of the ordering of the Handle/Use functions.
func (a *App) Use(mw ...Middleware) {
	a.mw = append(a.mw, mw...)
}

// Handle is our mechanism for mounting Handlers for a given HTTP verb and path
// pair, this makes for really easy, convenient routing.
func (a *App) Handle(verb, path string, handler Handler, mw ...Middleware) {

	// Wrap up the application-wide first, this will call the first function
	// of each middleware which will return a function of type Handler. Each
	// Handler will then be wrapped up with the other handlers from the chain.
	handler = wrapMiddleware(wrapMiddleware(handler, mw), a.mw)

	// The function to execute for each request.
	h := func(w http.ResponseWriter, r *http.Request, params map[string]string) {

		// Set the context with the required values to
		// process the request.
		v := Values{
			TraceID: uuid.New(),
			Now:     time.Now(),
		}
		ctx := context.WithValue(r.Context(), KeyValues, &v)

		// Set the trace id on the outgoing requests before any other header to
		// ensure that the trace id is ALWAYS added to the request regardless of
		// any error occuring or not.
		w.Header().Set(TraceIDHeader, v.TraceID)

		// Call the wrapped handler functions.
		handler(ctx, w, r, params)
	}

	// Add this handler for the specified verb and route.
	a.TreeMux.Handle(verb, path, h)
}

//==============================================================================

// Group allows a segment of middleware to be shared amongst handlers.
type Group struct {
	app *App
	mw  []Middleware
}

// Use adds the set of provided middleware onto the Application middleware chain.
func (g *Group) Use(mw ...Middleware) {
	g.mw = append(g.mw, mw...)
}

// Handle proxies the Handle function of the underlying App.
func (g *Group) Handle(verb, path string, handler Handler, mw ...Middleware) {

	// Wrap up the route specific middleware last because rememeber, the
	// middleware is wrapped backwards.
	handler = wrapMiddleware(handler, mw)

	// Wrap it with the App wrapper and additionally the group level middleware.
	g.app.Handle(verb, path, handler, g.mw...)
}

//==============================================================================

// Run is called to start the web service.
func Run(host string, routes http.Handler, readTimeout, writeTimeout time.Duration) error {

	// Create a new server and set timeout values.
	server := manners.NewWithServer(&http.Server{
		Addr:           host,
		Handler:        routes,
		ReadTimeout:    readTimeout,
		WriteTimeout:   writeTimeout,
		MaxHeaderBytes: 1 << 20,
	})

	go func() {

		// Listen for an interrupt signal from the OS.
		osSignals := make(chan os.Signal)
		signal.Notify(osSignals, os.Interrupt)

		sig := <-osSignals

		// Shut down the API server.
		server.Close()
	}()

	return server.ListenAndServe()
}

//==============================================================================

// wrapMiddleware wraps a handler with some middleware.
func wrapMiddleware(handler Handler, mw []Middleware) Handler {

	// Wrap with our group specific middleware.
	for i := len(mw) - 1; i >= 0; i-- {
		if mw[i] != nil {
			handler = mw[i](handler)
		}
	}

	return handler
}
<commit_msg>Removed variable because of logging.<commit_after>// Package web provides a thin layer of support for writing web services. It
// integrates with the ardanlabs kit repo to provide support for routing and
// application ctx. The base things you need to write a web service is
// provided.
package web

import (
	"context"
	"encoding/json"
	"io"
	"net/http"
	"os"
	"os/signal"
	"time"

	"github.com/braintree/manners"
	"github.com/dimfeld/httptreemux"
	"github.com/pborman/uuid"
	"gopkg.in/go-playground/validator.v8"
)

// TraceIDHeader is the header added to outgoing requests which adds the
// traceID to it.
const TraceIDHeader = "X-Trace-ID"

//==============================================================================

// validate provides a validator for checking models.
var validate = validator.New(&validator.Config{
	TagName:      "validate",
	FieldNameTag: "json",
})

// Unmarshal decodes the input to the struct type and checks the
// fields to verify the value is in a proper state.
func Unmarshal(r io.Reader, v interface{}) error {
	if err := json.NewDecoder(r).Decode(v); err != nil {
		return err
	}

	var inv InvalidError
	if fve := validate.Struct(v); fve != nil {
		for _, fe := range fve.(validator.ValidationErrors) {
			inv = append(inv, Invalid{Fld: fe.Field, Err: fe.Tag})
		}
		return inv
	}

	return nil
}

//==============================================================================

// Key represents the type of value for the context key.
type ctxKey int

// KeyValues is how request values or stored/retrieved.
const KeyValues ctxKey = 1

// Values represent state for each request.
type Values struct {
	TraceID    string
	Now        time.Time
	StatusCode int
}

//==============================================================================

// A Handler is a type that handles an http request within our own little mini
// framework.
type Handler func(ctx context.Context, w http.ResponseWriter, r *http.Request, params map[string]string) error

// A Middleware is a type that wraps a handler to remove boilerplate or other
// concerns not direct to any given Handler.
type Middleware func(Handler) Handler

//==============================================================================

// App is the entrypoint into our application and what configures our context
// object for each of our http handlers. Feel free to add any configuration
// data/logic on this App struct
type App struct {
	*httptreemux.TreeMux
	Values map[string]interface{}

	mw []Middleware
}

// New create an App value that handle a set of routes for the application.
// You can provide any number of middleware and they'll be used to wrap every
// request handler.
func New(mw ...Middleware) *App {
	return &App{
		TreeMux: httptreemux.New(),
		Values:  make(map[string]interface{}),
		mw:      mw,
	}
}

// Group creates a new App Group based on the current App and provided
// middleware.
func (a *App) Group(mw ...Middleware) *Group {
	return &Group{
		app: a,
		mw:  mw,
	}
}

// Use adds the set of provided middleware onto the Application middleware
// chain. Any route running off of this App will use all the middleware provided
// this way always regardless of the ordering of the Handle/Use functions.
func (a *App) Use(mw ...Middleware) {
	a.mw = append(a.mw, mw...)
}

// Handle is our mechanism for mounting Handlers for a given HTTP verb and path
// pair, this makes for really easy, convenient routing.
func (a *App) Handle(verb, path string, handler Handler, mw ...Middleware) {

	// Wrap up the application-wide first, this will call the first function
	// of each middleware which will return a function of type Handler. Each
	// Handler will then be wrapped up with the other handlers from the chain.
	handler = wrapMiddleware(wrapMiddleware(handler, mw), a.mw)

	// The function to execute for each request.
	h := func(w http.ResponseWriter, r *http.Request, params map[string]string) {

		// Set the context with the required values to
		// process the request.
		v := Values{
			TraceID: uuid.New(),
			Now:     time.Now(),
		}
		ctx := context.WithValue(r.Context(), KeyValues, &v)

		// Set the trace id on the outgoing requests before any other header to
		// ensure that the trace id is ALWAYS added to the request regardless of
		// any error occuring or not.
		w.Header().Set(TraceIDHeader, v.TraceID)

		// Call the wrapped handler functions.
		handler(ctx, w, r, params)
	}

	// Add this handler for the specified verb and route.
	a.TreeMux.Handle(verb, path, h)
}

//==============================================================================

// Group allows a segment of middleware to be shared amongst handlers.
type Group struct {
	app *App
	mw  []Middleware
}

// Use adds the set of provided middleware onto the Application middleware chain.
func (g *Group) Use(mw ...Middleware) {
	g.mw = append(g.mw, mw...)
}

// Handle proxies the Handle function of the underlying App.
func (g *Group) Handle(verb, path string, handler Handler, mw ...Middleware) {

	// Wrap up the route specific middleware last because rememeber, the
	// middleware is wrapped backwards.
	handler = wrapMiddleware(handler, mw)

	// Wrap it with the App wrapper and additionally the group level middleware.
	g.app.Handle(verb, path, handler, g.mw...)
}

//==============================================================================

// Run is called to start the web service.
func Run(host string, routes http.Handler, readTimeout, writeTimeout time.Duration) error {

	// Create a new server and set timeout values.
	server := manners.NewWithServer(&http.Server{
		Addr:           host,
		Handler:        routes,
		ReadTimeout:    readTimeout,
		WriteTimeout:   writeTimeout,
		MaxHeaderBytes: 1 << 20,
	})

	go func() {

		// Listen for an interrupt signal from the OS.
		osSignals := make(chan os.Signal)
		signal.Notify(osSignals, os.Interrupt)

		<-osSignals

		// Shut down the API server.
		server.Close()
	}()

	return server.ListenAndServe()
}

//==============================================================================

// wrapMiddleware wraps a handler with some middleware.
func wrapMiddleware(handler Handler, mw []Middleware) Handler {

	// Wrap with our group specific middleware.
	for i := len(mw) - 1; i >= 0; i-- {
		if mw[i] != nil {
			handler = mw[i](handler)
		}
	}

	return handler
}
<|endoftext|>


<commit_before>//go:build cgo && !ios

package rcmgr

import (
	"github.com/elastic/gosigar"
)

// DynamicLimit is a limit with dynamic memory values, based on available (free) memory
type DynamicLimit struct {
	BaseLimit

	// MinMemory is the minimum memory for this limit
	MinMemory int64
	// MaxMemory is the maximum memory for this limit
	MaxMemory int64
	// MemoryFraction is the fraction of available memory allowed for this limit,
	// bounded by [MinMemory, MaxMemory]
	MemoryFraction float64
}

var _ Limit = (*DynamicLimit)(nil)

func (l *DynamicLimit) GetMemoryLimit() int64 {
	var mem gosigar.Mem
	if err := mem.Get(); err != nil {
		panic(err)
	}

	limit := int64(float64(mem.ActualFree) * l.MemoryFraction)
	if limit < l.MinMemory {
		limit = l.MinMemory
	} else if limit > l.MaxMemory {
		limit = l.MaxMemory
	}

	return limit
}

// NewDynamicLimiter creates a limiter with default limits and a memory cap dynamically computed
// based on available memory. minMemory and maxMemory specify the system memory bounds,
// while memFraction specifies the fraction of available memory available for the system, within
// the specified bounds.
func NewDynamicLimiter(memFraction float64, minMemory, maxMemory int64) *BasicLimiter {
	system := &DynamicLimit{
		MinMemory:      minMemory,
		MaxMemory:      maxMemory,
		MemoryFraction: memFraction,
		BaseLimit:      DefaultSystemBaseLimit(),
	}
	transient := &DynamicLimit{
		MinMemory:      minMemory / 16,
		MaxMemory:      maxMemory / 16,
		MemoryFraction: memFraction / 16,
		BaseLimit:      DefaultTransientBaseLimit(),
	}
	svc := &DynamicLimit{
		MinMemory:      minMemory / 2,
		MaxMemory:      maxMemory / 2,
		MemoryFraction: memFraction / 2,
		BaseLimit:      DefaultServiceBaseLimit(),
	}
	proto := &DynamicLimit{
		MinMemory:      minMemory / 4,
		MaxMemory:      maxMemory / 4,
		MemoryFraction: memFraction / 4,
		BaseLimit:      DefaultProtocolBaseLimit(),
	}
	peer := &DynamicLimit{
		MinMemory:      minMemory / 16,
		MaxMemory:      maxMemory / 16,
		MemoryFraction: memFraction / 16,
		BaseLimit:      DefaultPeerBaseLimit(),
	}
	conn := &StaticLimit{
		Memory:    16 << 20,
		BaseLimit: ConnBaseLimit(),
	}
	stream := &StaticLimit{
		Memory:    16 << 20,
		BaseLimit: StreamBaseLimit(),
	}

	return &BasicLimiter{
		SystemLimits:          system,
		TransientLimits:       transient,
		DefaultServiceLimits:  svc,
		DefaultProtocolLimits: proto,
		DefaultPeerLimits:     peer,
		ConnLimits:            conn,
		StreamLimits:          stream,
	}
}
<commit_msg>fix build tags for v1.16 toolchain<commit_after>//go:build cgo && !ios
// +build cgo,!ios

package rcmgr

import (
	"github.com/elastic/gosigar"
)

// DynamicLimit is a limit with dynamic memory values, based on available (free) memory
type DynamicLimit struct {
	BaseLimit

	// MinMemory is the minimum memory for this limit
	MinMemory int64
	// MaxMemory is the maximum memory for this limit
	MaxMemory int64
	// MemoryFraction is the fraction of available memory allowed for this limit,
	// bounded by [MinMemory, MaxMemory]
	MemoryFraction float64
}

var _ Limit = (*DynamicLimit)(nil)

func (l *DynamicLimit) GetMemoryLimit() int64 {
	var mem gosigar.Mem
	if err := mem.Get(); err != nil {
		panic(err)
	}

	limit := int64(float64(mem.ActualFree) * l.MemoryFraction)
	if limit < l.MinMemory {
		limit = l.MinMemory
	} else if limit > l.MaxMemory {
		limit = l.MaxMemory
	}

	return limit
}

// NewDynamicLimiter creates a limiter with default limits and a memory cap dynamically computed
// based on available memory. minMemory and maxMemory specify the system memory bounds,
// while memFraction specifies the fraction of available memory available for the system, within
// the specified bounds.
func NewDynamicLimiter(memFraction float64, minMemory, maxMemory int64) *BasicLimiter {
	system := &DynamicLimit{
		MinMemory:      minMemory,
		MaxMemory:      maxMemory,
		MemoryFraction: memFraction,
		BaseLimit:      DefaultSystemBaseLimit(),
	}
	transient := &DynamicLimit{
		MinMemory:      minMemory / 16,
		MaxMemory:      maxMemory / 16,
		MemoryFraction: memFraction / 16,
		BaseLimit:      DefaultTransientBaseLimit(),
	}
	svc := &DynamicLimit{
		MinMemory:      minMemory / 2,
		MaxMemory:      maxMemory / 2,
		MemoryFraction: memFraction / 2,
		BaseLimit:      DefaultServiceBaseLimit(),
	}
	proto := &DynamicLimit{
		MinMemory:      minMemory / 4,
		MaxMemory:      maxMemory / 4,
		MemoryFraction: memFraction / 4,
		BaseLimit:      DefaultProtocolBaseLimit(),
	}
	peer := &DynamicLimit{
		MinMemory:      minMemory / 16,
		MaxMemory:      maxMemory / 16,
		MemoryFraction: memFraction / 16,
		BaseLimit:      DefaultPeerBaseLimit(),
	}
	conn := &StaticLimit{
		Memory:    16 << 20,
		BaseLimit: ConnBaseLimit(),
	}
	stream := &StaticLimit{
		Memory:    16 << 20,
		BaseLimit: StreamBaseLimit(),
	}

	return &BasicLimiter{
		SystemLimits:          system,
		TransientLimits:       transient,
		DefaultServiceLimits:  svc,
		DefaultProtocolLimits: proto,
		DefaultPeerLimits:     peer,
		ConnLimits:            conn,
		StreamLimits:          stream,
	}
}
<|endoftext|>


<commit_before><commit_msg>cache: use errors package instead of fmt.Errorf<commit_after><|endoftext|>


<commit_before>//
// Package webview implements Go bindings to https://github.com/zserge/webview C library.
//
// Bindings closely repeat the C APIs and include both, a simplified
// single-function API to just open a full-screen webview window, and a more
// advanced and featureful set of APIs, including Go-to-JavaScript bindings.
//
// The library uses gtk-webkit, Cocoa/Webkit and MSHTML (IE8..11) as a browser
// engine and supports Linux, MacOS and Windows 7..10 respectively.
//
package webview

/*
#cgo linux CFLAGS: -DWEBVIEW_GTK=1
#cgo linux pkg-config: gtk+-3.0 webkitgtk-3.0

#cgo windows CFLAGS: -DWEBVIEW_WINAPI=1
#cgo windows LDFLAGS: -lole32 -lcomctl32 -loleaut32 -luuid -mwindows

#cgo darwin CFLAGS: -DWEBVIEW_COCOA=1 -x objective-c
#cgo darwin LDFLAGS: -framework Cocoa -framework WebKit

#include <stdlib.h>
#include "webview.h"

extern void _webviewExternalInvokeCallback(void *, void *);

static inline void CgoWebViewFree(void *w) {
	free((void *)((struct webview *)w)->title);
	free((void *)((struct webview *)w)->url);
	free(w);
}

static inline void *CgoWebViewCreate(int width, int height, char *title, char *url, int resizable) {
	struct webview *w = (struct webview *) malloc(sizeof(*w));
	w->width = width;
	w->height = height;
	w->title = title;
	w->url = url;
	w->resizable = resizable;
	w->external_invoke_cb = (webview_external_invoke_cb_t) _webviewExternalInvokeCallback;
	if (webview_init(w) != 0) {
		CgoWebViewFree(w);
		return NULL;
	}
	return (void *)w;
}

static inline int CgoWebViewLoop(void *w, int blocking) {
	return webview_loop((struct webview *)w, blocking);
}

static inline void CgoWebViewTerminate(void *w) {
	webview_terminate((struct webview *)w);
}

static inline void CgoWebViewExit(void *w) {
	webview_exit((struct webview *)w);
}

static inline int CgoWebViewEval(void *w, char *js) {
	return webview_eval((struct webview *)w, js);
}

extern void _webviewDispatchGoCallback(void *);
static inline void _webview_dispatch_cb(struct webview *w, void *arg) {
	_webviewDispatchGoCallback(arg);
}
static inline void CgoWebViewDispatch(void *w, void *arg) {
	webview_dispatch((struct webview *)w, _webview_dispatch_cb, arg);
}
*/
import "C"
import (
	"errors"
	"sync"
	"unsafe"
)

// Open is a simplified API to open a single native window with a full-size webview in
// it. It can be helpful if you want to communicate with the core app using XHR
// or WebSockets (as opposed to using JavaScript bindings).
//
// Window appearance can be customized using title, width, height and resizable parameters.
// URL must be provided and can user either a http or https protocol, or be a
// local file:// URL. On some platforms "data:" URLs are also supported
// (Linux/MacOS).
func Open(title, url string, w, h int, resizable bool) error {
	titleStr := C.CString(title)
	defer C.free(unsafe.Pointer(titleStr))
	urlStr := C.CString(url)
	defer C.free(unsafe.Pointer(urlStr))
	resize := C.int(0)
	if resizable {
		resize = C.int(1)
	}

	r := C.webview(titleStr, urlStr, C.int(w), C.int(h), resize)
	if r != 0 {
		return errors.New("failed to create webview")
	}
	return nil
}

// ExternalInvokeCallbackFunc is a function type that is called every time
// "window.external.invoke_()" is called from JavaScript. Data is the only
// obligatory string parameter passed into the "invoke_(data)" function from
// JavaScript. To pass more complex data serialized JSON or base64 encoded
// string can be used.
type ExternalInvokeCallbackFunc func(w WebView, data string)

// Settings is a set of parameters to customize the initial WebView appearance
// and behavior. It is passed into the webview.New() constructor.
type Settings struct {
	// WebView main window title
	Title string
	// URL to open in a webview
	URL string
	// Window width in pixels
	Width int
	// Window height in pixels
	Height int
	// Allows/disallows window resizing
	Resizable bool
	// A callback that is executed when JavaScript calls "window.external.invoke_()"
	ExternalInvokeCallback ExternalInvokeCallbackFunc
}

// WebView is an interface that wraps the basic methods for controlling the UI
// loop, handling multithreading and providing JavaScript bindings.
type WebView interface {
	// Run() starts the main UI loop until the user closes the webview window or
	// Terminate() is called.
	Run()
	// Loop() runs a single iteration of the main UI.
	Loop(blocking bool) bool
	// Eval() evaluates an arbitrary JS code inside the webview. This method must
	// be called from the main thread only. See Dispatch() for more details.
	Eval(js string)
	// Terminate() breaks the main UI loop. This method must be called from the main thread
	// only. See Dispatch() for more details.
	Terminate()
	// Dispatch() schedules some arbitrary function to be executed on the main UI
	// thread. This may be helpful if you want to run some JavaScript from
	// background threads/goroutines, or to terminate the app.
	Dispatch(func())
	// Exit() closes the window and cleans up the resources. Use Terminate() to
	// forcefully break out of the main UI loop.
	Exit()
}

var (
	m     sync.Mutex
	index uintptr
	fns   = map[uintptr]func(){}
	cbs   = map[WebView]ExternalInvokeCallbackFunc{}
)

type webview struct {
	w unsafe.Pointer
}

var _ WebView = &webview{}

// New creates and opens a new webview window using the given settings. The
// returned object implements the WebView interface. This function returns nil
// if a window can not be created.
func New(settings Settings) WebView {
	if settings.Width == 0 {
		settings.Width = 640
	}
	if settings.Height == 0 {
		settings.Height = 480
	}
	if settings.Title == "" {
		settings.Title = "WebView"
	}
	if settings.URL == "" {
		return nil
	}
	resizable := 0
	if settings.Resizable {
		resizable = 1
	}
	w := &webview{}
	w.w = C.CgoWebViewCreate(C.int(settings.Width), C.int(settings.Height), C.CString(settings.Title), C.CString(settings.URL), C.int(resizable))
	if settings.ExternalInvokeCallback != nil {
		m.Lock()
		cbs[w] = settings.ExternalInvokeCallback
		m.Unlock()
	}
	return w
}

func (w *webview) Loop(blocking bool) bool {
	block := C.int(0)
	if blocking {
		block = 1
	}
	return C.CgoWebViewLoop(w.w, block) == 0
}

func (w *webview) Run() {
	for w.Loop(true) {
	}
}

func (w *webview) Exit() {
	C.CgoWebViewExit(w.w)
}

func (w *webview) Dispatch(f func()) {
	m.Lock()
	for ; fns[index] != nil; index++ {
	}
	fns[index] = f
	m.Unlock()
	C.CgoWebViewDispatch(w.w, unsafe.Pointer(index))
	m.Lock()
	delete(fns, index)
	m.Unlock()
}

func (w *webview) Eval(js string) {
	p := C.CString(js)
	defer C.free(unsafe.Pointer(p))
	C.CgoWebViewEval(w.w, p)
}

func (w *webview) Terminate() {
	C.CgoWebViewTerminate(w.w)
}

//export _webviewDispatchGoCallback
func _webviewDispatchGoCallback(index unsafe.Pointer) {
	m.Lock()
	defer m.Unlock()
	fns[uintptr(index)]()
}

//export _webviewExternalInvokeCallback
func _webviewExternalInvokeCallback(w unsafe.Pointer, data unsafe.Pointer) {
	m.Lock()
	var (
		cb ExternalInvokeCallbackFunc
		wv WebView
	)
	for wv, cb = range cbs {
		if wv.(*webview).w == w {
			break
		}
	}
	m.Unlock()
	cb(wv, C.GoString((*C.char)(data)))
}
<commit_msg>go: SetTitle API wrapper<commit_after>//
// Package webview implements Go bindings to https://github.com/zserge/webview C library.
//
// Bindings closely repeat the C APIs and include both, a simplified
// single-function API to just open a full-screen webview window, and a more
// advanced and featureful set of APIs, including Go-to-JavaScript bindings.
//
// The library uses gtk-webkit, Cocoa/Webkit and MSHTML (IE8..11) as a browser
// engine and supports Linux, MacOS and Windows 7..10 respectively.
//
package webview

/*
#cgo linux CFLAGS: -DWEBVIEW_GTK=1
#cgo linux pkg-config: gtk+-3.0 webkitgtk-3.0

#cgo windows CFLAGS: -DWEBVIEW_WINAPI=1
#cgo windows LDFLAGS: -lole32 -lcomctl32 -loleaut32 -luuid -mwindows

#cgo darwin CFLAGS: -DWEBVIEW_COCOA=1 -x objective-c
#cgo darwin LDFLAGS: -framework Cocoa -framework WebKit

#include <stdlib.h>
#include "webview.h"

extern void _webviewExternalInvokeCallback(void *, void *);

static inline void CgoWebViewFree(void *w) {
	free((void *)((struct webview *)w)->title);
	free((void *)((struct webview *)w)->url);
	free(w);
}

static inline void *CgoWebViewCreate(int width, int height, char *title, char *url, int resizable) {
	struct webview *w = (struct webview *) malloc(sizeof(*w));
	w->width = width;
	w->height = height;
	w->title = title;
	w->url = url;
	w->resizable = resizable;
	w->external_invoke_cb = (webview_external_invoke_cb_t) _webviewExternalInvokeCallback;
	if (webview_init(w) != 0) {
		CgoWebViewFree(w);
		return NULL;
	}
	return (void *)w;
}

static inline int CgoWebViewLoop(void *w, int blocking) {
	return webview_loop((struct webview *)w, blocking);
}

static inline void CgoWebViewTerminate(void *w) {
	webview_terminate((struct webview *)w);
}

static inline void CgoWebViewExit(void *w) {
	webview_exit((struct webview *)w);
}

static inline void CgoWebViewSetTitle(void *w, char *title) {
	webview_set_title((struct webview *)w, title);
}

static inline int CgoWebViewEval(void *w, char *js) {
	return webview_eval((struct webview *)w, js);
}

extern void _webviewDispatchGoCallback(void *);
static inline void _webview_dispatch_cb(struct webview *w, void *arg) {
	_webviewDispatchGoCallback(arg);
}
static inline void CgoWebViewDispatch(void *w, void *arg) {
	webview_dispatch((struct webview *)w, _webview_dispatch_cb, arg);
}
*/
import "C"
import (
	"errors"
	"sync"
	"unsafe"
)

// Open is a simplified API to open a single native window with a full-size webview in
// it. It can be helpful if you want to communicate with the core app using XHR
// or WebSockets (as opposed to using JavaScript bindings).
//
// Window appearance can be customized using title, width, height and resizable parameters.
// URL must be provided and can user either a http or https protocol, or be a
// local file:// URL. On some platforms "data:" URLs are also supported
// (Linux/MacOS).
func Open(title, url string, w, h int, resizable bool) error {
	titleStr := C.CString(title)
	defer C.free(unsafe.Pointer(titleStr))
	urlStr := C.CString(url)
	defer C.free(unsafe.Pointer(urlStr))
	resize := C.int(0)
	if resizable {
		resize = C.int(1)
	}

	r := C.webview(titleStr, urlStr, C.int(w), C.int(h), resize)
	if r != 0 {
		return errors.New("failed to create webview")
	}
	return nil
}

// ExternalInvokeCallbackFunc is a function type that is called every time
// "window.external.invoke_()" is called from JavaScript. Data is the only
// obligatory string parameter passed into the "invoke_(data)" function from
// JavaScript. To pass more complex data serialized JSON or base64 encoded
// string can be used.
type ExternalInvokeCallbackFunc func(w WebView, data string)

// Settings is a set of parameters to customize the initial WebView appearance
// and behavior. It is passed into the webview.New() constructor.
type Settings struct {
	// WebView main window title
	Title string
	// URL to open in a webview
	URL string
	// Window width in pixels
	Width int
	// Window height in pixels
	Height int
	// Allows/disallows window resizing
	Resizable bool
	// A callback that is executed when JavaScript calls "window.external.invoke_()"
	ExternalInvokeCallback ExternalInvokeCallbackFunc
}

// WebView is an interface that wraps the basic methods for controlling the UI
// loop, handling multithreading and providing JavaScript bindings.
type WebView interface {
	// Run() starts the main UI loop until the user closes the webview window or
	// Terminate() is called.
	Run()
	// Loop() runs a single iteration of the main UI.
	Loop(blocking bool) bool
	// SetTitle() changes window title. This method must be called from the main
	// thread only. See Dispatch() for more details.
	SetTitle(title string)
	// Eval() evaluates an arbitrary JS code inside the webview. This method must
	// be called from the main thread only. See Dispatch() for more details.
	Eval(js string)
	// Terminate() breaks the main UI loop. This method must be called from the main thread
	// only. See Dispatch() for more details.
	Terminate()
	// Dispatch() schedules some arbitrary function to be executed on the main UI
	// thread. This may be helpful if you want to run some JavaScript from
	// background threads/goroutines, or to terminate the app.
	Dispatch(func())
	// Exit() closes the window and cleans up the resources. Use Terminate() to
	// forcefully break out of the main UI loop.
	Exit()
}

var (
	m     sync.Mutex
	index uintptr
	fns   = map[uintptr]func(){}
	cbs   = map[WebView]ExternalInvokeCallbackFunc{}
)

type webview struct {
	w unsafe.Pointer
}

var _ WebView = &webview{}

// New creates and opens a new webview window using the given settings. The
// returned object implements the WebView interface. This function returns nil
// if a window can not be created.
func New(settings Settings) WebView {
	if settings.Width == 0 {
		settings.Width = 640
	}
	if settings.Height == 0 {
		settings.Height = 480
	}
	if settings.Title == "" {
		settings.Title = "WebView"
	}
	if settings.URL == "" {
		return nil
	}
	resizable := 0
	if settings.Resizable {
		resizable = 1
	}
	w := &webview{}
	w.w = C.CgoWebViewCreate(C.int(settings.Width), C.int(settings.Height), C.CString(settings.Title), C.CString(settings.URL), C.int(resizable))
	if settings.ExternalInvokeCallback != nil {
		m.Lock()
		cbs[w] = settings.ExternalInvokeCallback
		m.Unlock()
	}
	return w
}

func (w *webview) Loop(blocking bool) bool {
	block := C.int(0)
	if blocking {
		block = 1
	}
	return C.CgoWebViewLoop(w.w, block) == 0
}

func (w *webview) Run() {
	for w.Loop(true) {
	}
}

func (w *webview) Exit() {
	C.CgoWebViewExit(w.w)
}

func (w *webview) Dispatch(f func()) {
	m.Lock()
	for ; fns[index] != nil; index++ {
	}
	fns[index] = f
	m.Unlock()
	C.CgoWebViewDispatch(w.w, unsafe.Pointer(index))
	m.Lock()
	delete(fns, index)
	m.Unlock()
}

func (w *webview) SetTitle(title string) {
	p := C.CString(title)
	defer C.free(unsafe.Pointer(p))
	C.CgoWebViewSetTitle(w.w, p)
}

func (w *webview) Eval(js string) {
	p := C.CString(js)
	defer C.free(unsafe.Pointer(p))
	C.CgoWebViewEval(w.w, p)
}

func (w *webview) Terminate() {
	C.CgoWebViewTerminate(w.w)
}

//export _webviewDispatchGoCallback
func _webviewDispatchGoCallback(index unsafe.Pointer) {
	m.Lock()
	defer m.Unlock()
	fns[uintptr(index)]()
}

//export _webviewExternalInvokeCallback
func _webviewExternalInvokeCallback(w unsafe.Pointer, data unsafe.Pointer) {
	m.Lock()
	var (
		cb ExternalInvokeCallbackFunc
		wv WebView
	)
	for wv, cb = range cbs {
		if wv.(*webview).w == w {
			break
		}
	}
	m.Unlock()
	cb(wv, C.GoString((*C.char)(data)))
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"log"
	"net/http"
	"net/http/httputil"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/gin-gonic/gin"
	"github.com/go-redis/redis/v8"
)

var color = "green"
var version = os.Getenv("VERSION")
var redisUrl = os.Getenv("REDIS_URL")
var rdb = redis.NewClient(&redis.Options{
	Addr:     redisUrl,
	Password: "", // no password set
	DB:       0,  // use default DB
})
var rdbCtx = context.Background()

func main() {
	port := ":8080"

	flag.Parse()

	r := gin.Default()
	log.Printf("Backend version: %s\n", version)

	r.GET("/", handleIndex)
	r.GET("/version", handleVersion)
	r.GET("/healthz", handleHealthz)

	srv := &http.Server{
		Addr:    port,
		Handler: r,
	}

	go func() {
		if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
			log.Fatalf("listen: %s\n", err)
		}
	}()

	// Wait for interrupt signal to gracefully shutdown the server with
	// a timeout of 5 seconds.
	quit := make(chan os.Signal, 1)
	// kill (no param) default send syscall.SIGTERM
	// kill -2 is syscall.SIGINT
	// kill -9 is syscall.SIGKILL but can't be catch, so don't need add it
	signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
	<-quit
	log.Println("Shutting down server...")

	// The context is used to inform the server it has 5 seconds to finish
	// the request it is currently handling
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()
	if err := srv.Shutdown(ctx); err != nil {
		log.Fatal("Server forced to shutdown:", err)
	}

	log.Println("Server exiting")
}

func handleIndex(c *gin.Context) {
	log.Printf("Received request from %s at %s", c.Request.RemoteAddr, c.Request.URL.EscapedPath())
	p := PodMetadata{}
	counter, err := incrCounter(c)
	if err != nil {
		c.String(http.StatusInternalServerError, "%v", err)
		return
	}
	err = p.Populate(version, counter, color)
	if err != nil {
		c.String(http.StatusInternalServerError, "%v", err)
		return
	}
	raw, _ := httputil.DumpRequest(c.Request, true)
	p.RawRequest = string(raw)
	c.JSON(http.StatusOK, p)
}

func incrCounter(c *gin.Context) (string, error) {
	err := rdb.Incr(rdbCtx, "counter").Err()
	if err != nil {
		return "", err
	}
	val, err := rdb.Get(rdbCtx, "counter").Result()
	if err != nil {
		return "", err
	}
	return val, nil
}

func handleVersion(c *gin.Context) {
	c.String(http.StatusOK, "%s", c.Value("version"))
}

func handleHealthz(c *gin.Context) {
	c.String(http.StatusOK, "", "")
}
<commit_msg>Make it blue<commit_after>package main

import (
	"context"
	"flag"
	"log"
	"net/http"
	"net/http/httputil"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/gin-gonic/gin"
	"github.com/go-redis/redis/v8"
)

var color = "blue"
var version = os.Getenv("VERSION")
var redisUrl = os.Getenv("REDIS_URL")
var rdb = redis.NewClient(&redis.Options{
	Addr:     redisUrl,
	Password: "", // no password set
	DB:       0,  // use default DB
})
var rdbCtx = context.Background()

func main() {
	port := ":8080"

	flag.Parse()

	r := gin.Default()
	log.Printf("Backend version: %s\n", version)

	r.GET("/", handleIndex)
	r.GET("/version", handleVersion)
	r.GET("/healthz", handleHealthz)

	srv := &http.Server{
		Addr:    port,
		Handler: r,
	}

	go func() {
		if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
			log.Fatalf("listen: %s\n", err)
		}
	}()

	// Wait for interrupt signal to gracefully shutdown the server with
	// a timeout of 5 seconds.
	quit := make(chan os.Signal, 1)
	// kill (no param) default send syscall.SIGTERM
	// kill -2 is syscall.SIGINT
	// kill -9 is syscall.SIGKILL but can't be catch, so don't need add it
	signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
	<-quit
	log.Println("Shutting down server...")

	// The context is used to inform the server it has 5 seconds to finish
	// the request it is currently handling
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()
	if err := srv.Shutdown(ctx); err != nil {
		log.Fatal("Server forced to shutdown:", err)
	}

	log.Println("Server exiting")
}

func handleIndex(c *gin.Context) {
	log.Printf("Received request from %s at %s", c.Request.RemoteAddr, c.Request.URL.EscapedPath())
	p := PodMetadata{}
	counter, err := incrCounter(c)
	if err != nil {
		c.String(http.StatusInternalServerError, "%v", err)
		return
	}
	err = p.Populate(version, counter, color)
	if err != nil {
		c.String(http.StatusInternalServerError, "%v", err)
		return
	}
	raw, _ := httputil.DumpRequest(c.Request, true)
	p.RawRequest = string(raw)
	c.JSON(http.StatusOK, p)
}

func incrCounter(c *gin.Context) (string, error) {
	err := rdb.Incr(rdbCtx, "counter").Err()
	if err != nil {
		return "", err
	}
	val, err := rdb.Get(rdbCtx, "counter").Result()
	if err != nil {
		return "", err
	}
	return val, nil
}

func handleVersion(c *gin.Context) {
	c.String(http.StatusOK, "%s", c.Value("version"))
}

func handleHealthz(c *gin.Context) {
	c.String(http.StatusOK, "", "")
}
<|endoftext|>


<commit_before>package backend

import (
	"bufio"
	"fmt"
	"log"
	"runtime"

	"github.com/garyburd/redigo/redis"
	"github.com/go-pg/pg"
	"github.com/gocraft/work"
	"github.com/harrisbaird/dailyteedeals/config"
	"github.com/harrisbaird/dailyteedeals/models"
	"github.com/harrisbaird/dailyteedeals/utils"
)

const (
	jobNamespace       = "dailyteedeals"
	recheckRateSeconds = int64(5)
)

var (
	noRetryOptions = work.JobOptions{MaxFails: 1}
	minioConn      = utils.NewMinioConnection()
	enqueuer       *work.Enqueuer
	pool           *work.WorkerPool
)

type JobContext struct {
	DB *pg.DB
}

func Start(db *pg.DB) {
	log.Println("Starting backend")
	redisPool := &redis.Pool{Dial: func() (redis.Conn, error) { return redis.Dial("tcp", config.App.RedisAddr) }}
	pool = work.NewWorkerPool(JobContext{DB: db}, 10, jobNamespace, redisPool)
	enqueuer = work.NewEnqueuer(jobNamespace, redisPool)

	pool.Middleware(func(c *JobContext, job *work.Job, next work.NextMiddlewareFunc) error {
		c.DB = db
		return next()
	})
	pool.Middleware((*JobContext).Log)

	// Setup job scheduling
	if config.IsProduction() {
		log.Println("Creating periodic backend jobs")
		pool.PeriodicallyEnqueue("0 30 * * * *", "schedule_deal")
		// pool.PeriodicallyEnqueue("0 0 0 * * 1", "schedule_full")
		pool.PeriodicallyEnqueue("0 0 0 * * *", "update_exchange_rates")
	}

	pool.JobWithOptions("schedule_deal", noRetryOptions, (*JobContext).ScheduleDeal)
	pool.JobWithOptions("schedule_full", noRetryOptions, (*JobContext).ScheduleFull)
	pool.JobWithOptions("wait_for_scraper", noRetryOptions, (*JobContext).WaitForScraper)
	pool.JobWithOptions("parse_feed", noRetryOptions, (*JobContext).ParseFeed)
	pool.JobWithOptions("parse_item", noRetryOptions, (*JobContext).ParseItem)
	pool.JobWithOptions("update_exchange_rates", noRetryOptions, (*JobContext).UpdateExchangeRates)
	pool.Start()
}

func Stop() {
	log.Println("Stopping backend")
	pool.Stop()
}

func (c *JobContext) Log(job *work.Job, next work.NextMiddlewareFunc) error {
	log.Printf("Starting job: %s (%s)\n", job.Name, job.ID)

	err := next()
	if err != nil {
		_, fn, line, _ := runtime.Caller(1)
		log.Printf("[%s] Job errored: %s:%d %v\n", job.Name, fn, line, err)
	}

	return err
}

// ScheduleDeal schedules jobs for active sites which have
// deal_scraper enabled.
func (c *JobContext) ScheduleDeal(job *work.Job) error {
	return c.scheduleJobs(models.SiteDealJobType)
}

// ScheduleFull schedules jobs for active sites which have
// full_scraper enabled.
func (c *JobContext) ScheduleFull(job *work.Job) error {
	return c.scheduleJobs(models.SiteFullJobType)
}

// WaitForScraper waits until scrapyd reports that the job is finished
// and schedules a 'parse_feed' job, otherwise reschedules a 'wait_for_scraper
// job in 5 seconds.
func (c *JobContext) WaitForScraper(job *work.Job) error {
	spiderJob, err := models.FindSpiderJob(c.DB, int(job.ArgInt64("spider_job_id")))
	if err != nil {
		return err
	}
	finished, err := ScrapydIsFinished(spiderJob.ScrapydJobID)
	if err != nil {
		return err
	}

	if finished {
		// Scrapy job has finished, parse item feed.
		enqueuer.Enqueue("parse_feed", job.Args)
	} else {
		// Re-enqueue job with existing args.
		enqueuer.EnqueueUniqueIn("wait_for_scraper", recheckRateSeconds, job.Args)
	}

	return nil
}

// ParseFeed parses downloads and parses the item feed
// and creates a 'parse_item' job for each item.
func (c *JobContext) ParseFeed(job *work.Job) error {
	spiderJob, err := models.FindSpiderJob(c.DB, int(job.ArgInt64("spider_job_id")))
	if err != nil {
		return err
	}
	if err := models.MarkProductsInactive(c.DB, spiderJob.SiteID, spiderJob.JobType == models.SiteDealJobType.String()); err != nil {
		return err
	}

	feed, err := ScrapydDownloadFeed(spiderJob.ScrapydJobID)
	if err != nil {
		return err
	}

	scanner := bufio.NewScanner(feed)
	for scanner.Scan() {
		spiderItem, err := models.CreateSpiderItem(c.DB, int(spiderJob.ID), scanner.Text())
		if err != nil {
			fmt.Println("CreateSpiderItem: " + err.Error())
		}

		enqueuer.Enqueue("parse_item", work.Q{"spider_item_id": spiderItem.ID})
	}

	return scanner.Err()
}

// ParseItem parses the item data, creating the required database rows.
func (c *JobContext) ParseItem(job *work.Job) error {
	spiderItem, err := models.FindSpiderItem(c.DB, int(job.ArgInt64("spider_item_id")))
	if err != nil {
		return err
	}

	err = c.DB.RunInTransaction(func(tx *pg.Tx) error {
		err := spiderItem.ParseItemData(tx, minioConn)
		if err != nil {
			spiderItem.UpdateError(c.DB, err)
		}
		return err
	})
	return err
}

func (c *JobContext) UpdateExchangeRates(job *work.Job) error {
	return utils.UpdateRates()
}

func (c *JobContext) scheduleJobs(jobType models.SiteJobType) error {
	log.Printf("Starting jobs for %s sites\n", jobType.String())

	sites, err := models.ActiveSitesWithJobType(c.DB, jobType)
	if err != nil {
		return err
	}

	for _, site := range sites {
		scrapydJobID, err := ScrapydSchedule(site.Slug + "_" + jobType.String())
		if err != nil {
			log.Println("ScrapydSchedule: " + err.Error())
			continue
		}

		spiderJob, err := models.CreateSpiderJob(c.DB, site.ID, scrapydJobID, jobType.String())
		if err != nil {
			log.Println("CreateSpiderJob: " + err.Error())
			continue
		}

		_, err = enqueuer.EnqueueIn("wait_for_scraper", recheckRateSeconds,
			work.Q{"spider_job_id": spiderJob.ID})
		if err != nil {
			log.Println(err)
		}
	}

	return nil
}
<commit_msg>Write spider item error outside of transaction<commit_after>package backend

import (
	"bufio"
	"fmt"
	"log"
	"runtime"

	"github.com/garyburd/redigo/redis"
	"github.com/go-pg/pg"
	"github.com/gocraft/work"
	"github.com/harrisbaird/dailyteedeals/config"
	"github.com/harrisbaird/dailyteedeals/models"
	"github.com/harrisbaird/dailyteedeals/utils"
)

const (
	jobNamespace       = "dailyteedeals"
	recheckRateSeconds = int64(5)
)

var (
	noRetryOptions = work.JobOptions{MaxFails: 1}
	minioConn      = utils.NewMinioConnection()
	enqueuer       *work.Enqueuer
	pool           *work.WorkerPool
)

type JobContext struct {
	DB *pg.DB
}

func Start(db *pg.DB) {
	log.Println("Starting backend")
	redisPool := &redis.Pool{Dial: func() (redis.Conn, error) { return redis.Dial("tcp", config.App.RedisAddr) }}
	pool = work.NewWorkerPool(JobContext{DB: db}, 10, jobNamespace, redisPool)
	enqueuer = work.NewEnqueuer(jobNamespace, redisPool)

	pool.Middleware(func(c *JobContext, job *work.Job, next work.NextMiddlewareFunc) error {
		c.DB = db
		return next()
	})
	pool.Middleware((*JobContext).Log)

	// Setup job scheduling
	if config.IsProduction() {
		log.Println("Creating periodic backend jobs")
		pool.PeriodicallyEnqueue("0 30 * * * *", "schedule_deal")
		// pool.PeriodicallyEnqueue("0 0 0 * * 1", "schedule_full")
		pool.PeriodicallyEnqueue("0 0 0 * * *", "update_exchange_rates")
	}

	pool.JobWithOptions("schedule_deal", noRetryOptions, (*JobContext).ScheduleDeal)
	pool.JobWithOptions("schedule_full", noRetryOptions, (*JobContext).ScheduleFull)
	pool.JobWithOptions("wait_for_scraper", noRetryOptions, (*JobContext).WaitForScraper)
	pool.JobWithOptions("parse_feed", noRetryOptions, (*JobContext).ParseFeed)
	pool.JobWithOptions("parse_item", noRetryOptions, (*JobContext).ParseItem)
	pool.JobWithOptions("update_exchange_rates", noRetryOptions, (*JobContext).UpdateExchangeRates)
	pool.Start()
}

func Stop() {
	log.Println("Stopping backend")
	pool.Stop()
}

func (c *JobContext) Log(job *work.Job, next work.NextMiddlewareFunc) error {
	log.Printf("[%s] starting job\n", job.Name)

	err := next()
	if err != nil {
		_, fn, line, _ := runtime.Caller(1)
		log.Printf("[%s] encountered an error: %s:%d %v\n", job.Name, fn, line, err)
	} else {
		log.Printf("[%s] finished successfully\n", job.Name)
	}

	return err
}

// ScheduleDeal schedules jobs for active sites which have
// deal_scraper enabled.
func (c *JobContext) ScheduleDeal(job *work.Job) error {
	return c.scheduleJobs(models.SiteDealJobType)
}

// ScheduleFull schedules jobs for active sites which have
// full_scraper enabled.
func (c *JobContext) ScheduleFull(job *work.Job) error {
	return c.scheduleJobs(models.SiteFullJobType)
}

// WaitForScraper waits until scrapyd reports that the job is finished
// and schedules a 'parse_feed' job, otherwise reschedules a 'wait_for_scraper
// job in 5 seconds.
func (c *JobContext) WaitForScraper(job *work.Job) error {
	spiderJob, err := models.FindSpiderJob(c.DB, int(job.ArgInt64("spider_job_id")))
	if err != nil {
		return err
	}
	finished, err := ScrapydIsFinished(spiderJob.ScrapydJobID)
	if err != nil {
		return err
	}

	if finished {
		// Scrapy job has finished, parse item feed.
		enqueuer.Enqueue("parse_feed", job.Args)
	} else {
		// Re-enqueue job with existing args.
		enqueuer.EnqueueUniqueIn("wait_for_scraper", recheckRateSeconds, job.Args)
	}

	return nil
}

// ParseFeed parses downloads and parses the item feed
// and creates a 'parse_item' job for each item.
func (c *JobContext) ParseFeed(job *work.Job) error {
	spiderJob, err := models.FindSpiderJob(c.DB, int(job.ArgInt64("spider_job_id")))
	if err != nil {
		return err
	}
	if err := models.MarkProductsInactive(c.DB, spiderJob.SiteID, spiderJob.JobType == models.SiteDealJobType.String()); err != nil {
		return err
	}

	feed, err := ScrapydDownloadFeed(spiderJob.ScrapydJobID)
	if err != nil {
		return err
	}

	scanner := bufio.NewScanner(feed)
	for scanner.Scan() {
		spiderItem, err := models.CreateSpiderItem(c.DB, int(spiderJob.ID), scanner.Text())
		if err != nil {
			fmt.Println("CreateSpiderItem: " + err.Error())
		}

		enqueuer.Enqueue("parse_item", work.Q{"spider_item_id": spiderItem.ID})
	}

	return scanner.Err()
}

// ParseItem parses the item data, creating the required database rows.
func (c *JobContext) ParseItem(job *work.Job) error {
	var err error
	spiderItem, err := models.FindSpiderItem(c.DB, int(job.ArgInt64("spider_item_id")))
	if err != nil {
		return err
	}

	err = c.DB.RunInTransaction(func(tx *pg.Tx) error {
		return spiderItem.ParseItemData(tx, minioConn)
	})

	if err != nil {
		spiderItem.UpdateError(c.DB, err)
	}

	return err
}

func (c *JobContext) UpdateExchangeRates(job *work.Job) error {
	return utils.UpdateRates()
}

func (c *JobContext) scheduleJobs(jobType models.SiteJobType) error {
	log.Printf("Starting jobs for %s sites\n", jobType.String())

	sites, err := models.ActiveSitesWithJobType(c.DB, jobType)
	if err != nil {
		return err
	}

	for _, site := range sites {
		scrapydJobID, err := ScrapydSchedule(site.Slug + "_" + jobType.String())
		if err != nil {
			log.Println("ScrapydSchedule: " + err.Error())
			continue
		}

		spiderJob, err := models.CreateSpiderJob(c.DB, site.ID, scrapydJobID, jobType.String())
		if err != nil {
			log.Println("CreateSpiderJob: " + err.Error())
			continue
		}

		_, err = enqueuer.EnqueueIn("wait_for_scraper", recheckRateSeconds,
			work.Q{"spider_job_id": spiderJob.ID})
		if err != nil {
			log.Println(err)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package processor

import (
	"fmt"

	"github.com/nanobox-io/nanobox-boxfile"
	"github.com/nanobox-io/nanobox/models"
	"github.com/nanobox-io/nanobox/util"
	"github.com/nanobox-io/nanobox/util/data"
)

type devStop struct {
	config ProcessConfig
}

func init() {
	Register("dev_stop", devStopFunc)
}

func devStopFunc(config ProcessConfig) (Processor, error) {
	return devStop{config}, nil
}

func (self devStop) Results() ProcessConfig {
	return self.config
}

func (self devStop) Process() error {
	fmt.Println("-> stopping dev")
	app := models.App{}
	data.Get("apps", util.AppName(), app)

	if app.UsageCount == 0 {
		// remove all code containers
		boxfile := boxfile.New([]byte(self.config.Meta["boxfile"]))

		for _, codeName := range boxfile.Nodes("code") {

			self.config.Meta["name"] = codeName
			// dont catch errors because i dont
			// want destroy to fail
			err := Run("code_destroy", self.config)
			if err != nil {
				fmt.Printf("code_destroy (%s): %s\n", codeName, err.Error())
			}

		}
		// stop all data containers
		Run("service_stop_all", self.config)
	}

	//if nothing is running stop the provider
	keys, _ := data.Keys("apps")
	for _, key := range keys {
		data.Get("apps", key, &app)
		if app.UsageCount != 0 {
			// i found an app that is doing something
			// so dont shut down the provider
			fmt.Println("-> leaving provider running")
			return nil
		}
	}

	// this part only gets executed if we cant find any apps
	// doing anything in the loop above
	fmt.Println("-> stopping provider")
	return Run("provider_stop", self.config)
	// return nil
}
<commit_msg>dont stop apps containers if we dont need to<commit_after>package processor

import (
	"fmt"

	"github.com/nanobox-io/nanobox-boxfile"
	"github.com/nanobox-io/nanobox/models"
	"github.com/nanobox-io/nanobox/util"
	"github.com/nanobox-io/nanobox/util/data"
)

type devStop struct {
	config ProcessConfig
}

func init() {
	Register("dev_stop", devStopFunc)
}

func devStopFunc(config ProcessConfig) (Processor, error) {
	return devStop{config}, nil
}

func (self devStop) Results() ProcessConfig {
	return self.config
}

func (self devStop) Process() error {
	app := models.App{}
	data.Get("apps", util.AppName(), &app)

	if app.UsageCount == 0 {
		fmt.Println("-> stopping dev")
		// remove all code containers
		boxfile := boxfile.New([]byte(self.config.Meta["boxfile"]))

		for _, codeName := range boxfile.Nodes("code") {

			self.config.Meta["name"] = codeName
			// dont catch errors because i dont
			// want destroy to fail
			err := Run("code_destroy", self.config)
			if err != nil {
				fmt.Printf("code_destroy (%s): %s\n", codeName, err.Error())
			}

		}
		// stop all data containers
		Run("service_stop_all", self.config)
	}

	//if nothing is running stop the provider
	keys, _ := data.Keys("apps")
	for _, key := range keys {
		data.Get("apps", key, &app)
		if app.UsageCount != 0 {
			// i found an app that is doing something
			// so dont shut down the provider
			fmt.Println("-> leaving provider running")
			return nil
		}
	}

	// this part only gets executed if we cant find any apps
	// doing anything in the loop above
	fmt.Println("-> stopping provider")
	return Run("provider_stop", self.config)
	// return nil
}
<|endoftext|>


<commit_before>package beater

import (
	"fmt"

	"github.com/elastic/beats/libbeat/beat"
	"github.com/elastic/beats/libbeat/cfgfile"
	"github.com/elastic/beats/libbeat/logp"

	cfg "github.com/elastic/beats/filebeat/config"
	. "github.com/elastic/beats/filebeat/crawler"
	. "github.com/elastic/beats/filebeat/input"
)

// Beater object. Contains all objects needed to run the beat
type Filebeat struct {
	FbConfig *cfg.Config
	// Channel from harvesters to spooler
	publisherChan chan []*FileEvent
	spooler       *Spooler
	registrar     *Registrar
	cralwer       *Crawler
	done          chan struct{}
}

func New() *Filebeat {
	return &Filebeat{}
}

// Config setups up the filebeat configuration by fetch all additional config files
func (fb *Filebeat) Config(b *beat.Beat) error {

	// Load Base config
	err := cfgfile.Read(&fb.FbConfig, "")

	if err != nil {
		return fmt.Errorf("Error reading config file: %v", err)
	}

	// Check if optional config_dir is set to fetch additional prospector config files
	fb.FbConfig.FetchConfigs()

	return nil
}

func (fb *Filebeat) Setup(b *beat.Beat) error {
	fb.done = make(chan struct{})

	return nil
}

func (fb *Filebeat) Run(b *beat.Beat) error {

	var err error

	// Init channels
	fb.publisherChan = make(chan []*FileEvent, 1)

	// Setup registrar to persist state
	fb.registrar, err = NewRegistrar(fb.FbConfig.Filebeat.RegistryFile)
	if err != nil {
		logp.Err("Could not init registrar: %v", err)
		return err
	}

	fb.cralwer = &Crawler{
		Registrar: fb.registrar,
	}

	// Load the previous log file locations now, for use in prospector
	fb.registrar.LoadState()

	// Init and Start spooler: Harvesters dump events into the spooler.
	fb.spooler = NewSpooler(fb.FbConfig.Filebeat, fb.publisherChan)

	if err != nil {
		logp.Err("Could not init spooler: %v", err)
		return err
	}

	// Start up spooler
	fb.spooler.Start()

	// registrar records last acknowledged positions in all files.
	go fb.registrar.Run()

	err = fb.cralwer.Start(fb.FbConfig.Filebeat.Prospectors, fb.spooler.Channel)
	if err != nil {
		return err
	}

	// Publishes event to output
	pub := newPublisher(fb.FbConfig.Filebeat.PublishAsync,
		fb.publisherChan, fb.registrar.Channel, b.Events)
	pub.Start()

	// Blocks progressing
	select {
	case <-fb.done:
	}

	return nil
}

func (fb *Filebeat) Cleanup(b *beat.Beat) error {
	return nil
}

// Stop is called on exit for cleanup
func (fb *Filebeat) Stop() {

	logp.Info("Stopping filebeat")
	// Stop crawler -> stop prospectors -> stop harvesters
	fb.cralwer.Stop()

	// Stopping spooler will flush items
	fb.spooler.Stop()

	// Stopping registrar will write last state
	fb.registrar.Stop()

	// Stop Filebeat
	close(fb.done)
}
<commit_msg>Update Filebeat beater with GoLint styles<commit_after>package beater

import (
	"fmt"

	"github.com/elastic/beats/libbeat/beat"
	"github.com/elastic/beats/libbeat/cfgfile"
	"github.com/elastic/beats/libbeat/logp"

	cfg "github.com/elastic/beats/filebeat/config"
	. "github.com/elastic/beats/filebeat/crawler"
	. "github.com/elastic/beats/filebeat/input"
)

// Filebeat is a beater object. Contains all objects needed to run the beat
type Filebeat struct {
	FbConfig *cfg.Config
	// Channel from harvesters to spooler
	publisherChan chan []*FileEvent
	spooler       *Spooler
	registrar     *Registrar
	cralwer       *Crawler
	done          chan struct{}
}

// New creates a new Filebeat pointer instance.
func New() *Filebeat {
	return &Filebeat{}
}

// Config setups up the filebeat configuration by fetch all additional config files
func (fb *Filebeat) Config(b *beat.Beat) error {

	// Load Base config
	err := cfgfile.Read(&fb.FbConfig, "")

	if err != nil {
		return fmt.Errorf("Error reading config file: %v", err)
	}

	// Check if optional config_dir is set to fetch additional prospector config files
	fb.FbConfig.FetchConfigs()

	return nil
}

// Setup applies the minimum required setup to a new Filebeat instance for use.
func (fb *Filebeat) Setup(b *beat.Beat) error {
	fb.done = make(chan struct{})

	return nil
}

// Run allows the beater to be run as a beat.
func (fb *Filebeat) Run(b *beat.Beat) error {

	var err error

	// Init channels
	fb.publisherChan = make(chan []*FileEvent, 1)

	// Setup registrar to persist state
	fb.registrar, err = NewRegistrar(fb.FbConfig.Filebeat.RegistryFile)
	if err != nil {
		logp.Err("Could not init registrar: %v", err)
		return err
	}

	fb.cralwer = &Crawler{
		Registrar: fb.registrar,
	}

	// Load the previous log file locations now, for use in prospector
	fb.registrar.LoadState()

	// Init and Start spooler: Harvesters dump events into the spooler.
	fb.spooler = NewSpooler(fb.FbConfig.Filebeat, fb.publisherChan)

	if err != nil {
		logp.Err("Could not init spooler: %v", err)
		return err
	}

	// Start up spooler
	fb.spooler.Start()

	// registrar records last acknowledged positions in all files.
	go fb.registrar.Run()

	err = fb.cralwer.Start(fb.FbConfig.Filebeat.Prospectors, fb.spooler.Channel)
	if err != nil {
		return err
	}

	// Publishes event to output
	pub := newPublisher(fb.FbConfig.Filebeat.PublishAsync,
		fb.publisherChan, fb.registrar.Channel, b.Events)
	pub.Start()

	// Blocks progressing
	select {
	case <-fb.done:
	}

	return nil
}

// Cleanup removes any temporary files, data, or other items that were created by the Beat.
func (fb *Filebeat) Cleanup(b *beat.Beat) error {
	return nil
}

// Stop is called on exit to stop the crawling, spooling and registration processes.
func (fb *Filebeat) Stop() {

	logp.Info("Stopping filebeat")
	// Stop crawler -> stop prospectors -> stop harvesters
	fb.cralwer.Stop()

	// Stopping spooler will flush items
	fb.spooler.Stop()

	// Stopping registrar will write last state
	fb.registrar.Stop()

	// Stop Filebeat
	close(fb.done)
}
<|endoftext|>


<commit_before>package cmd

type instruction int

var (
    EXIT instruction
    PUSH1, PUSH2, PRINT instruction = 1, 2, 3
    NOP instruction = 4
    ADD, SUB, MUL instruction = 5, 6, 7;)

/* Two extra slots of buffer. */
type state struct { sp int; stk [66]int; }

func operate(op instruction, st state) state {

    if error := "Stack overflow!"; st.sp >= 64 {
        println(error); exit();
    }

    // I know, some cases are missing
    switch ; op {
        case PUSH1:
            st.stk[st.sp] = 1
            st.sp++
        case PUSH2:
            st.stk[st.sp] = 2
            st.sp++
        case ADD:
            if st.sp == 0 { break; }
            st.stk[st.sp - 1] += st.stk[st.sp]
            st.sp--
        case PRINT:
            println("[", st.sp, "] = ",
                     st.stk[st.sp])

    }

    return st
}

func exec(program [128]instruction) {

    var pc = 0
    var st struct { sp int; stk [66]int; }

    for pc < len(program) {
        if program[pc] == EXIT {
            return
        } else if program[pc] == NOP {
            continue
        } else {
            st = operate(program[pc], st)
        }

        pc += 1
    }

    end()
}

func end() { println("Hurrah!"); }
<commit_msg>fix scoping in cmd.go valid sample program<commit_after>package cmd

type instruction int

var (
    EXIT instruction
    PUSH1, PUSH2, PRINT instruction = 1, 2, 3
    NOP instruction = 4
    ADD, SUB, MUL instruction = 5, 6, 7;)

/* Two extra slots of buffer. */
type state struct { sp int; stk [66]int; }

func exit() {
	println("Actually exit is unsupported!");
}

func end() { println("Hurrah!"); }

func operate(op instruction, st state) state {

    if error := "Stack overflow!"; st.sp >= 64 {
        println(error); exit();
    }

    // I know, some cases are missing
    switch ; op {
        case PUSH1:
            st.stk[st.sp] = 1
            st.sp++
        case PUSH2:
            st.stk[st.sp] = 2
            st.sp++
        case ADD:
            if st.sp == 0 { break; }
            st.stk[st.sp - 1] += st.stk[st.sp]
            st.sp--
        case PRINT:
            println("[", st.sp, "] = ",
                     st.stk[st.sp])

    }

    return st
}

func exec(program [128]instruction) {

    var pc = 0
    var st struct { sp int; stk [66]int; }

    for pc < len(program) {
        if program[pc] == EXIT {
            return
        } else if program[pc] == NOP {
            continue
        } else {
            st = operate(program[pc], st)
        }

        pc += 1
    }

    end()
}
<|endoftext|>