<commit_before>/*
Copyright 2020 gRPC authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controllers

import (
	"context"
	"fmt"
	"time"

	"github.com/go-logr/logr"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/client"

	grpcv1 "github.com/grpc/test-infra/api/v1"
	"github.com/grpc/test-infra/pkg/defaults"
)

// reconcileTimeout specifies the maximum amount of time any set of API
// requests should take for a single invocation of the Reconcile method.
const reconcileTimeout = 1 * time.Minute

// cloneInitContainer holds the name of the init container that obtains a copy
// of the code at a specific point in time.
const cloneInitContainer = "clone"

// buildInitContainer holds the name of the init container that assembles a
// binary or other bundle required to run the tests.
const buildInitContainer = "build"

// runContainer holds the name of the main container where the test is executed.
const runContainer = "run"

// CloneRepoEnv specifies the name of the env variable that contains the git
// repository to clone.
const CloneRepoEnv = "CLONE_REPO"

// CloneGitRefEnv specifies the name of the env variable that contains the
// commit, tag or branch to checkout after cloning a git repository.
const CloneGitRefEnv = "CLONE_GIT_REF"

// LoadTestReconciler reconciles a LoadTest object
type LoadTestReconciler struct {
	client.Client
	Defaults *defaults.Defaults
	Log      logr.Logger
	Scheme   *runtime.Scheme
}

// +kubebuilder:rbac:groups=e2etest.grpc.io,resources=loadtests,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=e2etest.grpc.io,resources=loadtests/status,verbs=get;update;patch

// LoadTestMissing categorize missing components based on their roles at specific
// moment. The struct is a wrapper to help us get role information associate
// with components.
type LoadTestMissing struct {
	// Driver is the component that orchestrates the test. If Driver is not set
	// that means we already have the Driver running.
	Driver *grpcv1.Driver `json:"driver,omitempty"`

	// Servers are a list of components that receive traffic from. The list
	// indicates the Servers still in need.
	Servers []grpcv1.Server `json:"servers,omitempty"`

	// Clients are a list of components that send traffic to servers. The list
	// indicates the Clients still in need.
	Clients []grpcv1.Client `json:"clients,omitempty"`
}

// Reconcile attempts to bring the current state of the load test into agreement
// with its declared spec. This may mean provisioning resources, doing nothing
// or handling the termination of its pods.
func (r *LoadTestReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
	log := r.Log.WithValues("loadtest", req.NamespacedName)
	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
	defer cancel()

	// Fetch the current state of the world.

	var nodes corev1.NodeList
	if err := r.List(ctx, &nodes); err != nil {
		log.Error(err, "failed to list nodes")
		// attempt to requeue with exponential back-off
		return ctrl.Result{Requeue: true}, err
	}

	var pods corev1.PodList
	if err := r.List(ctx, &pods, client.InNamespace(req.Namespace)); err != nil {
		log.Error(err, "failed to list pods", "namespace", req.Namespace)
		// attempt to requeue with exponential back-off
		return ctrl.Result{Requeue: true}, err
	}

	var loadtests grpcv1.LoadTestList
	if err := r.List(ctx, &loadtests); err != nil {
		log.Error(err, "failed to list loadtests")
		// attempt to requeue with exponential back-off
		return ctrl.Result{Requeue: true}, err
	}

	var loadtest grpcv1.LoadTest
	if err := r.Get(ctx, req.NamespacedName, &loadtest); err != nil {
		log.Error(err, "failed to get loadtest", "name", req.NamespacedName)
		// do not requeue, may have been garbage collected
		return ctrl.Result{}, client.IgnoreNotFound(err)
	}

	// Check if the loadtest has terminated.

	// TODO: Do nothing if the loadtest has terminated.

	// Check the status of any running pods.

	// TODO: Add method to get list of owned pods and method to check their status.

	// Create any missing pods that the loadtest needs.

	// TODO: Add logic to schedule the next missing pod.

	// PLACEHOLDERS!
	_ = nodes
	_ = pods
	_ = loadtests
	_ = loadtest
	return ctrl.Result{}, nil
}

// checkMissingPods attempts to check if any required component is missing from
// the current load test. It takes reference of the current load test and a pod
// list that contains all running pods at the moment returns all missing
// components required from the current load test with their roles.
func checkMissingPods(currentLoadTest *grpcv1.LoadTest, allRunningPods *corev1.PodList) *LoadTestMissing {

	currentMissing := &LoadTestMissing{Servers: []grpcv1.Server{}, Clients: []grpcv1.Client{}}

	requiredClientMap := make(map[string]*grpcv1.Client)
	requiredServerMap := make(map[string]*grpcv1.Server)
	foundDriver := false

	for i := 0; i < len(currentLoadTest.Spec.Clients); i++ {
		requiredClientMap[*currentLoadTest.Spec.Clients[i].Name] = &currentLoadTest.Spec.Clients[i]
	}
	for i := 0; i < len(currentLoadTest.Spec.Servers); i++ {
		requiredServerMap[*currentLoadTest.Spec.Servers[i].Name] = &currentLoadTest.Spec.Servers[i]
	}

	if allRunningPods != nil {

		for _, eachPod := range allRunningPods.Items {

			if eachPod.Labels == nil {
				continue
			}

			loadTestLabel := eachPod.Labels[defaults.LoadTestLabel]
			roleLabel := eachPod.Labels[defaults.RoleLabel]
			componentNameLabel := eachPod.Labels[defaults.ComponentNameLabel]

			if loadTestLabel != currentLoadTest.Name {
				continue
			}
			if roleLabel == defaults.DriverRole {
				if *currentLoadTest.Spec.Driver.Component.Name == componentNameLabel {
					foundDriver = true
				}
			} else if roleLabel == defaults.ClientRole {
				if _, ok := requiredClientMap[componentNameLabel]; ok {
					delete(requiredClientMap, componentNameLabel)
				}
			} else if roleLabel == defaults.ServerRole {
				if _, ok := requiredServerMap[componentNameLabel]; ok {
					delete(requiredServerMap, componentNameLabel)
				}
			}
		}
	}

	for _, eachMissingClient := range requiredClientMap {
		currentMissing.Clients = append(currentMissing.Clients, *eachMissingClient)
	}

	for _, eachMissingServer := range requiredServerMap {
		currentMissing.Servers = append(currentMissing.Servers, *eachMissingServer)
	}

	if !foundDriver {
		currentMissing.Driver = currentLoadTest.Spec.Driver
	}

	return currentMissing
}

// SetupWithManager configures a controller-runtime manager.
func (r *LoadTestReconciler) SetupWithManager(mgr ctrl.Manager) error {
	return ctrl.NewControllerManagedBy(mgr).
		For(&grpcv1.LoadTest{}).
		Complete(r)
}

// newClientPod creates a client given a load test and a reference to its
// component. It returns an error if a pod cannot be constructed.
func newClientPod(loadtest *grpcv1.LoadTest, component *grpcv1.Component) (*corev1.Pod, error) {
	pod, err := newPod(loadtest, component, defaults.ClientRole)
	if err != nil {
		return nil, err
	}

	addDriverPort(&pod.Spec.Containers[0])

	return pod, nil
}

// newDriverPod creates a driver given a load test and a reference to its
// component. It returns an error if a pod cannot be constructed.
func newDriverPod(loadtest *grpcv1.LoadTest, component *grpcv1.Component) (*corev1.Pod, error) {
	pod, err := newPod(loadtest, component, defaults.DriverRole)
	if err != nil {
		return nil, err
	}

	addDriverPort(&pod.Spec.Containers[0])

	return pod, nil
}

// addDriverPort decorates a container with an additional port for the driver.
func addDriverPort(container *corev1.Container) {
	container.Ports = append(container.Ports, newContainerPort("driver", 10000))
}

// addServerPort decorates a container with an additional port for the server.
func addServerPort(container *corev1.Container) {
	container.Ports = append(container.Ports, newContainerPort("server", 10010))
}

// newContainerPort creates a Kubernetes ContainerPort object with the provided
// name and portNumber. The name should uniquely identify the port and the port
// number must be within the standard port range. The protocol is assumed to be
// TCP.
func newContainerPort(name string, portNumber int32) corev1.ContainerPort {
	return corev1.ContainerPort{
		Name:          name,
		Protocol:      corev1.ProtocolTCP,
		ContainerPort: portNumber,
	}
}

// newServerPod creates a server given a load test and a reference to its
// component. It returns an error if a pod cannot be constructed.
func newServerPod(loadtest *grpcv1.LoadTest, component *grpcv1.Component) (*corev1.Pod, error) {
	pod, err := newPod(loadtest, component, defaults.ServerRole)
	if err != nil {
		return nil, err
	}

	addDriverPort(&pod.Spec.Containers[0])
	addServerPort(&pod.Spec.Containers[0])

	return pod, nil
}

// newCloneContainer constructs a container given a grpcv1.Clone pointer. If
// the pointer is nil, an empty container is returned.
func newCloneContainer(clone *grpcv1.Clone) corev1.Container {
	if clone == nil {
		return corev1.Container{}
	}

	var env []corev1.EnvVar

	if clone.Repo != nil {
		env = append(env, corev1.EnvVar{Name: CloneRepoEnv, Value: *clone.Repo})
	}

	if clone.GitRef != nil {
		env = append(env, corev1.EnvVar{Name: CloneGitRefEnv, Value: *clone.GitRef})
	}

	return corev1.Container{
		Name:  cloneInitContainer,
		Image: safeStrUnwrap(clone.Image),
		Env:   env,
	}
}

// newBuildContainer constructs a container given a grpcv1.Build pointer. If
// the pointer is nil, an empty container is returned.
func newBuildContainer(build *grpcv1.Build) corev1.Container {
	if build == nil {
		return corev1.Container{}
	}

	return corev1.Container{
		Name:    buildInitContainer,
		Image:   *build.Image,
		Command: build.Command,
		Args:    build.Args,
		Env:     build.Env,
	}
}

// newRunContainer constructs a container given a grpcv1.Run object.
func newRunContainer(run grpcv1.Run) corev1.Container {
	return corev1.Container{
		Name:    runContainer,
		Image:   *run.Image,
		Command: run.Command,
		Args:    run.Args,
		Env:     run.Env,
	}
}

// newPod constructs a Kubernetes pod.
func newPod(loadtest *grpcv1.LoadTest, component *grpcv1.Component, role string) (*corev1.Pod, error) {
	var initContainers []corev1.Container

	if component.Clone != nil {
		initContainers = append(initContainers, newCloneContainer(component.Clone))
	}

	if component.Build != nil {
		initContainers = append(initContainers, newBuildContainer(component.Build))
	}

	return &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: fmt.Sprintf("%s-%s-%s", loadtest.Name, role, *component.Name),
			Labels: map[string]string{
				defaults.LoadTestLabel:      loadtest.Name,
				defaults.RoleLabel:          role,
				defaults.ComponentNameLabel: *component.Name,
			},
		},
		Spec: corev1.PodSpec{
			NodeSelector: map[string]string{
				"pool": *component.Pool,
			},
			InitContainers: initContainers,
			Containers:     []corev1.Container{newRunContainer(component.Run)},
			RestartPolicy:  corev1.RestartPolicyNever,
			Affinity: &corev1.Affinity{
				PodAntiAffinity: &corev1.PodAntiAffinity{
					RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{
						{
							LabelSelector: &metav1.LabelSelector{
								MatchExpressions: []metav1.LabelSelectorRequirement{
									{
										Key:      "generated",
										Operator: metav1.LabelSelectorOpExists,
									},
								},
							},
							TopologyKey: "kubernetes.io/hostname",
						},
					},
				},
			},
		},
	}, nil
}

// safeStrUnwrap accepts a string pointer, returning the dereferenced string or
// an empty string if the pointer is nil.
func safeStrUnwrap(strPtr *string) string {
	if strPtr == nil {
		return ""
	}

	return *strPtr
}
<commit_msg>Update controllers/loadtest_controller.go<commit_after>/*
Copyright 2020 gRPC authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controllers

import (
	"context"
	"fmt"
	"time"

	"github.com/go-logr/logr"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/client"

	grpcv1 "github.com/grpc/test-infra/api/v1"
	"github.com/grpc/test-infra/pkg/defaults"
)

// reconcileTimeout specifies the maximum amount of time any set of API
// requests should take for a single invocation of the Reconcile method.
const reconcileTimeout = 1 * time.Minute

// cloneInitContainer holds the name of the init container that obtains a copy
// of the code at a specific point in time.
const cloneInitContainer = "clone"

// buildInitContainer holds the name of the init container that assembles a
// binary or other bundle required to run the tests.
const buildInitContainer = "build"

// runContainer holds the name of the main container where the test is executed.
const runContainer = "run"

// CloneRepoEnv specifies the name of the env variable that contains the git
// repository to clone.
const CloneRepoEnv = "CLONE_REPO"

// CloneGitRefEnv specifies the name of the env variable that contains the
// commit, tag or branch to checkout after cloning a git repository.
const CloneGitRefEnv = "CLONE_GIT_REF"

// LoadTestReconciler reconciles a LoadTest object
type LoadTestReconciler struct {
	client.Client
	Defaults *defaults.Defaults
	Log      logr.Logger
	Scheme   *runtime.Scheme
}

// +kubebuilder:rbac:groups=e2etest.grpc.io,resources=loadtests,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=e2etest.grpc.io,resources=loadtests/status,verbs=get;update;patch

// LoadTestMissing categorize missing components based on their roles at specific
// moment. The struct is a wrapper to help us get role information associate
// with components.
type LoadTestMissing struct {
	// Driver is the component that orchestrates the test. If Driver is not set
	// that means we already have the Driver running.
	Driver *grpcv1.Driver `json:"driver,omitempty"`

	// Servers are a list of components that receive traffic from. The list
	// indicates the Servers still in need.
	Servers []grpcv1.Server `json:"servers,omitempty"`

	// Clients are a list of components that send traffic to servers. The list
	// indicates the Clients still in need.
	Clients []grpcv1.Client `json:"clients,omitempty"`
}

// Reconcile attempts to bring the current state of the load test into agreement
// with its declared spec. This may mean provisioning resources, doing nothing
// or handling the termination of its pods.
func (r *LoadTestReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
	log := r.Log.WithValues("loadtest", req.NamespacedName)
	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
	defer cancel()

	// Fetch the current state of the world.

	var nodes corev1.NodeList
	if err := r.List(ctx, &nodes); err != nil {
		log.Error(err, "failed to list nodes")
		// attempt to requeue with exponential back-off
		return ctrl.Result{Requeue: true}, err
	}

	var pods corev1.PodList
	if err := r.List(ctx, &pods, client.InNamespace(req.Namespace)); err != nil {
		log.Error(err, "failed to list pods", "namespace", req.Namespace)
		// attempt to requeue with exponential back-off
		return ctrl.Result{Requeue: true}, err
	}

	var loadtests grpcv1.LoadTestList
	if err := r.List(ctx, &loadtests); err != nil {
		log.Error(err, "failed to list loadtests")
		// attempt to requeue with exponential back-off
		return ctrl.Result{Requeue: true}, err
	}

	var loadtest grpcv1.LoadTest
	if err := r.Get(ctx, req.NamespacedName, &loadtest); err != nil {
		log.Error(err, "failed to get loadtest", "name", req.NamespacedName)
		// do not requeue, may have been garbage collected
		return ctrl.Result{}, client.IgnoreNotFound(err)
	}

	// Check if the loadtest has terminated.

	// TODO: Do nothing if the loadtest has terminated.

	// Check the status of any running pods.

	// TODO: Add method to get list of owned pods and method to check their status.

	// Create any missing pods that the loadtest needs.

	// TODO: Add logic to schedule the next missing pod.

	// PLACEHOLDERS!
	_ = nodes
	_ = pods
	_ = loadtests
	_ = loadtest
	return ctrl.Result{}, nil
}

// checkMissingPods attempts to check if any required component is missing from
// the current load test. It takes reference of the current load test and a pod
// list that contains all running pods at the moment, returning all missing
// components required from the current load test with their roles.
func checkMissingPods(currentLoadTest *grpcv1.LoadTest, allRunningPods *corev1.PodList) *LoadTestMissing {

	currentMissing := &LoadTestMissing{Servers: []grpcv1.Server{}, Clients: []grpcv1.Client{}}

	requiredClientMap := make(map[string]*grpcv1.Client)
	requiredServerMap := make(map[string]*grpcv1.Server)
	foundDriver := false

	for i := 0; i < len(currentLoadTest.Spec.Clients); i++ {
		requiredClientMap[*currentLoadTest.Spec.Clients[i].Name] = &currentLoadTest.Spec.Clients[i]
	}
	for i := 0; i < len(currentLoadTest.Spec.Servers); i++ {
		requiredServerMap[*currentLoadTest.Spec.Servers[i].Name] = &currentLoadTest.Spec.Servers[i]
	}

	if allRunningPods != nil {

		for _, eachPod := range allRunningPods.Items {

			if eachPod.Labels == nil {
				continue
			}

			loadTestLabel := eachPod.Labels[defaults.LoadTestLabel]
			roleLabel := eachPod.Labels[defaults.RoleLabel]
			componentNameLabel := eachPod.Labels[defaults.ComponentNameLabel]

			if loadTestLabel != currentLoadTest.Name {
				continue
			}
			if roleLabel == defaults.DriverRole {
				if *currentLoadTest.Spec.Driver.Component.Name == componentNameLabel {
					foundDriver = true
				}
			} else if roleLabel == defaults.ClientRole {
				if _, ok := requiredClientMap[componentNameLabel]; ok {
					delete(requiredClientMap, componentNameLabel)
				}
			} else if roleLabel == defaults.ServerRole {
				if _, ok := requiredServerMap[componentNameLabel]; ok {
					delete(requiredServerMap, componentNameLabel)
				}
			}
		}
	}

	for _, eachMissingClient := range requiredClientMap {
		currentMissing.Clients = append(currentMissing.Clients, *eachMissingClient)
	}

	for _, eachMissingServer := range requiredServerMap {
		currentMissing.Servers = append(currentMissing.Servers, *eachMissingServer)
	}

	if !foundDriver {
		currentMissing.Driver = currentLoadTest.Spec.Driver
	}

	return currentMissing
}

// SetupWithManager configures a controller-runtime manager.
func (r *LoadTestReconciler) SetupWithManager(mgr ctrl.Manager) error {
	return ctrl.NewControllerManagedBy(mgr).
		For(&grpcv1.LoadTest{}).
		Complete(r)
}

// newClientPod creates a client given a load test and a reference to its
// component. It returns an error if a pod cannot be constructed.
func newClientPod(loadtest *grpcv1.LoadTest, component *grpcv1.Component) (*corev1.Pod, error) {
	pod, err := newPod(loadtest, component, defaults.ClientRole)
	if err != nil {
		return nil, err
	}

	addDriverPort(&pod.Spec.Containers[0])

	return pod, nil
}

// newDriverPod creates a driver given a load test and a reference to its
// component. It returns an error if a pod cannot be constructed.
func newDriverPod(loadtest *grpcv1.LoadTest, component *grpcv1.Component) (*corev1.Pod, error) {
	pod, err := newPod(loadtest, component, defaults.DriverRole)
	if err != nil {
		return nil, err
	}

	addDriverPort(&pod.Spec.Containers[0])

	return pod, nil
}

// addDriverPort decorates a container with an additional port for the driver.
func addDriverPort(container *corev1.Container) {
	container.Ports = append(container.Ports, newContainerPort("driver", 10000))
}

// addServerPort decorates a container with an additional port for the server.
func addServerPort(container *corev1.Container) {
	container.Ports = append(container.Ports, newContainerPort("server", 10010))
}

// newContainerPort creates a Kubernetes ContainerPort object with the provided
// name and portNumber. The name should uniquely identify the port and the port
// number must be within the standard port range. The protocol is assumed to be
// TCP.
func newContainerPort(name string, portNumber int32) corev1.ContainerPort {
	return corev1.ContainerPort{
		Name:          name,
		Protocol:      corev1.ProtocolTCP,
		ContainerPort: portNumber,
	}
}

// newServerPod creates a server given a load test and a reference to its
// component. It returns an error if a pod cannot be constructed.
func newServerPod(loadtest *grpcv1.LoadTest, component *grpcv1.Component) (*corev1.Pod, error) {
	pod, err := newPod(loadtest, component, defaults.ServerRole)
	if err != nil {
		return nil, err
	}

	addDriverPort(&pod.Spec.Containers[0])
	addServerPort(&pod.Spec.Containers[0])

	return pod, nil
}

// newCloneContainer constructs a container given a grpcv1.Clone pointer. If
// the pointer is nil, an empty container is returned.
func newCloneContainer(clone *grpcv1.Clone) corev1.Container {
	if clone == nil {
		return corev1.Container{}
	}

	var env []corev1.EnvVar

	if clone.Repo != nil {
		env = append(env, corev1.EnvVar{Name: CloneRepoEnv, Value: *clone.Repo})
	}

	if clone.GitRef != nil {
		env = append(env, corev1.EnvVar{Name: CloneGitRefEnv, Value: *clone.GitRef})
	}

	return corev1.Container{
		Name:  cloneInitContainer,
		Image: safeStrUnwrap(clone.Image),
		Env:   env,
	}
}

// newBuildContainer constructs a container given a grpcv1.Build pointer. If
// the pointer is nil, an empty container is returned.
func newBuildContainer(build *grpcv1.Build) corev1.Container {
	if build == nil {
		return corev1.Container{}
	}

	return corev1.Container{
		Name:    buildInitContainer,
		Image:   *build.Image,
		Command: build.Command,
		Args:    build.Args,
		Env:     build.Env,
	}
}

// newRunContainer constructs a container given a grpcv1.Run object.
func newRunContainer(run grpcv1.Run) corev1.Container {
	return corev1.Container{
		Name:    runContainer,
		Image:   *run.Image,
		Command: run.Command,
		Args:    run.Args,
		Env:     run.Env,
	}
}

// newPod constructs a Kubernetes pod.
func newPod(loadtest *grpcv1.LoadTest, component *grpcv1.Component, role string) (*corev1.Pod, error) {
	var initContainers []corev1.Container

	if component.Clone != nil {
		initContainers = append(initContainers, newCloneContainer(component.Clone))
	}

	if component.Build != nil {
		initContainers = append(initContainers, newBuildContainer(component.Build))
	}

	return &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: fmt.Sprintf("%s-%s-%s", loadtest.Name, role, *component.Name),
			Labels: map[string]string{
				defaults.LoadTestLabel:      loadtest.Name,
				defaults.RoleLabel:          role,
				defaults.ComponentNameLabel: *component.Name,
			},
		},
		Spec: corev1.PodSpec{
			NodeSelector: map[string]string{
				"pool": *component.Pool,
			},
			InitContainers: initContainers,
			Containers:     []corev1.Container{newRunContainer(component.Run)},
			RestartPolicy:  corev1.RestartPolicyNever,
			Affinity: &corev1.Affinity{
				PodAntiAffinity: &corev1.PodAntiAffinity{
					RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{
						{
							LabelSelector: &metav1.LabelSelector{
								MatchExpressions: []metav1.LabelSelectorRequirement{
									{
										Key:      "generated",
										Operator: metav1.LabelSelectorOpExists,
									},
								},
							},
							TopologyKey: "kubernetes.io/hostname",
						},
					},
				},
			},
		},
	}, nil
}

// safeStrUnwrap accepts a string pointer, returning the dereferenced string or
// an empty string if the pointer is nil.
func safeStrUnwrap(strPtr *string) string {
	if strPtr == nil {
		return ""
	}

	return *strPtr
}
<|endoftext|>


<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package prjcfg

import (
	"context"
	"time"

	"google.golang.org/protobuf/proto"

	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/data/stringset"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/sync/parallel"
	"go.chromium.org/luci/server/tq"

	"go.chromium.org/luci/cv/internal/common"
)

// PM encapsulates Project Manager notified by the ConfigRefresher.
//
// In production, this will be prjmanager.Notifier.
type PM interface {
	Poke(ctx context.Context, luciProject string) error
	UpdateConfig(ctx context.Context, luciProject string) error
}

type Refresher struct {
	pm  PM
	tqd *tq.Dispatcher
	env *common.Env
}

// NewRefresher creates a new project config Refresher and registers its TQ tasks.
func NewRefresher(tqd *tq.Dispatcher, pm PM, env *common.Env) *Refresher {
	pcr := &Refresher{pm, tqd, env}
	pcr.tqd.RegisterTaskClass(tq.TaskClass{
		ID:           "refresh-project-config",
		Prototype:    &RefreshProjectConfigTask{},
		Queue:        "refresh-project-config",
		Kind:         tq.NonTransactional,
		Quiet:        true,
		QuietOnError: true,
		Handler: func(ctx context.Context, payload proto.Message) error {
			task := payload.(*RefreshProjectConfigTask)
			if err := pcr.refreshProject(ctx, task.GetProject(), task.GetDisable()); err != nil {
				// Never retry tasks because the refresh task is submitted every minute
				// by the AppEngine Cron.
				return common.TQIfy{NeverRetry: true}.Error(ctx, err)
			}
			return nil
		},
	})
	return pcr
}

// SubmitRefreshTasks submits tasks that update config for LUCI projects
// or disable projects that do not have CV config in LUCI Config.
//
// It's expected to be called by a cron.
func (r *Refresher) SubmitRefreshTasks(ctx context.Context) error {
	projects, err := ProjectsWithConfig(ctx)
	if err != nil {
		return err
	}

	// Consider only some projects, regardless of which projects are registered.
	// TODO(crbug/1158505): switch to -dev configs.
	if r.env.IsGAEDev {
		projects = []string{"infra", "cq-test"}
	} else {
		for i, p := range projects {
			if p == "cq-test" {
				projects = append(projects[:i], projects[i+1:]...)
				break
			}
		}
	}

	tasks := make([]*tq.Task, len(projects))
	for i, p := range projects {
		tasks[i] = &tq.Task{
			Title: "update/" + p,
			Payload: &RefreshProjectConfigTask{
				Project: p,
			},
		}
	}

	curEnabledProjects, err := GetAllProjectIDs(ctx, true)
	if err != nil {
		return err
	}
	projectsInLUCIConfig := stringset.NewFromSlice(projects...)
	for _, p := range curEnabledProjects {
		if !projectsInLUCIConfig.Has(p) {
			tasks = append(tasks, &tq.Task{
				Title: "disable/" + p,
				Payload: &RefreshProjectConfigTask{
					Project: p,
					Disable: true,
				},
			})
		}
	}

	err = parallel.WorkPool(32, func(workCh chan<- func() error) {
		for _, task := range tasks {
			task := task
			workCh <- func() (err error) {
				if err = r.tqd.AddTask(ctx, task); err != nil {
					logging.Errorf(ctx, "Failed to submit task for %q: %s", task.Title, err)
				}
				return
			}
		}
	})

	if err != nil {
		return err.(errors.MultiError).First()
	}
	return nil
}

func (r *Refresher) refreshProject(ctx context.Context, project string, disable bool) error {
	action, actionFn := "update", UpdateProject
	if disable {
		action, actionFn = "disable", DisableProject
	}
	err := actionFn(ctx, project, func(ctx context.Context) error {
		return r.pm.UpdateConfig(ctx, project)
	})
	if err != nil {
		return errors.Annotate(err, "failed to %s project %q", action, project).Err()
	}
	if !disable {
		return r.maybePokePM(ctx, project)
	}
	return nil
}

const pokePMInterval = 10 * time.Minute

func (r *Refresher) maybePokePM(ctx context.Context, project string) error {
	now := clock.Now(ctx).UTC()
	offset := common.DistributeOffset(pokePMInterval, "cron-poke", project)
	nextPokeETA := now.Truncate(pokePMInterval).Add(offset)
	if nextPokeETA.Before(now) {
		nextPokeETA = nextPokeETA.Add(pokePMInterval)
	}

	// Cron runs every minute on average and triggers RefreshProjectConfigTask,
	// which may be delayed, so send iff it's less than 1.5 minutes before next
	// poke. This will sometimes result in 2 pokes sent instead of 1, but pokes
	// are less likely to not be sent at all.
	if nextPokeETA.Sub(now) < 90*time.Second {
		return r.pm.Poke(ctx, project)
	}
	return nil
}
<commit_msg>[cv][dev] exclude `infra` from dev.<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package prjcfg

import (
	"context"
	"time"

	"google.golang.org/protobuf/proto"

	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/data/stringset"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/sync/parallel"
	"go.chromium.org/luci/server/tq"

	"go.chromium.org/luci/cv/internal/common"
)

// PM encapsulates Project Manager notified by the ConfigRefresher.
//
// In production, this will be prjmanager.Notifier.
type PM interface {
	Poke(ctx context.Context, luciProject string) error
	UpdateConfig(ctx context.Context, luciProject string) error
}

type Refresher struct {
	pm  PM
	tqd *tq.Dispatcher
	env *common.Env
}

// NewRefresher creates a new project config Refresher and registers its TQ tasks.
func NewRefresher(tqd *tq.Dispatcher, pm PM, env *common.Env) *Refresher {
	pcr := &Refresher{pm, tqd, env}
	pcr.tqd.RegisterTaskClass(tq.TaskClass{
		ID:           "refresh-project-config",
		Prototype:    &RefreshProjectConfigTask{},
		Queue:        "refresh-project-config",
		Kind:         tq.NonTransactional,
		Quiet:        true,
		QuietOnError: true,
		Handler: func(ctx context.Context, payload proto.Message) error {
			task := payload.(*RefreshProjectConfigTask)
			if err := pcr.refreshProject(ctx, task.GetProject(), task.GetDisable()); err != nil {
				// Never retry tasks because the refresh task is submitted every minute
				// by the AppEngine Cron.
				return common.TQIfy{NeverRetry: true}.Error(ctx, err)
			}
			return nil
		},
	})
	return pcr
}

// SubmitRefreshTasks submits tasks that update config for LUCI projects
// or disable projects that do not have CV config in LUCI Config.
//
// It's expected to be called by a cron.
func (r *Refresher) SubmitRefreshTasks(ctx context.Context) error {
	projects, err := ProjectsWithConfig(ctx)
	if err != nil {
		return err
	}

	// Consider only some projects, regardless of which projects are registered.
	// TODO(crbug/1158505): switch to -dev configs.
	if r.env.IsGAEDev {
		projects = []string{"cq-test"}
	} else {
		for i, p := range projects {
			if p == "cq-test" {
				projects = append(projects[:i], projects[i+1:]...)
				break
			}
		}
	}

	tasks := make([]*tq.Task, len(projects))
	for i, p := range projects {
		tasks[i] = &tq.Task{
			Title: "update/" + p,
			Payload: &RefreshProjectConfigTask{
				Project: p,
			},
		}
	}

	curEnabledProjects, err := GetAllProjectIDs(ctx, true)
	if err != nil {
		return err
	}
	projectsInLUCIConfig := stringset.NewFromSlice(projects...)
	for _, p := range curEnabledProjects {
		if !projectsInLUCIConfig.Has(p) {
			tasks = append(tasks, &tq.Task{
				Title: "disable/" + p,
				Payload: &RefreshProjectConfigTask{
					Project: p,
					Disable: true,
				},
			})
		}
	}

	err = parallel.WorkPool(32, func(workCh chan<- func() error) {
		for _, task := range tasks {
			task := task
			workCh <- func() (err error) {
				if err = r.tqd.AddTask(ctx, task); err != nil {
					logging.Errorf(ctx, "Failed to submit task for %q: %s", task.Title, err)
				}
				return
			}
		}
	})

	if err != nil {
		return err.(errors.MultiError).First()
	}
	return nil
}

func (r *Refresher) refreshProject(ctx context.Context, project string, disable bool) error {
	action, actionFn := "update", UpdateProject
	if disable {
		action, actionFn = "disable", DisableProject
	}
	err := actionFn(ctx, project, func(ctx context.Context) error {
		return r.pm.UpdateConfig(ctx, project)
	})
	if err != nil {
		return errors.Annotate(err, "failed to %s project %q", action, project).Err()
	}
	if !disable {
		return r.maybePokePM(ctx, project)
	}
	return nil
}

const pokePMInterval = 10 * time.Minute

func (r *Refresher) maybePokePM(ctx context.Context, project string) error {
	now := clock.Now(ctx).UTC()
	offset := common.DistributeOffset(pokePMInterval, "cron-poke", project)
	nextPokeETA := now.Truncate(pokePMInterval).Add(offset)
	if nextPokeETA.Before(now) {
		nextPokeETA = nextPokeETA.Add(pokePMInterval)
	}

	// Cron runs every minute on average and triggers RefreshProjectConfigTask,
	// which may be delayed, so send iff it's less than 1.5 minutes before next
	// poke. This will sometimes result in 2 pokes sent instead of 1, but pokes
	// are less likely to not be sent at all.
	if nextPokeETA.Sub(now) < 90*time.Second {
		return r.pm.Poke(ctx, project)
	}
	return nil
}
<|endoftext|>


<commit_before>package checkerFn

import (
	"github.com/admpub/nging/application/registry/upload/table"
	"github.com/webx-top/echo"
)

// Checker 验证并生成子文件夹名称和文件名称
type Checker func(echo.Context, table.TableInfoStorer) (subdir string, name string, err error)
<commit_msg>update<commit_after><|endoftext|>


<commit_before>// Copyright © 2017 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package migrate

import (
	"github.com/TheThingsNetwork/ttn/core/storage"
	redis "gopkg.in/redis.v5"
)

// AddPayloadFormat migration from 2.4.1 to 2.6.1
func AddPayloadFormat(prefix string) storage.MigrateFunction {
	return func(client *redis.Client, key string, obj map[string]string) (string, map[string]string, error) {
		obj["payload_format"] = "custom"
		if decoder, ok := obj["decoder"]; ok {
			delete(obj, "decoder")
			obj["custom_decoder"] = decoder
		}
		if converter, ok := obj["converter"]; ok {
			delete(obj, "converter")
			obj["custom_converter"] = converter
		}
		if validator, ok := obj["validator"]; ok {
			delete(obj, "validator")
			obj["custom_validator"] = validator
		}
		if encoder, ok := obj["encoder"]; ok {
			delete(obj, "encoder")
			obj["custom_encoder"] = encoder
		}
		return "2.6.2", obj, nil
	}
}

func init() {
	applicationMigrations["2.6.1"] = AddVersion
}
<commit_msg>Only set custom payload format if there's any JavaScript set<commit_after>// Copyright © 2017 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package migrate

import (
	"github.com/TheThingsNetwork/ttn/core/storage"
	redis "gopkg.in/redis.v5"
)

// AddPayloadFormat migration from 2.4.1 to 2.6.1
func AddPayloadFormat(prefix string) storage.MigrateFunction {
	return func(client *redis.Client, key string, obj map[string]string) (string, map[string]string, error) {
		any := false
		if decoder, ok := obj["decoder"]; ok {
			delete(obj, "decoder")
			obj["custom_decoder"] = decoder
			any = true
		}
		if converter, ok := obj["converter"]; ok {
			delete(obj, "converter")
			obj["custom_converter"] = converter
			any = true
		}
		if validator, ok := obj["validator"]; ok {
			delete(obj, "validator")
			obj["custom_validator"] = validator
			any = true
		}
		if encoder, ok := obj["encoder"]; ok {
			delete(obj, "encoder")
			obj["custom_encoder"] = encoder
			any = true
		}
		if any {
			obj["payload_format"] = "custom"
		}
		return "2.6.2", obj, nil
	}
}

func init() {
	applicationMigrations["2.6.1"] = AddVersion
}
<|endoftext|>


<commit_before>// Copyright 2011 Miek Gieben. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package dns

// Holds a bunch of helper functions for dealing with labels.

// SplitLabels splits a domainname string into its labels.
// www.miek.nl. returns []string{"www", "miek", "nl"}
// The root label (.) returns nil.
func SplitLabels(s string) []string {
	idx := Split(s)
	switch len(idx) {
	case 0:
		return nil
	case 1:
		return []string{s}
	default:
		begin := 0
		end := 0
		labels := make([]string, 0)
		for i := 1; i < len(idx); i++ {
			end = idx[i]
			labels = append(labels, s[begin:end])
			begin = end
		}
		return labels
	}
	panic("dns: not reached")
}

// CompareLabels compares the names s1 and s2 and
// returns how many labels they have in common starting from the right.
// The comparison stops at the first inequality. The labels are not downcased
// before the comparison.
//
// www.miek.nl. and miek.nl. have two labels in common: miek and nl
// www.miek.nl. and www.bla.nl. have one label in common: nl
func CompareLabels(s1, s2 string) (n int) {
	s1 = Fqdn(s1)
	s2 = Fqdn(s2)
	l1 := Split(s1)
	l2 := Split(s2)

	// the first check: root label
	if l1 == nil || l2 == nil {
		return
	}

	j1 := len(l1) - 1 // end
	i1 := len(l1) - 2 // start
	j2 := len(l2) - 1
	i2 := len(l2) - 2
	// the second check can be done here: last/only label
	// before we fall through into the for-loop below
	if s1[l1[j1]:] == s2[l2[j2]:] {
		n++
	} else {
		return
	}
	for {
		if i1 < 0 || i2 < 0 {
			break
		}
		if s1[l1[i1]:l1[j1]] == s2[l2[i2]:l2[j2]] {
			n++
		} else {
			break
		}
		j1--
		i1--
		j2--
		i2--
	}
	return
}

// LenLabels returns the number of labels in the string s
func LenLabels(s string) (labels int) {
	if s == "." {
		return
	}
	s = Fqdn(s) // TODO(miek): annoyed I need this
	off := 0
	end := false
	for {
		off, end = nextLabel(s, off)
		labels++
		if end {
			return
		}
	}

}

// Split splits a name s into its label indexes.
// www.miek.nl. returns []int{0, 4, 9}. The root name (.) returns nil.
func Split(s string) []int {
	if s == "." {
		return nil
	}
	s = Fqdn(s)     // Grrr!
	idx := []int{0} // TODO(miek): could allocate more (10) and then extend when needed
	off := 0
	end := false

	for {
		off, end = nextLabel(s, off)
		if end {
			return idx
		}
		idx = append(idx, off)
	}
}

// nextLabel returns the index of the start of the next label in the
// string s. The bool end is true when the end of the string has been
// reached.
func NextLabel(s string, offset int) (i int, end bool) {
	// The other label function are quite generous with memory,
	// this one does not allocate.
	quote := false
	for i = offset; i < len(s)-1; i++ {
		switch s[i] {
		case '\\':
			quote = !quote
		default:
			quote = false
		case '.':
			if quote {
				quote = !quote
				continue
			}
			return i + 1, false
		}
	}
	return i + 1, true
}
<commit_msg>Uh uppercase it here too<commit_after>// Copyright 2011 Miek Gieben. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package dns

// Holds a bunch of helper functions for dealing with labels.

// SplitLabels splits a domainname string into its labels.
// www.miek.nl. returns []string{"www", "miek", "nl"}
// The root label (.) returns nil.
func SplitLabels(s string) []string {
	idx := Split(s)
	switch len(idx) {
	case 0:
		return nil
	case 1:
		return []string{s}
	default:
		begin := 0
		end := 0
		labels := make([]string, 0)
		for i := 1; i < len(idx); i++ {
			end = idx[i]
			labels = append(labels, s[begin:end])
			begin = end
		}
		return labels
	}
	panic("dns: not reached")
}

// CompareLabels compares the names s1 and s2 and
// returns how many labels they have in common starting from the right.
// The comparison stops at the first inequality. The labels are not downcased
// before the comparison.
//
// www.miek.nl. and miek.nl. have two labels in common: miek and nl
// www.miek.nl. and www.bla.nl. have one label in common: nl
func CompareLabels(s1, s2 string) (n int) {
	s1 = Fqdn(s1)
	s2 = Fqdn(s2)
	l1 := Split(s1)
	l2 := Split(s2)

	// the first check: root label
	if l1 == nil || l2 == nil {
		return
	}

	j1 := len(l1) - 1 // end
	i1 := len(l1) - 2 // start
	j2 := len(l2) - 1
	i2 := len(l2) - 2
	// the second check can be done here: last/only label
	// before we fall through into the for-loop below
	if s1[l1[j1]:] == s2[l2[j2]:] {
		n++
	} else {
		return
	}
	for {
		if i1 < 0 || i2 < 0 {
			break
		}
		if s1[l1[i1]:l1[j1]] == s2[l2[i2]:l2[j2]] {
			n++
		} else {
			break
		}
		j1--
		i1--
		j2--
		i2--
	}
	return
}

// LenLabels returns the number of labels in the string s
func LenLabels(s string) (labels int) {
	if s == "." {
		return
	}
	s = Fqdn(s) // TODO(miek): annoyed I need this
	off := 0
	end := false
	for {
		off, end = NextLabel(s, off)
		labels++
		if end {
			return
		}
	}

}

// Split splits a name s into its label indexes.
// www.miek.nl. returns []int{0, 4, 9}. The root name (.) returns nil.
func Split(s string) []int {
	if s == "." {
		return nil
	}
	s = Fqdn(s)     // Grrr!
	idx := []int{0} // TODO(miek): could allocate more (10) and then extend when needed
	off := 0
	end := false

	for {
		off, end = NextLabel(s, off)
		if end {
			return idx
		}
		idx = append(idx, off)
	}
}

// NextLabel returns the index of the start of the next label in the
// string s. The bool end is true when the end of the string has been
// reached.
func NextLabel(s string, offset int) (i int, end bool) {
	// The other label function are quite generous with memory,
	// this one does not allocate.
	quote := false
	for i = offset; i < len(s)-1; i++ {
		switch s[i] {
		case '\\':
			quote = !quote
		default:
			quote = false
		case '.':
			if quote {
				quote = !quote
				continue
			}
			return i + 1, false
		}
	}
	return i + 1, true
}
<|endoftext|>


<commit_before>// Copyright ©2015 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package lapack

import "github.com/gonum/blas"

const None = 'N'

type Job byte

type Comp byte

// Complex128 defines the public complex128 LAPACK API supported by gonum/lapack.
type Complex128 interface{}

// Float64 defines the public float64 LAPACK API supported by gonum/lapack.
type Float64 interface {
	Dgecon(norm MatrixNorm, n int, a []float64, lda int, anorm float64, work []float64, iwork []int) float64
	Dgels(trans blas.Transpose, m, n, nrhs int, a []float64, lda int, b []float64, ldb int, work []float64, lwork int) bool
	Dgelqf(m, n int, a []float64, lda int, tau, work []float64, lwork int)
	Dgeqrf(m, n int, a []float64, lda int, tau, work []float64, lwork int)
	Dgesvd(jobU, jobVT SVDJob, m, n int, a []float64, lda int, s, u []float64, ldu int, vt []float64, ldvt int, work []float64, lwork int) (ok bool)
	Dgetrf(m, n int, a []float64, lda int, ipiv []int) (ok bool)
	Dgetri(n int, a []float64, lda int, ipiv []int, work []float64, lwork int) (ok bool)
	Dgetrs(trans blas.Transpose, n, nrhs int, a []float64, lda int, ipiv []int, b []float64, ldb int)
	Dlantr(norm MatrixNorm, uplo blas.Uplo, diag blas.Diag, m, n int, a []float64, lda int, work []float64) float64
	Dlange(norm MatrixNorm, m, n int, a []float64, lda int, work []float64) float64
	Dlansy(norm MatrixNorm, uplo blas.Uplo, n int, a []float64, lda int, work []float64) float64
	Dormqr(side blas.Side, trans blas.Transpose, m, n, k int, a []float64, lda int, tau, c []float64, ldc int, work []float64, lwork int)
	Dormlq(side blas.Side, trans blas.Transpose, m, n, k int, a []float64, lda int, tau, c []float64, ldc int, work []float64, lwork int)
	Dpocon(uplo blas.Uplo, n int, a []float64, lda int, anorm float64, work []float64, iwork []int) float64
	Dpotrf(ul blas.Uplo, n int, a []float64, lda int) (ok bool)
	Dsyev(jobz EigComp, uplo blas.Uplo, n int, a []float64, lda int, w, work []float64, lwork int) (ok bool)
	Dtrcon(norm MatrixNorm, uplo blas.Uplo, diag blas.Diag, n int, a []float64, lda int, work []float64, iwork []int) float64
	Dtrtri(uplo blas.Uplo, diag blas.Diag, n int, a []float64, lda int) (ok bool)
	Dtrtrs(uplo blas.Uplo, trans blas.Transpose, diag blas.Diag, n, nrhs int, a []float64, lda int, b []float64, ldb int) (ok bool)
}

// Direct specifies the direction of the multiplication for the Householder matrix.
type Direct byte

const (
	Forward  Direct = 'F' // Reflectors are right-multiplied, H_0 * H_1 * ... * H_{k-1}.
	Backward Direct = 'B' // Reflectors are left-multiplied, H_{k-1} * ... * H_1 * H_0.
)

// Sort is the sorting order.
type Sort byte

const (
	SortIncreasing Sort = 'I'
	SortDecreasing Sort = 'D'
)

// StoreV indicates the storage direction of elementary reflectors.
type StoreV byte

const (
	ColumnWise StoreV = 'C' // Reflector stored in a column of the matrix.
	RowWise    StoreV = 'R' // Reflector stored in a row of the matrix.
)

// MatrixNorm represents the kind of matrix norm to compute.
type MatrixNorm byte

const (
	MaxAbs       MatrixNorm = 'M' // max(abs(A(i,j)))  ('M')
	MaxColumnSum MatrixNorm = 'O' // Maximum column sum (one norm) ('1', 'O')
	MaxRowSum    MatrixNorm = 'I' // Maximum row sum (infinity norm) ('I', 'i')
	NormFrob     MatrixNorm = 'F' // Frobenius norm (sqrt of sum of squares) ('F', 'f', E, 'e')
)

// MatrixType represents the kind of matrix represented in the data.
type MatrixType byte

const (
	General  MatrixType = 'G' // A dense matrix (like blas64.General).
	UpperTri MatrixType = 'U' // An upper triangular matrix.
	LowerTri MatrixType = 'L' // A lower triangular matrix.
)

// Pivot specifies the pivot type for plane rotations
type Pivot byte

const (
	Variable Pivot = 'V'
	Top      Pivot = 'T'
	Bottom   Pivot = 'B'
)

type DecompUpdate byte

const (
	ApplyP DecompUpdate = 'P'
	ApplyQ DecompUpdate = 'Q'
)

// SVDJob specifies the singular vector computation type for SVD.
type SVDJob byte

const (
	SVDAll       SVDJob = 'A' // Compute all singular vectors
	SVDInPlace   SVDJob = 'S' // Compute the first singular vectors and store them in provided storage.
	SVDOverwrite SVDJob = 'O' // Compute the singular vectors and store them in input matrix
	SVDNone      SVDJob = 'N' // Do not compute singular vectors
)

// EigComp specifies the type of eigenvalue decomposition.
type EigComp byte

const (
	// EigValueOnly specifies to compute only the eigenvalues of the input matrix.
	EigValueOnly EigComp = 'N'
	// EigDecomp specifies to compute the eigenvalues and eigenvectors of the
	// full symmetric matrix.
	EigDecomp EigComp = 'V'
	// EigBoth specifies to compute both the eigenvalues and eigenvectors of the
	// input tridiagonal matrix.
	EigBoth EigComp = 'I'
)

// Jobs for Dgebal.
const (
	Permute      Job = 'P'
	Scale        Job = 'S'
	PermuteScale Job = 'B'
)
<commit_msg>lapack: add Job and Comp constants for Dhseqr<commit_after>// Copyright ©2015 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package lapack

import "github.com/gonum/blas"

const None = 'N'

type Job byte

type Comp byte

// Complex128 defines the public complex128 LAPACK API supported by gonum/lapack.
type Complex128 interface{}

// Float64 defines the public float64 LAPACK API supported by gonum/lapack.
type Float64 interface {
	Dgecon(norm MatrixNorm, n int, a []float64, lda int, anorm float64, work []float64, iwork []int) float64
	Dgels(trans blas.Transpose, m, n, nrhs int, a []float64, lda int, b []float64, ldb int, work []float64, lwork int) bool
	Dgelqf(m, n int, a []float64, lda int, tau, work []float64, lwork int)
	Dgeqrf(m, n int, a []float64, lda int, tau, work []float64, lwork int)
	Dgesvd(jobU, jobVT SVDJob, m, n int, a []float64, lda int, s, u []float64, ldu int, vt []float64, ldvt int, work []float64, lwork int) (ok bool)
	Dgetrf(m, n int, a []float64, lda int, ipiv []int) (ok bool)
	Dgetri(n int, a []float64, lda int, ipiv []int, work []float64, lwork int) (ok bool)
	Dgetrs(trans blas.Transpose, n, nrhs int, a []float64, lda int, ipiv []int, b []float64, ldb int)
	Dlantr(norm MatrixNorm, uplo blas.Uplo, diag blas.Diag, m, n int, a []float64, lda int, work []float64) float64
	Dlange(norm MatrixNorm, m, n int, a []float64, lda int, work []float64) float64
	Dlansy(norm MatrixNorm, uplo blas.Uplo, n int, a []float64, lda int, work []float64) float64
	Dormqr(side blas.Side, trans blas.Transpose, m, n, k int, a []float64, lda int, tau, c []float64, ldc int, work []float64, lwork int)
	Dormlq(side blas.Side, trans blas.Transpose, m, n, k int, a []float64, lda int, tau, c []float64, ldc int, work []float64, lwork int)
	Dpocon(uplo blas.Uplo, n int, a []float64, lda int, anorm float64, work []float64, iwork []int) float64
	Dpotrf(ul blas.Uplo, n int, a []float64, lda int) (ok bool)
	Dsyev(jobz EigComp, uplo blas.Uplo, n int, a []float64, lda int, w, work []float64, lwork int) (ok bool)
	Dtrcon(norm MatrixNorm, uplo blas.Uplo, diag blas.Diag, n int, a []float64, lda int, work []float64, iwork []int) float64
	Dtrtri(uplo blas.Uplo, diag blas.Diag, n int, a []float64, lda int) (ok bool)
	Dtrtrs(uplo blas.Uplo, trans blas.Transpose, diag blas.Diag, n, nrhs int, a []float64, lda int, b []float64, ldb int) (ok bool)
}

// Direct specifies the direction of the multiplication for the Householder matrix.
type Direct byte

const (
	Forward  Direct = 'F' // Reflectors are right-multiplied, H_0 * H_1 * ... * H_{k-1}.
	Backward Direct = 'B' // Reflectors are left-multiplied, H_{k-1} * ... * H_1 * H_0.
)

// Sort is the sorting order.
type Sort byte

const (
	SortIncreasing Sort = 'I'
	SortDecreasing Sort = 'D'
)

// StoreV indicates the storage direction of elementary reflectors.
type StoreV byte

const (
	ColumnWise StoreV = 'C' // Reflector stored in a column of the matrix.
	RowWise    StoreV = 'R' // Reflector stored in a row of the matrix.
)

// MatrixNorm represents the kind of matrix norm to compute.
type MatrixNorm byte

const (
	MaxAbs       MatrixNorm = 'M' // max(abs(A(i,j)))  ('M')
	MaxColumnSum MatrixNorm = 'O' // Maximum column sum (one norm) ('1', 'O')
	MaxRowSum    MatrixNorm = 'I' // Maximum row sum (infinity norm) ('I', 'i')
	NormFrob     MatrixNorm = 'F' // Frobenius norm (sqrt of sum of squares) ('F', 'f', E, 'e')
)

// MatrixType represents the kind of matrix represented in the data.
type MatrixType byte

const (
	General  MatrixType = 'G' // A dense matrix (like blas64.General).
	UpperTri MatrixType = 'U' // An upper triangular matrix.
	LowerTri MatrixType = 'L' // A lower triangular matrix.
)

// Pivot specifies the pivot type for plane rotations
type Pivot byte

const (
	Variable Pivot = 'V'
	Top      Pivot = 'T'
	Bottom   Pivot = 'B'
)

type DecompUpdate byte

const (
	ApplyP DecompUpdate = 'P'
	ApplyQ DecompUpdate = 'Q'
)

// SVDJob specifies the singular vector computation type for SVD.
type SVDJob byte

const (
	SVDAll       SVDJob = 'A' // Compute all singular vectors
	SVDInPlace   SVDJob = 'S' // Compute the first singular vectors and store them in provided storage.
	SVDOverwrite SVDJob = 'O' // Compute the singular vectors and store them in input matrix
	SVDNone      SVDJob = 'N' // Do not compute singular vectors
)

// EigComp specifies the type of eigenvalue decomposition.
type EigComp byte

const (
	// EigValueOnly specifies to compute only the eigenvalues of the input matrix.
	EigValueOnly EigComp = 'N'
	// EigDecomp specifies to compute the eigenvalues and eigenvectors of the
	// full symmetric matrix.
	EigDecomp EigComp = 'V'
	// EigBoth specifies to compute both the eigenvalues and eigenvectors of the
	// input tridiagonal matrix.
	EigBoth EigComp = 'I'
)

// Jobs for Dgebal.
const (
	Permute      Job = 'P'
	Scale        Job = 'S'
	PermuteScale Job = 'B'
)

// Jobs and Comps for Dhseqr.
const (
	EigenvaluesOnly     Job = 'E'
	EigenvaluesAndSchur Job = 'S'

	InitZ   Comp = 'I'
	UpdateZ Comp = 'V'
)
<|endoftext|>


<commit_before>package neural

import (
	"math/rand"

	"github.com/gonum/matrix/mat64"
)

type Layer interface {
	Forward(input []float64) []float64
	Backward(delta []float64) []float64
	SetWeights(weights *mat64.Dense, biases *mat64.Dense)
	UpdateWeights(weights *mat64.Dense, biases *mat64.Dense)
}

type simpleLayer struct {
	weights *mat64.Dense
	biases  *mat64.Dense
	inputs  int
	neurons int
}

func randomMatrix(rows, cols int) *mat64.Dense {
	data := make([]float64, rows*cols)
	for i := range data {
		data[i] = rand.Float64()
	}

	return mat64.NewDense(rows, cols, data)
}

func NewSimpleLayer(inputs, neurons int) Layer {
	return &simpleLayer{
		weights: randomMatrix(neurons, inputs),
		biases:  randomMatrix(neurons, 1),
		inputs:  inputs,
		neurons: neurons,
	}
}

func (s *simpleLayer) Forward(input []float64) []float64 {
	inputMat := mat64.NewDense(s.inputs, 1, input)
	outMat := mat64.NewDense(s.neurons, 1, nil)

	outMat.Mul(s.weights, inputMat)
	outMat.Add(outMat, s.biases)

	return outMat.RawMatrix().Data
}

func (s *simpleLayer) Backward(input []float64) []float64 {
	inputMat := mat64.NewDense(len(input), 1, input)
	outMat := mat64.NewDense(s.inputs, len(input), nil)

	outMat.Mul(s.weights.T(), inputMat)

	return outMat.RawMatrix().Data
}

func (s *simpleLayer) SetWeights(weights *mat64.Dense, biases *mat64.Dense) {
	s.weights.Clone(weights)
	s.biases.Clone(biases)
}

func (s *simpleLayer) UpdateWeights(weights *mat64.Dense, biases *mat64.Dense) {
	s.weights.Sub(s.weights, weights)
	s.biases.Sub(s.biases, biases)
}
<commit_msg>Adjust matrix dimensionality<commit_after>package neural

import (
	"math/rand"

	"github.com/gonum/matrix/mat64"
)

type Layer interface {
	Forward(input []float64) []float64
	Backward(delta []float64) []float64
	SetWeights(weights *mat64.Dense, biases *mat64.Dense)
	UpdateWeights(weights *mat64.Dense, biases *mat64.Dense)
}

type simpleLayer struct {
	weights *mat64.Dense
	biases  *mat64.Dense
	inputs  int
	neurons int
}

func randomMatrix(rows, cols int) *mat64.Dense {
	data := make([]float64, rows*cols)
	for i := range data {
		data[i] = rand.Float64()
	}

	return mat64.NewDense(rows, cols, data)
}

func NewSimpleLayer(inputs, neurons int) Layer {
	return &simpleLayer{
		weights: randomMatrix(neurons, inputs),
		biases:  randomMatrix(neurons, 1),
		inputs:  inputs,
		neurons: neurons,
	}
}

func (s *simpleLayer) Forward(input []float64) []float64 {
	inputMat := mat64.NewDense(s.inputs, 1, input)
	outMat := mat64.NewDense(s.neurons, 1, nil)

	outMat.Mul(s.weights, inputMat)
	outMat.Add(outMat, s.biases)

	return outMat.RawMatrix().Data
}

func (s *simpleLayer) Backward(input []float64) []float64 {
	inputMat := mat64.NewDense(len(input), 1, input)
	outMat := mat64.NewDense(s.inputs, 1, nil)

	outMat.Mul(s.weights.T(), inputMat)

	return outMat.RawMatrix().Data
}

func (s *simpleLayer) SetWeights(weights *mat64.Dense, biases *mat64.Dense) {
	s.weights.Clone(weights)
	s.biases.Clone(biases)
}

func (s *simpleLayer) UpdateWeights(weights *mat64.Dense, biases *mat64.Dense) {
	s.weights.Sub(s.weights, weights)
	s.biases.Sub(s.biases, biases)
}
<|endoftext|>


<commit_before>// +build linux darwin freebsd netbsd openbsd solaris dragonfly windows

package pb

import (
	"sync"
	"time"
)

// Create and start new pool with given bars
// You need call pool.Stop() after work
func StartPool(pbs ...*ProgressBar) (pool *Pool, err error) {
	pool = new(Pool)
	if err = pool.start(); err != nil {
		return
	}
	pool.add(pbs...)
	return
}

type Pool struct {
	RefreshRate time.Duration
	bars        []*ProgressBar
	quit        chan int
	finishOnce  sync.Once
}

func (p *Pool) add(pbs ...*ProgressBar) {
	for _, bar := range pbs {
		bar.ManualUpdate = true
		bar.NotPrint = true
		bar.Start()
		p.bars = append(p.bars, bar)
	}
}

func (p *Pool) start() (err error) {
	p.RefreshRate = DefaultRefreshRate
	quit, err := lockEcho()
	if err != nil {
		return
	}
	p.quit = make(chan int)
	go p.writer(quit)
	return
}

func (p *Pool) writer(finish chan int) {
	var first = true
	for {
		select {
		case <-time.After(p.RefreshRate):
			if p.print(first) {
				p.print(false)
				finish <- 1
				return
			}
			first = false
		case <-p.quit:
			finish <- 1
			return
		}
	}
}

// Restore terminal state and close pool
func (p *Pool) Stop() error {
	// Wait until one final refresh has passed.
	time.Sleep(p.RefreshRate)

	p.finishOnce.Do(func() {
		close(p.quit)
	})
	return unlockEcho()
}
<commit_msg>Public add method for Pool.<commit_after>// +build linux darwin freebsd netbsd openbsd solaris dragonfly windows

package pb

import (
	"sync"
	"time"
)

// Create and start new pool with given bars
// You need call pool.Stop() after work
func StartPool(pbs ...*ProgressBar) (pool *Pool, err error) {
	pool = new(Pool)
	if err = pool.start(); err != nil {
		return
	}
	pool.Add(pbs...)
	return
}

type Pool struct {
	RefreshRate time.Duration
	bars        []*ProgressBar
	quit        chan int
	finishOnce  sync.Once
}

// Add progress bars.
func (p *Pool) Add(pbs ...*ProgressBar) {
	for _, bar := range pbs {
		bar.ManualUpdate = true
		bar.NotPrint = true
		bar.Start()
		p.bars = append(p.bars, bar)
	}
}

func (p *Pool) start() (err error) {
	p.RefreshRate = DefaultRefreshRate
	quit, err := lockEcho()
	if err != nil {
		return
	}
	p.quit = make(chan int)
	go p.writer(quit)
	return
}

func (p *Pool) writer(finish chan int) {
	var first = true
	for {
		select {
		case <-time.After(p.RefreshRate):
			if p.print(first) {
				p.print(false)
				finish <- 1
				return
			}
			first = false
		case <-p.quit:
			finish <- 1
			return
		}
	}
}

// Restore terminal state and close pool
func (p *Pool) Stop() error {
	// Wait until one final refresh has passed.
	time.Sleep(p.RefreshRate)

	p.finishOnce.Do(func() {
		close(p.quit)
	})
	return unlockEcho()
}
<|endoftext|>


<commit_before>// Package pop3 implements the Post Office Protocol version 3.
package pop3

import (
	"errors"
	"fmt"
	"net"
	"strconv"
	"strings"
)

var (
	EOF = errors.New("skip the all mail remaining")
)

// MessageInfo has Number, Size, and Uid fields,
// and used as a return value of ListAll and UidlAll.
// When used as the return value of the method ListAll,
// MessageInfo contain only the Number and Size values.
// When used as the return value of the method UidlAll,
// MessageInfo contain only the Number and Uid values.
type MessageInfo struct {
	Number int
	Size   uint64
	Uid    string
}

// A Client represents a client connection to an POP server.
type Client struct {
	// Text is the pop3.Conn used by the Client.
	Text *Conn
	// keep a reference to the connection so it can be used to create a TLS
	// connection later
	conn net.Conn
}

// Dial returns a new Client connected to an POP server at addr.
// The addr must include a port number.
func Dial(addr string) (*Client, error) {
	conn, err := net.Dial("tcp", addr)

	if err != nil {
		return nil, err
	}

	return NewClient(conn)
}

// NewClient returns a new Client using an existing connection.
func NewClient(conn net.Conn) (*Client, error) {
	text := NewConn(conn)

	_, err := text.ReadResponse()

	if err != nil {
		return nil, err
	}

	return &Client{Text: text, conn: conn}, nil
}

// User issues a USER command to the server using the provided user name.
func (c *Client) User(user string) error {
	return c.cmdSimple("USER %s", user)
}

// Pass issues a PASS command to the server using the provided password.
func (c *Client) Pass(pass string) error {
	return c.cmdSimple("PASS %s", pass)
}

// Stat issues a STAT command to the server
// and returns mail count and total size.
func (c *Client) Stat() (int, uint64, error) {
	return c.cmdStatOrList("STAT", "STAT")
}

// Retr issues a RETR command to the server using the provided mail number
// and returns mail data.
func (c *Client) Retr(number int) (string, error) {
	var err error

	err = c.Text.WriteLine("RETR %d", number)

	if err != nil {
		return "", err
	}

	_, err = c.Text.ReadResponse()

	if err != nil {
		return "", err
	}

	return c.Text.ReadToPeriod()
}

// List issues a LIST command to the server using the provided mail number
// and returns mail number and size.
func (c *Client) List(number int) (int, uint64, error) {
	return c.cmdStatOrList("LIST", "LIST %d", number)
}

// List issues a LIST command to the server
// and returns array of MessageInfo.
func (c *Client) ListAll() ([]MessageInfo, error) {
	list := make([]MessageInfo, 0)

	err := c.cmdReadLines("LIST", func(line string) error {
		number, size, err := c.convertNumberAndSize(line)

		if err != nil {
			return err
		}

		list = append(list, MessageInfo{Number: number, Size: size})

		return nil
	})

	if err != nil {
		return nil, err
	}

	return list, nil
}

// Uidl issues a UIDL command to the server using the provided mail number
// and returns mail number and unique id.
func (c *Client) Uidl(number int) (int, string, error) {
	var err error

	err = c.Text.WriteLine("UIDL %d", number)

	if err != nil {
		return 0, "", err
	}

	var msg string

	msg, err = c.Text.ReadResponse()

	if err != nil {
		return 0, "", err
	}

	var val int
	var uid string

	val, uid, err = c.convertNumberAndUid(msg)

	if err != nil {
		return 0, "", err
	}

	return val, uid, nil
}

// Uidl issues a UIDL command to the server
// and returns array of MessageInfo.
func (c *Client) UidlAll() ([]MessageInfo, error) {
	list := make([]MessageInfo, 0)

	err := c.cmdReadLines("UIDL", func(line string) error {
		number, uid, err := c.convertNumberAndUid(line)

		if err != nil {
			return err
		}

		list = append(list, MessageInfo{Number: number, Uid: uid})

		return nil
	})

	if err != nil {
		return nil, err
	}

	return list, nil
}

// Dele issues a DELE command to the server using the provided mail number.
func (c *Client) Dele(number int) error {
	return c.cmdSimple("DELE %d", number)
}

// Noop issues a NOOP command to the server.
func (c *Client) Noop() error {
	return c.cmdSimple("NOOP")
}

// Rset issues a RSET command to the server.
func (c *Client) Rset() error {
	return c.cmdSimple("RSET")
}

// Quit issues a QUIT command to the server.
func (c *Client) Quit() error {
	return c.cmdSimple("QUIT")
}

// ReceiveMail connects to the server at addr,
// and authenticates with user and pass,
// and calling receiveFn for each mail.
func ReceiveMail(addr, user, pass string, receiveFn ReceiveMailFunc) error {
	c, err := Dial(addr)

	if err != nil {
		return err
	}

	defer func() {
		if err != nil && err != EOF {
			c.Rset()
		}

		c.Quit()
		c.Close()
	}()

	if err = c.User(user); err != nil {
		return err
	}

	if err = c.Pass(pass); err != nil {
		return err
	}

	var mis []MessageInfo

	if mis, err = c.UidlAll(); err != nil {
		return err
	}

	for _, mi := range mis {
		var data string

		data, err = c.Retr(mi.Number)

		del, err := receiveFn(mi.Number, mi.Uid, data, err)

		if err != nil && err != EOF {
			return err
		}

		if del {
			if err = c.Dele(mi.Number); err != nil {
				return err
			}
		}

		if err == EOF {
			break
		}
	}

	return nil
}

// ReceiveMailFunc is the type of the function called for each mail.
// Its arguments are mail's number, uid, data, and mail receiving error.
// if this function returns false value, the mail will be deleted,
// if its returns EOF, skip the all mail of remaining.
// (after deleting mail, if necessary)
type ReceiveMailFunc func(number int, uid, data string, err error) (bool, error)

func (c *Client) cmdSimple(format string, args ...interface{}) error {
	var err error

	err = c.Text.WriteLine(format, args...)

	if err != nil {
		return err
	}

	_, err = c.Text.ReadResponse()

	if err != nil {
		return err
	}

	return nil
}

func (c *Client) cmdStatOrList(name, format string, args ...interface{}) (int, uint64, error) {
	var err error

	err = c.Text.WriteLine(format, args...)

	if err != nil {
		return 0, 0, err
	}

	var msg string

	msg, err = c.Text.ReadResponse()

	if err != nil {
		return 0, 0, err
	}

	s := strings.Split(msg, " ")

	if len(s) < 2 {
		return 0, 0, ResponseError(fmt.Sprintf("invalid response format: %s", msg))
	}

	var val int
	var size uint64

	val, size, err = c.convertNumberAndSize(msg)

	if err != nil {
		return 0, 0, err
	}

	return val, size, nil
}

func (c *Client) cmdReadLines(cmnd string, lineFn lineFunc) error {
	var err error

	err = c.Text.WriteLine(cmnd)

	if err != nil {
		return err
	}

	_, err = c.Text.ReadResponse()

	if err != nil {
		return err
	}

	var lines []string

	lines, err = c.Text.ReadLines()

	if err != nil {
		return err
	}

	for _, line := range lines {
		err = lineFn(line)

		if err != nil {
			return err
		}
	}

	return nil
}

type lineFunc func(line string) error

func (c *Client) Close() error {
	return c.Text.Close()
}

func (c *Client) convertNumberAndSize(line string) (int, uint64, error) {
	var err error

	s := strings.Split(line, " ")

	if len(s) < 2 {
		return 0, 0, errors.New(fmt.Sprintf("the length of the array is less than 2: %s", line))
	}

	var val int
	var size uint64

	if val, err = strconv.Atoi(s[0]); err != nil {
		return 0, 0, errors.New(fmt.Sprintf("can not convert element[0] to int type: %s", line))
	}

	if size, err = strconv.ParseUint(s[1], 10, 64); err != nil {
		return 0, 0, errors.New(fmt.Sprintf("can not convert element[1] to uint64 type: %s", line))
	}

	return val, size, nil
}

func (c *Client) convertNumberAndUid(line string) (int, string, error) {
	var err error

	s := strings.Split(line, " ")

	if len(s) < 2 {
		return 0, "", errors.New(fmt.Sprintf("the length of the array is less than 2: %s", line))
	}

	var val int

	if val, err = strconv.Atoi(s[0]); err != nil {
		return 0, "", errors.New(fmt.Sprintf("can not convert element[0] to int type: %s", line))
	}

	return val, s[1], nil
}
<commit_msg>fix package comment.<commit_after>// Package pop3 provides simple POP3 Client.
package pop3

import (
	"errors"
	"fmt"
	"net"
	"strconv"
	"strings"
)

var (
	EOF = errors.New("skip the all mail remaining")
)

// MessageInfo has Number, Size, and Uid fields,
// and used as a return value of ListAll and UidlAll.
// When used as the return value of the method ListAll,
// MessageInfo contain only the Number and Size values.
// When used as the return value of the method UidlAll,
// MessageInfo contain only the Number and Uid values.
type MessageInfo struct {
	Number int
	Size   uint64
	Uid    string
}

// A Client represents a client connection to an POP server.
type Client struct {
	// Text is the pop3.Conn used by the Client.
	Text *Conn
	// keep a reference to the connection so it can be used to create a TLS
	// connection later
	conn net.Conn
}

// Dial returns a new Client connected to an POP server at addr.
// The addr must include a port number.
func Dial(addr string) (*Client, error) {
	conn, err := net.Dial("tcp", addr)

	if err != nil {
		return nil, err
	}

	return NewClient(conn)
}

// NewClient returns a new Client using an existing connection.
func NewClient(conn net.Conn) (*Client, error) {
	text := NewConn(conn)

	_, err := text.ReadResponse()

	if err != nil {
		return nil, err
	}

	return &Client{Text: text, conn: conn}, nil
}

// User issues a USER command to the server using the provided user name.
func (c *Client) User(user string) error {
	return c.cmdSimple("USER %s", user)
}

// Pass issues a PASS command to the server using the provided password.
func (c *Client) Pass(pass string) error {
	return c.cmdSimple("PASS %s", pass)
}

// Stat issues a STAT command to the server
// and returns mail count and total size.
func (c *Client) Stat() (int, uint64, error) {
	return c.cmdStatOrList("STAT", "STAT")
}

// Retr issues a RETR command to the server using the provided mail number
// and returns mail data.
func (c *Client) Retr(number int) (string, error) {
	var err error

	err = c.Text.WriteLine("RETR %d", number)

	if err != nil {
		return "", err
	}

	_, err = c.Text.ReadResponse()

	if err != nil {
		return "", err
	}

	return c.Text.ReadToPeriod()
}

// List issues a LIST command to the server using the provided mail number
// and returns mail number and size.
func (c *Client) List(number int) (int, uint64, error) {
	return c.cmdStatOrList("LIST", "LIST %d", number)
}

// List issues a LIST command to the server
// and returns array of MessageInfo.
func (c *Client) ListAll() ([]MessageInfo, error) {
	list := make([]MessageInfo, 0)

	err := c.cmdReadLines("LIST", func(line string) error {
		number, size, err := c.convertNumberAndSize(line)

		if err != nil {
			return err
		}

		list = append(list, MessageInfo{Number: number, Size: size})

		return nil
	})

	if err != nil {
		return nil, err
	}

	return list, nil
}

// Uidl issues a UIDL command to the server using the provided mail number
// and returns mail number and unique id.
func (c *Client) Uidl(number int) (int, string, error) {
	var err error

	err = c.Text.WriteLine("UIDL %d", number)

	if err != nil {
		return 0, "", err
	}

	var msg string

	msg, err = c.Text.ReadResponse()

	if err != nil {
		return 0, "", err
	}

	var val int
	var uid string

	val, uid, err = c.convertNumberAndUid(msg)

	if err != nil {
		return 0, "", err
	}

	return val, uid, nil
}

// Uidl issues a UIDL command to the server
// and returns array of MessageInfo.
func (c *Client) UidlAll() ([]MessageInfo, error) {
	list := make([]MessageInfo, 0)

	err := c.cmdReadLines("UIDL", func(line string) error {
		number, uid, err := c.convertNumberAndUid(line)

		if err != nil {
			return err
		}

		list = append(list, MessageInfo{Number: number, Uid: uid})

		return nil
	})

	if err != nil {
		return nil, err
	}

	return list, nil
}

// Dele issues a DELE command to the server using the provided mail number.
func (c *Client) Dele(number int) error {
	return c.cmdSimple("DELE %d", number)
}

// Noop issues a NOOP command to the server.
func (c *Client) Noop() error {
	return c.cmdSimple("NOOP")
}

// Rset issues a RSET command to the server.
func (c *Client) Rset() error {
	return c.cmdSimple("RSET")
}

// Quit issues a QUIT command to the server.
func (c *Client) Quit() error {
	return c.cmdSimple("QUIT")
}

// ReceiveMail connects to the server at addr,
// and authenticates with user and pass,
// and calling receiveFn for each mail.
func ReceiveMail(addr, user, pass string, receiveFn ReceiveMailFunc) error {
	c, err := Dial(addr)

	if err != nil {
		return err
	}

	defer func() {
		if err != nil && err != EOF {
			c.Rset()
		}

		c.Quit()
		c.Close()
	}()

	if err = c.User(user); err != nil {
		return err
	}

	if err = c.Pass(pass); err != nil {
		return err
	}

	var mis []MessageInfo

	if mis, err = c.UidlAll(); err != nil {
		return err
	}

	for _, mi := range mis {
		var data string

		data, err = c.Retr(mi.Number)

		del, err := receiveFn(mi.Number, mi.Uid, data, err)

		if err != nil && err != EOF {
			return err
		}

		if del {
			if err = c.Dele(mi.Number); err != nil {
				return err
			}
		}

		if err == EOF {
			break
		}
	}

	return nil
}

// ReceiveMailFunc is the type of the function called for each mail.
// Its arguments are mail's number, uid, data, and mail receiving error.
// if this function returns false value, the mail will be deleted,
// if its returns EOF, skip the all mail of remaining.
// (after deleting mail, if necessary)
type ReceiveMailFunc func(number int, uid, data string, err error) (bool, error)

func (c *Client) cmdSimple(format string, args ...interface{}) error {
	var err error

	err = c.Text.WriteLine(format, args...)

	if err != nil {
		return err
	}

	_, err = c.Text.ReadResponse()

	if err != nil {
		return err
	}

	return nil
}

func (c *Client) cmdStatOrList(name, format string, args ...interface{}) (int, uint64, error) {
	var err error

	err = c.Text.WriteLine(format, args...)

	if err != nil {
		return 0, 0, err
	}

	var msg string

	msg, err = c.Text.ReadResponse()

	if err != nil {
		return 0, 0, err
	}

	s := strings.Split(msg, " ")

	if len(s) < 2 {
		return 0, 0, ResponseError(fmt.Sprintf("invalid response format: %s", msg))
	}

	var val int
	var size uint64

	val, size, err = c.convertNumberAndSize(msg)

	if err != nil {
		return 0, 0, err
	}

	return val, size, nil
}

func (c *Client) cmdReadLines(cmnd string, lineFn lineFunc) error {
	var err error

	err = c.Text.WriteLine(cmnd)

	if err != nil {
		return err
	}

	_, err = c.Text.ReadResponse()

	if err != nil {
		return err
	}

	var lines []string

	lines, err = c.Text.ReadLines()

	if err != nil {
		return err
	}

	for _, line := range lines {
		err = lineFn(line)

		if err != nil {
			return err
		}
	}

	return nil
}

type lineFunc func(line string) error

func (c *Client) Close() error {
	return c.Text.Close()
}

func (c *Client) convertNumberAndSize(line string) (int, uint64, error) {
	var err error

	s := strings.Split(line, " ")

	if len(s) < 2 {
		return 0, 0, errors.New(fmt.Sprintf("the length of the array is less than 2: %s", line))
	}

	var val int
	var size uint64

	if val, err = strconv.Atoi(s[0]); err != nil {
		return 0, 0, errors.New(fmt.Sprintf("can not convert element[0] to int type: %s", line))
	}

	if size, err = strconv.ParseUint(s[1], 10, 64); err != nil {
		return 0, 0, errors.New(fmt.Sprintf("can not convert element[1] to uint64 type: %s", line))
	}

	return val, size, nil
}

func (c *Client) convertNumberAndUid(line string) (int, string, error) {
	var err error

	s := strings.Split(line, " ")

	if len(s) < 2 {
		return 0, "", errors.New(fmt.Sprintf("the length of the array is less than 2: %s", line))
	}

	var val int

	if val, err = strconv.Atoi(s[0]); err != nil {
		return 0, "", errors.New(fmt.Sprintf("can not convert element[0] to int type: %s", line))
	}

	return val, s[1], nil
}
<|endoftext|>


<commit_before>/*
Package proj transforms coordinates with libproj4.

	// proj by EPSG code
	wgs84, err := proj.NewEPSG(4326)
	if err != nil {
		log.Fatal(err)
	}

	// proj by proj4 definition string
	utm32, err := proj.New("+proj=utm +zone=32 +ellps=GRS80 +towgs84=0,0,0,0,0,0,0 +units=m +no_defs")
	if err != nil {
		log.Fatal(err)
	}

	xs := []float64{8.15, 9.12}
	ys := []float64{53.2, 52.32}

	// transform all coordinates to UTM 32 (in-place)
	if err := wgs84.Transform(utm32, xs, ys); err != nil {
		log.Fatal(err)
	}

	// transformer from src to dst projection
	transf, err := proj.NewTransformer("+init=epsg:25832", "+init=epsg:3857")
	if err != nil {
		log.Fatal(err)
	}
	if err := transf.Transform(xs, ys); err != nil {
		log.Fatal(err)
	}
*/
package proj

// #cgo LDFLAGS: -lproj
// #include <proj_api.h>
// #include <stdlib.h>
// extern char *go_proj_finder_wrapper(char *name);
import "C"

import (
	"errors"
	"fmt"
	"math"
	"os"
	"path/filepath"
	"runtime"
	"unsafe"
)

type Proj struct {
	p   C.projPJ
	ctx C.projCtx
}

const deg2Rad = math.Pi / 180.0
const rad2Deg = 180.0 / math.Pi

// NewEPSG initializes a new projection by the numeric EPSG code.
func NewEPSG(epsgCode int) (*Proj, error) {
	return New(fmt.Sprintf("+init=epsg:%d", epsgCode))
}

// New initializes new projection with a full proj4 init string (e.g. "+proj=longlat +datum=WGS84 +no_defs").
func New(init string) (*Proj, error) {
	ctx := C.pj_ctx_alloc()
	if ctx == nil {
		errnoRef := C.pj_get_errno_ref()
		if errnoRef == nil {
			return nil, errors.New("unknown error on pj_ctx_alloc")
		}
		return nil, errors.New(C.GoString(C.pj_strerrno(*errnoRef)))
	}

	c := C.CString(init)
	defer C.free(unsafe.Pointer(c))
	proj := C.pj_init_plus_ctx(ctx, c)
	if proj == nil {
		errno := C.pj_ctx_get_errno(ctx)
		return nil, errors.New(C.GoString(C.pj_strerrno(errno)))
	}

	p := &Proj{proj, ctx}
	runtime.SetFinalizer(p, free)
	return p, nil
}

func free(p *Proj) {
	p.Free()
}

// Free deallocates the projection immediately. Proj will be deallocated on garbage collection otherwise.
func (p *Proj) Free() {
	if p.p != nil {
		C.pj_free(p.p)
		p.p = nil
	}
	if p.ctx != nil {
		C.pj_ctx_free(p.ctx)
		p.ctx = nil
	}
}

// Transform coordinates to dst projection. Transforms coordinates in-place.
func (p *Proj) Transform(dst *Proj, xs, ys []float64) error {
	if p == nil {
		return errors.New("missing/invalid projection")
	}
	if dst == nil {
		return errors.New("missing/invalid dst projection")
	}
	if len(xs) != len(ys) {
		return errors.New("number of x and y coordinates differs")
	}
	if xs == nil || ys == nil {
		return nil
	}

	if C.pj_is_latlong(p.p) != 0 {
		for i := range xs {
			xs[i] *= deg2Rad
		}
		for i := range ys {
			ys[i] *= deg2Rad
		}
	}
	r := C.pj_transform(p.p, dst.p, C.long(len(xs)), 0,
		(*C.double)(unsafe.Pointer(&xs[0])),
		(*C.double)(unsafe.Pointer(&ys[0])),
		nil)

	if r != 0 {
		errnoRef := C.pj_get_errno_ref()
		if errnoRef == nil {
			return errors.New("unknown error")
		}
		return errors.New(C.GoString(C.pj_strerrno(*errnoRef)))
	}

	if C.pj_is_latlong(dst.p) != 0 {
		for i := range xs {
			xs[i] *= rad2Deg
		}
		for i := range ys {
			ys[i] *= rad2Deg
		}
	}
	return nil
}

// IsLatLong returns whether the projection uses lat/long coordinates, instead projected.
func (p *Proj) IsLatLong() bool {
	return C.pj_is_latlong(p.p) != 0
}

type Transformer struct {
	Src *Proj
	Dst *Proj
}

func (t *Transformer) Transform(xs, ys []float64) error {
	return t.Src.Transform(t.Dst, xs, ys)
}

func NewTransformer(initSrc, initDst string) (Transformer, error) {
	src, err := New(initSrc)
	if err != nil {
		return Transformer{}, err
	}
	dst, err := New(initDst)
	if err != nil {
		return Transformer{}, err
	}
	return Transformer{Src: src, Dst: dst}, nil
}

func NewEPSGTransformer(srcEPSG, dstEPSG int) (Transformer, error) {
	src, err := NewEPSG(srcEPSG)
	if err != nil {
		return Transformer{}, err
	}
	dst, err := NewEPSG(dstEPSG)
	if err != nil {
		return Transformer{}, err
	}
	return Transformer{Src: src, Dst: dst}, nil
}

var searchPaths []string
var finderResults map[string]*C.char

//export go_proj_finder
func go_proj_finder(cname *C.char) *C.char {
	name := C.GoString(cname)
	path, ok := finderResults[name]
	if !ok {
		for _, p := range searchPaths {
			p = filepath.Join(p, name)
			_, err := os.Stat(p)
			if err == nil {
				path = C.CString(p)
				break
			}
		}
		// cache result, even if it is nil
		finderResults[name] = path
	}
	return path
}

// SetSearchPaths add one or more directories to search for proj definition files.
// Multiple calls overwrite the previous search paths.
func SetSearchPaths(paths []string) {
	finderResults = make(map[string]*C.char)
	searchPaths = paths
	C.pj_set_finder((*[0]byte)(unsafe.Pointer(C.go_proj_finder_wrapper)))
}
<commit_msg>document transformer<commit_after>/*
Package proj transforms coordinates with libproj4.

	// proj by EPSG code
	wgs84, err := proj.NewEPSG(4326)
	if err != nil {
		log.Fatal(err)
	}

	// proj by proj4 definition string
	utm32, err := proj.New("+proj=utm +zone=32 +ellps=GRS80 +towgs84=0,0,0,0,0,0,0 +units=m +no_defs")
	if err != nil {
		log.Fatal(err)
	}

	xs := []float64{8.15, 9.12}
	ys := []float64{53.2, 52.32}

	// transform all coordinates to UTM 32 (in-place)
	if err := wgs84.Transform(utm32, xs, ys); err != nil {
		log.Fatal(err)
	}

	// transformer from src to dst projection
	transf, err := proj.NewTransformer("+init=epsg:25832", "+init=epsg:3857")
	if err != nil {
		log.Fatal(err)
	}
	if err := transf.Transform(xs, ys); err != nil {
		log.Fatal(err)
	}
*/
package proj

// #cgo LDFLAGS: -lproj
// #include <proj_api.h>
// #include <stdlib.h>
// extern char *go_proj_finder_wrapper(char *name);
import "C"

import (
	"errors"
	"fmt"
	"math"
	"os"
	"path/filepath"
	"runtime"
	"unsafe"
)

type Proj struct {
	p   C.projPJ
	ctx C.projCtx
}

const deg2Rad = math.Pi / 180.0
const rad2Deg = 180.0 / math.Pi

// NewEPSG initializes a new projection by the numeric EPSG code.
func NewEPSG(epsgCode int) (*Proj, error) {
	return New(fmt.Sprintf("+init=epsg:%d", epsgCode))
}

// New initializes new projection with a full proj4 init string (e.g. "+proj=longlat +datum=WGS84 +no_defs").
func New(init string) (*Proj, error) {
	ctx := C.pj_ctx_alloc()
	if ctx == nil {
		errnoRef := C.pj_get_errno_ref()
		if errnoRef == nil {
			return nil, errors.New("unknown error on pj_ctx_alloc")
		}
		return nil, errors.New(C.GoString(C.pj_strerrno(*errnoRef)))
	}

	c := C.CString(init)
	defer C.free(unsafe.Pointer(c))
	proj := C.pj_init_plus_ctx(ctx, c)
	if proj == nil {
		errno := C.pj_ctx_get_errno(ctx)
		return nil, errors.New(C.GoString(C.pj_strerrno(errno)))
	}

	p := &Proj{proj, ctx}
	runtime.SetFinalizer(p, free)
	return p, nil
}

func free(p *Proj) {
	p.Free()
}

// Free deallocates the projection immediately. Proj will be deallocated on garbage collection otherwise.
func (p *Proj) Free() {
	if p.p != nil {
		C.pj_free(p.p)
		p.p = nil
	}
	if p.ctx != nil {
		C.pj_ctx_free(p.ctx)
		p.ctx = nil
	}
}

// Transform coordinates to dst projection. Transforms coordinates in-place.
func (p *Proj) Transform(dst *Proj, xs, ys []float64) error {
	if p == nil {
		return errors.New("missing/invalid projection")
	}
	if dst == nil {
		return errors.New("missing/invalid dst projection")
	}
	if len(xs) != len(ys) {
		return errors.New("number of x and y coordinates differs")
	}
	if xs == nil || ys == nil {
		return nil
	}

	if C.pj_is_latlong(p.p) != 0 {
		for i := range xs {
			xs[i] *= deg2Rad
		}
		for i := range ys {
			ys[i] *= deg2Rad
		}
	}
	r := C.pj_transform(p.p, dst.p, C.long(len(xs)), 0,
		(*C.double)(unsafe.Pointer(&xs[0])),
		(*C.double)(unsafe.Pointer(&ys[0])),
		nil)

	if r != 0 {
		errnoRef := C.pj_get_errno_ref()
		if errnoRef == nil {
			return errors.New("unknown error")
		}
		return errors.New(C.GoString(C.pj_strerrno(*errnoRef)))
	}

	if C.pj_is_latlong(dst.p) != 0 {
		for i := range xs {
			xs[i] *= rad2Deg
		}
		for i := range ys {
			ys[i] *= rad2Deg
		}
	}
	return nil
}

// IsLatLong returns whether the projection uses lat/long coordinates, instead projected.
func (p *Proj) IsLatLong() bool {
	return C.pj_is_latlong(p.p) != 0
}

type Transformer struct {
	Src *Proj
	Dst *Proj
}

// Transform coordinates fron src to dst projection. Transforms coordinates in-place.
func (t *Transformer) Transform(xs, ys []float64) error {
	return t.Src.Transform(t.Dst, xs, ys)
}

// NewTransformer initializes new transformer with src and dst projection with
// a full proj4 init string (e.g. "+proj=longlat +datum=WGS84 +no_defs").
func NewTransformer(initSrc, initDst string) (Transformer, error) {
	src, err := New(initSrc)
	if err != nil {
		return Transformer{}, err
	}
	dst, err := New(initDst)
	if err != nil {
		return Transformer{}, err
	}
	return Transformer{Src: src, Dst: dst}, nil
}

// NewEPSGTransformer initializes a new transformer with src and dst projection by the numeric EPSG code.
func NewEPSGTransformer(srcEPSG, dstEPSG int) (Transformer, error) {
	src, err := NewEPSG(srcEPSG)
	if err != nil {
		return Transformer{}, err
	}
	dst, err := NewEPSG(dstEPSG)
	if err != nil {
		return Transformer{}, err
	}
	return Transformer{Src: src, Dst: dst}, nil
}

var searchPaths []string
var finderResults map[string]*C.char

//export go_proj_finder
func go_proj_finder(cname *C.char) *C.char {
	name := C.GoString(cname)
	path, ok := finderResults[name]
	if !ok {
		for _, p := range searchPaths {
			p = filepath.Join(p, name)
			_, err := os.Stat(p)
			if err == nil {
				path = C.CString(p)
				break
			}
		}
		// cache result, even if it is nil
		finderResults[name] = path
	}
	return path
}

// SetSearchPaths add one or more directories to search for proj definition files.
// Multiple calls overwrite the previous search paths.
func SetSearchPaths(paths []string) {
	finderResults = make(map[string]*C.char)
	searchPaths = paths
	C.pj_set_finder((*[0]byte)(unsafe.Pointer(C.go_proj_finder_wrapper)))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"os/signal"
	"sync"
	"syscall"
	"time"

	"github.com/doozr/guac"
	"github.com/doozr/jot"
	"github.com/doozr/qbot/command"
	"github.com/doozr/qbot/dispatch"
	"github.com/doozr/qbot/notification"
	"github.com/doozr/qbot/queue"
	"github.com/doozr/qbot/usercache"
)

// Version is the current release version
var Version string

func main() {
	if len(os.Args) < 3 {
		fmt.Println("Usage: qbot <token> <data file>")
		os.Exit(1)
	}

	if Version != "" {
		log.Printf("Qbot version %s", Version)
	} else {
		log.Printf("Qbot <unversioned build>")
	}

	// Get command line parameters
	token := os.Args[1]
	filename := os.Args[2]

	// Turn on jot if required
	if os.Getenv("QBOT_DEBUG") == "true" {
		jot.Enable()
	}

	// Synchronisation primitives
	waitGroup := sync.WaitGroup{}
	done := make(chan struct{})

	// Connect to Slack
	client, err := guac.New(token).RealTime()
	if err != nil {
		log.Fatal("Error connecting to Slack ", err)
	}
	log.Print("Connected to slack as ", client.Name())

	// Instantiate state
	userCache := getUserList(client.WebClient)
	name := client.Name()
	jot.Print("qbot: name is ", name)
	q := loadQueue(filename)

	// Set up command and response processors
	notifications := notification.New(userCache)
	commands := command.New(notifications, userCache)

	// Create channels
	messageChan := make(dispatch.MessageChan, 100)
	saveChan := make(dispatch.SaveChan, 5)
	notifyChan := make(dispatch.NotifyChan, 5)
	userChan := make(dispatch.UserChan, 5)

	// Start goroutines
	waitGroup.Add(4)
	go dispatch.Message(name, q, commands, messageChan, saveChan, notifyChan, &waitGroup)
	go dispatch.Save(filename, saveChan, &waitGroup)
	go dispatch.Notify(client, notifyChan, &waitGroup)
	go dispatch.User(userCache, userChan, &waitGroup)

	// Dispatch incoming events
	jot.Println("qbot: ready to receive events")
	abort := listen(name, client, 60*time.Second, messageChan, userChan, done, &waitGroup)

	// Wait for signals to stop
	sig := make(chan os.Signal, 1)
	signal.Notify(sig, syscall.SIGINT)
	signal.Notify(sig, syscall.SIGTERM)
	signal.Notify(sig, syscall.SIGKILL)

	// Wait for a signal
	select {
	case <-abort:
		log.Print("Execution aborted - shutting down")
	case s := <-sig:
		log.Printf("Received %s signal - shutting down", s)
	}

	jot.Print("qbot: closing done channel")
	close(done)

	jot.Print("qbot: closing connection")
	client.Close()

	jot.Println("qbot: closing dispatch channels")
	close(messageChan)
	close(saveChan)
	close(notifyChan)
	close(userChan)

	jot.Print("qbot: waiting for dispatch to terminate")
	waitGroup.Wait()

	jot.Print("qbot: shutdown complete")
}

func getUserList(client guac.WebClient) (userCache *usercache.UserCache) {
	log.Println("Getting user list")
	users, err := client.UsersList()
	if err != nil {
		log.Fatal(err)
	}
	userCache = usercache.New(users)
	jot.Print("loaded user list: ", userCache)
	return
}

func loadQueue(filename string) (q queue.Queue) {
	q, err := queue.Load(filename)
	if err != nil {
		log.Fatalf("Error loading queue: %s", err)
	}
	log.Printf("Loaded queue from %s", filename)
	return
}
<commit_msg>Do not poke internals<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"os/signal"
	"sync"
	"syscall"
	"time"

	"github.com/doozr/guac"
	"github.com/doozr/jot"
	"github.com/doozr/qbot/command"
	"github.com/doozr/qbot/dispatch"
	"github.com/doozr/qbot/notification"
	"github.com/doozr/qbot/queue"
	"github.com/doozr/qbot/usercache"
)

// Version is the current release version
var Version string

func main() {
	if len(os.Args) < 3 {
		fmt.Println("Usage: qbot <token> <data file>")
		os.Exit(1)
	}

	if Version != "" {
		log.Printf("Qbot version %s", Version)
	} else {
		log.Printf("Qbot <unversioned build>")
	}

	// Get command line parameters
	token := os.Args[1]
	filename := os.Args[2]

	// Turn on jot if required
	if os.Getenv("QBOT_DEBUG") == "true" {
		jot.Enable()
	}

	// Synchronisation primitives
	waitGroup := sync.WaitGroup{}
	done := make(chan struct{})

	// Connect to Slack
	client, err := guac.New(token).RealTime()
	if err != nil {
		log.Fatal("Error connecting to Slack ", err)
	}
	log.Print("Connected to slack as ", client.Name())

	// Instantiate state
	userCache := getUserList(client)
	name := client.Name()
	jot.Print("qbot: name is ", name)
	q := loadQueue(filename)

	// Set up command and response processors
	notifications := notification.New(userCache)
	commands := command.New(notifications, userCache)

	// Create channels
	messageChan := make(dispatch.MessageChan, 100)
	saveChan := make(dispatch.SaveChan, 5)
	notifyChan := make(dispatch.NotifyChan, 5)
	userChan := make(dispatch.UserChan, 5)

	// Start goroutines
	waitGroup.Add(4)
	go dispatch.Message(name, q, commands, messageChan, saveChan, notifyChan, &waitGroup)
	go dispatch.Save(filename, saveChan, &waitGroup)
	go dispatch.Notify(client, notifyChan, &waitGroup)
	go dispatch.User(userCache, userChan, &waitGroup)

	// Dispatch incoming events
	jot.Println("qbot: ready to receive events")
	abort := listen(name, client, 60*time.Second, messageChan, userChan, done, &waitGroup)

	// Wait for signals to stop
	sig := make(chan os.Signal, 1)
	signal.Notify(sig, syscall.SIGINT)
	signal.Notify(sig, syscall.SIGTERM)
	signal.Notify(sig, syscall.SIGKILL)

	// Wait for a signal
	select {
	case <-abort:
		log.Print("Execution aborted - shutting down")
	case s := <-sig:
		log.Printf("Received %s signal - shutting down", s)
	}

	jot.Print("qbot: closing done channel")
	close(done)

	jot.Print("qbot: closing connection")
	client.Close()

	jot.Println("qbot: closing dispatch channels")
	close(messageChan)
	close(saveChan)
	close(notifyChan)
	close(userChan)

	jot.Print("qbot: waiting for dispatch to terminate")
	waitGroup.Wait()

	jot.Print("qbot: shutdown complete")
}

func getUserList(client guac.WebClient) (userCache *usercache.UserCache) {
	log.Println("Getting user list")
	users, err := client.UsersList()
	if err != nil {
		log.Fatal(err)
	}
	userCache = usercache.New(users)
	jot.Print("loaded user list: ", userCache)
	return
}

func loadQueue(filename string) (q queue.Queue) {
	q, err := queue.Load(filename)
	if err != nil {
		log.Fatalf("Error loading queue: %s", err)
	}
	log.Printf("Loaded queue from %s", filename)
	return
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"sync"
	"syscall"
	"time"

	"github.com/doozr/guac"
	"github.com/doozr/jot"
	"github.com/doozr/qbot/command"
	"github.com/doozr/qbot/notification"
	"github.com/doozr/qbot/queue"
	"github.com/doozr/qbot/usercache"
)

// Version is the current release version
var Version = "<unversioned build>"

// DoneChan is a channel used for informing go routines to shut down
type DoneChan chan struct{}

func main() {
	log.Printf("Qbot version %s", Version)

	// Turn on jot if required
	if os.Getenv("QBOT_DEBUG") == "true" {
		jot.Enable()
	}

	token, filename := parseArgs()

	// Synchronisation primitives
	waitGroup := sync.WaitGroup{}
	done := make(DoneChan)

	// Connect to Slack
	client := connectToSlack(token)
	log.Print("Connected to slack as ", client.Name())

	// Instantiate state
	userCache := getUserList(client)
	q := loadQueue(filename)

	// Set up command and response processors
	notifications := notification.New(userCache)
	commands := command.New(notifications, userCache)

	// Create dispatchers
	notify := createNotifier(client)
	persist := createPersister(filename)
	messageHandler := createMessageHandler(client.ID(), client.Name(), q, commands, notify, persist)
	userChangeHandler := createUserChangeHandler(userCache)

	// start keepalive
	startKeepAlive(client, done, &waitGroup)

	// Receive incoming events
	receiver := createReceiver(client)
	events := receive(receiver, done, &waitGroup)

	// Dispatch incoming events
	jot.Println("qbot: ready to receive events")
	dispatcher := createDispatcher(client, 1*time.Minute, messageHandler, userChangeHandler)
	abort := dispatch(dispatcher, events, done, &waitGroup)

	// Wait for signals to stop
	sig := addSignalHandler()

	// Wait for a signal or an error to kill the process
	wait(sig, abort)

	// Shut it down
	close(done)
	client.Close()
	waitGroup.Wait()

	jot.Print("qbot: shutdown complete")
}

func parseArgs() (token, filename string) {
	if len(os.Args) < 3 {
		fmt.Println("Usage: qbot <token> <data file>")
		os.Exit(1)
	}
	// Get command line parameters
	token = os.Args[1]
	filename = os.Args[2]
	return
}

func connectToSlack(token string) guac.RealTimeClient {
	client, err := guac.New(token).RealTime()
	if err != nil {
		log.Fatal(err)
	}
	return client
}

func getUserList(client guac.WebClient) (userCache *usercache.UserCache) {
	log.Println("Getting user list")
	users, err := client.UsersList()
	if err != nil {
		log.Fatal(err)
	}
	userCache = usercache.New(users)
	jot.Print("loaded user list: ", userCache)
	return
}

func loadQueue(filename string) (q queue.Queue) {
	q = queue.Queue{}
	if _, err := os.Stat(filename); err == nil {
		dat, err := ioutil.ReadFile(filename)
		if err != nil {
			log.Fatalf("Error loading queue: %s", err)
		}
		json.Unmarshal(dat, &q)
		jot.Printf("loadQueue: read queue from %s: %v", filename, q)
		log.Printf("Loaded queue from %s", filename)
	}
	return q
}

func addSignalHandler() chan os.Signal {
	sig := make(chan os.Signal, 1)
	signal.Notify(sig, syscall.SIGINT)
	signal.Notify(sig, syscall.SIGTERM)
	signal.Notify(sig, syscall.SIGKILL)
	return sig
}

func wait(sig chan os.Signal, abort chan error) {
	select {
	case err := <-abort:
		if err != nil {
			log.Print("Error: ", err)
		}
		log.Print("Execution terminated - shutting down")
	case s := <-sig:
		log.Printf("Received %s signal - shutting down", s)
	}
}
<commit_msg>OR DIE!<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"sync"
	"syscall"
	"time"

	"github.com/doozr/guac"
	"github.com/doozr/jot"
	"github.com/doozr/qbot/command"
	"github.com/doozr/qbot/notification"
	"github.com/doozr/qbot/queue"
	"github.com/doozr/qbot/usercache"
)

// Version is the current release version
var Version = "<unversioned build>"

// DoneChan is a channel used for informing go routines to shut down
type DoneChan chan struct{}

func main() {
	log.Printf("Qbot version %s", Version)

	// Turn on jot if required
	if os.Getenv("QBOT_DEBUG") == "true" {
		jot.Enable()
	}

	token, filename := parseCLI()

	waitGroup := sync.WaitGroup{}
	done := make(DoneChan)

	q := loadQueueOrDie(filename)

	client := connectToSlackOrDie(token)

	userCache := getUserListOrDie(client)

	notifications := notification.New(userCache)
	commands := command.New(notifications, userCache)

	notify := createNotifier(client)
	persist := createPersister(filename)
	messageHandler := createMessageHandler(client.ID(), client.Name(), q, commands, notify, persist)
	userChangeHandler := createUserChangeHandler(userCache)

	receiver := createReceiver(client)
	events := receive(receiver, done, &waitGroup)

	dispatcher := createDispatcher(client, 1*time.Minute, messageHandler, userChangeHandler)
	abort := dispatch(dispatcher, events, done, &waitGroup)

	sig := addSignalHandler()

	startKeepAlive(client, done, &waitGroup)

	wait(sig, abort)

	close(done)
	client.Close()
	waitGroup.Wait()

	jot.Print("qbot: shutdown complete")
}

func parseCLI() (token, filename string) {
	if len(os.Args) < 3 {
		fmt.Println("Usage: qbot <token> <data file>")
		os.Exit(1)
	}
	token = os.Args[1]
	filename = os.Args[2]
	return
}

func connectToSlackOrDie(token string) guac.RealTimeClient {
	client, err := guac.New(token).RealTime()
	if err != nil {
		log.Fatal(err)
	}
	log.Print("Connected to slack as ", client.Name())
	return client
}

func loadQueueOrDie(filename string) (q queue.Queue) {
	q = queue.Queue{}
	if _, err := os.Stat(filename); err == nil {
		dat, err := ioutil.ReadFile(filename)
		if err != nil {
			log.Fatalf("Error loading queue: %s", err)
		}
		json.Unmarshal(dat, &q)
		jot.Printf("loadQueue: read queue from %s: %v", filename, q)
		log.Printf("Loaded queue from %s", filename)
	}
	return q
}

func getUserListOrDie(client guac.WebClient) (userCache *usercache.UserCache) {
	log.Println("Getting user list")
	users, err := client.UsersList()
	if err != nil {
		log.Fatal(err)
	}
	userCache = usercache.New(users)
	jot.Print("loaded user list: ", userCache)
	return
}

func addSignalHandler() chan os.Signal {
	sig := make(chan os.Signal, 1)
	signal.Notify(sig, syscall.SIGINT)
	signal.Notify(sig, syscall.SIGTERM)
	signal.Notify(sig, syscall.SIGKILL)
	return sig
}

func wait(sig chan os.Signal, abort chan error) {
	select {
	case err := <-abort:
		if err != nil {
			log.Print("Error: ", err)
		}
		log.Print("Execution terminated - shutting down")
	case s := <-sig:
		log.Printf("Received %s signal - shutting down", s)
	}
}
<|endoftext|>