<commit_before>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main

import (
	"context"
	"flag"
	"fmt"
	"net"
	"os"
	"path/filepath"
	"sync"

	log "github.com/golang/glog"
	"github.com/google/kne/deploy"
	cpb "github.com/google/kne/proto/controller"
	tpb "github.com/google/kne/proto/topo"
	"github.com/google/kne/topo"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/credentials/alts"
	"google.golang.org/grpc/status"
	"google.golang.org/protobuf/encoding/prototext"
	"k8s.io/client-go/util/homedir"
)

var (
	defaultKubeCfg            = ""
	defaultTopoBasePath       = ""
	defaultMetallbManifestDir = ""
	defaultMeshnetManifestDir = ""
	// Flags.
	port = flag.Int("port", 50051, "Controller server port")
)

func init() {
	if home := homedir.HomeDir(); home != "" {
		defaultKubeCfg = filepath.Join(home, ".kube", "config")
		defaultTopoBasePath = filepath.Join(home, "kne", "examples")
		defaultMeshnetManifestDir = filepath.Join(home, "kne", "manifests", "meshnet", "base")
		defaultMetallbManifestDir = filepath.Join(home, "kne", "manifests", "metallb")
	}
}

type server struct {
	cpb.UnimplementedTopologyManagerServer

	muDeploy    sync.Mutex // guards deployements map
	deployments map[string]*deploy.Deployment
	muTopo      sync.Mutex        // guards topos map
	topos       map[string][]byte // stores the topology protobuf from the initial topology creation request
}

func newServer() *server {
	return &server{
		deployments: map[string]*deploy.Deployment{},
		topos:       map[string][]byte{},
	}
}

func newDeployment(req *cpb.CreateClusterRequest) (*deploy.Deployment, error) {
	d := &deploy.Deployment{}
	switch kind := req.ClusterSpec.(type) {
	case *cpb.CreateClusterRequest_Kind:
		d.Cluster = &deploy.KindSpec{
			Name:                     req.GetKind().Name,
			Recycle:                  req.GetKind().Recycle,
			Version:                  req.GetKind().Version,
			Image:                    req.GetKind().Image,
			Retain:                   req.GetKind().Retain,
			GoogleArtifactRegistries: req.GetKind().GoogleArtifactRegistries,
			ContainerImages:          req.GetKind().ContainerImages,
		}
	default:
		return nil, fmt.Errorf("cluster type not supported: %T", kind)
	}
	switch metallb := req.IngressSpec.(type) {
	case *cpb.CreateClusterRequest_Metallb:
		l := &deploy.MetalLBSpec{}
		path := defaultMetallbManifestDir
		if req.GetMetallb().ManifestDir != "" {
			path = req.GetMetallb().ManifestDir
		}
		p, err := validatePath(path)
		if err != nil {
			return nil, fmt.Errorf("failed to validate path %q", path)
		}
		l.ManifestDir = p
		l.IPCount = int(req.GetMetallb().IpCount)
		l.Version = req.GetMetallb().Version
		d.Ingress = l
	default:
		return nil, fmt.Errorf("ingress spec not supported: %T", metallb)
	}
	switch meshnet := req.CniSpec.(type) {
	case *cpb.CreateClusterRequest_Meshnet:
		m := &deploy.MeshnetSpec{}
		path := defaultMeshnetManifestDir
		if req.GetMeshnet().ManifestDir != "" {
			path = req.GetMeshnet().ManifestDir
		}
		p, err := validatePath(path)
		if err != nil {
			return nil, fmt.Errorf("failed to validate path %q", path)
		}
		m.Image = req.GetMeshnet().Image
		m.ManifestDir = p
		d.CNI = m
	default:
		return nil, fmt.Errorf("cni type not supported: %T", meshnet)
	}
	return d, nil
}

func (s *server) CreateCluster(ctx context.Context, req *cpb.CreateClusterRequest) (*cpb.CreateClusterResponse, error) {
	log.Infof("Received CreateCluster request: %v", req)
	d, err := newDeployment(req)
	if err != nil {
		return nil, status.Errorf(codes.InvalidArgument, "unable to parse request: %v", err)
	}
	log.Infof("Parsed request into deployment: %v", d)
	s.muDeploy.Lock()
	defer s.muDeploy.Unlock()
	if _, ok := s.deployments[d.Cluster.GetName()]; ok { // if OK
		return nil, status.Errorf(codes.AlreadyExists, "cluster %q already exists", d.Cluster.GetName())
	}
	if err := d.Deploy(ctx, defaultKubeCfg); err != nil {
		return nil, status.Errorf(codes.Internal, "failed to deploy cluster: %v", err)
	}
	s.deployments[d.Cluster.GetName()] = d
	log.Infof("Cluster %q deployed and ready for topology", d.Cluster.GetName())
	resp := &cpb.CreateClusterResponse{
		Name:  d.Cluster.GetName(),
		State: cpb.ClusterState_CLUSTER_STATE_RUNNING,
	}
	return resp, nil
}

func (s *server) DeleteCluster(ctx context.Context, req *cpb.DeleteClusterRequest) (*cpb.DeleteClusterResponse, error) {
	log.Infof("Received DeleteCluster request: %v", req)
	s.muDeploy.Lock()
	defer s.muDeploy.Unlock()
	d, ok := s.deployments[req.GetName()]
	if !ok {
		return nil, status.Errorf(codes.NotFound, "cluster %q not found, can only delete clusters created using TopologyManager", req.GetName())
	}
	if err := d.Delete(); err != nil {
		return nil, status.Errorf(codes.Internal, "failed to delete cluster: %v", err)
	}
	delete(s.deployments, req.GetName())
	log.Infof("Deleted cluster %q", d.Cluster.GetName())
	return &cpb.DeleteClusterResponse{}, nil
}

func (s *server) ShowCluster(ctx context.Context, req *cpb.ShowClusterRequest) (*cpb.ShowClusterResponse, error) {
	log.Infof("Received ShowCluster request: %v", req)
	s.muDeploy.Lock()
	defer s.muDeploy.Unlock()
	d, ok := s.deployments[req.GetName()]
	if !ok {
		return nil, status.Errorf(codes.NotFound, "cluster %q not found, can only show clusters created using TopologyManager", req.GetName())
	}
	if err := d.Healthy(ctx); err != nil {
		return &cpb.ShowClusterResponse{State: cpb.ClusterState_CLUSTER_STATE_ERROR}, nil
	}
	return &cpb.ShowClusterResponse{State: cpb.ClusterState_CLUSTER_STATE_RUNNING}, nil
}

func (s *server) CreateTopology(ctx context.Context, req *cpb.CreateTopologyRequest) (*cpb.CreateTopologyResponse, error) {
	log.Infof("Received CreateTopology request: %v", req)
	topoPb := req.GetTopology()
	if topoPb == nil {
		return nil, status.Errorf(codes.InvalidArgument, "invalid request: missing topology protobuf")
	}
	if topoPb.GetName() == "" {
		return nil, status.Errorf(codes.InvalidArgument, "missing topology name")
	}

	s.muTopo.Lock()
	defer s.muTopo.Unlock()
	if _, ok := s.topos[topoPb.GetName()]; ok {
		return nil, status.Errorf(codes.AlreadyExists, "topology %q already exists", req.Topology.GetName())
	}

	for _, node := range topoPb.Nodes {
		if node.GetConfig() == nil || node.GetConfig().GetFile() == "" {
			// A config section is not required: you are allowed to bring up a
			// topology with no initial config.
			continue
		}
		path := node.GetConfig().GetFile()
		if !filepath.IsAbs(path) {
			path = filepath.Join(defaultTopoBasePath, path)
		}
		log.Infof("Checking config path: %q", path)
		if _, err := validatePath(path); err != nil {
			return nil, status.Errorf(codes.InvalidArgument, "config file not found for node %q: %v", node.GetName(), err)
		}
		node.GetConfig().ConfigData = &tpb.Config_File{File: path}
	}
	// Saves the original topology protobuf.
	txtPb, err := prototext.Marshal(topoPb)
	if err != nil {
		return nil, status.Errorf(codes.Internal, "invalid topology protobuf: %v", err)
	}
	path := defaultKubeCfg
	if req.Kubecfg != "" {
		path = req.Kubecfg
	}
	kcfg, err := validatePath(path)
	if err != nil {
		return nil, status.Errorf(codes.InvalidArgument, "kubecfg %q does not exist: %v", path, err)
	}
	if err := topo.CreateTopology(ctx, topo.TopologyParams{
		TopoNewOptions: []topo.Option{topo.WithTopology(topoPb)},
		Kubecfg:        kcfg,
	}); err != nil {
		return nil, status.Errorf(codes.Internal, "failed to create topology: %v", err)
	}

	s.topos[topoPb.GetName()] = txtPb
	return &cpb.CreateTopologyResponse{
		TopologyName: req.Topology.GetName(),
		State:        cpb.TopologyState_TOPOLOGY_STATE_RUNNING,
	}, nil
}

func (s *server) DeleteTopology(ctx context.Context, req *cpb.DeleteTopologyRequest) (*cpb.DeleteTopologyResponse, error) {
	log.Infof("Received DeleteTopology request: %v", req)
	s.muTopo.Lock()
	defer s.muTopo.Unlock()
	txtPb, ok := s.topos[req.GetTopologyName()]
	if !ok {
		return nil, status.Errorf(codes.NotFound, "topology %q not found", req.GetTopologyName())
	}
	topoPb := &tpb.Topology{}
	if err := prototext.Unmarshal(txtPb, topoPb); err != nil {
		return nil, status.Errorf(codes.Internal, "invalid topology protobuf: %v", err)
	}
	kcfg, err := validatePath(defaultKubeCfg)
	if err != nil {
		return nil, status.Errorf(codes.Internal, "default kubecfg %q does not exist: %v", defaultKubeCfg, err)
	}
	if err := topo.DeleteTopology(ctx, topo.TopologyParams{
		TopoNewOptions: []topo.Option{topo.WithTopology(topoPb)},
		Kubecfg:        kcfg,
	}); err != nil {
		return nil, status.Errorf(codes.Internal, "failed to delete topology: %v", err)
	}
	return &cpb.DeleteTopologyResponse{}, nil
}

func (s *server) ShowTopology(ctx context.Context, req *cpb.ShowTopologyRequest) (*cpb.ShowTopologyResponse, error) {
	log.Infof("Received ShowTopology request: %v", req)
	s.muTopo.Lock()
	defer s.muTopo.Unlock()
	txtPb, ok := s.topos[req.GetTopologyName()]
	if !ok {
		return nil, status.Errorf(codes.NotFound, "topology %q not found", req.GetTopologyName())
	}
	topoPb := &tpb.Topology{}
	if err := prototext.Unmarshal(txtPb, topoPb); err != nil {
		return nil, status.Errorf(codes.Internal, "invalid topology protobuf: %v", err)
	}
	kcfg, err := validatePath(defaultKubeCfg)
	if err != nil {
		return nil, status.Errorf(codes.Internal, "default kubecfg %q does not exist: %v", defaultKubeCfg, err)
	}
	resp, err := topo.GetTopologyServices(ctx, topo.TopologyParams{
		TopoNewOptions: []topo.Option{topo.WithTopology(topoPb)},
		Kubecfg:        kcfg,
	})
	if err != nil {
		return nil, status.Errorf(codes.Internal, "failed to show topology: %v", err)
	}
	return resp, nil
}

func validatePath(path string) (string, error) {
	path, err := filepath.Abs(path)
	if err != nil {
		return "", fmt.Errorf("failed to evaluate absolute path: %q", path)
	}
	if _, err := os.Stat(path); err != nil {
		return "", fmt.Errorf("path %q does not exist", path)
	}
	return path, nil
}

func main() {
	flag.Parse()
	addr := fmt.Sprintf(":%d", *port)
	lis, err := net.Listen("tcp6", addr)
	if err != nil {
		log.Fatalf("failed to listen: %v", err)
	}
	creds := alts.NewServerCreds(alts.DefaultServerOptions())
	s := grpc.NewServer(grpc.Creds(creds))
	cpb.RegisterTopologyManagerServer(s, newServer())
	log.Infof("Controller server listening at %v", lis.Addr())
	if err := s.Serve(lis); err != nil {
		log.Fatalf("failed to serve: %v", err)
	}
}
<commit_msg>modify keepalive enforcement<commit_after>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main

import (
	"context"
	"flag"
	"fmt"
	"net"
	"os"
	"path/filepath"
	"sync"

	log "github.com/golang/glog"
	"github.com/google/kne/deploy"
	cpb "github.com/google/kne/proto/controller"
	tpb "github.com/google/kne/proto/topo"
	"github.com/google/kne/topo"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/credentials/alts"
	"google.golang.org/grpc/keepalive"
	"google.golang.org/grpc/status"
	"google.golang.org/protobuf/encoding/prototext"
	"k8s.io/client-go/util/homedir"
)

var (
	defaultKubeCfg            = ""
	defaultTopoBasePath       = ""
	defaultMetallbManifestDir = ""
	defaultMeshnetManifestDir = ""
	// Flags.
	port = flag.Int("port", 50051, "Controller server port")
)

func init() {
	if home := homedir.HomeDir(); home != "" {
		defaultKubeCfg = filepath.Join(home, ".kube", "config")
		defaultTopoBasePath = filepath.Join(home, "kne", "examples")
		defaultMeshnetManifestDir = filepath.Join(home, "kne", "manifests", "meshnet", "base")
		defaultMetallbManifestDir = filepath.Join(home, "kne", "manifests", "metallb")
	}
}

type server struct {
	cpb.UnimplementedTopologyManagerServer

	muDeploy    sync.Mutex // guards deployements map
	deployments map[string]*deploy.Deployment
	muTopo      sync.Mutex        // guards topos map
	topos       map[string][]byte // stores the topology protobuf from the initial topology creation request
}

func newServer() *server {
	return &server{
		deployments: map[string]*deploy.Deployment{},
		topos:       map[string][]byte{},
	}
}

func newDeployment(req *cpb.CreateClusterRequest) (*deploy.Deployment, error) {
	d := &deploy.Deployment{}
	switch kind := req.ClusterSpec.(type) {
	case *cpb.CreateClusterRequest_Kind:
		d.Cluster = &deploy.KindSpec{
			Name:                     req.GetKind().Name,
			Recycle:                  req.GetKind().Recycle,
			Version:                  req.GetKind().Version,
			Image:                    req.GetKind().Image,
			Retain:                   req.GetKind().Retain,
			GoogleArtifactRegistries: req.GetKind().GoogleArtifactRegistries,
			ContainerImages:          req.GetKind().ContainerImages,
		}
	default:
		return nil, fmt.Errorf("cluster type not supported: %T", kind)
	}
	switch metallb := req.IngressSpec.(type) {
	case *cpb.CreateClusterRequest_Metallb:
		l := &deploy.MetalLBSpec{}
		path := defaultMetallbManifestDir
		if req.GetMetallb().ManifestDir != "" {
			path = req.GetMetallb().ManifestDir
		}
		p, err := validatePath(path)
		if err != nil {
			return nil, fmt.Errorf("failed to validate path %q", path)
		}
		l.ManifestDir = p
		l.IPCount = int(req.GetMetallb().IpCount)
		l.Version = req.GetMetallb().Version
		d.Ingress = l
	default:
		return nil, fmt.Errorf("ingress spec not supported: %T", metallb)
	}
	switch meshnet := req.CniSpec.(type) {
	case *cpb.CreateClusterRequest_Meshnet:
		m := &deploy.MeshnetSpec{}
		path := defaultMeshnetManifestDir
		if req.GetMeshnet().ManifestDir != "" {
			path = req.GetMeshnet().ManifestDir
		}
		p, err := validatePath(path)
		if err != nil {
			return nil, fmt.Errorf("failed to validate path %q", path)
		}
		m.Image = req.GetMeshnet().Image
		m.ManifestDir = p
		d.CNI = m
	default:
		return nil, fmt.Errorf("cni type not supported: %T", meshnet)
	}
	return d, nil
}

func (s *server) CreateCluster(ctx context.Context, req *cpb.CreateClusterRequest) (*cpb.CreateClusterResponse, error) {
	log.Infof("Received CreateCluster request: %v", req)
	d, err := newDeployment(req)
	if err != nil {
		return nil, status.Errorf(codes.InvalidArgument, "unable to parse request: %v", err)
	}
	log.Infof("Parsed request into deployment: %v", d)
	s.muDeploy.Lock()
	defer s.muDeploy.Unlock()
	if _, ok := s.deployments[d.Cluster.GetName()]; ok { // if OK
		return nil, status.Errorf(codes.AlreadyExists, "cluster %q already exists", d.Cluster.GetName())
	}
	if err := d.Deploy(ctx, defaultKubeCfg); err != nil {
		return nil, status.Errorf(codes.Internal, "failed to deploy cluster: %v", err)
	}
	s.deployments[d.Cluster.GetName()] = d
	log.Infof("Cluster %q deployed and ready for topology", d.Cluster.GetName())
	resp := &cpb.CreateClusterResponse{
		Name:  d.Cluster.GetName(),
		State: cpb.ClusterState_CLUSTER_STATE_RUNNING,
	}
	return resp, nil
}

func (s *server) DeleteCluster(ctx context.Context, req *cpb.DeleteClusterRequest) (*cpb.DeleteClusterResponse, error) {
	log.Infof("Received DeleteCluster request: %v", req)
	s.muDeploy.Lock()
	defer s.muDeploy.Unlock()
	d, ok := s.deployments[req.GetName()]
	if !ok {
		return nil, status.Errorf(codes.NotFound, "cluster %q not found, can only delete clusters created using TopologyManager", req.GetName())
	}
	if err := d.Delete(); err != nil {
		return nil, status.Errorf(codes.Internal, "failed to delete cluster: %v", err)
	}
	delete(s.deployments, req.GetName())
	log.Infof("Deleted cluster %q", d.Cluster.GetName())
	return &cpb.DeleteClusterResponse{}, nil
}

func (s *server) ShowCluster(ctx context.Context, req *cpb.ShowClusterRequest) (*cpb.ShowClusterResponse, error) {
	log.Infof("Received ShowCluster request: %v", req)
	s.muDeploy.Lock()
	defer s.muDeploy.Unlock()
	d, ok := s.deployments[req.GetName()]
	if !ok {
		return nil, status.Errorf(codes.NotFound, "cluster %q not found, can only show clusters created using TopologyManager", req.GetName())
	}
	if err := d.Healthy(ctx); err != nil {
		return &cpb.ShowClusterResponse{State: cpb.ClusterState_CLUSTER_STATE_ERROR}, nil
	}
	return &cpb.ShowClusterResponse{State: cpb.ClusterState_CLUSTER_STATE_RUNNING}, nil
}

func (s *server) CreateTopology(ctx context.Context, req *cpb.CreateTopologyRequest) (*cpb.CreateTopologyResponse, error) {
	log.Infof("Received CreateTopology request: %v", req)
	topoPb := req.GetTopology()
	if topoPb == nil {
		return nil, status.Errorf(codes.InvalidArgument, "invalid request: missing topology protobuf")
	}
	if topoPb.GetName() == "" {
		return nil, status.Errorf(codes.InvalidArgument, "missing topology name")
	}

	s.muTopo.Lock()
	defer s.muTopo.Unlock()
	if _, ok := s.topos[topoPb.GetName()]; ok {
		return nil, status.Errorf(codes.AlreadyExists, "topology %q already exists", req.Topology.GetName())
	}

	for _, node := range topoPb.Nodes {
		if node.GetConfig() == nil || node.GetConfig().GetFile() == "" {
			// A config section is not required: you are allowed to bring up a
			// topology with no initial config.
			continue
		}
		path := node.GetConfig().GetFile()
		if !filepath.IsAbs(path) {
			path = filepath.Join(defaultTopoBasePath, path)
		}
		log.Infof("Checking config path: %q", path)
		if _, err := validatePath(path); err != nil {
			return nil, status.Errorf(codes.InvalidArgument, "config file not found for node %q: %v", node.GetName(), err)
		}
		node.GetConfig().ConfigData = &tpb.Config_File{File: path}
	}
	// Saves the original topology protobuf.
	txtPb, err := prototext.Marshal(topoPb)
	if err != nil {
		return nil, status.Errorf(codes.Internal, "invalid topology protobuf: %v", err)
	}
	path := defaultKubeCfg
	if req.Kubecfg != "" {
		path = req.Kubecfg
	}
	kcfg, err := validatePath(path)
	if err != nil {
		return nil, status.Errorf(codes.InvalidArgument, "kubecfg %q does not exist: %v", path, err)
	}
	if err := topo.CreateTopology(ctx, topo.TopologyParams{
		TopoNewOptions: []topo.Option{topo.WithTopology(topoPb)},
		Kubecfg:        kcfg,
	}); err != nil {
		return nil, status.Errorf(codes.Internal, "failed to create topology: %v", err)
	}

	s.topos[topoPb.GetName()] = txtPb
	return &cpb.CreateTopologyResponse{
		TopologyName: req.Topology.GetName(),
		State:        cpb.TopologyState_TOPOLOGY_STATE_RUNNING,
	}, nil
}

func (s *server) DeleteTopology(ctx context.Context, req *cpb.DeleteTopologyRequest) (*cpb.DeleteTopologyResponse, error) {
	log.Infof("Received DeleteTopology request: %v", req)
	s.muTopo.Lock()
	defer s.muTopo.Unlock()
	txtPb, ok := s.topos[req.GetTopologyName()]
	if !ok {
		return nil, status.Errorf(codes.NotFound, "topology %q not found", req.GetTopologyName())
	}
	topoPb := &tpb.Topology{}
	if err := prototext.Unmarshal(txtPb, topoPb); err != nil {
		return nil, status.Errorf(codes.Internal, "invalid topology protobuf: %v", err)
	}
	kcfg, err := validatePath(defaultKubeCfg)
	if err != nil {
		return nil, status.Errorf(codes.Internal, "default kubecfg %q does not exist: %v", defaultKubeCfg, err)
	}
	if err := topo.DeleteTopology(ctx, topo.TopologyParams{
		TopoNewOptions: []topo.Option{topo.WithTopology(topoPb)},
		Kubecfg:        kcfg,
	}); err != nil {
		return nil, status.Errorf(codes.Internal, "failed to delete topology: %v", err)
	}
	return &cpb.DeleteTopologyResponse{}, nil
}

func (s *server) ShowTopology(ctx context.Context, req *cpb.ShowTopologyRequest) (*cpb.ShowTopologyResponse, error) {
	log.Infof("Received ShowTopology request: %v", req)
	s.muTopo.Lock()
	defer s.muTopo.Unlock()
	txtPb, ok := s.topos[req.GetTopologyName()]
	if !ok {
		return nil, status.Errorf(codes.NotFound, "topology %q not found", req.GetTopologyName())
	}
	topoPb := &tpb.Topology{}
	if err := prototext.Unmarshal(txtPb, topoPb); err != nil {
		return nil, status.Errorf(codes.Internal, "invalid topology protobuf: %v", err)
	}
	kcfg, err := validatePath(defaultKubeCfg)
	if err != nil {
		return nil, status.Errorf(codes.Internal, "default kubecfg %q does not exist: %v", defaultKubeCfg, err)
	}
	resp, err := topo.GetTopologyServices(ctx, topo.TopologyParams{
		TopoNewOptions: []topo.Option{topo.WithTopology(topoPb)},
		Kubecfg:        kcfg,
	})
	if err != nil {
		return nil, status.Errorf(codes.Internal, "failed to show topology: %v", err)
	}
	return resp, nil
}

func validatePath(path string) (string, error) {
	path, err := filepath.Abs(path)
	if err != nil {
		return "", fmt.Errorf("failed to evaluate absolute path: %q", path)
	}
	if _, err := os.Stat(path); err != nil {
		return "", fmt.Errorf("path %q does not exist", path)
	}
	return path, nil
}

func main() {
	flag.Parse()
	addr := fmt.Sprintf(":%d", *port)
	lis, err := net.Listen("tcp6", addr)
	if err != nil {
		log.Fatalf("failed to listen: %v", err)
	}
	creds := alts.NewServerCreds(alts.DefaultServerOptions())
	s := grpc.NewServer(
		grpc.Creds(creds),
		grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{PermitWithoutStream: true}),
	)
	cpb.RegisterTopologyManagerServer(s, newServer())
	log.Infof("Controller server listening at %v", lis.Addr())
	if err := s.Serve(lis); err != nil {
		log.Fatalf("failed to serve: %v", err)
	}
}
<|endoftext|>


<commit_before>package controllers

import (
  "seeme/db"

  "net/http"
  "encoding/json"
)

func GetUserNotesController(w http.ResponseWriter, r *http.Request) {
  if _, err := db.GetUser(r.FormValue("username")); err != nil {
    w.Write([]byte("User Not Found!"))
    return
  }

  userNotes, err := db.GetNotesList(r.FormValue("username"))
  if err != nil {
    w.Write([]byte(err.Error()))
    return
  }
  js, err := json.Marshal(userNotes)
  if err != nil {
    w.Write([]byte("Notes Processing Error!"))
    return
  }

  w.Header().Set("Content-Type", "application/json")
  w.Write(js)
}<commit_msg>added notes obj to return json for get_notes<commit_after>package controllers

import (
  "seeme/db"
  "seeme/models"

  "net/http"
  "encoding/json"
)

func GetUserNotesController(w http.ResponseWriter, r *http.Request) {
  if _, err := db.GetUser(r.FormValue("username")); err != nil {
    w.Write([]byte("User Not Found!"))
    return
  }

  userNotes, err := db.GetNotesList(r.FormValue("username"))
  if err != nil {
    w.Write([]byte(err.Error()))
    return
  }

  var notes models.NotesJSON
  notes.Notes = userNotes
  js, err := json.Marshal([]models.NotesJSON{notes})
  if err != nil {
    w.Write([]byte("Notes Processing Error!"))
    return
  }

  w.Header().Set("Content-Type", "application/json")
  w.Write(js)
}<|endoftext|>


<commit_before>// this file contains a pre-processor to pull some stuff out of the markdown file before parsing it

package gnosis

import (
	"bufio"
	"bytes"
	"errors"
	"html/template"
	"os"
)

type PageMetadata struct {
	Keywords map[string]bool
	Tags     map[string]bool
	Loaded   bool
	Page     []byte
}

func (pdata *PageMetadata) LoadPage(pageName string) error {
	f, err := os.Open(pageName)
	reader := bufio.NewReader(f)
	upperLine, fullLine, err := reader.ReadLine()

	// inspect the first line you read
	if err != nil {
		return err
	} else if !fullLine {
		return errors.New("first line I read wasn't a full line")
	} else if pdata.lineIsTitle(upperLine) {
		return errors.New("first line looks an awful lot like the underside of the title o.O")
	}

	lowerLine, fullLine, err := reader.ReadLine()

	// inspect the lower line
	if err != nil {
		return err
	} else if !fullLine {
		return errors.New("second line I read wasn't a full line")
	} else if pdata.lineIsTitle(lowerLine) {

		// read the rest of the page
		var restOfPage []byte
		_, err = reader.Read(restOfPage)
		if err != nil {
			return err
		}
		// if the second line is a title, read the rest of the page in
		// you don't have any metadata to work with here, move on
		pdata.Page = bytes.Join([][]byte{upperLine, lowerLine, restOfPage}, []byte("\n"))

		// you've successfully loaded the page - so return nothing
		pdata.Loaded = true
		return nil
	}

	// if you're at this point, the first line is metadata
	// you gotta process it and work with the next line
	// so let's just read through the file until we hit the title
	for !pdata.lineIsTitle(lowerLine) {
		// process the line
		pdata.processMetadata(upperLine)
		// shift the lower line up
		upperLine = lowerLine
		// read in a new lower line
		lowerLine, fullLine, err := reader.ReadLine()
		if err != nil {
			return err
		} else if !fullLine {
			return errors.New("I filled my buffer with a line")
		}
	}

	// by this point, I should have read everything in - let's read the rest and just return it
	var restOfPage []byte
	_, err = reader.Read(restOfPage)
	pdata.Page = bytes.Join([][]byte{upperLine, lowerLine, restOfPage}, []byte("\n"))

	return err
}

// takes a single line of input and determines if it's a top level markdown header
func (pdata *PageMetadata) lineIsTitle(line []byte) bool {
	finalLength := len(line)
	i := 0

	// if the row doesn't start with tabs, spaces, or ='s
	if (line[i] != ' ' && line[i] != '=') && line[i] != '\t' {
		return false
	}

	// skip any spaces or tabs at the start
	for line[i] == ' ' || line[i] == '\t' {
		i++
	}

	// if the next item's not a =, bail out
	if line[i] != '=' {
		return false
	}

	// run through all of the ='s
	for line[i] == '=' {
		i++
	}

	if line[i] != ' ' && line[i] != '\t' && line[i] != '\n' {
		return false
	}

	//ditch all spaces after any ='s
	for line[i] == ' ' || line[i] == '\t' {
		i++
	}

	if finalLength == i+1 {
		return true
	} else {
		return false
	}
}

func (pdata *PageMetadata) checkMatch(input []byte, looking []byte, tracker map[string]bool) {
	// trim off any blank spaces at the start of the line
	value := bytes.Trim(input, " \t")

	// should be a substring match based on the start of the array
	if bytes.Equal(input[:len(looking)], looking) {
		// trim off the target from the []byte
		value = input[len(looking):]

		// trim spaces at the start and at the end
		value = bytes.Trim(value, " \t\n")

		if value[0] == ':' || value[0] == '=' {
			value = bytes.Trim(value, " \t\n=:")
		}

		// replace any spaces in the middle with -'s
		bytes.Replace(value, []byte(" "), []byte("-"), -1)

		// suppress any double dashes
		for i := 0; i < len(value)-1; i++ {
			if value[i] == '-' && value[i+1] == '-' {
				value = append(value[:i], value[i+1:]...)
			}
		}

		// now just add the value to the array that you're tracking
		tracker[string(value)] = true
	}
}

// returns all the tags within a list as an array of strings
func (pdata *PageMetadata) ListMeta() ([]string, []string) {
	topics := []string{}
	for oneTag, _ := range pdata.Tags {
		topics = append(topics[:], oneTag)
	}

	keywords := []string{}
	for oneKeyword, _ := range pdata.Keywords {
		keywords = append(keywords[:], oneKeyword)
	}

	return topics, keywords
}

// return the bytes to display the tags on the page
// takes the prefix for the tags
func (pdata *PageMetadata) PrintTags(tagPrefix string) template.HTML {
	response := new([]byte)
	for oneTag, _ := range pdata.Tags {
		response = bytes.Join([][]byte{"
", tagPrefix, oneTag, "
"}, []byte(""))
	}
	return template.HTML(response)
}

// returns the bytes to add the keywrods to the html output
func (pdata *PageMetadata) PrintKeywords() template.HTML {
	response := []byte("<meta name='keywords' content='")
	for oneKeyword, _ := range pdata.Keywords {
		response = bytes.Join([][]byte{response, oneKeyword}, []byte(","))
	}
	// clean up the end of the string and add the ending tag
	response = response.TrimSuffix(response, ',')
	response = append(response, "'>")

	return template.HTML(response)
}

// runs through all restricted tags, and looks for a match
// if matched, returns true, otherwise false
func (pdata *PageMetadata) MatchedTag(checkTags []string) bool {
	for _, tag := range checkTags {
		if pdata.Tags[tag] == true {
			return true
		}
	}
	return false
}

func (pdata *PageMetadata) processMetadata(line []byte) error {
	pdata.checkMatch(line, "tag", pdata.Tags)
	pdata.checkMatch(line, "topic", pdata.Tags)
	pdata.checkMatch(line, "category", pdata.Tags)

	pdata.checkMatch(line, "keyword", pdata.Keywords)
	pdata.checkMatch(line, "keywords", pdata.Keywords)
	pdata.checkMatch(line, "meta", pdata.Keywords)
}
<commit_msg>fixed up PrintTags<commit_after>// this file contains a pre-processor to pull some stuff out of the markdown file before parsing it

package gnosis

import (
	"bufio"
	"bytes"
	"errors"
	"html/template"
	"os"
)

type PageMetadata struct {
	Keywords map[string]bool
	Tags     map[string]bool
	Loaded   bool
	Page     []byte
}

func (pdata *PageMetadata) LoadPage(pageName string) error {
	f, err := os.Open(pageName)
	reader := bufio.NewReader(f)
	upperLine, fullLine, err := reader.ReadLine()

	// inspect the first line you read
	if err != nil {
		return err
	} else if !fullLine {
		return errors.New("first line I read wasn't a full line")
	} else if pdata.lineIsTitle(upperLine) {
		return errors.New("first line looks an awful lot like the underside of the title o.O")
	}

	lowerLine, fullLine, err := reader.ReadLine()

	// inspect the lower line
	if err != nil {
		return err
	} else if !fullLine {
		return errors.New("second line I read wasn't a full line")
	} else if pdata.lineIsTitle(lowerLine) {

		// read the rest of the page
		var restOfPage []byte
		_, err = reader.Read(restOfPage)
		if err != nil {
			return err
		}
		// if the second line is a title, read the rest of the page in
		// you don't have any metadata to work with here, move on
		pdata.Page = bytes.Join([][]byte{upperLine, lowerLine, restOfPage}, []byte("\n"))

		// you've successfully loaded the page - so return nothing
		pdata.Loaded = true
		return nil
	}

	// if you're at this point, the first line is metadata
	// you gotta process it and work with the next line
	// so let's just read through the file until we hit the title
	for !pdata.lineIsTitle(lowerLine) {
		// process the line
		pdata.processMetadata(upperLine)
		// shift the lower line up
		upperLine = lowerLine
		// read in a new lower line
		lowerLine, fullLine, err := reader.ReadLine()
		if err != nil {
			return err
		} else if !fullLine {
			return errors.New("I filled my buffer with a line")
		}
	}

	// by this point, I should have read everything in - let's read the rest and just return it
	var restOfPage []byte
	_, err = reader.Read(restOfPage)
	pdata.Page = bytes.Join([][]byte{upperLine, lowerLine, restOfPage}, []byte("\n"))

	return err
}

// takes a single line of input and determines if it's a top level markdown header
func (pdata *PageMetadata) lineIsTitle(line []byte) bool {
	finalLength := len(line)
	i := 0

	// if the row doesn't start with tabs, spaces, or ='s
	if (line[i] != ' ' && line[i] != '=') && line[i] != '\t' {
		return false
	}

	// skip any spaces or tabs at the start
	for line[i] == ' ' || line[i] == '\t' {
		i++
	}

	// if the next item's not a =, bail out
	if line[i] != '=' {
		return false
	}

	// run through all of the ='s
	for line[i] == '=' {
		i++
	}

	if line[i] != ' ' && line[i] != '\t' && line[i] != '\n' {
		return false
	}

	//ditch all spaces after any ='s
	for line[i] == ' ' || line[i] == '\t' {
		i++
	}

	if finalLength == i+1 {
		return true
	} else {
		return false
	}
}

func (pdata *PageMetadata) checkMatch(input []byte, looking []byte, tracker map[string]bool) {
	// trim off any blank spaces at the start of the line
	value := bytes.Trim(input, " \t")

	// should be a substring match based on the start of the array
	if bytes.Equal(input[:len(looking)], looking) {
		// trim off the target from the []byte
		value = input[len(looking):]

		// trim spaces at the start and at the end
		value = bytes.Trim(value, " \t\n")

		if value[0] == ':' || value[0] == '=' {
			value = bytes.Trim(value, " \t\n=:")
		}

		// replace any spaces in the middle with -'s
		bytes.Replace(value, []byte(" "), []byte("-"), -1)

		// suppress any double dashes
		for i := 0; i < len(value)-1; i++ {
			if value[i] == '-' && value[i+1] == '-' {
				value = append(value[:i], value[i+1:]...)
			}
		}

		// now just add the value to the array that you're tracking
		tracker[string(value)] = true
	}
}

// returns all the tags within a list as an array of strings
func (pdata *PageMetadata) ListMeta() ([]string, []string) {
	topics := []string{}
	for oneTag, _ := range pdata.Tags {
		topics = append(topics[:], oneTag)
	}

	keywords := []string{}
	for oneKeyword, _ := range pdata.Keywords {
		keywords = append(keywords[:], oneKeyword)
	}

	return topics, keywords
}

// return the bytes to display the tags on the page
// takes the prefix for the tags
func (pdata *PageMetadata) PrintTags(tagPrefix string) template.HTML {
	response := []byte{}
	openingTag := []byte("
")
	closingTag := []byte("
")
	for oneTag, _ := range pdata.Tags {
		response = bytes.Join([][]byte{openingTag, []byte(tagPrefix), []byte(oneTag), closingTag}, []byte(""))
	}
	return template.HTML(response)
}

// returns the bytes to add the keywrods to the html output
func (pdata *PageMetadata) PrintKeywords() template.HTML {
	response := []byte("<meta name='keywords' content='")
	for oneKeyword, _ := range pdata.Keywords {
		response = bytes.Join([][]byte{response, oneKeyword}, []byte(","))
	}
	// clean up the end of the string and add the ending tag
	response = response.TrimSuffix(response, ',')
	response = append(response, "'>")

	return template.HTML(response)
}

// runs through all restricted tags, and looks for a match
// if matched, returns true, otherwise false
func (pdata *PageMetadata) MatchedTag(checkTags []string) bool {
	for _, tag := range checkTags {
		if pdata.Tags[tag] == true {
			return true
		}
	}
	return false
}

func (pdata *PageMetadata) processMetadata(line []byte) error {
	pdata.checkMatch(line, "tag", pdata.Tags)
	pdata.checkMatch(line, "topic", pdata.Tags)
	pdata.checkMatch(line, "category", pdata.Tags)

	pdata.checkMatch(line, "keyword", pdata.Keywords)
	pdata.checkMatch(line, "keywords", pdata.Keywords)
	pdata.checkMatch(line, "meta", pdata.Keywords)
}
<|endoftext|>


<commit_before>package metafora

import (
	"math/rand"
	"sort"
	"sync"
	"time"
)

var (

	// balance calls are randomized and this is the upper bound of the random
	// amount
	balanceJitterMax = 10 * int64(time.Second)

	//FIXME should probably be improved, see usage in Run()
	consumerRetryDelay = 10 * time.Second
)

type runningTask struct {
	h Handler
	c chan struct{}
}

// Consumer is the core Metafora task runner.
type Consumer struct {
	// Func to create new handlers
	handler HandlerFunc

	// Map of task:Handler
	running map[string]runningTask

	// Mutex to protect access to running
	runL sync.Mutex

	// WaitGroup for running handlers and consumer goroutines
	hwg sync.WaitGroup

	bal      Balancer
	balEvery time.Duration
	coord    Coordinator
	logger   *logger
	stop     chan struct{} // closed by Shutdown to cause Run to exit

	// ticked on each loop of the main loop to enforce sequential interaction
	// with coordinator and balancer
	tick chan int

	watch chan string // channel for watcher to send tasks to main loop

	// Set by command handler, read anywhere via Consumer.frozen()
	freezeL sync.Mutex
	freeze  bool
}

// NewConsumer returns a new consumer and calls Init on the Balancer and Coordinator.
func NewConsumer(coord Coordinator, h HandlerFunc, b Balancer) (*Consumer, error) {
	c := &Consumer{
		running:  make(map[string]runningTask),
		handler:  h,
		bal:      b,
		balEvery: 15 * time.Minute, //TODO make balance wait configurable
		coord:    coord,
		logger:   stdoutLogger(),
		stop:     make(chan struct{}),
		tick:     make(chan int),
		watch:    make(chan string),
	}

	// initialize balancer with the consumer and a prefixed logger
	b.Init(&struct {
		*Consumer
		Logger
	}{Consumer: c, Logger: c.logger})

	if err := coord.Init(&coordinatorContext{Consumer: c, Logger: c.logger}); err != nil {
		return nil, err
	}
	return c, nil
}

// SetLogger assigns the logger to use as well as a level
//
// The logger parameter is an interface that requires the following
// method to be implemented (such as the the stdlib log.Logger):
//
//    Output(calldepth int, s string)
//
func (c *Consumer) SetLogger(l logOutputter, lvl LogLevel) {
	c.logger.l = l
	c.logger.lvl = lvl
}

// Run is the core run loop of Metafora. It is responsible for calling into the
// Coordinator to claim work and Balancer to rebalance work.
//
// Run blocks until Shutdown is called or an internal error occurs.
func (c *Consumer) Run() {
	c.logger.Log(LogLevelDebug, "Starting consumer")

	// chans for core goroutines to communicate with main loop
	balance := make(chan bool)
	cmdChan := make(chan Command)

	// Balance is called by the main loop when the balance channel is ticked
	c.hwg.Add(1)
	go func() {
		defer c.hwg.Done()
		randInt := rand.New(rand.NewSource(time.Now().UnixNano())).Int63n
		for {
			select {
			case <-c.stop:
				// Shutdown has been called.
				return
			case <-time.After(c.balEvery + time.Duration(randInt(balanceJitterMax))):
				c.logger.Log(LogLevelInfo, "Balancing")
				select {
				case balance <- true:
					// Ticked balance
				case <-c.stop:
					// Shutdown has been called.
					return
				}
			}
			// Wait for main loop to signal balancing is done
			select {
			case <-c.stop:
				return
			case <-c.tick:
			}
		}
	}()

	// Watch for new tasks in a goroutine
	c.hwg.Add(1)
	go c.watcher()

	// Watch for new commands in a goroutine
	c.hwg.Add(1)
	go func() {
		defer func() {
			close(cmdChan)
			c.hwg.Done()
		}()
		for {
			cmd, err := c.coord.Command()
			if err != nil {
				//FIXME add more sophisticated error handling
				c.logger.Log(LogLevelError, "Coordinator returned an error during command, waiting and retrying. %v", err)
				select {
				case <-c.stop:
					return
				case <-time.After(consumerRetryDelay):
				}
				continue
			}
			if cmd == nil {
				c.logger.Log(LogLevelDebug, "Command coordinator exited")
				return
			}
			// Send command to watcher (or shutdown)
			select {
			case <-c.stop:
				return
			case cmdChan <- cmd:
			}
			// Wait for main loop to signal command has been handled
			select {
			case <-c.stop:
				return
			case <-c.tick:
			}
		}
	}()

	// Make sure Run() cleans up on exit (stops coordinator, releases tasks, etc)
	defer c.shutdown()

	// Main Loop ensures events are processed synchronously
	for {
		if c.frozen() {
			// Only recv commands while frozen
			select {
			case <-c.stop:
				// Shutdown has been called.
				return
			case cmd, ok := <-cmdChan:
				if !ok {
					c.logger.Log(LogLevelDebug, "Command channel closed. Exiting main loop.")
					return
				}
				c.logger.Log(LogLevelDebug, "Received command: %s", cmd)
				c.handleCommand(cmd)
			}
			// Must send tick whenever main loop restarts
			select {
			case <-c.stop:
				return
			case c.tick <- 1:
			}
			continue
		}

		select {
		case <-c.stop:
			// Shutdown has been called.
			return
		case <-balance:
			c.balance()
		case task, ok := <-c.watch:
			if !ok {
				c.logger.Log(LogLevelDebug, "Watch channel closed. Exiting main loop.")
				return
			}
			if !c.bal.CanClaim(task) {
				c.logger.Log(LogLevelInfo, "Balancer rejected task %s", task)
				break
			}
			if !c.coord.Claim(task) {
				c.logger.Log(LogLevelInfo, "Coordinator unable to claim task %s", task)
				break
			}
			c.claimed(task)
		case cmd, ok := <-cmdChan:
			if !ok {
				c.logger.Log(LogLevelDebug, "Command channel closed. Exiting main loop.")
				return
			}
			c.handleCommand(cmd)
		}
		// Signal that main loop is restarting after handling an event
		select {
		case <-c.stop:
			return
		case c.tick <- 1:
		}
	}
}

func (c *Consumer) watcher() {
	defer func() {
		close(c.watch)
		c.hwg.Done()
	}()
	c.logger.Log(LogLevelDebug, "Consumer watching")

	for {
		task, err := c.coord.Watch()
		if err != nil {
			//FIXME add more sophisticated error handling
			c.logger.Log(LogLevelError, "Coordinator returned an error during watch, waiting and retrying: %v", err)
			select {
			case <-c.stop:
				return
			case <-time.After(consumerRetryDelay):
			}
			continue
		}
		if task == "" {
			c.logger.Log(LogLevelInfo, "Coordinator has closed, no longer watching for tasks.")
			return
		}
		// Send task to watcher (or shutdown)
		select {
		case <-c.stop:
			return
		case c.watch <- task:
		}
		// Wait for main loop to signal task has been handled
		select {
		case <-c.stop:
			return
		case <-c.tick:
		}
	}
}

func (c *Consumer) balance() {
	for _, task := range c.bal.Balance() {
		//TODO Release tasks asynchronously as their shutdown might be slow?
		c.release(task)
	}
}

// shutdown is the actual shutdown logic called when Run() exits.
func (c *Consumer) shutdown() {
	c.logger.Log(LogLevelDebug, "Closing Coordinator")
	c.coord.Close()

	// Build list of of currently running tasks
	tasks := c.Tasks()
	c.logger.Log(LogLevelInfo, "Sending stop signal to %d handler(s)", len(tasks))

	// Concurrently shutdown handlers as they may take a while to shutdown
	for _, id := range tasks {
		go c.release(id)
	}

	c.logger.Log(LogLevelInfo, "Waiting for handlers to exit")
	c.hwg.Wait()
}

// Shutdown stops the main Run loop, calls Stop on all handlers, and calls
// Close on the Coordinator. Running tasks will be released for other nodes to
// claim.
func (c *Consumer) Shutdown() {
	select {
	case <-c.stop:
		// already stopped
	default:
		c.logger.Log(LogLevelDebug, "Stopping Run loop")
		close(c.stop)
	}
	c.hwg.Wait()
}

// Tasks returns a sorted list of running Task IDs.
func (c *Consumer) Tasks() []string {
	c.runL.Lock()
	defer c.runL.Unlock()
	t := make([]string, len(c.running))
	i := 0
	for id, _ := range c.running {
		t[i] = id
		i++
	}
	sort.Strings(t)
	return t
}

// claimed starts a handler for a claimed task. It is the only method to
// manipulate c.running and closes the runningTask channel when a handler's Run
// method exits.
func (c *Consumer) claimed(taskID string) {
	h := c.handler()

	c.logger.Log(LogLevelDebug, "Attempting to start task "+taskID)
	// Associate handler with taskID
	// **This is the only place tasks should be added to c.running**
	c.runL.Lock()
	c.running[taskID] = runningTask{h: h, c: make(chan struct{})}
	c.runL.Unlock()

	c.hwg.Add(1)
	// Start handler in its own goroutine
	go func() {
		c.logger.Log(LogLevelInfo, "Task started: %s", taskID)
		defer c.logger.Log(LogLevelInfo, "Task exited: %s", taskID)
		defer func() {
			if err := recover(); err != nil {
				c.logger.Log(LogLevelError, "Handler %s panic()'d: %v", taskID, err)
				// panics are considered fatal errors. Make sure the task isn't
				// rescheduled.
				c.coord.Done(taskID)
			}
			// **This is the only place tasks should be removed from c.running**
			c.runL.Lock()
			close(c.running[taskID].c)
			delete(c.running, taskID)
			c.runL.Unlock()
			c.hwg.Done()
		}()

		// Run the task
		c.logger.Log(LogLevelDebug, "Calling run for task %s", taskID)
		if err := h.Run(taskID); err != nil {
			if ferr, ok := err.(FatalError); ok && ferr.Fatal() {
				c.logger.Log(LogLevelError, "Handler for %s exited with fatal error: %v", taskID, err)
			} else {
				c.logger.Log(LogLevelError, "Handler for %s exited with error: %v", taskID, err)
				// error was non-fatal, release and let another node try
				c.coord.Release(taskID)
				return
			}
		}
		c.coord.Done(taskID)
	}()
}

// release stops and Coordinator.Release()s a task if it's running.
//
// release blocks until the task handler stops running.
func (c *Consumer) release(taskID string) {
	// Stop task...
	if c.stopTask(taskID) {
		// ...instruct the coordinator to release it
		c.coord.Release(taskID)
	}
}

// stopTask returns true if the task was running and stopped successfully.
func (c *Consumer) stopTask(taskID string) bool {
	c.runL.Lock()
	task, ok := c.running[taskID]
	c.runL.Unlock()

	if !ok {
		// This can happen if a task completes during Balance() and is not an error.
		c.logger.Log(LogLevelWarn, "Tried to release a non-running task: %s", taskID)
		return false
	}

	// all handler methods must be wrapped in a recover to prevent a misbehaving
	// handler from crashing the entire consumer
	func() {
		defer func() {
			if err := recover(); err != nil {
				c.logger.Log(LogLevelError, "Handler %s panic()'d on Stop: %v", taskID, err)
			}
		}()

		task.h.Stop()
	}()

	// Once the handler is stopped...
	//FIXME should there be a timeout here?
	<-task.c
	return true
}

func (c *Consumer) frozen() bool {
	c.freezeL.Lock()
	r := c.freeze
	c.freezeL.Unlock()
	return r
}

func (c *Consumer) handleCommand(cmd Command) {
	switch cmd.Name() {
	case cmdFreeze:
		if c.frozen() {
			c.logger.Log(LogLevelInfo, "Ignoring freeze command: already frozen")
			return
		}
		c.logger.Log(LogLevelInfo, "Freezing")
		c.freezeL.Lock()
		c.freeze = true
		c.freezeL.Unlock()
	case cmdUnfreeze:
		if !c.frozen() {
			c.logger.Log(LogLevelInfo, "Ignoring unfreeze command: not frozen")
			return
		}
		c.logger.Log(LogLevelInfo, "Unfreezing")
		c.freezeL.Lock()
		c.freeze = false
		c.freezeL.Unlock()
	case cmdBalance:
		c.logger.Log(LogLevelInfo, "Balancing due to command")
		c.balance()
		c.logger.Log(LogLevelDebug, "Finished balancing due to command")
	case cmdReleaseTask:
		taskI, ok := cmd.Parameters()["task"]
		task, ok2 := taskI.(string)
		if !ok || !ok2 {
			c.logger.Log(LogLevelError, "Release task command didn't contain a valid task")
			return
		}
		c.logger.Log(LogLevelInfo, "Releasing task %s due to command", task)
		c.release(task)
	case cmdStopTask:
		taskI, ok := cmd.Parameters()["task"]
		task, ok2 := taskI.(string)
		if !ok || !ok2 {
			c.logger.Log(LogLevelError, "Stop task command didn't contain a valid task")
			return
		}
		c.logger.Log(LogLevelInfo, "Stopping task %s due to command", task)
		c.stopTask(task)
	default:
		c.logger.Log(LogLevelWarn, "Discarding unknown command: %s", cmd.Name())
	}
}
<commit_msg>Revert making shutdown wait on Watch/Command/Balance<commit_after>package metafora

import (
	"math/rand"
	"sort"
	"sync"
	"time"
)

var (

	// balance calls are randomized and this is the upper bound of the random
	// amount
	balanceJitterMax = 10 * int64(time.Second)

	//FIXME should probably be improved, see usage in Run()
	consumerRetryDelay = 10 * time.Second
)

type runningTask struct {
	h Handler
	c chan struct{}
}

// Consumer is the core Metafora task runner.
type Consumer struct {
	// Func to create new handlers
	handler HandlerFunc

	// Map of task:Handler
	running map[string]runningTask

	// Mutex to protect access to running
	runL sync.Mutex

	// WaitGroup for running handlers and consumer goroutines
	hwg sync.WaitGroup

	bal      Balancer
	balEvery time.Duration
	coord    Coordinator
	logger   *logger
	stop     chan struct{} // closed by Shutdown to cause Run to exit

	// ticked on each loop of the main loop to enforce sequential interaction
	// with coordinator and balancer
	tick chan int

	watch chan string // channel for watcher to send tasks to main loop

	// Set by command handler, read anywhere via Consumer.frozen()
	freezeL sync.Mutex
	freeze  bool
}

// NewConsumer returns a new consumer and calls Init on the Balancer and Coordinator.
func NewConsumer(coord Coordinator, h HandlerFunc, b Balancer) (*Consumer, error) {
	c := &Consumer{
		running:  make(map[string]runningTask),
		handler:  h,
		bal:      b,
		balEvery: 15 * time.Minute, //TODO make balance wait configurable
		coord:    coord,
		logger:   stdoutLogger(),
		stop:     make(chan struct{}),
		tick:     make(chan int),
		watch:    make(chan string),
	}

	// initialize balancer with the consumer and a prefixed logger
	b.Init(&struct {
		*Consumer
		Logger
	}{Consumer: c, Logger: c.logger})

	if err := coord.Init(&coordinatorContext{Consumer: c, Logger: c.logger}); err != nil {
		return nil, err
	}
	return c, nil
}

// SetLogger assigns the logger to use as well as a level
//
// The logger parameter is an interface that requires the following
// method to be implemented (such as the the stdlib log.Logger):
//
//    Output(calldepth int, s string)
//
func (c *Consumer) SetLogger(l logOutputter, lvl LogLevel) {
	c.logger.l = l
	c.logger.lvl = lvl
}

// Run is the core run loop of Metafora. It is responsible for calling into the
// Coordinator to claim work and Balancer to rebalance work.
//
// Run blocks until Shutdown is called or an internal error occurs.
func (c *Consumer) Run() {
	c.logger.Log(LogLevelDebug, "Starting consumer")

	// chans for core goroutines to communicate with main loop
	balance := make(chan bool)
	cmdChan := make(chan Command)

	// Balance is called by the main loop when the balance channel is ticked
	go func() {
		randInt := rand.New(rand.NewSource(time.Now().UnixNano())).Int63n
		for {
			select {
			case <-c.stop:
				// Shutdown has been called.
				return
			case <-time.After(c.balEvery + time.Duration(randInt(balanceJitterMax))):
				c.logger.Log(LogLevelInfo, "Balancing")
				select {
				case balance <- true:
					// Ticked balance
				case <-c.stop:
					// Shutdown has been called.
					return
				}
			}
			// Wait for main loop to signal balancing is done
			select {
			case <-c.stop:
				return
			case <-c.tick:
			}
		}
	}()

	// Watch for new tasks in a goroutine
	go c.watcher()

	// Watch for new commands in a goroutine
	go func() {
		defer close(cmdChan)
		for {
			cmd, err := c.coord.Command()
			if err != nil {
				//FIXME add more sophisticated error handling
				c.logger.Log(LogLevelError, "Coordinator returned an error during command, waiting and retrying. %v", err)
				select {
				case <-c.stop:
					return
				case <-time.After(consumerRetryDelay):
				}
				continue
			}
			if cmd == nil {
				c.logger.Log(LogLevelDebug, "Command coordinator exited")
				return
			}
			// Send command to watcher (or shutdown)
			select {
			case <-c.stop:
				return
			case cmdChan <- cmd:
			}
			// Wait for main loop to signal command has been handled
			select {
			case <-c.stop:
				return
			case <-c.tick:
			}
		}
	}()

	// Make sure Run() cleans up on exit (stops coordinator, releases tasks, etc)
	defer c.shutdown()

	// Main Loop ensures events are processed synchronously
	for {
		if c.frozen() {
			// Only recv commands while frozen
			select {
			case <-c.stop:
				// Shutdown has been called.
				return
			case cmd, ok := <-cmdChan:
				if !ok {
					c.logger.Log(LogLevelDebug, "Command channel closed. Exiting main loop.")
					return
				}
				c.logger.Log(LogLevelDebug, "Received command: %s", cmd)
				c.handleCommand(cmd)
			}
			// Must send tick whenever main loop restarts
			select {
			case <-c.stop:
				return
			case c.tick <- 1:
			}
			continue
		}

		select {
		case <-c.stop:
			// Shutdown has been called.
			return
		case <-balance:
			c.balance()
		case task, ok := <-c.watch:
			if !ok {
				c.logger.Log(LogLevelDebug, "Watch channel closed. Exiting main loop.")
				return
			}
			if !c.bal.CanClaim(task) {
				c.logger.Log(LogLevelInfo, "Balancer rejected task %s", task)
				break
			}
			if !c.coord.Claim(task) {
				c.logger.Log(LogLevelInfo, "Coordinator unable to claim task %s", task)
				break
			}
			c.claimed(task)
		case cmd, ok := <-cmdChan:
			if !ok {
				c.logger.Log(LogLevelDebug, "Command channel closed. Exiting main loop.")
				return
			}
			c.handleCommand(cmd)
		}
		// Signal that main loop is restarting after handling an event
		select {
		case <-c.stop:
			return
		case c.tick <- 1:
		}
	}
}

func (c *Consumer) watcher() {
	defer close(c.watch)
	c.logger.Log(LogLevelDebug, "Consumer watching")

	for {
		task, err := c.coord.Watch()
		if err != nil {
			//FIXME add more sophisticated error handling
			c.logger.Log(LogLevelError, "Coordinator returned an error during watch, waiting and retrying: %v", err)
			select {
			case <-c.stop:
				return
			case <-time.After(consumerRetryDelay):
			}
			continue
		}
		if task == "" {
			c.logger.Log(LogLevelInfo, "Coordinator has closed, no longer watching for tasks.")
			return
		}
		// Send task to watcher (or shutdown)
		select {
		case <-c.stop:
			return
		case c.watch <- task:
		}
		// Wait for main loop to signal task has been handled
		select {
		case <-c.stop:
			return
		case <-c.tick:
		}
	}
}

func (c *Consumer) balance() {
	for _, task := range c.bal.Balance() {
		//TODO Release tasks asynchronously as their shutdown might be slow?
		c.release(task)
	}
}

// shutdown is the actual shutdown logic called when Run() exits.
func (c *Consumer) shutdown() {
	c.logger.Log(LogLevelDebug, "Closing Coordinator")
	c.coord.Close()

	// Build list of of currently running tasks
	tasks := c.Tasks()
	c.logger.Log(LogLevelInfo, "Sending stop signal to %d handler(s)", len(tasks))

	// Concurrently shutdown handlers as they may take a while to shutdown
	for _, id := range tasks {
		go c.release(id)
	}

	c.logger.Log(LogLevelInfo, "Waiting for handlers to exit")
	c.hwg.Wait()
}

// Shutdown stops the main Run loop, calls Stop on all handlers, and calls
// Close on the Coordinator. Running tasks will be released for other nodes to
// claim.
func (c *Consumer) Shutdown() {
	select {
	case <-c.stop:
		// already stopped
	default:
		c.logger.Log(LogLevelDebug, "Stopping Run loop")
		close(c.stop)
	}
	c.hwg.Wait()
}

// Tasks returns a sorted list of running Task IDs.
func (c *Consumer) Tasks() []string {
	c.runL.Lock()
	defer c.runL.Unlock()
	t := make([]string, len(c.running))
	i := 0
	for id, _ := range c.running {
		t[i] = id
		i++
	}
	sort.Strings(t)
	return t
}

// claimed starts a handler for a claimed task. It is the only method to
// manipulate c.running and closes the runningTask channel when a handler's Run
// method exits.
func (c *Consumer) claimed(taskID string) {
	h := c.handler()

	c.logger.Log(LogLevelDebug, "Attempting to start task "+taskID)
	// Associate handler with taskID
	// **This is the only place tasks should be added to c.running**
	c.runL.Lock()
	c.running[taskID] = runningTask{h: h, c: make(chan struct{})}
	c.runL.Unlock()

	c.hwg.Add(1)
	// Start handler in its own goroutine
	go func() {
		c.logger.Log(LogLevelInfo, "Task started: %s", taskID)
		defer c.logger.Log(LogLevelInfo, "Task exited: %s", taskID)
		defer func() {
			if err := recover(); err != nil {
				c.logger.Log(LogLevelError, "Handler %s panic()'d: %v", taskID, err)
				// panics are considered fatal errors. Make sure the task isn't
				// rescheduled.
				c.coord.Done(taskID)
			}
			// **This is the only place tasks should be removed from c.running**
			c.runL.Lock()
			close(c.running[taskID].c)
			delete(c.running, taskID)
			c.runL.Unlock()
			c.hwg.Done()
		}()

		// Run the task
		c.logger.Log(LogLevelDebug, "Calling run for task %s", taskID)
		if err := h.Run(taskID); err != nil {
			if ferr, ok := err.(FatalError); ok && ferr.Fatal() {
				c.logger.Log(LogLevelError, "Handler for %s exited with fatal error: %v", taskID, err)
			} else {
				c.logger.Log(LogLevelError, "Handler for %s exited with error: %v", taskID, err)
				// error was non-fatal, release and let another node try
				c.coord.Release(taskID)
				return
			}
		}
		c.coord.Done(taskID)
	}()
}

// release stops and Coordinator.Release()s a task if it's running.
//
// release blocks until the task handler stops running.
func (c *Consumer) release(taskID string) {
	// Stop task...
	if c.stopTask(taskID) {
		// ...instruct the coordinator to release it
		c.coord.Release(taskID)
	}
}

// stopTask returns true if the task was running and stopped successfully.
func (c *Consumer) stopTask(taskID string) bool {
	c.runL.Lock()
	task, ok := c.running[taskID]
	c.runL.Unlock()

	if !ok {
		// This can happen if a task completes during Balance() and is not an error.
		c.logger.Log(LogLevelWarn, "Tried to release a non-running task: %s", taskID)
		return false
	}

	// all handler methods must be wrapped in a recover to prevent a misbehaving
	// handler from crashing the entire consumer
	func() {
		defer func() {
			if err := recover(); err != nil {
				c.logger.Log(LogLevelError, "Handler %s panic()'d on Stop: %v", taskID, err)
			}
		}()

		task.h.Stop()
	}()

	// Once the handler is stopped...
	//FIXME should there be a timeout here?
	<-task.c
	return true
}

func (c *Consumer) frozen() bool {
	c.freezeL.Lock()
	r := c.freeze
	c.freezeL.Unlock()
	return r
}

func (c *Consumer) handleCommand(cmd Command) {
	switch cmd.Name() {
	case cmdFreeze:
		if c.frozen() {
			c.logger.Log(LogLevelInfo, "Ignoring freeze command: already frozen")
			return
		}
		c.logger.Log(LogLevelInfo, "Freezing")
		c.freezeL.Lock()
		c.freeze = true
		c.freezeL.Unlock()
	case cmdUnfreeze:
		if !c.frozen() {
			c.logger.Log(LogLevelInfo, "Ignoring unfreeze command: not frozen")
			return
		}
		c.logger.Log(LogLevelInfo, "Unfreezing")
		c.freezeL.Lock()
		c.freeze = false
		c.freezeL.Unlock()
	case cmdBalance:
		c.logger.Log(LogLevelInfo, "Balancing due to command")
		c.balance()
		c.logger.Log(LogLevelDebug, "Finished balancing due to command")
	case cmdReleaseTask:
		taskI, ok := cmd.Parameters()["task"]
		task, ok2 := taskI.(string)
		if !ok || !ok2 {
			c.logger.Log(LogLevelError, "Release task command didn't contain a valid task")
			return
		}
		c.logger.Log(LogLevelInfo, "Releasing task %s due to command", task)
		c.release(task)
	case cmdStopTask:
		taskI, ok := cmd.Parameters()["task"]
		task, ok2 := taskI.(string)
		if !ok || !ok2 {
			c.logger.Log(LogLevelError, "Stop task command didn't contain a valid task")
			return
		}
		c.logger.Log(LogLevelInfo, "Stopping task %s due to command", task)
		c.stopTask(task)
	default:
		c.logger.Log(LogLevelWarn, "Discarding unknown command: %s", cmd.Name())
	}
}
<|endoftext|>


<commit_before>package tapalcatl

import (
	"archive/zip"
	"fmt"
	"io"
)

type TileCoord struct {
	Z, X, Y int
	Format  string
}

func (t TileCoord) FileName() string {
	return fmt.Sprintf("%d/%d/%d.%s", t.Z, t.X, t.Y, t.Format)
}

// isPowerOfTwo return true when the given integer is a power of two.
// See https://graphics.stanford.edu/~seander/bithacks.html#DetermineIfPowerOf2
// for details.
func isPowerOfTwo(i int) bool {
	if i > 0 {
		return (i & (i - 1)) == 0
	}
	return false
}

// sizeToZoom returns the zoom equivalent to a metatile size, meaning that
// there are size * size tiles at this zoom level. Input size must be a
// power of two.
//
// Algorithm from: https://graphics.stanford.edu/~seander/bithacks.html#IntegerLog
// as Go seems to lack an integer Log2 in its math library.
func sizeToZoom(v uint) uint {
	var b = [...]uint{0x2, 0xC, 0xF0, 0xFF00, 0xFFFF0000}
	var S = [...]uint{1, 2, 3, 4, 16}
	var r uint = 0
	var i int

	for i = 4; i >= 0; i-- {
		if v & b[i] != 0 {
			v >>= S[i]
			r |= S[i]
		}
	}

	return r
}

// MetaAndOffset returns the metatile coordinate and the offset within it for
// this TileCoord object. The argument metaSize indicates the size of the
// metatile and tileSize indicates the size of the tile within the metatile
// that you want to extract, both in units of "standard" 256px tiles.
//
// For example, to extract a 1x1 regular 256px tile from a 2x2 metatile, one
// would call MetaAndOffset(2, 1). To extract the 512px tile from the same,
// call MetaAndOffset(2, 2).
func (t TileCoord) MetaAndOffset(metaSize, tileSize int) (meta, offset TileCoord, err error) {
	// check that sizes are powers of two before proceeding.
	if !isPowerOfTwo(metaSize) {
		err = fmt.Errorf("Metatile size is required to be a power of two, but %d is not.", metaSize)
		return
	}
	if !isPowerOfTwo(tileSize) {
		err = fmt.Errorf("Tile size is required to be a power of two, but %d is not.", tileSize)
		return
	}

	// now we can calculate the delta in zoom level, knowing that both must be
	// powers of two, and hence positive.
	metaZoom := sizeToZoom(uint(metaSize))
	tileZoom := sizeToZoom(uint(tileSize))
	if tileZoom > metaZoom {
		err = fmt.Errorf("Tile size must not be greater than metatile size, but %d > %d.", tileSize, metaSize)
		return
	}
	deltaZ := metaZoom - tileZoom

	// note that the uint->int conversion is technically a narrowing, but cannot
	// overflow because we know it contains the difference of two Log2s, which
	// cannot be larger than 32.
	meta.Z = t.Z - int(deltaZ)
	meta.X = t.X >> deltaZ
	meta.Y = t.Y >> deltaZ
	meta.Format = "zip"

	offset.Z = t.Z - meta.Z
	offset.X = t.X - (meta.X << deltaZ)
	offset.Y = t.Y - (meta.Y << deltaZ)
	offset.Format = t.Format

	return
}

func NewMetatileReader(t TileCoord, r io.ReaderAt, size int64) (io.ReadCloser, uint64, error) {
	z, err := zip.NewReader(r, size)
	if err != nil {
		return nil, 0, err
	}

	target := t.FileName()

	for _, f := range z.File {
		if f.Name == target {
			result, err := f.Open()
			return result, f.UncompressedSize64, err
		}
	}

	return nil, 0, fmt.Errorf("Unable to find relative tile offset %#v in metatile.", target)
}
<commit_msg>Add brackets to prevent ugly gofmt.<commit_after>package tapalcatl

import (
	"archive/zip"
	"fmt"
	"io"
)

type TileCoord struct {
	Z, X, Y int
	Format  string
}

func (t TileCoord) FileName() string {
	return fmt.Sprintf("%d/%d/%d.%s", t.Z, t.X, t.Y, t.Format)
}

// isPowerOfTwo return true when the given integer is a power of two.
// See https://graphics.stanford.edu/~seander/bithacks.html#DetermineIfPowerOf2
// for details.
func isPowerOfTwo(i int) bool {
	if i > 0 {
		return (i & (i - 1)) == 0
	}
	return false
}

// sizeToZoom returns the zoom equivalent to a metatile size, meaning that
// there are size * size tiles at this zoom level. Input size must be a
// power of two.
//
// Algorithm from: https://graphics.stanford.edu/~seander/bithacks.html#IntegerLog
// as Go seems to lack an integer Log2 in its math library.
func sizeToZoom(v uint) uint {
	var b = [...]uint{0x2, 0xC, 0xF0, 0xFF00, 0xFFFF0000}
	var S = [...]uint{1, 2, 3, 4, 16}
	var r uint = 0
	var i int

	for i = 4; i >= 0; i-- {
		if (v & b[i]) != 0 {
			v >>= S[i]
			r |= S[i]
		}
	}

	return r
}

// MetaAndOffset returns the metatile coordinate and the offset within it for
// this TileCoord object. The argument metaSize indicates the size of the
// metatile and tileSize indicates the size of the tile within the metatile
// that you want to extract, both in units of "standard" 256px tiles.
//
// For example, to extract a 1x1 regular 256px tile from a 2x2 metatile, one
// would call MetaAndOffset(2, 1). To extract the 512px tile from the same,
// call MetaAndOffset(2, 2).
func (t TileCoord) MetaAndOffset(metaSize, tileSize int) (meta, offset TileCoord, err error) {
	// check that sizes are powers of two before proceeding.
	if !isPowerOfTwo(metaSize) {
		err = fmt.Errorf("Metatile size is required to be a power of two, but %d is not.", metaSize)
		return
	}
	if !isPowerOfTwo(tileSize) {
		err = fmt.Errorf("Tile size is required to be a power of two, but %d is not.", tileSize)
		return
	}

	// now we can calculate the delta in zoom level, knowing that both must be
	// powers of two, and hence positive.
	metaZoom := sizeToZoom(uint(metaSize))
	tileZoom := sizeToZoom(uint(tileSize))
	if tileZoom > metaZoom {
		err = fmt.Errorf("Tile size must not be greater than metatile size, but %d > %d.", tileSize, metaSize)
		return
	}
	deltaZ := metaZoom - tileZoom

	// note that the uint->int conversion is technically a narrowing, but cannot
	// overflow because we know it contains the difference of two Log2s, which
	// cannot be larger than 32.
	meta.Z = t.Z - int(deltaZ)
	meta.X = t.X >> deltaZ
	meta.Y = t.Y >> deltaZ
	meta.Format = "zip"

	offset.Z = t.Z - meta.Z
	offset.X = t.X - (meta.X << deltaZ)
	offset.Y = t.Y - (meta.Y << deltaZ)
	offset.Format = t.Format

	return
}

func NewMetatileReader(t TileCoord, r io.ReaderAt, size int64) (io.ReadCloser, uint64, error) {
	z, err := zip.NewReader(r, size)
	if err != nil {
		return nil, 0, err
	}

	target := t.FileName()

	for _, f := range z.File {
		if f.Name == target {
			result, err := f.Open()
			return result, f.UncompressedSize64, err
		}
	}

	return nil, 0, fmt.Errorf("Unable to find relative tile offset %#v in metatile.", target)
}
<|endoftext|>


<commit_before>package http

import (
	"github.com/Cepave/open-falcon-backend/modules/fe/g"
	"github.com/Cepave/open-falcon-backend/modules/fe/http/boss"
	"github.com/Cepave/open-falcon-backend/modules/fe/http/dashboard"
	"github.com/Cepave/open-falcon-backend/modules/fe/http/home"
	"github.com/Cepave/open-falcon-backend/modules/fe/http/portal"
	"github.com/Cepave/open-falcon-backend/modules/fe/http/uic"
	uic_model "github.com/Cepave/open-falcon-backend/modules/fe/model/uic"
	log "github.com/Sirupsen/logrus"
	"github.com/astaxie/beego"
	"github.com/astaxie/beego/plugins/cors"
)

func Start() {
	if !g.Config().Http.Enabled {
		return
	}

	addr := g.Config().Http.Listen
	if addr == "" {
		return
	}

	home.ConfigRoutes()
	uic.ConfigRoutes()
	dashboard.ConfigRoutes()
	portal.ConfigRoutes()
	boss.ConfigRoutes()

	beego.AddFuncMap("member", uic_model.MembersByTeamId)
	beego.InsertFilter("*", beego.BeforeRouter, cors.Allow(&cors.Options{
		AllowAllOrigins: true,
	}))
	if g.Config().Http.ViewPath != "" {
		log.Infof("set http view_path in %v", g.Config().Http.ViewPath)
		beego.SetViewsPath(g.Config().Http.ViewPath)
	}
	if g.Config().Http.StaticPath != "" {
		log.Infof("set http static_path in %v", g.Config().Http.StaticPath)
		beego.SetStaticPath("/static", g.Config().Http.StaticPath)
	}
	beego.Run(addr)
}
<commit_msg>add logging level set up for beego<commit_after>package http

import (
	"strings"

	"github.com/Cepave/open-falcon-backend/modules/fe/g"
	"github.com/Cepave/open-falcon-backend/modules/fe/http/boss"
	"github.com/Cepave/open-falcon-backend/modules/fe/http/dashboard"
	"github.com/Cepave/open-falcon-backend/modules/fe/http/home"
	"github.com/Cepave/open-falcon-backend/modules/fe/http/portal"
	"github.com/Cepave/open-falcon-backend/modules/fe/http/uic"
	uic_model "github.com/Cepave/open-falcon-backend/modules/fe/model/uic"
	log "github.com/Sirupsen/logrus"
	"github.com/astaxie/beego"
	"github.com/astaxie/beego/plugins/cors"
)

func Start() {
	if !g.Config().Http.Enabled {
		return
	}

	addr := g.Config().Http.Listen
	if addr == "" {
		return
	}

	switch strings.ToLower(g.Config().Log) {
	case "info":
		beego.SetLevel(beego.LevelInformational)
	case "debug":
		beego.SetLevel(beego.LevelDebug)
	case "warn":
		beego.SetLevel(beego.LevelWarning)
	case "error":
		beego.SetLevel(beego.LevelError)
	}

	home.ConfigRoutes()
	uic.ConfigRoutes()
	dashboard.ConfigRoutes()
	portal.ConfigRoutes()
	boss.ConfigRoutes()

	beego.AddFuncMap("member", uic_model.MembersByTeamId)
	beego.InsertFilter("*", beego.BeforeRouter, cors.Allow(&cors.Options{
		AllowAllOrigins: true,
	}))
	if g.Config().Http.ViewPath != "" {
		log.Infof("set http view_path in %v", g.Config().Http.ViewPath)
		beego.SetViewsPath(g.Config().Http.ViewPath)
	}
	if g.Config().Http.StaticPath != "" {
		log.Infof("set http static_path in %v", g.Config().Http.StaticPath)
		beego.SetStaticPath("/static", g.Config().Http.StaticPath)
	}
	beego.Run(addr)
}
<|endoftext|>


<commit_before>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package graph

import (
	"errors"
	"sync"

	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
)

// A visitor in a directed graph whose nodes are identified by strings.
type Visitor interface {
	// Process the supplied node and return a list of direct successors.
	Visit(ctx context.Context, node string) (adjacent []string, err error)
}

// Invoke v.Visit on each node reachable from the supplied search roots,
// including the roots themselves. Use the supplied degree of parallelism.
//
// It is guaranteed that if a node N is fed to v.Visit, then either:
//
//  *  N is an element of roots, or
//  *  There exists a direct predecessor N' of N such that v.Visit(N') was
//     called and returned successfully.
//
// In particular, if the graph is a rooted tree and searching starts at the
// root, then parents will be successfully visited before children are visited.
// However note that in arbitrary DAGs it is *not* guaranteed that all of a
// node's predecessors have been visited before it is.
func Traverse(
	ctx context.Context,
	parallelism int,
	roots []string,
	v Visitor) (err error) {
	b := syncutil.NewBundle(ctx)
	defer func() { err = b.Join() }()

	// Set up initial state.
	ts := &traverseState{
		admitted: make(map[string]struct{}),
	}

	ts.mu = syncutil.NewInvariantMutex(ts.checkInvariants)
	ts.cond.L = &ts.mu

	for _, r := range roots {
		ts.admitted[r] = struct{}{}
		ts.toVisit = append(ts.toVisit, r)
	}

	// Ensure that ts.cancelled is set when the context is eventually cancelled.
	go watchForCancel(ctx, ts)

	// Run the appropriate number of workers.
	for i := 0; i < parallelism; i++ {
		b.Add(func(ctx context.Context) (err error) {
			err = traverse(ctx, ts, v)
			return
		})
	}

	return
}

// State shared by each traverse worker.
type traverseState struct {
	mu syncutil.InvariantMutex

	// All nodes that have ever been seen. If a node is in this map, it will
	// eventually be visted (barring errors returned by the visitor).
	//
	// GUARDED_BY(mu)
	admitted map[string]struct{}

	// Admitted nodes that have yet to be visted.
	//
	// INVARIANT: For each n in toVisit, n is a key of admitted.
	//
	// GUARDED_BY(mu)
	toVisit []string

	// Set to true if the context has been cancelled. All workers should return
	// when this happens.
	//
	// GUARDED_BY(mu)
	cancelled bool

	// The number of workers that are doing something besides waiting on a node
	// to visit. If this hits zero with toVisit empty, it means that there is
	// nothing further to do.
	//
	// GUARDED_BY(mu)
	busyWorkers int

	// Signalled with mu held when any of the following state changes:
	//
	//  *  toVisit
	//  *  cancelled
	//  *  busyWorkers
	//
	// GUARDED_BY(mu)
	cond sync.Cond
}

// LOCKS_REQUIRED(ts.mu)
func (ts *traverseState) checkInvariants() {
	panic("TODO")
}

// A single traverse worker.
func traverse(
	ctx context.Context,
	ts *traverseState,
	v Visitor) (err error) {
	err = errors.New("TODO")
	return
}

// Bridge context cancellation with traverseState.cancelled.
func watchForCancel(
	ctx context.Context,
	ts *traverseState) {
	panic("TODO")
}
<commit_msg>traverseState.checkInvariants<commit_after>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package graph

import (
	"errors"
	"fmt"
	"sync"

	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
)

// A visitor in a directed graph whose nodes are identified by strings.
type Visitor interface {
	// Process the supplied node and return a list of direct successors.
	Visit(ctx context.Context, node string) (adjacent []string, err error)
}

// Invoke v.Visit on each node reachable from the supplied search roots,
// including the roots themselves. Use the supplied degree of parallelism.
//
// It is guaranteed that if a node N is fed to v.Visit, then either:
//
//  *  N is an element of roots, or
//  *  There exists a direct predecessor N' of N such that v.Visit(N') was
//     called and returned successfully.
//
// In particular, if the graph is a rooted tree and searching starts at the
// root, then parents will be successfully visited before children are visited.
// However note that in arbitrary DAGs it is *not* guaranteed that all of a
// node's predecessors have been visited before it is.
func Traverse(
	ctx context.Context,
	parallelism int,
	roots []string,
	v Visitor) (err error) {
	b := syncutil.NewBundle(ctx)
	defer func() { err = b.Join() }()

	// Set up initial state.
	ts := &traverseState{
		admitted: make(map[string]struct{}),
	}

	ts.mu = syncutil.NewInvariantMutex(ts.checkInvariants)
	ts.cond.L = &ts.mu

	for _, r := range roots {
		ts.admitted[r] = struct{}{}
		ts.toVisit = append(ts.toVisit, r)
	}

	// Ensure that ts.cancelled is set when the context is eventually cancelled.
	go watchForCancel(ctx, ts)

	// Run the appropriate number of workers.
	for i := 0; i < parallelism; i++ {
		b.Add(func(ctx context.Context) (err error) {
			err = traverse(ctx, ts, v)
			return
		})
	}

	return
}

// State shared by each traverse worker.
type traverseState struct {
	mu syncutil.InvariantMutex

	// All nodes that have ever been seen. If a node is in this map, it will
	// eventually be visted (barring errors returned by the visitor).
	//
	// GUARDED_BY(mu)
	admitted map[string]struct{}

	// Admitted nodes that have yet to be visted.
	//
	// INVARIANT: For each n in toVisit, n is a key of admitted.
	//
	// GUARDED_BY(mu)
	toVisit []string

	// Set to true if the context has been cancelled. All workers should return
	// when this happens.
	//
	// GUARDED_BY(mu)
	cancelled bool

	// The number of workers that are doing something besides waiting on a node
	// to visit. If this hits zero with toVisit empty, it means that there is
	// nothing further to do.
	//
	// GUARDED_BY(mu)
	busyWorkers int

	// Signalled with mu held when any of the following state changes:
	//
	//  *  toVisit
	//  *  cancelled
	//  *  busyWorkers
	//
	// GUARDED_BY(mu)
	cond sync.Cond
}

// LOCKS_REQUIRED(ts.mu)
func (ts *traverseState) checkInvariants() {
	// INVARIANT: For each n in toVisit, n is a key of admitted.
	for _, n := range ts.toVisit {
		if _, ok := ts.admitted[n]; !ok {
			panic(fmt.Sprintf("Expected %q to be in admitted map", n))
		}
	}
}

// A single traverse worker.
func traverse(
	ctx context.Context,
	ts *traverseState,
	v Visitor) (err error) {
	err = errors.New("TODO")
	return
}

// Bridge context cancellation with traverseState.cancelled.
func watchForCancel(
	ctx context.Context,
	ts *traverseState) {
	panic("TODO")
}
<|endoftext|>


<commit_before>package mixpanel

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"
)

const (
	apiBaseURL = "https://api.mixpanel.com"
	library    = "vizzlo/mixpanel"
)

// actionSource describes where the call is originating from.
// This determines whether or not the location properties on a profile should be updated.
type actionSource int

const (
	// sourceUser flags the action as having originated with the user in question.
	sourceUser actionSource = iota
	// sourceScript means that the action originated in a backend script.
	// The IP should not be tracked in this case.
	sourceScript
)

// Client is a client to talk to the API
type Client struct {
	Token   string
	BaseUrl string
	Client  http.Client
}

// Properties are key=value pairs that decorate an event or a profile.
type Properties map[string]interface{}

// Operation is an action performed on a user profile.
// Typically this is $set or $unset, but others are available.
type Operation struct {
	Name   string
	Values Properties
}

// New returns a configured client.
func New(token string) *Client {
	return &Client{
		Token:   token,
		BaseUrl: apiBaseURL,
		Client: http.Client{
			Timeout: 60 * time.Second,
		},
	}
}

// Track sends event data with optional metadata.
func (m *Client) Track(distinctID string, event string, props Properties) error {
	if distinctID != "" {
		props["distinct_id"] = distinctID
	}
	props["token"] = m.Token
	props["mp_lib"] = library

	data := map[string]interface{}{"event": event, "properties": props}
	return m.makeRequestWithData("GET", "track", data, sourceUser)
}

func (m *Client) TrackAsScript(distinctID string, event string, props Properties) error {
	if distinctID != "" {
		props["distinct_id"] = distinctID
	}
	props["token"] = m.Token
	props["mp_lib"] = library

	data := map[string]interface{}{"event": event, "properties": props}
	return m.makeRequestWithData("GET", "track", data, sourceScript)
}

// Engage updates profile data.
// This will update the IP and related data on the profile.
// If you don't have the IP address of the user, then use the UpdateProperties method instead,
// otherwise the user's location will be set to wherever the script was run from.
func (m *Client) Engage(distinctID string, props Properties, op *Operation) error {
	return m.engage(distinctID, props, op, sourceUser)
}

// EngageAsScript calls the engage endpoint, but doesn't set IP, city, country, on the profile.
func (m *Client) EngageAsScript(distinctID string, props Properties, op *Operation) error {
	return m.engage(distinctID, props, op, sourceScript)
}

func (m *Client) engage(distinctID string, props Properties, op *Operation, as actionSource) error {
	if distinctID != "" {
		props["$distinct_id"] = distinctID
	}
	props["$token"] = m.Token
	props["mp_lib"] = library
	if op.Name == "$unset" {
		keys := []interface{}{}
		for key := range op.Values {
			keys = append(keys, key)
		}
		props[op.Name] = keys
	} else {
		props[op.Name] = op.Values
	}

	return m.makeRequestWithData("GET", "engage", props, as)
}

// TrackingPixel returns a url that, when clicked, will track the given data and then redirect to provided url.
func (m *Client) TrackingPixel(distinctID, event string, props Properties) (string, error) {
	if distinctID != "" {
		props["$distinct_id"] = distinctID
	}
	props["$token"] = m.Token
	props["mp_lib"] = library

	data := map[string]interface{}{"event": event, "properties": props}
	json, err := json.Marshal(data)
	if err != nil {
		return "", err
	}

	params := map[string]string{
		"data": base64.StdEncoding.EncodeToString(json),
		"img":  "1",
	}
	query := url.Values{}
	for k, v := range params {
		query[k] = []string{v}
	}
	return fmt.Sprintf("%s/%s?%s", m.BaseUrl, "track", query.Encode()), nil
}

// RedirectURL returns a url that, when clicked, will track the given data and then redirect to provided url.
func (m *Client) RedirectURL(distinctID, event, uri string, props Properties) (string, error) {
	if distinctID != "" {
		props["$distinct_id"] = distinctID
	}
	props["$token"] = m.Token
	props["mp_lib"] = library

	data := map[string]interface{}{"event": event, "properties": props}
	b, err := json.Marshal(data)
	if err != nil {
		return "", err
	}

	params := map[string]string{
		"data":     base64.StdEncoding.EncodeToString(b),
		"redirect": uri,
	}
	query := url.Values{}
	for k, v := range params {
		query[k] = []string{v}
	}
	return fmt.Sprintf("%s/%s?%s", m.BaseUrl, "track", query.Encode()), nil
}

func (m *Client) makeRequest(method string, endpoint string, paramMap map[string]string) error {
	var (
		err error
		req *http.Request
		r   io.Reader
	)

	if endpoint == "" {
		return fmt.Errorf("endpoint missing")
	}

	endpoint = fmt.Sprintf("%s/%s", m.BaseUrl, endpoint)

	if paramMap == nil {
		paramMap = map[string]string{}
	}

	params := url.Values{}
	for k, v := range paramMap {
		params[k] = []string{v}
	}

	switch method {
	case "GET":
		enc := params.Encode()
		if enc != "" {
			endpoint = endpoint + "?" + enc
		}
	case "POST":
		r = strings.NewReader(params.Encode())
	default:
		return fmt.Errorf("method not supported: %v", method)
	}

	req, err = http.NewRequest(method, endpoint, r)
	if err != nil {
		return err
	}

	resp, err := m.Client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	// The API documentation states that success will be reported with either "1" or "1\n".
	if strings.Trim(string(b), "\n") != "1" {
		return fmt.Errorf("request failed - %s", b)
	}
	return nil
}

func (m *Client) makeRequestWithData(method string, endpoint string, data Properties, as actionSource) error {
	b, err := json.Marshal(data)
	if err != nil {
		return err
	}

	params := map[string]string{
		"data": base64.StdEncoding.EncodeToString(b),
	}

	if as == sourceScript {
		params["ip"] = "0"
	}

	return m.makeRequest(method, endpoint, params)
}
<commit_msg>Rename BaseUrl --> BaseURL.<commit_after>package mixpanel

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"
)

const (
	apiBaseURL = "https://api.mixpanel.com"
	library    = "vizzlo/mixpanel"
)

// actionSource describes where the call is originating from.
// This determines whether or not the location properties on a profile should be updated.
type actionSource int

const (
	// sourceUser flags the action as having originated with the user in question.
	sourceUser actionSource = iota
	// sourceScript means that the action originated in a backend script.
	// The IP should not be tracked in this case.
	sourceScript
)

// Client is a client to talk to the API
type Client struct {
	Token   string
	BaseURL string
	Client  http.Client
}

// Properties are key=value pairs that decorate an event or a profile.
type Properties map[string]interface{}

// Operation is an action performed on a user profile.
// Typically this is $set or $unset, but others are available.
type Operation struct {
	Name   string
	Values Properties
}

// New returns a configured client.
func New(token string) *Client {
	return &Client{
		Token:   token,
		BaseURL: apiBaseURL,
		Client: http.Client{
			Timeout: 60 * time.Second,
		},
	}
}

// Track sends event data with optional metadata.
func (m *Client) Track(distinctID string, event string, props Properties) error {
	if distinctID != "" {
		props["distinct_id"] = distinctID
	}
	props["token"] = m.Token
	props["mp_lib"] = library

	data := map[string]interface{}{"event": event, "properties": props}
	return m.makeRequestWithData("GET", "track", data, sourceUser)
}

func (m *Client) TrackAsScript(distinctID string, event string, props Properties) error {
	if distinctID != "" {
		props["distinct_id"] = distinctID
	}
	props["token"] = m.Token
	props["mp_lib"] = library

	data := map[string]interface{}{"event": event, "properties": props}
	return m.makeRequestWithData("GET", "track", data, sourceScript)
}

// Engage updates profile data.
// This will update the IP and related data on the profile.
// If you don't have the IP address of the user, then use the UpdateProperties method instead,
// otherwise the user's location will be set to wherever the script was run from.
func (m *Client) Engage(distinctID string, props Properties, op *Operation) error {
	return m.engage(distinctID, props, op, sourceUser)
}

// EngageAsScript calls the engage endpoint, but doesn't set IP, city, country, on the profile.
func (m *Client) EngageAsScript(distinctID string, props Properties, op *Operation) error {
	return m.engage(distinctID, props, op, sourceScript)
}

func (m *Client) engage(distinctID string, props Properties, op *Operation, as actionSource) error {
	if distinctID != "" {
		props["$distinct_id"] = distinctID
	}
	props["$token"] = m.Token
	props["mp_lib"] = library
	if op.Name == "$unset" {
		keys := []interface{}{}
		for key := range op.Values {
			keys = append(keys, key)
		}
		props[op.Name] = keys
	} else {
		props[op.Name] = op.Values
	}

	return m.makeRequestWithData("GET", "engage", props, as)
}

// TrackingPixel returns a url that, when clicked, will track the given data and then redirect to provided url.
func (m *Client) TrackingPixel(distinctID, event string, props Properties) (string, error) {
	if distinctID != "" {
		props["$distinct_id"] = distinctID
	}
	props["$token"] = m.Token
	props["mp_lib"] = library

	data := map[string]interface{}{"event": event, "properties": props}
	json, err := json.Marshal(data)
	if err != nil {
		return "", err
	}

	params := map[string]string{
		"data": base64.StdEncoding.EncodeToString(json),
		"img":  "1",
	}
	query := url.Values{}
	for k, v := range params {
		query[k] = []string{v}
	}
	return fmt.Sprintf("%s/%s?%s", m.BaseURL, "track", query.Encode()), nil
}

// RedirectURL returns a url that, when clicked, will track the given data and then redirect to provided url.
func (m *Client) RedirectURL(distinctID, event, uri string, props Properties) (string, error) {
	if distinctID != "" {
		props["$distinct_id"] = distinctID
	}
	props["$token"] = m.Token
	props["mp_lib"] = library

	data := map[string]interface{}{"event": event, "properties": props}
	b, err := json.Marshal(data)
	if err != nil {
		return "", err
	}

	params := map[string]string{
		"data":     base64.StdEncoding.EncodeToString(b),
		"redirect": uri,
	}
	query := url.Values{}
	for k, v := range params {
		query[k] = []string{v}
	}
	return fmt.Sprintf("%s/%s?%s", m.BaseURL, "track", query.Encode()), nil
}

func (m *Client) makeRequest(method string, endpoint string, paramMap map[string]string) error {
	var (
		err error
		req *http.Request
		r   io.Reader
	)

	if endpoint == "" {
		return fmt.Errorf("endpoint missing")
	}

	endpoint = fmt.Sprintf("%s/%s", m.BaseURL, endpoint)

	if paramMap == nil {
		paramMap = map[string]string{}
	}

	params := url.Values{}
	for k, v := range paramMap {
		params[k] = []string{v}
	}

	switch method {
	case "GET":
		enc := params.Encode()
		if enc != "" {
			endpoint = endpoint + "?" + enc
		}
	case "POST":
		r = strings.NewReader(params.Encode())
	default:
		return fmt.Errorf("method not supported: %v", method)
	}

	req, err = http.NewRequest(method, endpoint, r)
	if err != nil {
		return err
	}

	resp, err := m.Client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	// The API documentation states that success will be reported with either "1" or "1\n".
	if strings.Trim(string(b), "\n") != "1" {
		return fmt.Errorf("request failed - %s", b)
	}
	return nil
}

func (m *Client) makeRequestWithData(method string, endpoint string, data Properties, as actionSource) error {
	b, err := json.Marshal(data)
	if err != nil {
		return err
	}

	params := map[string]string{
		"data": base64.StdEncoding.EncodeToString(b),
	}

	if as == sourceScript {
		params["ip"] = "0"
	}

	return m.makeRequest(method, endpoint, params)
}
<|endoftext|>