<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

// This file contains code for supporting local sockets for the Cloud SQL Proxy.

import (
	"bytes"
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/GoogleCloudPlatform/cloudsql-proxy/logging"
	"github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/fuse"
	"github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/proxy"
	"github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/util"
	sqladmin "google.golang.org/api/sqladmin/v1beta4"
)

// WatchInstances handles the lifecycle of local sockets used for proxying
// local connections.  Values received from the updates channel are
// interpretted as a comma-separated list of instances.  The set of sockets in
// 'dir' is the union of 'instances' and the most recent list from 'updates'.
func WatchInstances(dir string, cfgs []instanceConfig, updates <-chan string, cl *http.Client) (<-chan proxy.Conn, error) {
	ch := make(chan proxy.Conn, 1)

	// Instances specified statically (e.g. as flags to the binary) will always
	// be available. They are ignored if also returned by the GCE metadata since
	// the socket will already be open.
	staticInstances := make(map[string]net.Listener, len(cfgs))
	for _, v := range cfgs {
		l, err := listenInstance(ch, v)
		if err != nil {
			return nil, err
		}
		staticInstances[v.Instance] = l
	}

	if updates != nil {
		go watchInstancesLoop(dir, ch, updates, staticInstances, cl)
	}
	return ch, nil
}

func watchInstancesLoop(dir string, dst chan<- proxy.Conn, updates <-chan string, static map[string]net.Listener, cl *http.Client) {
	dynamicInstances := make(map[string]net.Listener)
	for instances := range updates {
		list, err := parseInstanceConfigs(dir, strings.Split(instances, ","), cl)
		if err != nil {
			logging.Errorf("%v", err)
		}

		stillOpen := make(map[string]net.Listener)
		for _, cfg := range list {
			instance := cfg.Instance

			// If the instance is specified in the static list don't do anything:
			// it's already open and should stay open forever.
			if _, ok := static[instance]; ok {
				continue
			}

			if l, ok := dynamicInstances[instance]; ok {
				delete(dynamicInstances, instance)
				stillOpen[instance] = l
				continue
			}

			l, err := listenInstance(dst, cfg)
			if err != nil {
				logging.Errorf("Couldn't open socket for %q: %v", instance, err)
				continue
			}
			stillOpen[instance] = l
		}

		// Any instance in dynamicInstances was not in the most recent metadata
		// update. Clean up those instances' sockets by closing them; note that
		// this does not affect any existing connections instance.
		for instance, listener := range dynamicInstances {
			logging.Infof("Closing socket for instance %v", instance)
			listener.Close()
		}

		dynamicInstances = stillOpen
	}

	for _, v := range static {
		if err := v.Close(); err != nil {
			logging.Errorf("Error closing %q: %v", v.Addr(), err)
		}
	}
	for _, v := range dynamicInstances {
		if err := v.Close(); err != nil {
			logging.Errorf("Error closing %q: %v", v.Addr(), err)
		}
	}
}

func remove(path string) {
	if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
		logging.Infof("Remove(%q) error: %v", path, err)
	}
}

// listenInstance starts listening on a new unix socket in dir to connect to the
// specified instance. New connections to this socket are sent to dst.
func listenInstance(dst chan<- proxy.Conn, cfg instanceConfig) (net.Listener, error) {
	unix := cfg.Network == "unix"
	if unix {
		remove(cfg.Address)
	}
	l, err := net.Listen(cfg.Network, cfg.Address)
	if err != nil {
		return nil, err
	}
	if unix {
		if err := os.Chmod(cfg.Address, 0777|os.ModeSocket); err != nil {
			logging.Errorf("couldn't update permissions for socket file %q: %v; other users may not be unable to connect", cfg.Address, err)
		}
	}

	go func() {
		for {
			start := time.Now()
			c, err := l.Accept()
			if err != nil {
				logging.Errorf("Error in accept for %q on %v: %v", cfg, cfg.Address, err)
				if nerr, ok := err.(net.Error); ok && nerr.Temporary() {
					d := 10*time.Millisecond - time.Since(start)
					if d > 0 {
						time.Sleep(d)
					}
					continue
				}
				l.Close()
				return
			}
			logging.Verbosef("New connection for %q", cfg.Instance)
			dst <- proxy.Conn{cfg.Instance, c}
		}
	}()

	logging.Infof("Listening on %s for %s", cfg.Address, cfg.Instance)
	return l, nil
}

type instanceConfig struct {
	Instance         string
	Network, Address string
}

// loopbackForNet maps a network (e.g. tcp6) to the loopback address for that
// network. It is updated during the initialization of validNets to include a
// valid loopback address for "tcp".
var loopbackForNet = map[string]string{
	"tcp4": "127.0.0.1",
	"tcp6": "[::1]",
}

// validNets tracks the networks that are valid for this platform and machine.
var validNets = func() map[string]bool {
	m := map[string]bool{
		"unix": runtime.GOOS != "windows",
	}

	anyTCP := false
	for _, n := range []string{"tcp4", "tcp6"} {
		addr, ok := loopbackForNet[n]
		if !ok {
			// This is effectively a compile-time error.
			panic(fmt.Sprintf("no loopback address found for %v", n))
		}
		// Open any port to see if the net is valid.
		x, err := net.Listen(n, addr+":0")
		if err != nil {
			// Error is too verbose to be useful.
			continue
		}
		x.Close()
		m[n] = true

		if !anyTCP {
			anyTCP = true
			// Set the loopback value for generic tcp if it hasn't already been
			// set. (If both tcp4/tcp6 are supported the first one in the list
			// (tcp4's 127.0.0.1) is used.
			loopbackForNet["tcp"] = addr
		}
	}
	if anyTCP {
		m["tcp"] = true
	}
	return m
}()

func parseInstanceConfig(dir, instance string, cl *http.Client) (instanceConfig, error) {
	var ret instanceConfig
	eq := strings.Index(instance, "=")
	if eq != -1 {
		spl := strings.SplitN(instance[eq+1:], ":", 3)
		ret.Instance = instance[:eq]

		switch len(spl) {
		default:
			return ret, fmt.Errorf("invalid %q: expected 'project:instance=tcp:port'", instance)
		case 2:
			// No "host" part of the address. Be safe and assume that they want a
			// loopback address.
			ret.Network = spl[0]
			addr, ok := loopbackForNet[spl[0]]
			if !ok {
				return ret, fmt.Errorf("invalid %q: unrecognized network %v", instance, spl[0])
			}
			ret.Address = fmt.Sprintf("%s:%s", addr, spl[1])
		case 3:
			// User provided a host and port; use that.
			ret.Network = spl[0]
			ret.Address = fmt.Sprintf("%s:%s", spl[1], spl[2])
		}
	} else {
		sql, err := sqladmin.New(cl)
		if err != nil {
			return instanceConfig{}, err
		}

		ret.Instance = instance
		// Default to unix socket.
		ret.Network = "unix"

		proj, _, name := util.SplitName(instance)
		if proj == "" || name == "" {
			return instanceConfig{}, fmt.Errorf("invalid instance name: must be in the form `project:region:instance-name`; invalid name was %q", instance)
		}
		// We allow people to omit the region due to historical reasons. It'll
		// fail later in the code if this isn't allowed, so just assume it's
		// allowed until we actually need the region in this API call.
		in, err := sql.Instances.Get(proj, name).Do()
		if err != nil {
			return instanceConfig{}, err
		}
		if strings.HasPrefix(strings.ToLower(in.DatabaseVersion), "postgres") {
			path := filepath.Join(dir, instance)
			if err := os.MkdirAll(path, 0755); err != nil {
				return instanceConfig{}, err
			}
			ret.Address = filepath.Join(path, ".s.PGSQL.5432")
		} else {
			ret.Address = filepath.Join(dir, instance)
		}
	}

	if !validNets[ret.Network] {
		return ret, fmt.Errorf("invalid %q: unsupported network: %v", instance, ret.Network)
	}
	return ret, nil
}

// parseInstanceConfigs calls parseInstanceConfig for each instance in the
// provided slice, collecting errors along the way. There may be valid
// instanceConfigs returned even if there's an error.
func parseInstanceConfigs(dir string, instances []string, cl *http.Client) ([]instanceConfig, error) {
	errs := new(bytes.Buffer)
	var cfg []instanceConfig
	for _, v := range instances {
		if v == "" {
			continue
		}
		if c, err := parseInstanceConfig(dir, v, cl); err != nil {
			fmt.Fprintf(errs, "\n\t%v", err)
		} else {
			cfg = append(cfg, c)
		}
	}

	var err error
	if errs.Len() > 0 {
		err = fmt.Errorf("errors parsing config:%s", errs)
	}
	return cfg, err
}

// CreateInstanceConfigs verifies that the parameters passed to it are valid
// for the proxy for the platform and system and then returns a slice of valid
// instanceConfig.
func CreateInstanceConfigs(dir string, useFuse bool, instances []string, instancesSrc string, cl *http.Client) ([]instanceConfig, error) {
	if useFuse && !fuse.Supported() {
		return nil, errors.New("FUSE not supported on this system")
	}

	cfgs, err := parseInstanceConfigs(dir, instances, cl)
	if err != nil {
		logging.Errorf("%v", err)
	}

	if dir == "" {
		// Reasons to set '-dir':
		//    - Using -fuse
		//    - Using the metadata to get a list of instances
		//    - Having an instance that uses a 'unix' network
		if useFuse {
			return nil, errors.New("must set -dir because -fuse was set")
		} else if instancesSrc != "" {
			return nil, errors.New("must set -dir because -instances_metadata was set")
		} else {
			for _, v := range cfgs {
				if v.Network == "unix" {
					return nil, fmt.Errorf("must set -dir: using a unix socket for %v", v.Instance)
				}
			}
		}
		// Otherwise it's safe to not set -dir
	}

	if useFuse {
		if len(instances) != 0 || instancesSrc != "" {
			return nil, errors.New("-fuse is not compatible with -projects, -instances, or -instances_metadata")
		}
		return nil, nil
	}
	// FUSE disabled.
	if len(instances) == 0 && instancesSrc == "" {
		if fuse.Supported() {
			return nil, errors.New("must specify -projects, -fuse, or -instances")
		}
		return nil, errors.New("must specify -projects or -instances")
	}
	return cfgs, nil
}
<commit_msg>Update proxy.go<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

// This file contains code for supporting local sockets for the Cloud SQL Proxy.

import (
	"bytes"
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/GoogleCloudPlatform/cloudsql-proxy/logging"
	"github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/fuse"
	"github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/proxy"
	"github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/util"
	sqladmin "google.golang.org/api/sqladmin/v1beta4"
)

// WatchInstances handles the lifecycle of local sockets used for proxying
// local connections.  Values received from the updates channel are
// interpretted as a comma-separated list of instances.  The set of sockets in
// 'dir' is the union of 'instances' and the most recent list from 'updates'.
func WatchInstances(dir string, cfgs []instanceConfig, updates <-chan string, cl *http.Client) (<-chan proxy.Conn, error) {
	ch := make(chan proxy.Conn, 1)

	// Instances specified statically (e.g. as flags to the binary) will always
	// be available. They are ignored if also returned by the GCE metadata since
	// the socket will already be open.
	staticInstances := make(map[string]net.Listener, len(cfgs))
	for _, v := range cfgs {
		l, err := listenInstance(ch, v)
		if err != nil {
			return nil, err
		}
		staticInstances[v.Instance] = l
	}

	if updates != nil {
		go watchInstancesLoop(dir, ch, updates, staticInstances, cl)
	}
	return ch, nil
}

func watchInstancesLoop(dir string, dst chan<- proxy.Conn, updates <-chan string, static map[string]net.Listener, cl *http.Client) {
	dynamicInstances := make(map[string]net.Listener)
	for instances := range updates {
		list, err := parseInstanceConfigs(dir, strings.Split(instances, ","), cl)
		if err != nil {
			logging.Errorf("%v", err)
		}

		stillOpen := make(map[string]net.Listener)
		for _, cfg := range list {
			instance := cfg.Instance

			// If the instance is specified in the static list don't do anything:
			// it's already open and should stay open forever.
			if _, ok := static[instance]; ok {
				continue
			}

			if l, ok := dynamicInstances[instance]; ok {
				delete(dynamicInstances, instance)
				stillOpen[instance] = l
				continue
			}

			l, err := listenInstance(dst, cfg)
			if err != nil {
				logging.Errorf("Couldn't open socket for %q: %v", instance, err)
				continue
			}
			stillOpen[instance] = l
		}

		// Any instance in dynamicInstances was not in the most recent metadata
		// update. Clean up those instances' sockets by closing them; note that
		// this does not affect any existing connections instance.
		for instance, listener := range dynamicInstances {
			logging.Infof("Closing socket for instance %v", instance)
			listener.Close()
		}

		dynamicInstances = stillOpen
	}

	for _, v := range static {
		if err := v.Close(); err != nil {
			logging.Errorf("Error closing %q: %v", v.Addr(), err)
		}
	}
	for _, v := range dynamicInstances {
		if err := v.Close(); err != nil {
			logging.Errorf("Error closing %q: %v", v.Addr(), err)
		}
	}
}

func remove(path string) {
	if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
		logging.Infof("Remove(%q) error: %v", path, err)
	}
}

// listenInstance starts listening on a new unix socket in dir to connect to the
// specified instance. New connections to this socket are sent to dst.
func listenInstance(dst chan<- proxy.Conn, cfg instanceConfig) (net.Listener, error) {
	unix := cfg.Network == "unix"
	if unix {
		remove(cfg.Address)
	}
	l, err := net.Listen(cfg.Network, cfg.Address)
	if err != nil {
		return nil, err
	}
	if unix {
		if err := os.Chmod(cfg.Address, 0777|os.ModeSocket); err != nil {
			logging.Errorf("couldn't update permissions for socket file %q: %v; other users may not be unable to connect", cfg.Address, err)
		}
	}

	go func() {
		for {
			start := time.Now()
			c, err := l.Accept()
			if err != nil {
				logging.Errorf("Error in accept for %q on %v: %v", cfg, cfg.Address, err)
				if nerr, ok := err.(net.Error); ok && nerr.Temporary() {
					d := 10*time.Millisecond - time.Since(start)
					if d > 0 {
						time.Sleep(d)
					}
					continue
				}
				l.Close()
				return
			}
			logging.Verbosef("New connection for %q", cfg.Instance)
			dst <- proxy.Conn{cfg.Instance, c}
		}
	}()

	logging.Infof("Listening on %s for %s", cfg.Address, cfg.Instance)
	return l, nil
}

type instanceConfig struct {
	Instance         string
	Network, Address string
}

// loopbackForNet maps a network (e.g. tcp6) to the loopback address for that
// network. It is updated during the initialization of validNets to include a
// valid loopback address for "tcp".
var loopbackForNet = map[string]string{
	"tcp4": "127.0.0.1",
	"tcp6": "[::1]",
}

// validNets tracks the networks that are valid for this platform and machine.
var validNets = func() map[string]bool {
	m := map[string]bool{
		"unix": runtime.GOOS != "windows",
	}

	anyTCP := false
	for _, n := range []string{"tcp4", "tcp6"} {
		addr, ok := loopbackForNet[n]
		if !ok {
			// This is effectively a compile-time error.
			panic(fmt.Sprintf("no loopback address found for %v", n))
		}
		// Open any port to see if the net is valid.
		x, err := net.Listen(n, addr+":0")
		if err != nil {
			// Error is too verbose to be useful.
			continue
		}
		x.Close()
		m[n] = true

		if !anyTCP {
			anyTCP = true
			// Set the loopback value for generic tcp if it hasn't already been
			// set. (If both tcp4/tcp6 are supported the first one in the list
			// (tcp4's 127.0.0.1) is used.
			loopbackForNet["tcp"] = addr
		}
	}
	if anyTCP {
		m["tcp"] = true
	}
	return m
}()

func parseInstanceConfig(dir, instance string, cl *http.Client) (instanceConfig, error) {
	var ret instanceConfig
	eq := strings.Index(instance, "=")
	if eq != -1 {
		spl := strings.SplitN(instance[eq+1:], ":", 3)
		ret.Instance = instance[:eq]

		switch len(spl) {
		default:
			return ret, fmt.Errorf("invalid %q: expected 'project:instance=tcp:port'", instance)
		case 2:
			// No "host" part of the address. Be safe and assume that they want a
			// loopback address.
			ret.Network = spl[0]
			addr, ok := loopbackForNet[spl[0]]
			if !ok {
				return ret, fmt.Errorf("invalid %q: unrecognized network %v", instance, spl[0])
			}
			ret.Address = fmt.Sprintf("%s:%s", addr, spl[1])
		case 3:
			// User provided a host and port; use that.
			ret.Network = spl[0]
			ret.Address = fmt.Sprintf("%s:%s", spl[1], spl[2])
		}
	} else {
		sql, err := sqladmin.New(cl)
		if err != nil {
			return instanceConfig{}, err
		}

		ret.Instance = instance
		// Default to unix socket.
		ret.Network = "unix"

		proj, _, name := util.SplitName(instance)
		if proj == "" || name == "" {
			return instanceConfig{}, fmt.Errorf("invalid instance name: must be in the form `project:region:instance-name`; invalid name was %q", instance)
		}
		// We allow people to omit the region due to historical reasons. It'll
		// fail later in the code if this isn't allowed, so just assume it's
		// allowed until we actually need the region in this API call.
		in, err := sql.Instances.Get(proj, name).Do()
		if err != nil {
			return instanceConfig{}, err
		}
		if strings.HasPrefix(strings.ToLower(in.DatabaseVersion), "postgres") {
			path := filepath.Join(dir, instance)
			if err := os.MkdirAll(path, 0755); err != nil {
				return instanceConfig{}, err
			}
			ret.Address = filepath.Join(path, ".s.PGSQL.5432")
		} else {
			ret.Address = filepath.Join(dir, instance)
		}
	}

	if !validNets[ret.Network] {
		return ret, fmt.Errorf("invalid %q: unsupported network: %v", instance, ret.Network)
	}
	return ret, nil
}

// parseInstanceConfigs calls parseInstanceConfig for each instance in the
// provided slice, collecting errors along the way. There may be valid
// instanceConfigs returned even if there's an error.
func parseInstanceConfigs(dir string, instances []string, cl *http.Client) ([]instanceConfig, error) {
	errs := new(bytes.Buffer)
	var cfg []instanceConfig
	for _, v := range instances {
		if v == "" {
			continue
		}
		if c, err := parseInstanceConfig(dir, v, cl); err != nil {
			fmt.Fprintf(errs, "\n\t%v", err)
		} else {
			cfg = append(cfg, c)
		}
	}

	var err error
	if errs.Len() > 0 {
		err = fmt.Errorf("errors parsing config:%s", errs)
	}
	return cfg, err
}

// CreateInstanceConfigs verifies that the parameters passed to it are valid
// for the proxy for the platform and system and then returns a slice of valid
// instanceConfig.
func CreateInstanceConfigs(dir string, useFuse bool, instances []string, instancesSrc string, cl *http.Client) ([]instanceConfig, error) {
	if useFuse && !fuse.Supported() {
		return nil, errors.New("FUSE not supported on this system")
	}

	cfgs, err := parseInstanceConfigs(dir, instances, cl)
	if err != nil {
		logging.Errorf("%v", err)
	}

	if dir == "" {
		// Reasons to set '-dir':
		//    - Using -fuse
		//    - Using the metadata to get a list of instances
		//    - Having an instance that uses a 'unix' network
		if useFuse {
			return nil, errors.New("must set -dir because -fuse was set")
		} else if instancesSrc != "" {
			return nil, errors.New("must set -dir because -instances_metadata was set")
		} else {
			for _, v := range cfgs {
				if v.Network == "unix" {
					return nil, fmt.Errorf("must set -dir: using a unix socket for %v", v.Instance)
				}
			}
		}
		// Otherwise it's safe to not set -dir
	}

	if useFuse {
		if len(instances) != 0 || instancesSrc != "" {
			return nil, errors.New("-fuse is not compatible with -projects, -instances, or -instances_metadata")
		}
		return nil, nil
	}
	// FUSE disabled.
        if len(instances) == 0 && instancesSrc == "" {
                var flags string
                if fuse.Supported() {
                        flags = "-projects, -fuse, or -instances"
                } else {
                        flags = "-projects or -instances"
                }

                errStr := fmt.Sprintf("no instance selected because none of %s is specified", flags)
                if gcloudErrStr != "" {
                        errStr = fmt.Sprintf("%s and %s", errStr, gcloudErrStr)
                }
                return nil, errors.New(errStr)
        }
	return cfgs, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package app

import (
	"fmt"
	"os"
	"time"

	"github.com/golang/glog"
	"github.com/kubernetes-sigs/kube-batch/cmd/kube-batch/app/options"
	"github.com/kubernetes-sigs/kube-batch/pkg/scheduler"
	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/uuid"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/scheme"
	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
	"k8s.io/client-go/rest"
	restclient "k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/client-go/tools/leaderelection"
	"k8s.io/client-go/tools/leaderelection/resourcelock"
	"k8s.io/client-go/tools/record"
)

const (
	leaseDuration = 15 * time.Second
	renewDeadline = 10 * time.Second
	retryPeriod   = 5 * time.Second
)

func buildConfig(master, kubeconfig string) (*rest.Config, error) {
	if master != "" || kubeconfig != "" {
		return clientcmd.BuildConfigFromFlags(master, kubeconfig)
	}
	return rest.InClusterConfig()
}

func Run(opt *options.ServerOption) error {
	config, err := buildConfig(opt.Master, opt.Kubeconfig)
	if err != nil {
		return err
	}

	neverStop := make(chan struct{})

	// Start policy controller to allocate resources.
	sched, err := scheduler.NewScheduler(config, opt.SchedulerName, opt.SchedulerConf, opt.NamespaceAsQueue)
	if err != nil {
		panic(err)
	}

	run := func(stopCh <-chan struct{}) {
		sched.Run(stopCh)
		<-stopCh
	}

	if !opt.EnableLeaderElection {
		run(neverStop)
		return fmt.Errorf("finished without leader elect")
	}

	leaderElectionClient, err := clientset.NewForConfig(restclient.AddUserAgent(config, "leader-election"))
	if err != nil {
		return err
	}

	// Prepare event clients.
	broadcaster := record.NewBroadcaster()
	broadcaster.StartRecordingToSink(&corev1.EventSinkImpl{Interface: leaderElectionClient.CoreV1().Events(opt.LockObjectNamespace)})
	eventRecorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "kar-scheduler"})

	hostname, err := os.Hostname()
	if err != nil {
		return fmt.Errorf("unable to get hostname: %v", err)
	}
	// add a uniquifier so that two processes on the same host don't accidentally both become active
	id := hostname + "_" + string(uuid.NewUUID())

	rl, err := resourcelock.New(resourcelock.ConfigMapsResourceLock,
		opt.LockObjectNamespace,
		"kar-scheduler",
		leaderElectionClient.CoreV1(),
		resourcelock.ResourceLockConfig{
			Identity:      id,
			EventRecorder: eventRecorder,
		})
	if err != nil {
		return fmt.Errorf("couldn't create resource lock: %v", err)
	}

	leaderelection.RunOrDie(leaderelection.LeaderElectionConfig{
		Lock:          rl,
		LeaseDuration: leaseDuration,
		RenewDeadline: renewDeadline,
		RetryPeriod:   retryPeriod,
		Callbacks: leaderelection.LeaderCallbacks{
			OnStartedLeading: run,
			OnStoppedLeading: func() {
				glog.Fatalf("leaderelection lost")
			},
		},
	})
	return fmt.Errorf("lost lease")
}
<commit_msg>change kar-scheduler to kube-batch<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package app

import (
	"fmt"
	"os"
	"time"

	"github.com/golang/glog"
	"github.com/kubernetes-sigs/kube-batch/cmd/kube-batch/app/options"
	"github.com/kubernetes-sigs/kube-batch/pkg/scheduler"
	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/uuid"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/scheme"
	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
	"k8s.io/client-go/rest"
	restclient "k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/client-go/tools/leaderelection"
	"k8s.io/client-go/tools/leaderelection/resourcelock"
	"k8s.io/client-go/tools/record"
)

const (
	leaseDuration = 15 * time.Second
	renewDeadline = 10 * time.Second
	retryPeriod   = 5 * time.Second
)

func buildConfig(master, kubeconfig string) (*rest.Config, error) {
	if master != "" || kubeconfig != "" {
		return clientcmd.BuildConfigFromFlags(master, kubeconfig)
	}
	return rest.InClusterConfig()
}

func Run(opt *options.ServerOption) error {
	config, err := buildConfig(opt.Master, opt.Kubeconfig)
	if err != nil {
		return err
	}

	neverStop := make(chan struct{})

	// Start policy controller to allocate resources.
	sched, err := scheduler.NewScheduler(config, opt.SchedulerName, opt.SchedulerConf, opt.NamespaceAsQueue)
	if err != nil {
		panic(err)
	}

	run := func(stopCh <-chan struct{}) {
		sched.Run(stopCh)
		<-stopCh
	}

	if !opt.EnableLeaderElection {
		run(neverStop)
		return fmt.Errorf("finished without leader elect")
	}

	leaderElectionClient, err := clientset.NewForConfig(restclient.AddUserAgent(config, "leader-election"))
	if err != nil {
		return err
	}

	// Prepare event clients.
	broadcaster := record.NewBroadcaster()
	broadcaster.StartRecordingToSink(&corev1.EventSinkImpl{Interface: leaderElectionClient.CoreV1().Events(opt.LockObjectNamespace)})
	eventRecorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "kube-batch"})

	hostname, err := os.Hostname()
	if err != nil {
		return fmt.Errorf("unable to get hostname: %v", err)
	}
	// add a uniquifier so that two processes on the same host don't accidentally both become active
	id := hostname + "_" + string(uuid.NewUUID())

	rl, err := resourcelock.New(resourcelock.ConfigMapsResourceLock,
		opt.LockObjectNamespace,
		"kube-batch",
		leaderElectionClient.CoreV1(),
		resourcelock.ResourceLockConfig{
			Identity:      id,
			EventRecorder: eventRecorder,
		})
	if err != nil {
		return fmt.Errorf("couldn't create resource lock: %v", err)
	}

	leaderelection.RunOrDie(leaderelection.LeaderElectionConfig{
		Lock:          rl,
		LeaseDuration: leaseDuration,
		RenewDeadline: renewDeadline,
		RetryPeriod:   retryPeriod,
		Callbacks: leaderelection.LeaderCallbacks{
			OnStartedLeading: run,
			OnStoppedLeading: func() {
				glog.Fatalf("leaderelection lost")
			},
		},
	})
	return fmt.Errorf("lost lease")
}
<|endoftext|>


<commit_before>package testrunner

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"time"

	"code.cloudfoundry.org/rep/cmd/rep/config"

	"github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
)

type Runner struct {
	binPath           string
	Session           *gexec.Session
	StartCheck        string
	repConfig         config.RepConfig
	repConfigFilePath string
}
type Duration time.Duration

func New(binPath string, repConfig config.RepConfig) *Runner {
	return &Runner{
		binPath:    binPath,
		StartCheck: "rep.started",
		repConfig:  repConfig,
	}
}

func (d *Duration) MarshalJSON() ([]byte, error) {
	t := time.Duration(*d)
	return []byte(fmt.Sprintf(`"%s"`, t.String())), nil
}

func (r *Runner) Start() {
	if r.Session != nil && r.Session.ExitCode() == -1 {
		panic("starting more than one rep!!!")
	}

	f, err := ioutil.TempFile("", "rep")
	Expect(err).NotTo(HaveOccurred())

	encoder := json.NewEncoder(f)

	err = encoder.Encode(r.repConfig)
	Expect(err).NotTo(HaveOccurred())

	r.repConfigFilePath = f.Name()

	args := []string{
		"--config", r.repConfigFilePath,
	}

	repSession, err := gexec.Start(
		exec.Command(
			r.binPath,
			args...,
		),
		gexec.NewPrefixedWriter("\x1b[32m[o]\x1b[32m[rep]\x1b[0m ", ginkgo.GinkgoWriter),
		gexec.NewPrefixedWriter("\x1b[91m[e]\x1b[32m[rep]\x1b[0m ", ginkgo.GinkgoWriter),
	)

	Expect(err).NotTo(HaveOccurred())
	r.Session = repSession

	Eventually(r.Session.Buffer(), 2).Should(gbytes.Say(r.StartCheck))
}

func (r *Runner) Stop() {
	err := os.RemoveAll(r.repConfigFilePath)
	Expect(err).NotTo(HaveOccurred())

	if r.Session != nil {
		r.Session.Interrupt().Wait(5 * time.Second)
	}
}

func (r *Runner) KillWithFire() {
	err := os.RemoveAll(r.repConfigFilePath)
	Expect(err).NotTo(HaveOccurred())

	if r.Session != nil {
		r.Session.Kill().Wait(5 * time.Second)
	}
}
<commit_msg>Delete unused type Duration<commit_after>package testrunner

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"os/exec"
	"time"

	"code.cloudfoundry.org/rep/cmd/rep/config"

	"github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
)

type Runner struct {
	binPath           string
	Session           *gexec.Session
	StartCheck        string
	repConfig         config.RepConfig
	repConfigFilePath string
}

func New(binPath string, repConfig config.RepConfig) *Runner {
	return &Runner{
		binPath:    binPath,
		StartCheck: "rep.started",
		repConfig:  repConfig,
	}
}

func (r *Runner) Start() {
	if r.Session != nil && r.Session.ExitCode() == -1 {
		panic("starting more than one rep!!!")
	}

	f, err := ioutil.TempFile("", "rep")
	Expect(err).NotTo(HaveOccurred())

	encoder := json.NewEncoder(f)

	err = encoder.Encode(r.repConfig)
	Expect(err).NotTo(HaveOccurred())

	r.repConfigFilePath = f.Name()

	args := []string{
		"--config", r.repConfigFilePath,
	}

	repSession, err := gexec.Start(
		exec.Command(
			r.binPath,
			args...,
		),
		gexec.NewPrefixedWriter("\x1b[32m[o]\x1b[32m[rep]\x1b[0m ", ginkgo.GinkgoWriter),
		gexec.NewPrefixedWriter("\x1b[91m[e]\x1b[32m[rep]\x1b[0m ", ginkgo.GinkgoWriter),
	)

	Expect(err).NotTo(HaveOccurred())
	r.Session = repSession

	Eventually(r.Session.Buffer(), 2).Should(gbytes.Say(r.StartCheck))
}

func (r *Runner) Stop() {
	err := os.RemoveAll(r.repConfigFilePath)
	Expect(err).NotTo(HaveOccurred())

	if r.Session != nil {
		r.Session.Interrupt().Wait(5 * time.Second)
	}
}

func (r *Runner) KillWithFire() {
	err := os.RemoveAll(r.repConfigFilePath)
	Expect(err).NotTo(HaveOccurred())

	if r.Session != nil {
		r.Session.Kill().Wait(5 * time.Second)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"strings"
	"syscall"

	"github.com/Masterminds/semver/v3"
	"github.com/go-semantic-release/semantic-release/v2/pkg/config"
	"github.com/go-semantic-release/semantic-release/v2/pkg/generator"
	"github.com/go-semantic-release/semantic-release/v2/pkg/hooks"
	"github.com/go-semantic-release/semantic-release/v2/pkg/plugin/manager"
	"github.com/go-semantic-release/semantic-release/v2/pkg/provider"
	"github.com/go-semantic-release/semantic-release/v2/pkg/semrel"
	"github.com/spf13/cobra"
)

// SRVERSION is the semantic-release version (added at compile time)
var SRVERSION string

var exitHandler func()

func errorHandler(logger *log.Logger) func(error, ...int) {
	return func(err error, exitCode ...int) {
		if err != nil {
			logger.Println(err)
			if exitHandler != nil {
				exitHandler()
			}
			if len(exitCode) == 1 {
				os.Exit(exitCode[0])
				return
			}
			os.Exit(1)
		}
	}
}

func main() {
	cmd := &cobra.Command{
		Use:     "semantic-release",
		Short:   "semantic-release - fully automated package/module/image publishing",
		Run:     cliHandler,
		Version: SRVERSION,
	}

	err := config.InitConfig(cmd)
	if err != nil {
		fmt.Printf("\nConfig error: %s\n", err.Error())
		os.Exit(1)
		return
	}
	err = cmd.Execute()
	if err != nil {
		fmt.Printf("\n%s\n", err.Error())
		os.Exit(1)
	}
}

func cliHandler(cmd *cobra.Command, args []string) {
	logger := log.New(os.Stderr, "[go-semantic-release]: ", 0)
	exitIfError := errorHandler(logger)

	logger.Printf("version: %s\n", SRVERSION)

	conf, err := config.NewConfig(cmd)
	exitIfError(err)

	pluginManager, err := manager.New(conf)
	exitIfError(err)
	exitHandler = func() {
		logger.Println("stopping plugins...")
		pluginManager.Stop()
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
	go func() {
		<-c
		exitIfError(errors.New("terminating..."))
	}()

	if conf.DownloadPlugins {
		exitIfError(pluginManager.FetchAllPlugins())
		logger.Println("all plugins are downloaded")
		os.Exit(0)
	}

	ci, err := pluginManager.GetCICondition()
	exitIfError(err)
	logger.Printf("ci-condition plugin: %s@%s\n", ci.Name(), ci.Version())

	prov, err := pluginManager.GetProvider()
	exitIfError(err)
	logger.Printf("provider plugin: %s@%s\n", prov.Name(), prov.Version())

	if conf.ProviderOpts["token"] == "" {
		conf.ProviderOpts["token"] = conf.Token
	}
	err = prov.Init(conf.ProviderOpts)
	exitIfError(err)

	logger.Println("getting default branch...")
	repoInfo, err := prov.GetInfo()
	exitIfError(err)
	logger.Println("found default branch: " + repoInfo.DefaultBranch)
	if repoInfo.Private {
		logger.Println("repo is private")
	}

	currentBranch := ci.GetCurrentBranch()
	if currentBranch == "" {
		exitIfError(fmt.Errorf("current branch not found"))
	}
	logger.Println("found current branch: " + currentBranch)

	if conf.MaintainedVersion != "" && currentBranch == repoInfo.DefaultBranch {
		exitIfError(fmt.Errorf("maintained version not allowed on default branch"))
	}

	if conf.MaintainedVersion != "" {
		logger.Println("found maintained version: " + conf.MaintainedVersion)
		repoInfo.DefaultBranch = "*"
	}

	currentSha := ci.GetCurrentSHA()
	logger.Println("found current sha: " + currentSha)

	hooksExecutor, err := pluginManager.GetChainedHooksExecutor()
	exitIfError(err)

	hooksNames := hooksExecutor.GetNameVersionPairs()
	if len(hooksNames) > 0 {
		logger.Printf("hooks plugins: %s\n", strings.Join(hooksNames, ", "))
	}

	exitIfError(hooksExecutor.Init(conf.HooksOpts))

	if !conf.NoCI {
		logger.Println("running CI condition...")
		conditionConfig := map[string]string{
			"token":         conf.Token,
			"defaultBranch": repoInfo.DefaultBranch,
			"private":       fmt.Sprintf("%t", repoInfo.Private),
		}
		for k, v := range conf.CIConditionOpts {
			conditionConfig[k] = v
		}
		err = ci.RunCondition(conditionConfig)
		if err != nil {
			herr := hooksExecutor.NoRelease(&hooks.NoReleaseConfig{
				Reason:  hooks.NoReleaseReason_CONDITION,
				Message: err.Error(),
			})
			if herr != nil {
				logger.Printf("there was an error executing the hooks plugins: %s", herr.Error())
			}
			exitIfError(err, 66)
		}

	}

	logger.Println("getting latest release...")
	matchRegex := ""
	match := strings.TrimSpace(conf.Match)
	if match != "" {
		logger.Printf("getting latest release matching %s...", match)
		matchRegex = "^" + match
	}
	releases, err := prov.GetReleases(matchRegex)
	exitIfError(err)
	release, err := semrel.GetLatestReleaseFromReleases(releases, conf.MaintainedVersion)
	exitIfError(err)
	logger.Println("found version: " + release.Version)

	if strings.Contains(conf.MaintainedVersion, "-") && semver.MustParse(release.Version).Prerelease() == "" {
		exitIfError(fmt.Errorf("no pre-release for this version possible"))
	}

	logger.Println("getting commits...")
	rawCommits, err := prov.GetCommits(release.SHA, currentSha)
	exitIfError(err)

	logger.Println("analyzing commits...")
	commitAnalyzer, err := pluginManager.GetCommitAnalyzer()
	exitIfError(err)
	logger.Printf("commit-analyzer plugin: %s@%s\n", commitAnalyzer.Name(), commitAnalyzer.Version())
	exitIfError(commitAnalyzer.Init(conf.ChangelogGeneratorOpts))

	commits := commitAnalyzer.Analyze(rawCommits)

	logger.Println("calculating new version...")
	newVer := semrel.GetNewVersion(conf, commits, release)
	if newVer == "" {
		herr := hooksExecutor.NoRelease(&hooks.NoReleaseConfig{
			Reason:  hooks.NoReleaseReason_NO_CHANGE,
			Message: "",
		})
		if herr != nil {
			logger.Printf("there was an error executing the hooks plugins: %s", herr.Error())
		}
		errNoChange := errors.New("no change")
		if conf.AllowNoChanges {
			exitIfError(errNoChange, 0)
		} else {
			exitIfError(errNoChange, 65)
		}
	}
	logger.Println("new version: " + newVer)

	logger.Println("generating changelog...")
	changelogGenerator, err := pluginManager.GetChangelogGenerator()
	exitIfError(err)
	logger.Printf("changelog-generator plugin: %s@%s\n", changelogGenerator.Name(), changelogGenerator.Version())
	exitIfError(changelogGenerator.Init(conf.ChangelogGeneratorOpts))

	changelogRes := changelogGenerator.Generate(&generator.ChangelogGeneratorConfig{
		Commits:       commits,
		LatestRelease: release,
		NewVersion:    newVer,
	})
	if conf.Changelog != "" {
		oldFile := make([]byte, 0)
		if conf.PrependChangelog {
			oldFileData, err := ioutil.ReadFile(conf.Changelog)
			if err == nil {
				oldFile = append([]byte("\n"), oldFileData...)
			}
		}
		changelogData := append([]byte(changelogRes), oldFile...)
		exitIfError(ioutil.WriteFile(conf.Changelog, changelogData, 0644))
	}

	if conf.Dry {
		if conf.VersionFile {
			exitIfError(ioutil.WriteFile(".version-unreleased", []byte(newVer), 0644))
		}
		exitIfError(errors.New("DRY RUN: no release was created"), 0)
	}

	logger.Println("creating release...")
	newRelease := &provider.CreateReleaseConfig{
		Changelog:  changelogRes,
		NewVersion: newVer,
		Prerelease: conf.Prerelease,
		Branch:     currentBranch,
		SHA:        currentSha,
	}
	exitIfError(prov.CreateRelease(newRelease))

	if conf.Ghr {
		exitIfError(ioutil.WriteFile(".ghr", []byte(fmt.Sprintf("-u %s -r %s v%s", repoInfo.Owner, repoInfo.Repo, newVer)), 0644))
	}

	if conf.VersionFile {
		exitIfError(ioutil.WriteFile(".version", []byte(newVer), 0644))
	}

	if len(conf.UpdateFiles) > 0 {
		logger.Println("updating files...")
		updater, err := pluginManager.GetChainedUpdater()
		exitIfError(err)
		logger.Printf("files-updater plugins: %s\n", strings.Join(updater.GetNameVersionPairs(), ", "))
		exitIfError(updater.Init(conf.FilesUpdaterOpts))

		for _, f := range conf.UpdateFiles {
			exitIfError(updater.Apply(f, newVer))
		}
	}

	herr := hooksExecutor.Success(&hooks.SuccessHookConfig{
		Commits:     commits,
		PrevRelease: release,
		NewRelease: &semrel.Release{
			SHA:     currentSha,
			Version: newVer,
		},
		Changelog: changelogRes,
		RepoInfo:  repoInfo,
	})

	if herr != nil {
		logger.Printf("there was an error executing the hooks plugins: %s", herr.Error())
	}

	logger.Println("done.")
}
<commit_msg>fix: stop plugins after successful run<commit_after>package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"strings"
	"syscall"

	"github.com/Masterminds/semver/v3"
	"github.com/go-semantic-release/semantic-release/v2/pkg/config"
	"github.com/go-semantic-release/semantic-release/v2/pkg/generator"
	"github.com/go-semantic-release/semantic-release/v2/pkg/hooks"
	"github.com/go-semantic-release/semantic-release/v2/pkg/plugin/manager"
	"github.com/go-semantic-release/semantic-release/v2/pkg/provider"
	"github.com/go-semantic-release/semantic-release/v2/pkg/semrel"
	"github.com/spf13/cobra"
)

// SRVERSION is the semantic-release version (added at compile time)
var SRVERSION string

var exitHandler func()

func errorHandler(logger *log.Logger) func(error, ...int) {
	return func(err error, exitCode ...int) {
		if err != nil {
			logger.Println(err)
			if exitHandler != nil {
				exitHandler()
			}
			if len(exitCode) == 1 {
				os.Exit(exitCode[0])
				return
			}
			os.Exit(1)
		}
	}
}

func main() {
	cmd := &cobra.Command{
		Use:     "semantic-release",
		Short:   "semantic-release - fully automated package/module/image publishing",
		Run:     cliHandler,
		Version: SRVERSION,
	}

	err := config.InitConfig(cmd)
	if err != nil {
		fmt.Printf("\nConfig error: %s\n", err.Error())
		os.Exit(1)
		return
	}
	err = cmd.Execute()
	if err != nil {
		fmt.Printf("\n%s\n", err.Error())
		os.Exit(1)
	}
}

func cliHandler(cmd *cobra.Command, args []string) {
	logger := log.New(os.Stderr, "[go-semantic-release]: ", 0)
	exitIfError := errorHandler(logger)

	logger.Printf("version: %s\n", SRVERSION)

	conf, err := config.NewConfig(cmd)
	exitIfError(err)

	pluginManager, err := manager.New(conf)
	exitIfError(err)
	exitHandler = func() {
		logger.Println("stopping plugins...")
		pluginManager.Stop()
	}
	defer exitHandler()

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
	go func() {
		<-c
		exitIfError(errors.New("terminating..."))
	}()

	if conf.DownloadPlugins {
		exitIfError(pluginManager.FetchAllPlugins())
		logger.Println("all plugins are downloaded")
		os.Exit(0)
	}

	ci, err := pluginManager.GetCICondition()
	exitIfError(err)
	logger.Printf("ci-condition plugin: %s@%s\n", ci.Name(), ci.Version())

	prov, err := pluginManager.GetProvider()
	exitIfError(err)
	logger.Printf("provider plugin: %s@%s\n", prov.Name(), prov.Version())

	if conf.ProviderOpts["token"] == "" {
		conf.ProviderOpts["token"] = conf.Token
	}
	err = prov.Init(conf.ProviderOpts)
	exitIfError(err)

	logger.Println("getting default branch...")
	repoInfo, err := prov.GetInfo()
	exitIfError(err)
	logger.Println("found default branch: " + repoInfo.DefaultBranch)
	if repoInfo.Private {
		logger.Println("repo is private")
	}

	currentBranch := ci.GetCurrentBranch()
	if currentBranch == "" {
		exitIfError(fmt.Errorf("current branch not found"))
	}
	logger.Println("found current branch: " + currentBranch)

	if conf.MaintainedVersion != "" && currentBranch == repoInfo.DefaultBranch {
		exitIfError(fmt.Errorf("maintained version not allowed on default branch"))
	}

	if conf.MaintainedVersion != "" {
		logger.Println("found maintained version: " + conf.MaintainedVersion)
		repoInfo.DefaultBranch = "*"
	}

	currentSha := ci.GetCurrentSHA()
	logger.Println("found current sha: " + currentSha)

	hooksExecutor, err := pluginManager.GetChainedHooksExecutor()
	exitIfError(err)

	hooksNames := hooksExecutor.GetNameVersionPairs()
	if len(hooksNames) > 0 {
		logger.Printf("hooks plugins: %s\n", strings.Join(hooksNames, ", "))
	}

	exitIfError(hooksExecutor.Init(conf.HooksOpts))

	if !conf.NoCI {
		logger.Println("running CI condition...")
		conditionConfig := map[string]string{
			"token":         conf.Token,
			"defaultBranch": repoInfo.DefaultBranch,
			"private":       fmt.Sprintf("%t", repoInfo.Private),
		}
		for k, v := range conf.CIConditionOpts {
			conditionConfig[k] = v
		}
		err = ci.RunCondition(conditionConfig)
		if err != nil {
			herr := hooksExecutor.NoRelease(&hooks.NoReleaseConfig{
				Reason:  hooks.NoReleaseReason_CONDITION,
				Message: err.Error(),
			})
			if herr != nil {
				logger.Printf("there was an error executing the hooks plugins: %s", herr.Error())
			}
			exitIfError(err, 66)
		}

	}

	logger.Println("getting latest release...")
	matchRegex := ""
	match := strings.TrimSpace(conf.Match)
	if match != "" {
		logger.Printf("getting latest release matching %s...", match)
		matchRegex = "^" + match
	}
	releases, err := prov.GetReleases(matchRegex)
	exitIfError(err)
	release, err := semrel.GetLatestReleaseFromReleases(releases, conf.MaintainedVersion)
	exitIfError(err)
	logger.Println("found version: " + release.Version)

	if strings.Contains(conf.MaintainedVersion, "-") && semver.MustParse(release.Version).Prerelease() == "" {
		exitIfError(fmt.Errorf("no pre-release for this version possible"))
	}

	logger.Println("getting commits...")
	rawCommits, err := prov.GetCommits(release.SHA, currentSha)
	exitIfError(err)

	logger.Println("analyzing commits...")
	commitAnalyzer, err := pluginManager.GetCommitAnalyzer()
	exitIfError(err)
	logger.Printf("commit-analyzer plugin: %s@%s\n", commitAnalyzer.Name(), commitAnalyzer.Version())
	exitIfError(commitAnalyzer.Init(conf.ChangelogGeneratorOpts))

	commits := commitAnalyzer.Analyze(rawCommits)

	logger.Println("calculating new version...")
	newVer := semrel.GetNewVersion(conf, commits, release)
	if newVer == "" {
		herr := hooksExecutor.NoRelease(&hooks.NoReleaseConfig{
			Reason:  hooks.NoReleaseReason_NO_CHANGE,
			Message: "",
		})
		if herr != nil {
			logger.Printf("there was an error executing the hooks plugins: %s", herr.Error())
		}
		errNoChange := errors.New("no change")
		if conf.AllowNoChanges {
			exitIfError(errNoChange, 0)
		} else {
			exitIfError(errNoChange, 65)
		}
	}
	logger.Println("new version: " + newVer)

	logger.Println("generating changelog...")
	changelogGenerator, err := pluginManager.GetChangelogGenerator()
	exitIfError(err)
	logger.Printf("changelog-generator plugin: %s@%s\n", changelogGenerator.Name(), changelogGenerator.Version())
	exitIfError(changelogGenerator.Init(conf.ChangelogGeneratorOpts))

	changelogRes := changelogGenerator.Generate(&generator.ChangelogGeneratorConfig{
		Commits:       commits,
		LatestRelease: release,
		NewVersion:    newVer,
	})
	if conf.Changelog != "" {
		oldFile := make([]byte, 0)
		if conf.PrependChangelog {
			oldFileData, err := ioutil.ReadFile(conf.Changelog)
			if err == nil {
				oldFile = append([]byte("\n"), oldFileData...)
			}
		}
		changelogData := append([]byte(changelogRes), oldFile...)
		exitIfError(ioutil.WriteFile(conf.Changelog, changelogData, 0644))
	}

	if conf.Dry {
		if conf.VersionFile {
			exitIfError(ioutil.WriteFile(".version-unreleased", []byte(newVer), 0644))
		}
		exitIfError(errors.New("DRY RUN: no release was created"), 0)
	}

	logger.Println("creating release...")
	newRelease := &provider.CreateReleaseConfig{
		Changelog:  changelogRes,
		NewVersion: newVer,
		Prerelease: conf.Prerelease,
		Branch:     currentBranch,
		SHA:        currentSha,
	}
	exitIfError(prov.CreateRelease(newRelease))

	if conf.Ghr {
		exitIfError(ioutil.WriteFile(".ghr", []byte(fmt.Sprintf("-u %s -r %s v%s", repoInfo.Owner, repoInfo.Repo, newVer)), 0644))
	}

	if conf.VersionFile {
		exitIfError(ioutil.WriteFile(".version", []byte(newVer), 0644))
	}

	if len(conf.UpdateFiles) > 0 {
		logger.Println("updating files...")
		updater, err := pluginManager.GetChainedUpdater()
		exitIfError(err)
		logger.Printf("files-updater plugins: %s\n", strings.Join(updater.GetNameVersionPairs(), ", "))
		exitIfError(updater.Init(conf.FilesUpdaterOpts))

		for _, f := range conf.UpdateFiles {
			exitIfError(updater.Apply(f, newVer))
		}
	}

	herr := hooksExecutor.Success(&hooks.SuccessHookConfig{
		Commits:     commits,
		PrevRelease: release,
		NewRelease: &semrel.Release{
			SHA:     currentSha,
			Version: newVer,
		},
		Changelog: changelogRes,
		RepoInfo:  repoInfo,
	})

	if herr != nil {
		logger.Printf("there was an error executing the hooks plugins: %s", herr.Error())
	}

	logger.Println("done.")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"os"
	"sync"

	"github.com/thomersch/grandine/lib/cugdf"
	"github.com/thomersch/grandine/lib/spatial"

	"github.com/thomersch/gosmparse"
)

type nd struct {
	Lat, Lon float64
	Tags     map[string]interface{}
}
type wy struct {
	ID      int64
	NodeIDs []int64
	Tags    map[string]interface{}
}
type rl struct {
	Members []gosmparse.RelationMember
	Tags    map[string]interface{}
}

type dataHandler struct {
	conds []condition

	ec *elemCache

	nodes    []nd
	nodesMtx sync.Mutex
	ways     []wy
	waysMtx  sync.Mutex
	rels     []rl
	relsMtx  sync.Mutex
}

func (d *dataHandler) ReadNode(n gosmparse.Node) {
	for _, cond := range d.conds {
		if cond.Matches(n.Tags) {
			d.nodesMtx.Lock()
			d.nodes = append(d.nodes, nd{
				Lat:  n.Lat,
				Lon:  n.Lon,
				Tags: cond.Map(n.Tags),
			})
			d.nodesMtx.Unlock()
		}
	}
}

func (d *dataHandler) ReadWay(w gosmparse.Way) {
	for _, cond := range d.conds {
		if cond.Matches(w.Tags) {
			d.ec.AddNodes(w.NodeIDs...)
			d.ec.setMembers(w.ID, w.NodeIDs)

			d.waysMtx.Lock()
			d.ways = append(d.ways, wy{
				ID:      w.ID,
				NodeIDs: w.NodeIDs,
				Tags:    cond.Map(w.Tags),
			})
			d.waysMtx.Unlock()
		}
	}
}

func (d *dataHandler) ReadRelation(r gosmparse.Relation) {
	for _, cond := range d.conds {
		if cond.Matches(r.Tags) {
			d.relsMtx.Lock()
			d.rels = append(d.rels, rl{
				Members: r.Members,
				Tags:    cond.Map(r.Tags),
			})
			d.relsMtx.Unlock()

			for _, memb := range r.Members {
				switch memb.Type {
				case gosmparse.WayType:
					d.ec.AddWay(memb.ID)
				} // TODO: check if relations of nodes/relations are necessary
			}
		}
	}
}

type elemCache struct {
	nodes    map[int64]spatial.Point
	nodesMtx sync.Mutex
	ways     map[int64][]int64
	waysMtx  sync.Mutex
}

func NewElemCache() *elemCache {
	return &elemCache{
		nodes: map[int64]spatial.Point{},
		ways:  map[int64][]int64{},
	}
}

func (d *elemCache) AddNodes(nIDs ...int64) {
	d.nodesMtx.Lock()
	for _, nID := range nIDs {
		d.nodes[nID] = spatial.Point{}
	}
	d.nodesMtx.Unlock()
}

func (d *elemCache) AddWay(wID int64) {
	d.waysMtx.Lock()
	d.ways[wID] = []int64{}
	d.waysMtx.Unlock()
}

func (d *elemCache) SetCoord(nID int64, coord spatial.Point) {
	d.nodesMtx.Lock()
	d.nodes[nID] = coord
	d.nodesMtx.Unlock()
}

func (d *elemCache) setMembers(wID int64, members []int64) {
	d.waysMtx.Lock()
	d.ways[wID] = members
	d.waysMtx.Unlock()
}

func (d *elemCache) ReadWay(w gosmparse.Way) {
	d.waysMtx.Lock()
	_, ok := d.ways[w.ID]
	d.waysMtx.Unlock()
	if ok {
		d.setMembers(w.ID, w.NodeIDs)
		d.AddNodes(w.NodeIDs...)
	}
}

func (d *elemCache) Line(wID int64) spatial.Line {
	// check if mutex is needed
	membs, ok := d.ways[wID]
	if !ok {
		log.Fatalf("missing referenced way: %v", wID)
	}

	var l spatial.Line
	for _, memb := range membs {
		l = append(l, d.nodes[memb])
	}
	return l
}

// Interface enforces this. Probably I should change the behavior.
func (d *elemCache) ReadNode(n gosmparse.Node)         {}
func (d *elemCache) ReadRelation(r gosmparse.Relation) {}

type nodeCollector struct {
	ec *elemCache
}

func (d *nodeCollector) ReadNode(n gosmparse.Node) {
	d.ec.SetCoord(n.ID, spatial.Point{float64(n.Lon), float64(n.Lat)})
}
func (d *nodeCollector) ReadWay(w gosmparse.Way)           {}
func (d *nodeCollector) ReadRelation(r gosmparse.Relation) {}

type tagMapFn func(map[string]string) map[string]interface{}

type mapper struct {
	cond     condition
	mapper   tagMapFn
	elemType spatial.GeomType
}

type condition struct {
	// TODO: make it possible to specify condition type (node/way/rel)
	key    string
	value  string
	mapper tagMapFn
}

func (c *condition) Matches(kv map[string]string) bool {
	if v, ok := kv[c.key]; ok {
		if len(c.value) == 0 || c.value == v {
			return true
		}
	}
	return false
}

func (c *condition) Map(kv map[string]string) map[string]interface{} {
	return c.mapper(kv)
}

func main() {
	transportationMapFn := func(kv map[string]string) map[string]interface{} {
		var cl string
		if class, ok := kv["highway"]; ok {
			cl = class
		}
		return map[string]interface{}{
			"@layer": "transportation",
			"class":  cl,
		}
	}

	landuseMapFn := func(kv map[string]string) map[string]interface{} {
		return map[string]interface{}{
			"__type":    "area",
			"landcover": "wood",
		}
	}

	conds := []condition{
		condition{"highway", "primary", transportationMapFn},
		condition{"highway", "secondary", transportationMapFn},
		condition{"highway", "tertiary", transportationMapFn},
		condition{"railway", "rail", transportationMapFn},
		condition{"landuse", "forest", landuseMapFn},
	}

	source := flag.String("src", "osm.pbf", "")
	outfile := flag.String("out", "osm.cugdf", "")
	flag.Parse()

	f, err := os.Open(*source)
	if err != nil {
		log.Fatal(err)
	}
	dec := gosmparse.NewDecoder(f)

	// First pass
	ec := NewElemCache()
	dh := dataHandler{
		conds: conds,
		ec:    ec,
	}
	log.Println("Starting 3 step parsing")
	log.Println("Reading data (1/3)...")
	err = dec.Parse(&dh)
	if err != nil {
		log.Fatal(err)
	}

	_, err = f.Seek(0, 0) // jumps to beginning of file
	if err != nil {
		log.Fatal(err)
	}

	// Second pass
	log.Println("Collecting nodes (2/3)...")
	err = dec.Parse(ec)
	if err != nil {
		log.Fatal(err)
	}

	_, err = f.Seek(0, 0)
	if err != nil {
		log.Fatal(err)
	}

	// Third pass
	log.Println("Resolving dependent objects (3/3)...")
	rc := nodeCollector{
		ec: ec,
	}
	err = dec.Parse(&rc)
	if err != nil {
		log.Fatal(err)
	}

	var fc []spatial.Feature

	log.Println("Parsing completed.")

	log.Println("Collecting points...")
	for _, pt := range dh.nodes {
		props := map[string]interface{}{}
		for k, v := range pt.Tags {
			props[k] = v
		}
		fc = append(fc, spatial.Feature{
			Props:    props,
			Geometry: spatial.MustNewGeom(spatial.Point{float64(pt.Lon), float64(pt.Lat)}),
		})
	}

	log.Println("Assembling ways...")
	// TODO: auto-detect if linestring or polygon, based on tags
	for _, wy := range dh.ways {
		var (
			area  bool
			props = map[string]interface{}{}
			geom  interface{}
		)
		for k, v := range wy.Tags {
			if k == "__type" && v == "area" {
				area = true
				continue
			}
			props[k] = v
		}
		ln := ec.Line(wy.ID)
		if !ln.Clockwise() {
			ln.Reverse()
		}
		if area {
			geom = spatial.Polygon{ln}
		} else {
			geom = ln
		}

		fc = append(fc, spatial.Feature{
			Props:    props,
			Geometry: spatial.MustNewGeom(geom),
		})
	}

	log.Println("Assembling relations...")
	for _, rl := range dh.rels {
		if v, ok := rl.Tags["type"]; !ok || v != "multipolygon" {
			continue
		}
		var poly spatial.Polygon

		for _, memb := range rl.Members {
			if memb.Role == "outer" || memb.Role == "inner" {
				ring := ec.Line(memb.ID)
				if (memb.Role == "outer" && !ring.Clockwise()) || (memb.Role == "inner" && ring.Clockwise()) {
					ring.Reverse()
				}
				poly = append(poly, ring)
			}
		}
	}

	log.Println("Writing out")
	of, err := os.Create(*outfile)
	if err != nil {
		log.Fatal(err)
	}
	err = cugdf.Marshal(fc, of)
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>cmd/spatialize: openmaptiles scheme implementation, zoom level tags<commit_after>package main

import (
	"flag"
	"log"
	"os"
	"sync"

	"github.com/thomersch/grandine/lib/cugdf"
	"github.com/thomersch/grandine/lib/spatial"

	"github.com/thomersch/gosmparse"
)

type nd struct {
	Lat, Lon float64
	Tags     map[string]interface{}
}
type wy struct {
	ID      int64
	NodeIDs []int64
	Tags    map[string]interface{}
}
type rl struct {
	Members []gosmparse.RelationMember
	Tags    map[string]interface{}
}

type dataHandler struct {
	conds []condition

	ec *elemCache

	nodes    []nd
	nodesMtx sync.Mutex
	ways     []wy
	waysMtx  sync.Mutex
	rels     []rl
	relsMtx  sync.Mutex
}

func (d *dataHandler) ReadNode(n gosmparse.Node) {
	for _, cond := range d.conds {
		if cond.Matches(n.Tags) {
			d.nodesMtx.Lock()
			d.nodes = append(d.nodes, nd{
				Lat:  n.Lat,
				Lon:  n.Lon,
				Tags: cond.Map(n.Tags),
			})
			d.nodesMtx.Unlock()
		}
	}
}

func (d *dataHandler) ReadWay(w gosmparse.Way) {
	for _, cond := range d.conds {
		if cond.Matches(w.Tags) {
			d.ec.AddNodes(w.NodeIDs...)
			d.ec.setMembers(w.ID, w.NodeIDs)

			d.waysMtx.Lock()
			d.ways = append(d.ways, wy{
				ID:      w.ID,
				NodeIDs: w.NodeIDs,
				Tags:    cond.Map(w.Tags),
			})
			d.waysMtx.Unlock()
		}
	}
}

func (d *dataHandler) ReadRelation(r gosmparse.Relation) {
	for _, cond := range d.conds {
		if cond.Matches(r.Tags) {
			d.relsMtx.Lock()
			d.rels = append(d.rels, rl{
				Members: r.Members,
				Tags:    cond.Map(r.Tags),
			})
			d.relsMtx.Unlock()

			for _, memb := range r.Members {
				switch memb.Type {
				case gosmparse.WayType:
					d.ec.AddWay(memb.ID)
				} // TODO: check if relations of nodes/relations are necessary
			}
		}
	}
}

type elemCache struct {
	nodes    map[int64]spatial.Point
	nodesMtx sync.Mutex
	ways     map[int64][]int64
	waysMtx  sync.Mutex
}

func NewElemCache() *elemCache {
	return &elemCache{
		nodes: map[int64]spatial.Point{},
		ways:  map[int64][]int64{},
	}
}

func (d *elemCache) AddNodes(nIDs ...int64) {
	d.nodesMtx.Lock()
	for _, nID := range nIDs {
		d.nodes[nID] = spatial.Point{}
	}
	d.nodesMtx.Unlock()
}

func (d *elemCache) AddWay(wID int64) {
	d.waysMtx.Lock()
	d.ways[wID] = []int64{}
	d.waysMtx.Unlock()
}

func (d *elemCache) SetCoord(nID int64, coord spatial.Point) {
	d.nodesMtx.Lock()
	d.nodes[nID] = coord
	d.nodesMtx.Unlock()
}

func (d *elemCache) setMembers(wID int64, members []int64) {
	d.waysMtx.Lock()
	d.ways[wID] = members
	d.waysMtx.Unlock()
}

func (d *elemCache) ReadWay(w gosmparse.Way) {
	d.waysMtx.Lock()
	_, ok := d.ways[w.ID]
	d.waysMtx.Unlock()
	if ok {
		d.setMembers(w.ID, w.NodeIDs)
		d.AddNodes(w.NodeIDs...)
	}
}

func (d *elemCache) Line(wID int64) spatial.Line {
	// check if mutex is needed
	membs, ok := d.ways[wID]
	if !ok {
		log.Fatalf("missing referenced way: %v", wID)
	}

	var l spatial.Line
	for _, memb := range membs {
		l = append(l, d.nodes[memb])
	}
	return l
}

// Interface enforces this. Probably I should change the behavior.
func (d *elemCache) ReadNode(n gosmparse.Node)         {}
func (d *elemCache) ReadRelation(r gosmparse.Relation) {}

type nodeCollector struct {
	ec *elemCache
}

func (d *nodeCollector) ReadNode(n gosmparse.Node) {
	d.ec.SetCoord(n.ID, spatial.Point{float64(n.Lon), float64(n.Lat)})
}
func (d *nodeCollector) ReadWay(w gosmparse.Way)           {}
func (d *nodeCollector) ReadRelation(r gosmparse.Relation) {}

type tagMapFn func(map[string]string) map[string]interface{}

type mapper struct {
	cond     condition
	mapper   tagMapFn
	elemType spatial.GeomType
}

type condition struct {
	// TODO: make it possible to specify condition type (node/way/rel)
	key    string
	value  string
	mapper tagMapFn
}

func (c *condition) Matches(kv map[string]string) bool {
	if v, ok := kv[c.key]; ok {
		if len(c.value) == 0 || c.value == v {
			return true
		}
	}
	return false
}

func (c *condition) Map(kv map[string]string) map[string]interface{} {
	return c.mapper(kv)
}

func main() {
	transportationMapFn := func(kv map[string]string) map[string]interface{} {
		var cl string
		if class, ok := kv["highway"]; ok {
			cl = class
		}
		return map[string]interface{}{
			"@layer": "transportation",
			"class":  cl,
		}
	}

	landuseMapFn := func(kv map[string]string) map[string]interface{} {
		return map[string]interface{}{
			"__type": "area",
			"@layer": "landcover",
			"class":  "wood",
		}
	}

	aerowayMapFn := func(kv map[string]string) map[string]interface{} {
		var cl string
		if class, ok := kv["aeroway"]; ok {
			cl = class
		}
		return map[string]interface{}{
			"@layer": "aeroway",
			"class":  cl,
		}
	}

	buildingMapFn := func(kv map[string]string) map[string]interface{} {
		return map[string]interface{}{
			"@layer":    "building",
			"@zoom:min": 14,
		}
	}

	waterwayMapFn := func(kv map[string]string) map[string]interface{} {
		var cl string
		if class, ok := kv["waterway"]; ok {
			cl = class
		}
		return map[string]interface{}{
			"@layer": "waterway",
			"class":  cl,
		}
	}

	conds := []condition{
		condition{"aeroway", "aerodrome", aerowayMapFn},
		condition{"aeroway", "apron", aerowayMapFn},
		condition{"aeroway", "heliport", aerowayMapFn},
		condition{"aeroway", "runway", aerowayMapFn},
		condition{"aeroway", "helipad", aerowayMapFn},
		condition{"aeroway", "taxiway", aerowayMapFn},
		condition{"highway", "primary", transportationMapFn},
		condition{"highway", "secondary", transportationMapFn},
		condition{"highway", "tertiary", transportationMapFn},
		condition{"building", "", buildingMapFn},
		condition{"landuse", "forest", landuseMapFn},
		condition{"railway", "rail", transportationMapFn},
		condition{"waterway", "river", waterwayMapFn},
	}

	source := flag.String("src", "osm.pbf", "")
	outfile := flag.String("out", "osm.cugdf", "")
	flag.Parse()

	f, err := os.Open(*source)
	if err != nil {
		log.Fatal(err)
	}
	dec := gosmparse.NewDecoder(f)

	// First pass
	ec := NewElemCache()
	dh := dataHandler{
		conds: conds,
		ec:    ec,
	}
	log.Println("Starting 3 step parsing")
	log.Println("Reading data (1/3)...")
	err = dec.Parse(&dh)
	if err != nil {
		log.Fatal(err)
	}

	_, err = f.Seek(0, 0) // jumps to beginning of file
	if err != nil {
		log.Fatal(err)
	}

	// Second pass
	log.Println("Collecting nodes (2/3)...")
	err = dec.Parse(ec)
	if err != nil {
		log.Fatal(err)
	}

	_, err = f.Seek(0, 0)
	if err != nil {
		log.Fatal(err)
	}

	// Third pass
	log.Println("Resolving dependent objects (3/3)...")
	rc := nodeCollector{
		ec: ec,
	}
	err = dec.Parse(&rc)
	if err != nil {
		log.Fatal(err)
	}

	var fc []spatial.Feature

	log.Println("Parsing completed.")

	log.Println("Collecting points...")
	for _, pt := range dh.nodes {
		props := map[string]interface{}{}
		for k, v := range pt.Tags {
			props[k] = v
		}
		fc = append(fc, spatial.Feature{
			Props:    props,
			Geometry: spatial.MustNewGeom(spatial.Point{float64(pt.Lon), float64(pt.Lat)}),
		})
	}

	log.Println("Assembling ways...")
	// TODO: auto-detect if linestring or polygon, based on tags
	for _, wy := range dh.ways {
		var (
			area  bool
			props = map[string]interface{}{}
			geom  interface{}
		)
		for k, v := range wy.Tags {
			if k == "__type" && v == "area" {
				area = true
				continue
			}
			props[k] = v
		}
		ln := ec.Line(wy.ID)
		if !ln.Clockwise() {
			ln.Reverse()
		}
		if area {
			geom = spatial.Polygon{ln}
		} else {
			geom = ln
		}

		fc = append(fc, spatial.Feature{
			Props:    props,
			Geometry: spatial.MustNewGeom(geom),
		})
	}

	log.Println("Assembling relations...")
	for _, rl := range dh.rels {
		if v, ok := rl.Tags["type"]; !ok || v != "multipolygon" {
			continue
		}
		var poly spatial.Polygon

		for _, memb := range rl.Members {
			if memb.Role == "outer" || memb.Role == "inner" {
				ring := ec.Line(memb.ID)
				if (memb.Role == "outer" && !ring.Clockwise()) || (memb.Role == "inner" && ring.Clockwise()) {
					ring.Reverse()
				}
				poly = append(poly, ring)
			}
		}
	}

	log.Println("Writing out")
	of, err := os.Create(*outfile)
	if err != nil {
		log.Fatal(err)
	}
	err = cugdf.Marshal(fc, of)
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main_test

import (
	"fmt"
	"net"
	"os/exec"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
)

const (
	BACKEND_IP = "localhost"
)

func startMainWithArgs(args ...string) *gexec.Session {
	command := exec.Command(switchboardBinPath, args...)
	session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
	Expect(err).NotTo(HaveOccurred())
	Eventually(session).Should(gbytes.Say("started on port"))
	fmt.Printf("Switchboard started with args:%v\n", args)
	return session
}

func startBackendWithArgs(args ...string) *gexec.Session {
	command := exec.Command(dummyListenerBinPath, args...)
	session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
	Expect(err).NotTo(HaveOccurred())
	Eventually(session).Should(gbytes.Say("Backend listening on"))
	fmt.Printf("Backend started with args:%v\n", args)
	return session
}

var _ = Describe("Switchboard", func() {
	Context("with a single backend node", func() {
		It("forwards multiple client connections to the backend", func() {

			backendSession := startBackendWithArgs([]string{
				fmt.Sprintf("-port=%d", backendPort),
			}...)
			defer backendSession.Terminate()

			session := startMainWithArgs([]string{
				fmt.Sprintf("-port=%d", switchboardPort),
				fmt.Sprintf("-backendIp=%s", BACKEND_IP),
				fmt.Sprintf("-backendPort=%d", backendPort),
			}...)
			defer session.Terminate()

			count := 10
			buffers := make([][]byte, count)
			conns := make([]net.Conn, count)

			for i := 0; i < count; i++ {
				var conn net.Conn
				Eventually(func() error {
					var err error
					conn, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
					return err
				}, 1*time.Second, 10*time.Millisecond).ShouldNot(HaveOccurred())

				buffers[i] = make([]byte, 1024)
				conns[i] = conn
			}

			for i, conn := range conns {
				// Run the clients in parallel via goroutines
				go func(i int, conn net.Conn) {
					data := buffers[i]
					var n int
					var err error

					conn.Write([]byte(fmt.Sprintf("test%d", i)))
					// Read is a blocking method so we background it in a goroutine.
					go func() {
						n, err = conn.Read(data)
					}()

					// Read is asynchronous so we need to use Eventually
					Eventually(func() string {
						return string(data[:n])
					}).Should(ContainSubstring(fmt.Sprintf("Echo: test%d", i)))
				}(i, conn)
			}
		})
	})
})
<commit_msg>Switchboard maintains a long-lived connection when other clients disconnect<commit_after>package main_test

import (
	"fmt"
	"net"
	"os/exec"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
)

const (
	BACKEND_IP = "localhost"
)

func startMainWithArgs(args ...string) *gexec.Session {
	command := exec.Command(switchboardBinPath, args...)
	session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
	Expect(err).NotTo(HaveOccurred())
	Eventually(session).Should(gbytes.Say("started on port"))
	return session
}

func startBackendWithArgs(args ...string) *gexec.Session {
	command := exec.Command(dummyListenerBinPath, args...)
	session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
	Expect(err).NotTo(HaveOccurred())
	Eventually(session).Should(gbytes.Say("Backend listening on"))
	return session
}

var _ = Describe("Switchboard", func() {
	Context("with a single backend node", func() {
		It("forwards multiple client connections to the backend", func() {

			backendSession := startBackendWithArgs([]string{
				fmt.Sprintf("-port=%d", backendPort),
			}...)
			defer backendSession.Terminate()

			session := startMainWithArgs([]string{
				fmt.Sprintf("-port=%d", switchboardPort),
				fmt.Sprintf("-backendIp=%s", BACKEND_IP),
				fmt.Sprintf("-backendPort=%d", backendPort),
			}...)
			defer session.Terminate()

			count := 10
			buffers := make([][]byte, count)
			conns := make([]net.Conn, count)

			for i := 0; i < count; i++ {
				var conn net.Conn
				Eventually(func() error {
					var err error
					conn, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
					return err
				}, 1*time.Second, 10*time.Millisecond).ShouldNot(HaveOccurred())

				buffers[i] = make([]byte, 1024)
				conns[i] = conn
			}

			for i, conn := range conns {
				// Run the clients in parallel via goroutines
				go func(i int, conn net.Conn) {
					data := buffers[i]
					var n int
					var err error

					conn.Write([]byte(fmt.Sprintf("test%d", i)))
					// Read is a blocking method so we background it in a goroutine.
					go func() {
						n, err = conn.Read(data)
					}()

					// Read is asynchronous so we need to use Eventually
					Eventually(func() string {
						return string(data[:n])
					}).Should(ContainSubstring(fmt.Sprintf("Echo: test%d", i)))
				}(i, conn)
			}
		})

		It("can maintain a long-lived connection when other clients disconnect", func() {

			backendSession := startBackendWithArgs([]string{
				fmt.Sprintf("-port=%d", backendPort),
			}...)
			defer backendSession.Terminate()

			session := startMainWithArgs([]string{
				fmt.Sprintf("-port=%d", switchboardPort),
				fmt.Sprintf("-backendIp=%s", BACKEND_IP),
				fmt.Sprintf("-backendPort=%d", backendPort),
			}...)
			defer session.Terminate()

			var longConnection net.Conn
			var shortConnection net.Conn

			Eventually(func() error {
				var err error
				longConnection, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
				return err
			}, 1*time.Second, 10*time.Millisecond).ShouldNot(HaveOccurred())

			Eventually(func() error {
				var err error
				shortConnection, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
				return err
			}, 1*time.Second, 10*time.Millisecond).ShouldNot(HaveOccurred())

			longBuffer := make([]byte, 1024)
			shortBuffer := make([]byte, 1024)

			longConnection.Write([]byte("longdata"))
			var n int
			var err error
			go func() {
				n, err = longConnection.Read(longBuffer)
			}()

			Eventually(func() string {
				return string(longBuffer[:n])
			}).Should(ContainSubstring("longdata"))

			shortConnection.Write([]byte("shortdata"))
			go func() {
				n, err = shortConnection.Read(shortBuffer)
			}()

			Eventually(func() string {
				return string(longBuffer[:n])
			}).Should(ContainSubstring("longdata"))

			shortConnection.Close()

			longConnection.Write([]byte("longdata1"))
			go func() {
				n, err = longConnection.Read(longBuffer)
			}()

			Eventually(func() string {
				return string(longBuffer[:n])
			}).Should(ContainSubstring("longdata1"))
		})
	})
})
<|endoftext|>


<commit_before>package main

import (
	"github.com/toomore/gogrs/twse"
	"github.com/toomore/gogrs/utils"
)

type checkGroup interface {
	String() string
	CheckFunc(...*twse.Data) bool
	Mindata() int
}

type check01 struct{}

func (check01) String() string {
	return "MA 3 > 6 > 18"
}

func (check01) Mindata() int {
	return 18
}

func (check01) CheckFunc(b ...*twse.Data) bool {
	if !prepareData(b...)[0] {
		return false
	}
	var ma3 = b[0].MA(3)
	if days, ok := utils.CountCountineFloat64(utils.DeltaFloat64(ma3)); !ok || days == 0 {
		return false
	}
	var ma6 = b[0].MA(6)
	if days, ok := utils.CountCountineFloat64(utils.DeltaFloat64(ma6)); !ok || days == 0 {
		return false
	}
	var ma18 = b[0].MA(18)
	if days, ok := utils.CountCountineFloat64(utils.DeltaFloat64(ma18)); !ok || days == 0 {
		return false
	}
	//log.Println(ma3[len(ma3)-1], ma6[len(ma6)-1], ma18[len(ma18)-1])
	if ma3[len(ma3)-1] > ma6[len(ma6)-1] && ma6[len(ma6)-1] > ma18[len(ma18)-1] {
		return true
	}
	return false
}

type check02 struct{}

func (check02) String() string {
	return "量大於前三天 K 線收紅"
}

func (check02) Mindata() int {
	return 4
}

func (check02) CheckFunc(b ...*twse.Data) bool {
	if !prepareData(b...)[0] {
		return false
	}
	return utils.ThanSumPastUint64((*b[0]).GetVolumeList(), 3, true) && ((*b[0]).IsRed() || (*b[0]).IsThanYesterday())
}

type check03 struct{}

func (check03) String() string {
	return "量或價走平 45 天"
}

func (check03) Mindata() int {
	return 45
}

func (check03) CheckFunc(b ...*twse.Data) bool {
	if !prepareData(b...)[0] {
		return false
	}

	var (
		price  = b[0].GetPriceList()
		volume = b[0].GetVolumeList()
	)

	return price[len(price)-1] > 10 &&
		(utils.SD(price[len(price)-45:]) < 0.25 ||
			utils.SDUint64(volume[len(volume)-45:]) < 0.25)
}

type check04 struct{}

func (check04) String() string {
	return "(MA3 < MA6) > MA18 and MA3UP(1)"
}

func (check04) Mindata() int {
	return 18
}

func (check04) CheckFunc(b ...*twse.Data) bool {
	if !prepareData(b...)[0] {
		return false
	}
	var ma3 = b[0].MA(3)
	if days, up := utils.CountCountineFloat64(utils.DeltaFloat64(ma3)); up && days == 1 {
		var (
			ma6      = b[0].MA(6)
			ma18     = b[0].MA(18)
			ma3Last  = len(ma3) - 1
			ma6Last  = len(ma6) - 1
			ma18Last = len(ma18) - 1
		)
		return (ma3[ma3Last] > ma18[ma18Last] && ma6[ma6Last] > ma18[ma18Last]) && ma3[ma3Last] < ma6[ma6Last]
	}
	return false
}

type check05 struct{}

func (check05) String() string {
	return "三日內最大量 K 線收紅 收在 MA18 之上"
}

func (check05) Mindata() int {
	return 18
}

func (check05) CheckFunc(b ...*twse.Data) bool {
	if !prepareData(b...)[0] {
		return false
	}
	var (
		vols        = b[0].GetVolumeList()
		volsFloat64 = make([]float64, 3)
	)
	for i, v := range vols[len(vols)-3:] {
		volsFloat64[i] = float64(v)
	}
	if days, up := utils.CountCountineFloat64(utils.DeltaFloat64(volsFloat64)); up && days >= 1 && b[0].IsRed() {
		var (
			ma18      = b[0].MA(18)
			priceList = b[0].GetPriceList()
		)

		if priceList[len(priceList)-1] > ma18[len(ma18)-1] {
			return true
		}
	}
	return false
}

type check06 struct{}

func (check06) String() string {
	return "漲幅 7% 以上"
}

func (check06) Mindata() int {
	return 1
}

func (check06) CheckFunc(b ...*twse.Data) bool {
	if !prepareData(b...)[0] {
		return false
	}

	priceList := b[0].GetPriceList()
	openList := b[0].GetOpenList()
	price := priceList[len(priceList)-1]
	open := openList[len(openList)-1]

	if price > open && (price-open)/open > 0.068 {
		return true
	}

	return false
}

func prepareData(b ...*twse.Data) []bool {
	var (
		result  []bool
		mindata int
	)

	for i := range ckList {
		if ckList[i].Mindata() > mindata {
			mindata = ckList[i].Mindata()
		}
	}

	for i := range b {
		result = make([]bool, len(b))
		b[i].Get()
		if b[i].Len() < mindata {
			start := b[i].Len()
			for {
				b[i].PlusData()
				if b[i].Len() > mindata {
					result[i] = true
					break
				}
				if b[i].Len() == start {
					break
				}
				start = b[i].Len()
			}
			if b[i].Len() < mindata {
				result[i] = false
			}
		} else {
			result[i] = true
		}
	}
	return result
}

func init() {
	ckList.Add(checkGroup(check01{}))
	ckList.Add(checkGroup(check02{}))
	ckList.Add(checkGroup(check03{}))
	ckList.Add(checkGroup(check04{}))
	ckList.Add(checkGroup(check05{}))
	ckList.Add(checkGroup(check06{}))
}
<commit_msg>Add `check07` in twsereport.<commit_after>package main

import (
	"github.com/toomore/gogrs/twse"
	"github.com/toomore/gogrs/utils"
)

type checkGroup interface {
	String() string
	CheckFunc(...*twse.Data) bool
	Mindata() int
}

type check01 struct{}

func (check01) String() string {
	return "MA 3 > 6 > 18"
}

func (check01) Mindata() int {
	return 18
}

func (check01) CheckFunc(b ...*twse.Data) bool {
	if !prepareData(b...)[0] {
		return false
	}
	var ma3 = b[0].MA(3)
	if days, ok := utils.CountCountineFloat64(utils.DeltaFloat64(ma3)); !ok || days == 0 {
		return false
	}
	var ma6 = b[0].MA(6)
	if days, ok := utils.CountCountineFloat64(utils.DeltaFloat64(ma6)); !ok || days == 0 {
		return false
	}
	var ma18 = b[0].MA(18)
	if days, ok := utils.CountCountineFloat64(utils.DeltaFloat64(ma18)); !ok || days == 0 {
		return false
	}
	//log.Println(ma3[len(ma3)-1], ma6[len(ma6)-1], ma18[len(ma18)-1])
	if ma3[len(ma3)-1] > ma6[len(ma6)-1] && ma6[len(ma6)-1] > ma18[len(ma18)-1] {
		return true
	}
	return false
}

type check02 struct{}

func (check02) String() string {
	return "量大於前三天 K 線收紅"
}

func (check02) Mindata() int {
	return 4
}

func (check02) CheckFunc(b ...*twse.Data) bool {
	if !prepareData(b...)[0] {
		return false
	}
	return utils.ThanSumPastUint64((*b[0]).GetVolumeList(), 3, true) && ((*b[0]).IsRed() || (*b[0]).IsThanYesterday())
}

type check03 struct{}

func (check03) String() string {
	return "量或價走平 45 天"
}

func (check03) Mindata() int {
	return 45
}

func (check03) CheckFunc(b ...*twse.Data) bool {
	if !prepareData(b...)[0] {
		return false
	}

	var (
		price  = b[0].GetPriceList()
		volume = b[0].GetVolumeList()
	)

	return price[len(price)-1] > 10 &&
		(utils.SD(price[len(price)-45:]) < 0.25 ||
			utils.SDUint64(volume[len(volume)-45:]) < 0.25)
}

type check04 struct{}

func (check04) String() string {
	return "(MA3 < MA6) > MA18 and MA3UP(1)"
}

func (check04) Mindata() int {
	return 18
}

func (check04) CheckFunc(b ...*twse.Data) bool {
	if !prepareData(b...)[0] {
		return false
	}
	var ma3 = b[0].MA(3)
	if days, up := utils.CountCountineFloat64(utils.DeltaFloat64(ma3)); up && days == 1 {
		var (
			ma6      = b[0].MA(6)
			ma18     = b[0].MA(18)
			ma3Last  = len(ma3) - 1
			ma6Last  = len(ma6) - 1
			ma18Last = len(ma18) - 1
		)
		return (ma3[ma3Last] > ma18[ma18Last] && ma6[ma6Last] > ma18[ma18Last]) && ma3[ma3Last] < ma6[ma6Last]
	}
	return false
}

type check05 struct{}

func (check05) String() string {
	return "三日內最大量 K 線收紅 收在 MA18 之上"
}

func (check05) Mindata() int {
	return 18
}

func (check05) CheckFunc(b ...*twse.Data) bool {
	if !prepareData(b...)[0] {
		return false
	}
	var (
		vols        = b[0].GetVolumeList()
		volsFloat64 = make([]float64, 3)
	)
	for i, v := range vols[len(vols)-3:] {
		volsFloat64[i] = float64(v)
	}
	if days, up := utils.CountCountineFloat64(utils.DeltaFloat64(volsFloat64)); up && days >= 1 && b[0].IsRed() {
		var (
			ma18      = b[0].MA(18)
			priceList = b[0].GetPriceList()
		)

		if priceList[len(priceList)-1] > ma18[len(ma18)-1] {
			return true
		}
	}
	return false
}

type check06 struct{}

func (check06) String() string {
	return "漲幅 7% 以上"
}

func (check06) Mindata() int {
	return 1
}

func (check06) CheckFunc(b ...*twse.Data) bool {
	if !prepareData(b...)[0] {
		return false
	}

	priceList := b[0].GetPriceList()
	openList := b[0].GetOpenList()
	price := priceList[len(priceList)-1]
	open := openList[len(openList)-1]

	if price > open && (price-open)/open > 0.068 {
		return true
	}

	return false
}

type check07 struct{}

func (check07) String() string {
	return "多方力道 > 0.75"
}

func (check07) Mindata() int {
	return 1
}

func (check07) CheckFunc(b ...*twse.Data) bool {
	if !prepareData(b...)[0] {
		return false
	}

	var power []float64
	power = utils.CalLHPower(b[0].GetPriceList(), b[0].GetLowList(), b[0].GetHighList())

	if power[len(power)-1] > 0.75 {
		return true
	}

	return false
}

func prepareData(b ...*twse.Data) []bool {
	var (
		result  []bool
		mindata int
	)

	for i := range ckList {
		if ckList[i].Mindata() > mindata {
			mindata = ckList[i].Mindata()
		}
	}

	for i := range b {
		result = make([]bool, len(b))
		b[i].Get()
		if b[i].Len() < mindata {
			start := b[i].Len()
			for {
				b[i].PlusData()
				if b[i].Len() > mindata {
					result[i] = true
					break
				}
				if b[i].Len() == start {
					break
				}
				start = b[i].Len()
			}
			if b[i].Len() < mindata {
				result[i] = false
			}
		} else {
			result[i] = true
		}
	}
	return result
}

func init() {
	ckList.Add(checkGroup(check01{}))
	ckList.Add(checkGroup(check02{}))
	ckList.Add(checkGroup(check03{}))
	ckList.Add(checkGroup(check04{}))
	ckList.Add(checkGroup(check05{}))
	ckList.Add(checkGroup(check06{}))
	ckList.Add(checkGroup(check07{}))
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"runtime"
	"syscall"
	"time"

	"github.com/bitly/nsq/nsqd"
	"github.com/codegangsta/cli"

	tcli "github.com/toorop/tmail/cli"
	"github.com/toorop/tmail/core"
	"github.com/toorop/tmail/rest"
)

const (
	// TmailVersion version of tmail
	TmailVersion = "0.0.11"
)

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	var err error
	if err = core.Bootstrap(); err != nil {
		log.Fatalln(err)
	}
	core.Version = TmailVersion

	// Check base path structure
	requiredPaths := []string{"db", "nsq", "ssl"}
	for _, p := range requiredPaths {
		if err = os.MkdirAll(path.Join(core.GetBasePath(), p), 0700); err != nil {
			log.Fatalln("Unable to create path "+path.Join(core.GetBasePath(), p), " - ", err.Error())
		}
	}

	// TODO: if clusterMode check if nsqlookupd is available

	// check DB
	// TODO: do check in CLI call (raise error & ask for user to run tmail initdb|checkdb)
	if !core.IsOkDB(core.DB) {
		var r []byte
		for {
			fmt.Printf("Database 'driver: %s, source: %s' misses some tables.\r\nShould i create them ? (y/n):", core.Cfg.GetDbDriver(), core.Cfg.GetDbSource())
			r, _, _ = bufio.NewReader(os.Stdin).ReadLine()
			if r[0] == 110 || r[0] == 121 {
				break
			}
		}
		if r[0] == 121 {
			if err = core.InitDB(core.DB); err != nil {
				log.Fatalln(err)
			}
		} else {
			log.Println("See you soon...")
			os.Exit(0)
		}
	}
	// sync tables from structs
	if err := core.AutoMigrateDB(core.DB); err != nil {
		log.Fatalln(err)
	}

	// init rand seed
	rand.Seed(time.Now().UTC().UnixNano())

	// Dovecot support
	if core.Cfg.GetDovecotSupportEnabled() {
		_, err := exec.LookPath(core.Cfg.GetDovecotLda())
		if err != nil {
			log.Fatalln("Unable to find Dovecot LDA binary, checks your config poarameter TMAIL_DOVECOT_LDA ", err)
		}
	}
}

// MAIN
func main() {
	app := cli.NewApp()
	app.Name = "tmail"
	app.Usage = "SMTP server"
	app.Author = "Stéphane Depierrepont aka toorop"
	app.Email = "toorop@tmail.io"
	app.Version = TmailVersion
	app.Commands = tcli.CliCommands
	// no know command ? Launch server
	app.Action = func(c *cli.Context) {
		if len(c.Args()) != 0 {
			cli.ShowAppHelp(c)
		} else {
			// if there is nothing to do then... do nothing
			if !core.Cfg.GetLaunchDeliverd() && !core.Cfg.GetLaunchSmtpd() {
				log.Fatalln("I have nothing to do, so i do nothing. Bye.")
			}

			// Loop
			sigChan := make(chan os.Signal, 1)
			signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)

			// TODO
			// Chanel to comunicate between all elements
			//daChan := make(chan string)

			// init and launch nsqd
			opts := nsqd.NewNSQDOptions()
			opts.Logger = log.New(ioutil.Discard, "", 0)
			if core.Cfg.GetDebugEnabled() {
				opts.Logger = core.Log
			}
			opts.Verbose = core.Cfg.GetDebugEnabled()
			opts.DataPath = core.GetBasePath() + "/nsq"
			// if cluster get lookupd addresses
			if core.Cfg.GetClusterModeEnabled() {
				opts.NSQLookupdTCPAddresses = core.Cfg.GetNSQLookupdTcpAddresses()
			}

			// deflate (compression)
			opts.DeflateEnabled = true

			// if a message timeout it returns to the queue: https://groups.google.com/d/msg/nsq-users/xBQF1q4srUM/kX22TIoIs-QJ
			// msg timeout : base time to wait from consummer before requeuing a message
			// note: deliverd consumer return immediatly (message is handled in a go routine)
			// Ce qui est au dessus est faux malgres la go routine il attends toujours a la réponse
			// et c'est normal car le message est toujours "in flight"
			// En fait ce timeout c'est le temps durant lequel le message peut rester dans le state "in flight"
			// autrement dit c'est le temps maxi que peu prendre deliverd.processMsg
			opts.MsgTimeout = 10 * time.Minute

			// maximum duration before a message will timeout
			opts.MaxMsgTimeout = 15 * time.Hour

			// maximum requeuing timeout for a message
			// si le client ne demande pas de requeue dans ce delais alors
			// le message et considéré comme traité
			opts.MaxReqTimeout = 1 * time.Hour

			// Number of message in RAM before synching to disk
			opts.MemQueueSize = 0

			nsqd := nsqd.NewNSQD(opts)
			nsqd.LoadMetadata()
			err := nsqd.PersistMetadata()
			if err != nil {
				log.Fatalf("ERROR: failed to persist metadata - %s", err.Error())
			}
			nsqd.Main()

			// smtpd
			if core.Cfg.GetLaunchSmtpd() {
				// clamav ?
				if core.Cfg.GetSmtpdClamavEnabled() {
					if err = core.NewClamav().Ping(); err != nil {
						log.Fatalln("Unable to connect to clamd -", err)
					}
				}

				smtpdDsns, err := core.GetDsnsFromString(core.Cfg.GetSmtpdDsns())
				if err != nil {
					log.Fatalln("unable to parse smtpd dsn -", err)
				}
				for _, dsn := range smtpdDsns {
					go core.NewSmtpd(dsn).ListenAndServe()
					core.Log.Info("smtpd " + dsn.String() + " launched.")
				}
			}

			// deliverd
			go core.LaunchDeliverd()

			// HTTP REST server
			if core.Cfg.GetRestServerLaunch() {
				go rest.LaunchServer()
			}

			<-sigChan
			core.Log.Info("Exiting...")

			// close NsqQueueProducer if exists
			core.NsqQueueProducer.Stop()

			// flush nsqd memory to disk
			nsqd.Exit()

			// exit
			os.Exit(0)
		}
	}
	app.Run(os.Args)

}
<commit_msg>Fixed compilation issues creating new nsqd objects<commit_after>package main

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"runtime"
	"syscall"
	"time"

	"github.com/bitly/nsq/nsqd"
	"github.com/codegangsta/cli"

	tcli "github.com/toorop/tmail/cli"
	"github.com/toorop/tmail/core"
	"github.com/toorop/tmail/rest"
)

const (
	// TmailVersion version of tmail
	TmailVersion = "0.0.11"
)

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	var err error
	if err = core.Bootstrap(); err != nil {
		log.Fatalln(err)
	}
	core.Version = TmailVersion

	// Check base path structure
	requiredPaths := []string{"db", "nsq", "ssl"}
	for _, p := range requiredPaths {
		if err = os.MkdirAll(path.Join(core.GetBasePath(), p), 0700); err != nil {
			log.Fatalln("Unable to create path "+path.Join(core.GetBasePath(), p), " - ", err.Error())
		}
	}

	// TODO: if clusterMode check if nsqlookupd is available

	// check DB
	// TODO: do check in CLI call (raise error & ask for user to run tmail initdb|checkdb)
	if !core.IsOkDB(core.DB) {
		var r []byte
		for {
			fmt.Printf("Database 'driver: %s, source: %s' misses some tables.\r\nShould i create them ? (y/n):", core.Cfg.GetDbDriver(), core.Cfg.GetDbSource())
			r, _, _ = bufio.NewReader(os.Stdin).ReadLine()
			if r[0] == 110 || r[0] == 121 {
				break
			}
		}
		if r[0] == 121 {
			if err = core.InitDB(core.DB); err != nil {
				log.Fatalln(err)
			}
		} else {
			log.Println("See you soon...")
			os.Exit(0)
		}
	}
	// sync tables from structs
	if err := core.AutoMigrateDB(core.DB); err != nil {
		log.Fatalln(err)
	}

	// init rand seed
	rand.Seed(time.Now().UTC().UnixNano())

	// Dovecot support
	if core.Cfg.GetDovecotSupportEnabled() {
		_, err := exec.LookPath(core.Cfg.GetDovecotLda())
		if err != nil {
			log.Fatalln("Unable to find Dovecot LDA binary, checks your config poarameter TMAIL_DOVECOT_LDA ", err)
		}
	}
}

// MAIN
func main() {
	app := cli.NewApp()
	app.Name = "tmail"
	app.Usage = "SMTP server"
	app.Author = "Stéphane Depierrepont aka toorop"
	app.Email = "toorop@tmail.io"
	app.Version = TmailVersion
	app.Commands = tcli.CliCommands
	// no know command ? Launch server
	app.Action = func(c *cli.Context) {
		if len(c.Args()) != 0 {
			cli.ShowAppHelp(c)
		} else {
			// if there is nothing to do then... do nothing
			if !core.Cfg.GetLaunchDeliverd() && !core.Cfg.GetLaunchSmtpd() {
				log.Fatalln("I have nothing to do, so i do nothing. Bye.")
			}

			// Loop
			sigChan := make(chan os.Signal, 1)
			signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)

			// TODO
			// Chanel to comunicate between all elements
			//daChan := make(chan string)

			// init and launch nsqd
			opts := nsqd.NewOptions()
			opts.Logger = log.New(ioutil.Discard, "", 0)
			if core.Cfg.GetDebugEnabled() {
				opts.Logger = core.Log
			}
			opts.Verbose = core.Cfg.GetDebugEnabled()
			opts.DataPath = core.GetBasePath() + "/nsq"
			// if cluster get lookupd addresses
			if core.Cfg.GetClusterModeEnabled() {
				opts.NSQLookupdTCPAddresses = core.Cfg.GetNSQLookupdTcpAddresses()
			}

			// deflate (compression)
			opts.DeflateEnabled = true

			// if a message timeout it returns to the queue: https://groups.google.com/d/msg/nsq-users/xBQF1q4srUM/kX22TIoIs-QJ
			// msg timeout : base time to wait from consummer before requeuing a message
			// note: deliverd consumer return immediatly (message is handled in a go routine)
			// Ce qui est au dessus est faux malgres la go routine il attends toujours a la réponse
			// et c'est normal car le message est toujours "in flight"
			// En fait ce timeout c'est le temps durant lequel le message peut rester dans le state "in flight"
			// autrement dit c'est le temps maxi que peu prendre deliverd.processMsg
			opts.MsgTimeout = 10 * time.Minute

			// maximum duration before a message will timeout
			opts.MaxMsgTimeout = 15 * time.Hour

			// maximum requeuing timeout for a message
			// si le client ne demande pas de requeue dans ce delais alors
			// le message et considéré comme traité
			opts.MaxReqTimeout = 1 * time.Hour

			// Number of message in RAM before synching to disk
			opts.MemQueueSize = 0

			nsqd := nsqd.New(opts)
			nsqd.LoadMetadata()
			err := nsqd.PersistMetadata()
			if err != nil {
				log.Fatalf("ERROR: failed to persist metadata - %s", err.Error())
			}
			nsqd.Main()

			// smtpd
			if core.Cfg.GetLaunchSmtpd() {
				// clamav ?
				if core.Cfg.GetSmtpdClamavEnabled() {
					if err = core.NewClamav().Ping(); err != nil {
						log.Fatalln("Unable to connect to clamd -", err)
					}
				}

				smtpdDsns, err := core.GetDsnsFromString(core.Cfg.GetSmtpdDsns())
				if err != nil {
					log.Fatalln("unable to parse smtpd dsn -", err)
				}
				for _, dsn := range smtpdDsns {
					go core.NewSmtpd(dsn).ListenAndServe()
					core.Log.Info("smtpd " + dsn.String() + " launched.")
				}
			}

			// deliverd
			go core.LaunchDeliverd()

			// HTTP REST server
			if core.Cfg.GetRestServerLaunch() {
				go rest.LaunchServer()
			}

			<-sigChan
			core.Log.Info("Exiting...")

			// close NsqQueueProducer if exists
			core.NsqQueueProducer.Stop()

			// flush nsqd memory to disk
			nsqd.Exit()

			// exit
			os.Exit(0)
		}
	}
	app.Run(os.Args)

}
<|endoftext|>