<commit_before>// The MIT License (MIT)
//
// Copyright (c) 2014 wandoulabs
// Copyright (c) 2014 siddontang
//
// Permission is hereby granted, free of charge, to any person obtaining a copy of
// this software and associated documentation files (the "Software"), to deal in
// the Software without restriction, including without limitation the rights to
// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
// the Software, and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.

// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"encoding/json"
	"math/rand"
	"net"
	"net/http"
	"sync"
	"sync/atomic"
	"time"
	// For pprof
	_ "net/http/pprof"

	"github.com/juju/errors"
	"github.com/ngaut/log"
	"github.com/pingcap/tidb/mysql"
	"github.com/pingcap/tidb/terror"
	"github.com/pingcap/tidb/util/arena"
	"github.com/pingcap/tidb/util/printer"
	"github.com/prometheus/client_golang/prometheus"
)

var (
	baseConnID uint32 = 10000
)

var (
	errUnknownFieldType  = terror.ClassServer.New(codeUnknownFieldType, "unknown field type")
	errInvalidPayloadLen = terror.ClassServer.New(codeInvalidPayloadLen, "invalid payload length")
	errInvalidSequence   = terror.ClassServer.New(codeInvalidSequence, "invalid sequence")
	errInvalidType       = terror.ClassServer.New(codeInvalidType, "invalid type")
	errNotAllowedCommand = terror.ClassServer.New(codeNotAllowedCommand,
		"the used command is not allowed with this TiDB version")
)

// Server is the MySQL protocol server
type Server struct {
	cfg               *Config
	driver            IDriver
	listener          net.Listener
	rwlock            *sync.RWMutex
	concurrentLimiter *TokenLimiter
	clients           map[uint32]*clientConn
}

// ConnectionCount gets current connection count.
func (s *Server) ConnectionCount() int {
	var cnt int
	s.rwlock.RLock()
	cnt = len(s.clients)
	s.rwlock.RUnlock()
	return cnt
}

func (s *Server) getToken() *Token {
	return s.concurrentLimiter.Get()
}

func (s *Server) releaseToken(token *Token) {
	s.concurrentLimiter.Put(token)
}

// Generate a random string using ASCII characters but avoid separator character.
// See https://github.com/mysql/mysql-server/blob/5.7/mysys_ssl/crypt_genhash_impl.cc#L435
func randomBuf(size int) []byte {
	buf := make([]byte, size)
	for i := 0; i < size; i++ {
		buf[i] = byte(rand.Intn(127))
		if buf[i] == 0 || buf[i] == byte('$') {
			buf[i]++
		}
	}
	return buf
}

// newConn creates a new *clientConn from a net.Conn.
// It allocates a connection ID and random salt data for authentication.
func (s *Server) newConn(conn net.Conn) *clientConn {
	log.Info("newConn", conn.RemoteAddr().String())
	cc := &clientConn{
		conn:         conn,
		pkt:          newPacketIO(conn),
		server:       s,
		connectionID: atomic.AddUint32(&baseConnID, 1),
		collation:    mysql.DefaultCollationID,
		alloc:        arena.NewAllocator(32 * 1024),
	}
	cc.salt = randomBuf(20)
	return cc
}

func (s *Server) skipAuth() bool {
	return s.cfg.SkipAuth
}

// NewServer creates a new Server.
func NewServer(cfg *Config, driver IDriver) (*Server, error) {
	s := &Server{
		cfg:               cfg,
		driver:            driver,
		concurrentLimiter: NewTokenLimiter(100),
		rwlock:            &sync.RWMutex{},
		clients:           make(map[uint32]*clientConn),
	}

	var err error
	if cfg.Socket != "" {
		cfg.SkipAuth = true
		s.listener, err = net.Listen("unix", cfg.Socket)
	} else {
		s.listener, err = net.Listen("tcp", s.cfg.Addr)
	}
	if err != nil {
		return nil, errors.Trace(err)
	}

	// Init rand seed for randomBuf()
	rand.Seed(time.Now().UTC().UnixNano())
	log.Infof("Server run MySQL Protocol Listen at [%s]", s.cfg.Addr)
	return s, nil
}

// Run runs the server.
func (s *Server) Run() error {

	// Start http api to report tidb info such as tps.
	if s.cfg.ReportStatus {
		s.startStatusHTTP()
	}

	for {
		conn, err := s.listener.Accept()
		if err != nil {
			if opErr, ok := err.(*net.OpError); ok {
				if opErr.Err.Error() == "use of closed network connection" {
					return nil
				}
			}
			log.Errorf("accept error %s", err.Error())
			return errors.Trace(err)
		}

		go s.onConn(conn)
	}
}

// Close closes the server.
func (s *Server) Close() {
	s.rwlock.Lock()
	defer s.rwlock.Unlock()

	if s.listener != nil {
		s.listener.Close()
		s.listener = nil
	}
}

// onConn runs in its own goroutine, handles queries from this connection.
func (s *Server) onConn(c net.Conn) {
	conn := s.newConn(c)
	if err := conn.handshake(); err != nil {
		log.Errorf("handshake error %s", errors.ErrorStack(err))
		c.Close()
		return
	}
	defer func() {
		log.Infof("close %s", conn)
	}()

	s.rwlock.Lock()
	s.clients[conn.connectionID] = conn
	connections := len(s.clients)
	s.rwlock.Unlock()
	connGauge.Set(float64(connections))

	conn.Run()
}

var once sync.Once

const defaultStatusAddr = ":10080"

func (s *Server) startStatusHTTP() {
	once.Do(func() {
		go func() {
			http.HandleFunc("/status", func(w http.ResponseWriter, req *http.Request) {
				w.Header().Set("Content-Type", "application/json")
				s := status{
					Connections: s.ConnectionCount(),
					Version:     mysql.ServerVersion,
					GitHash:     printer.TiDBGitHash,
				}
				js, err := json.Marshal(s)
				if err != nil {
					w.WriteHeader(http.StatusInternalServerError)
					log.Error("Encode json error", err)
				} else {
					w.Write(js)
				}

			})
			// HTTP path for prometheus.
			http.Handle("/metrics", prometheus.Handler())
			addr := s.cfg.StatusAddr
			if len(addr) == 0 {
				addr = defaultStatusAddr
			}
			log.Infof("Listening on %v for status and metrics report.", addr)
			err := http.ListenAndServe(addr, nil)
			if err != nil {
				log.Fatal(err)
			}
		}()
	})
}

// TiDB status
type status struct {
	Connections int    `json:"connections"`
	Version     string `json:"version"`
	GitHash     string `json:"git_hash"`
}

// Server error codes.
const (
	codeUnknownFieldType  = 1
	codeInvalidPayloadLen = 2
	codeInvalidSequence   = 3
	codeInvalidType       = 4

	codeNotAllowedCommand = 1148
)

func init() {
	serverMySQLErrCodes := map[terror.ErrCode]uint16{
		codeNotAllowedCommand: mysql.ErrNotAllowedCommand,
	}
	terror.ErrClassToMySQLCodes[terror.ClassServer] = serverMySQLErrCodes
}
<commit_msg>server: Change handshake error log level (#1839)<commit_after>// The MIT License (MIT)
//
// Copyright (c) 2014 wandoulabs
// Copyright (c) 2014 siddontang
//
// Permission is hereby granted, free of charge, to any person obtaining a copy of
// this software and associated documentation files (the "Software"), to deal in
// the Software without restriction, including without limitation the rights to
// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
// the Software, and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.

// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"encoding/json"
	"math/rand"
	"net"
	"net/http"
	"sync"
	"sync/atomic"
	"time"
	// For pprof
	_ "net/http/pprof"

	"github.com/juju/errors"
	"github.com/ngaut/log"
	"github.com/pingcap/tidb/mysql"
	"github.com/pingcap/tidb/terror"
	"github.com/pingcap/tidb/util/arena"
	"github.com/pingcap/tidb/util/printer"
	"github.com/prometheus/client_golang/prometheus"
)

var (
	baseConnID uint32 = 10000
)

var (
	errUnknownFieldType  = terror.ClassServer.New(codeUnknownFieldType, "unknown field type")
	errInvalidPayloadLen = terror.ClassServer.New(codeInvalidPayloadLen, "invalid payload length")
	errInvalidSequence   = terror.ClassServer.New(codeInvalidSequence, "invalid sequence")
	errInvalidType       = terror.ClassServer.New(codeInvalidType, "invalid type")
	errNotAllowedCommand = terror.ClassServer.New(codeNotAllowedCommand,
		"the used command is not allowed with this TiDB version")
)

// Server is the MySQL protocol server
type Server struct {
	cfg               *Config
	driver            IDriver
	listener          net.Listener
	rwlock            *sync.RWMutex
	concurrentLimiter *TokenLimiter
	clients           map[uint32]*clientConn
}

// ConnectionCount gets current connection count.
func (s *Server) ConnectionCount() int {
	var cnt int
	s.rwlock.RLock()
	cnt = len(s.clients)
	s.rwlock.RUnlock()
	return cnt
}

func (s *Server) getToken() *Token {
	return s.concurrentLimiter.Get()
}

func (s *Server) releaseToken(token *Token) {
	s.concurrentLimiter.Put(token)
}

// Generate a random string using ASCII characters but avoid separator character.
// See https://github.com/mysql/mysql-server/blob/5.7/mysys_ssl/crypt_genhash_impl.cc#L435
func randomBuf(size int) []byte {
	buf := make([]byte, size)
	for i := 0; i < size; i++ {
		buf[i] = byte(rand.Intn(127))
		if buf[i] == 0 || buf[i] == byte('$') {
			buf[i]++
		}
	}
	return buf
}

// newConn creates a new *clientConn from a net.Conn.
// It allocates a connection ID and random salt data for authentication.
func (s *Server) newConn(conn net.Conn) *clientConn {
	log.Info("newConn", conn.RemoteAddr().String())
	cc := &clientConn{
		conn:         conn,
		pkt:          newPacketIO(conn),
		server:       s,
		connectionID: atomic.AddUint32(&baseConnID, 1),
		collation:    mysql.DefaultCollationID,
		alloc:        arena.NewAllocator(32 * 1024),
	}
	cc.salt = randomBuf(20)
	return cc
}

func (s *Server) skipAuth() bool {
	return s.cfg.SkipAuth
}

// NewServer creates a new Server.
func NewServer(cfg *Config, driver IDriver) (*Server, error) {
	s := &Server{
		cfg:               cfg,
		driver:            driver,
		concurrentLimiter: NewTokenLimiter(100),
		rwlock:            &sync.RWMutex{},
		clients:           make(map[uint32]*clientConn),
	}

	var err error
	if cfg.Socket != "" {
		cfg.SkipAuth = true
		s.listener, err = net.Listen("unix", cfg.Socket)
	} else {
		s.listener, err = net.Listen("tcp", s.cfg.Addr)
	}
	if err != nil {
		return nil, errors.Trace(err)
	}

	// Init rand seed for randomBuf()
	rand.Seed(time.Now().UTC().UnixNano())
	log.Infof("Server run MySQL Protocol Listen at [%s]", s.cfg.Addr)
	return s, nil
}

// Run runs the server.
func (s *Server) Run() error {

	// Start http api to report tidb info such as tps.
	if s.cfg.ReportStatus {
		s.startStatusHTTP()
	}

	for {
		conn, err := s.listener.Accept()
		if err != nil {
			if opErr, ok := err.(*net.OpError); ok {
				if opErr.Err.Error() == "use of closed network connection" {
					return nil
				}
			}
			log.Errorf("accept error %s", err.Error())
			return errors.Trace(err)
		}

		go s.onConn(conn)
	}
}

// Close closes the server.
func (s *Server) Close() {
	s.rwlock.Lock()
	defer s.rwlock.Unlock()

	if s.listener != nil {
		s.listener.Close()
		s.listener = nil
	}
}

// onConn runs in its own goroutine, handles queries from this connection.
func (s *Server) onConn(c net.Conn) {
	conn := s.newConn(c)
	if err := conn.handshake(); err != nil {
		// Some keep alive services will send request to TiDB and disconnect immediately.
		// So we use info log level.
		log.Infof("handshake error %s", errors.ErrorStack(err))
		c.Close()
		return
	}
	defer func() {
		log.Infof("close %s", conn)
	}()

	s.rwlock.Lock()
	s.clients[conn.connectionID] = conn
	connections := len(s.clients)
	s.rwlock.Unlock()
	connGauge.Set(float64(connections))

	conn.Run()
}

var once sync.Once

const defaultStatusAddr = ":10080"

func (s *Server) startStatusHTTP() {
	once.Do(func() {
		go func() {
			http.HandleFunc("/status", func(w http.ResponseWriter, req *http.Request) {
				w.Header().Set("Content-Type", "application/json")
				s := status{
					Connections: s.ConnectionCount(),
					Version:     mysql.ServerVersion,
					GitHash:     printer.TiDBGitHash,
				}
				js, err := json.Marshal(s)
				if err != nil {
					w.WriteHeader(http.StatusInternalServerError)
					log.Error("Encode json error", err)
				} else {
					w.Write(js)
				}

			})
			// HTTP path for prometheus.
			http.Handle("/metrics", prometheus.Handler())
			addr := s.cfg.StatusAddr
			if len(addr) == 0 {
				addr = defaultStatusAddr
			}
			log.Infof("Listening on %v for status and metrics report.", addr)
			err := http.ListenAndServe(addr, nil)
			if err != nil {
				log.Fatal(err)
			}
		}()
	})
}

// TiDB status
type status struct {
	Connections int    `json:"connections"`
	Version     string `json:"version"`
	GitHash     string `json:"git_hash"`
}

// Server error codes.
const (
	codeUnknownFieldType  = 1
	codeInvalidPayloadLen = 2
	codeInvalidSequence   = 3
	codeInvalidType       = 4

	codeNotAllowedCommand = 1148
)

func init() {
	serverMySQLErrCodes := map[terror.ErrCode]uint16{
		codeNotAllowedCommand: mysql.ErrNotAllowedCommand,
	}
	terror.ErrClassToMySQLCodes[terror.ClassServer] = serverMySQLErrCodes
}
<|endoftext|>


<commit_before>// gorewind is an event store server written in Python that talks ZeroMQ.
// Copyright (C) 2013  Jens Rantil
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

// Contains the server loop. Deals with incoming requests and delegates
// them to the event store.
package server

import (
	"bytes"
	"errors"
	"fmt"
	"container/list"
	zmq "github.com/alecthomas/gozmq"
)

type InitParams struct {
	Store *EventStore
	CommandSocketZPath *string
	EvPubSocketZPath *string
}

// Check all required initialization parameters are set.
func checkAllInitParamsSet(p *InitParams) error {
	if p.Store == nil {
		return errors.New("Missing param: Store")
	}
	if p.CommandSocketZPath == nil {
		return errors.New("Missing param: CommandSocketZPath")
	}
	if p.EvPubSocketZPath == nil {
		return errors.New("Missing param: EvPubSocketZPath")
	}
	return nil
}

// A server instance. Can be run.
type Server struct {
	params InitParams

	evpubsock *zmq.Socket
	commandsock *zmq.Socket
	context *zmq.Context
}

// Initialize a new event store server and return a handle to it. The
// event store is not started. It's up to the caller to execute Run()
// on the server handle.
func New(params *InitParams) (*Server, error) {
	if params == nil {
		return nil, errors.New("Missing init params")
	}
	if err := checkAllInitParamsSet(params); err != nil {
		return nil, err
	}

	server := Server{
		params: *params,
	}

	var allOkay *bool = new(bool)
	*allOkay = false
	defer func() {
		if (!*allOkay) {
			server.Close()
		}
	}()

	context, err := zmq.NewContext()
	if err != nil {
		return nil, err
	}
	server.context = &context

	commandsock, err := context.NewSocket(zmq.ROUTER)
	if err != nil {
		return nil, err
	}
	server.commandsock = &commandsock
	err = commandsock.Bind(*params.CommandSocketZPath)
	if err != nil {
		return nil, err
	}

	evpubsock, err := context.NewSocket(zmq.PUB)
	if err != nil {
		return nil, err
	}
	server.evpubsock = &evpubsock
	if binderr := evpubsock.Bind(*params.EvPubSocketZPath); binderr != nil {
		return nil, err
	}

	*allOkay = true

	return &server, nil
}

func (v *Server) Close() {
	(*v.evpubsock).Close()
	v.evpubsock = nil
	(*v.commandsock).Close()
	v.commandsock = nil
	(*v.context).Close()
	v.context = nil
}

// Runs the server that distributes requests to workers.
// Panics on error since it is an essential piece of code required to
// run the application correctly.
func (v *Server) Run() {
	loopServer((*v).params.Store, *(*v).evpubsock, *(*v).commandsock)
}

// The result of an asynchronous zmq.Poll call.
type zmqPollResult struct {
	nbrOfChanges int
	err error
}

// Polls a bunch of ZeroMQ sockets and notifies the result through a
// channel. This makes it possible to combine ZeroMQ polling with Go's
// own built-in channels.
func asyncPoll(notifier chan zmqPollResult, items zmq.PollItems) {
	a, b := zmq.Poll(items, -1)
	notifier <- zmqPollResult{a, b}
}

// The core ZeroMQ messaging loop. Handles requests and responses
// asynchronously using the router socket. Every request is delegated to
// a goroutine for maximum concurrency.
//
// `gozmq` does currently not support copy-free messages/frames. This
// means that every message passing through this function needs to be
// copied in-memory. If this becomes a bottleneck in the future,
// multiple router sockets can be hooked to this final router to scale
// message copying.
func loopServer(estore *EventStore, evpubsock, frontend zmq.Socket) {
	toPoll := zmq.PollItems{
		zmq.PollItem{Socket: frontend, zmq.Events: zmq.POLLIN},
	}

	pubchan := make(chan StoredEvent)
	estore.RegisterPublishedEventsChannel(pubchan)
	go publishAllSavedEvents(pubchan, evpubsock)

	pollchan := make(chan zmqPollResult)
	respchan := make(chan [][]byte)
	go asyncPoll(pollchan, toPoll)
	for {
		select {
		case <- pollchan:
			if toPoll[0].REvents&zmq.POLLIN != 0 {
				msg, _ := toPoll[0].Socket.RecvMultipart(0)
				go handleRequest(respchan, estore, msg)
			}
			go asyncPoll(pollchan, toPoll)
		case frames := <-respchan:
			if err := frontend.SendMultipart(frames, 0); err != nil {
				fmt.Println(err)
			}
		}
	}
}

// Publishes stored events to event listeners.
//
// Pops previously stored messages off a channel and published them to a
// ZeroMQ socket.
func publishAllSavedEvents(toPublish chan StoredEvent, evpub zmq.Socket) {
	msg := make([][]byte, 3)
	for {
		event := <-toPublish

		msg[0] = event.Stream
		msg[1] = event.Id
		msg[2] = event.Data

		if err := evpub.SendMultipart(msg, 0); err != nil {
			// TODO: Use logger
			fmt.Println(err)
		}
	}
}

// Handles a single ZeroMQ RES/REQ loop synchronously.
//
// The full request message stored in `msg` and the full ZeroMQ response
// is pushed to `respchan`. The function does not return any error
// because it is expected to be called asynchronously as a goroutine.
func handleRequest(respchan chan [][]byte, estore *EventStore, msg [][]byte) {

	// TODO: Rename to 'framelist'
	// TODO: Avoid this creation and move it into copyList.
	parts := list.New()
	for _, msgpart := range msg {
		parts.PushBack(msgpart)
	}

	// TODO: Possibly wrap ZeroMQ router frames into a Type before
	// calling this method. That would yield a nicer API without
	// nitty gritty ZeroMQ details.
	resptemplate := list.New()
	emptyFrame := []byte("")
	for true {
		resptemplate.PushBack(parts.Remove(parts.Front()))

		if bytes.Equal(parts.Front().Value.([]byte), emptyFrame) {
			break
		}
	}

	if parts.Len() == 0 {
		errstr := "Incoming command was empty. Ignoring it."
		// TODO: Migrate to logging system
		fmt.Println(errstr)
		response := copyList(resptemplate)
		response.PushBack([]byte("ERROR " + errstr))
		respchan <- listToFrames(response)
		return
	}

	command := string(parts.Front().Value.([]byte))
	switch command {
	case "PUBLISH":
		parts.Remove(parts.Front())
		if parts.Len() != 2 {
			errstr := "Wrong number of frames for PUBLISH."
			// TODO: Migrate to logging system
			fmt.Println(errstr)
			response := copyList(resptemplate)
			response.PushBack([]byte("ERROR " + errstr))
			respchan <- listToFrames(response)
		} else {
			estream := parts.Remove(parts.Front())
			data := parts.Remove(parts.Front())
			newevent := UnstoredEvent{
				Stream: estream.([]byte),
				Data: data.([]byte),
			}
			newId, err := estore.Add(newevent)
			if err != nil {
				// TODO: Migrate to logging system
				sErr := err.Error()
				fmt.Println(sErr)

				response := copyList(resptemplate)
				response.PushBack([]byte("ERROR " + sErr))
				respchan <- listToFrames(response)
			} else {
				// the event was added
				response := copyList(resptemplate)
				response.PushBack([]byte("PUBLISHED"))
				response.PushBack([]byte(newId))
				respchan <- listToFrames(response)
			}
		}
	case "QUERY":
		parts.Remove(parts.Front())
		if parts.Len() != 3 {
			errstr := "Wrong number of frames for QUERY."
			// TODO: Migrate to logging system
			fmt.Println(errstr)
			response := copyList(resptemplate)
			response.PushBack([]byte("ERROR " + errstr))
			respchan <- listToFrames(response)
		} else {
			estreamprefix := parts.Remove(parts.Front())
			fromid := parts.Remove(parts.Front())
			toid := parts.Remove(parts.Front())

			events := make(chan StoredEvent)
			req := QueryRequest{
				Stream: estreamprefix.([]byte),
				FromId: fromid.([]byte),
				ToId: toid.([]byte),
			}
			go estore.Query(req, events)
			for eventdata := range(events) {
				response := copyList(resptemplate)
				response.PushBack(eventdata.Stream)
				response.PushBack(eventdata.Id)
				response.PushBack(eventdata.Data)

				respchan <- listToFrames(response)
			}
			response := copyList(resptemplate)
			response.PushBack([]byte("END"))
			respchan <- listToFrames(response)
		}
	default:
		// TODO: Move these error strings out as constants of
		//       this package.

		// TODO: Move the chunk of code below into a separate
		// function and reuse for similar piece of code above.
		errstr := "Unknown request type."
		// TODO: Migrate to logging system
		fmt.Println(errstr)
		response := copyList(resptemplate)
		response.PushBack([]byte("ERROR " + errstr))
		respchan <- listToFrames(response)
	}
}

// Convert a doubly linked list of message frames to a slice of message
// fram
func listToFrames(l *list.List) [][]byte {
	frames := make([][]byte, l.Len())
	i := 0
	for e := l.Front(); e != nil; e = e.Next() {
		frames[i] = e.Value.([]byte)
	}
	return frames
}

// Helper function for copying a doubly linked list.
func copyList(l *list.List) *list.List {
	replica := list.New()
	replica.PushBackList(l)
	return replica
}

<commit_msg>Removing an unfixable TODO<commit_after>// gorewind is an event store server written in Python that talks ZeroMQ.
// Copyright (C) 2013  Jens Rantil
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

// Contains the server loop. Deals with incoming requests and delegates
// them to the event store.
package server

import (
	"bytes"
	"errors"
	"fmt"
	"container/list"
	zmq "github.com/alecthomas/gozmq"
)

type InitParams struct {
	Store *EventStore
	CommandSocketZPath *string
	EvPubSocketZPath *string
}

// Check all required initialization parameters are set.
func checkAllInitParamsSet(p *InitParams) error {
	if p.Store == nil {
		return errors.New("Missing param: Store")
	}
	if p.CommandSocketZPath == nil {
		return errors.New("Missing param: CommandSocketZPath")
	}
	if p.EvPubSocketZPath == nil {
		return errors.New("Missing param: EvPubSocketZPath")
	}
	return nil
}

// A server instance. Can be run.
type Server struct {
	params InitParams

	evpubsock *zmq.Socket
	commandsock *zmq.Socket
	context *zmq.Context
}

// Initialize a new event store server and return a handle to it. The
// event store is not started. It's up to the caller to execute Run()
// on the server handle.
func New(params *InitParams) (*Server, error) {
	if params == nil {
		return nil, errors.New("Missing init params")
	}
	if err := checkAllInitParamsSet(params); err != nil {
		return nil, err
	}

	server := Server{
		params: *params,
	}

	var allOkay *bool = new(bool)
	*allOkay = false
	defer func() {
		if (!*allOkay) {
			server.Close()
		}
	}()

	context, err := zmq.NewContext()
	if err != nil {
		return nil, err
	}
	server.context = &context

	commandsock, err := context.NewSocket(zmq.ROUTER)
	if err != nil {
		return nil, err
	}
	server.commandsock = &commandsock
	err = commandsock.Bind(*params.CommandSocketZPath)
	if err != nil {
		return nil, err
	}

	evpubsock, err := context.NewSocket(zmq.PUB)
	if err != nil {
		return nil, err
	}
	server.evpubsock = &evpubsock
	if binderr := evpubsock.Bind(*params.EvPubSocketZPath); binderr != nil {
		return nil, err
	}

	*allOkay = true

	return &server, nil
}

func (v *Server) Close() {
	(*v.evpubsock).Close()
	v.evpubsock = nil
	(*v.commandsock).Close()
	v.commandsock = nil
	(*v.context).Close()
	v.context = nil
}

// Runs the server that distributes requests to workers.
// Panics on error since it is an essential piece of code required to
// run the application correctly.
func (v *Server) Run() {
	loopServer((*v).params.Store, *(*v).evpubsock, *(*v).commandsock)
}

// The result of an asynchronous zmq.Poll call.
type zmqPollResult struct {
	nbrOfChanges int
	err error
}

// Polls a bunch of ZeroMQ sockets and notifies the result through a
// channel. This makes it possible to combine ZeroMQ polling with Go's
// own built-in channels.
func asyncPoll(notifier chan zmqPollResult, items zmq.PollItems) {
	a, b := zmq.Poll(items, -1)
	notifier <- zmqPollResult{a, b}
}

// The core ZeroMQ messaging loop. Handles requests and responses
// asynchronously using the router socket. Every request is delegated to
// a goroutine for maximum concurrency.
//
// `gozmq` does currently not support copy-free messages/frames. This
// means that every message passing through this function needs to be
// copied in-memory. If this becomes a bottleneck in the future,
// multiple router sockets can be hooked to this final router to scale
// message copying.
func loopServer(estore *EventStore, evpubsock, frontend zmq.Socket) {
	toPoll := zmq.PollItems{
		zmq.PollItem{Socket: frontend, zmq.Events: zmq.POLLIN},
	}

	pubchan := make(chan StoredEvent)
	estore.RegisterPublishedEventsChannel(pubchan)
	go publishAllSavedEvents(pubchan, evpubsock)

	pollchan := make(chan zmqPollResult)
	respchan := make(chan [][]byte)
	go asyncPoll(pollchan, toPoll)
	for {
		select {
		case <- pollchan:
			if toPoll[0].REvents&zmq.POLLIN != 0 {
				msg, _ := toPoll[0].Socket.RecvMultipart(0)
				go handleRequest(respchan, estore, msg)
			}
			go asyncPoll(pollchan, toPoll)
		case frames := <-respchan:
			if err := frontend.SendMultipart(frames, 0); err != nil {
				fmt.Println(err)
			}
		}
	}
}

// Publishes stored events to event listeners.
//
// Pops previously stored messages off a channel and published them to a
// ZeroMQ socket.
func publishAllSavedEvents(toPublish chan StoredEvent, evpub zmq.Socket) {
	msg := make([][]byte, 3)
	for {
		event := <-toPublish

		msg[0] = event.Stream
		msg[1] = event.Id
		msg[2] = event.Data

		if err := evpub.SendMultipart(msg, 0); err != nil {
			// TODO: Use logger
			fmt.Println(err)
		}
	}
}

// Handles a single ZeroMQ RES/REQ loop synchronously.
//
// The full request message stored in `msg` and the full ZeroMQ response
// is pushed to `respchan`. The function does not return any error
// because it is expected to be called asynchronously as a goroutine.
func handleRequest(respchan chan [][]byte, estore *EventStore, msg [][]byte) {

	// TODO: Rename to 'framelist'
	parts := list.New()
	for _, msgpart := range msg {
		parts.PushBack(msgpart)
	}

	// TODO: Possibly wrap ZeroMQ router frames into a Type before
	// calling this method. That would yield a nicer API without
	// nitty gritty ZeroMQ details.
	resptemplate := list.New()
	emptyFrame := []byte("")
	for true {
		resptemplate.PushBack(parts.Remove(parts.Front()))

		if bytes.Equal(parts.Front().Value.([]byte), emptyFrame) {
			break
		}
	}

	if parts.Len() == 0 {
		errstr := "Incoming command was empty. Ignoring it."
		// TODO: Migrate to logging system
		fmt.Println(errstr)
		response := copyList(resptemplate)
		response.PushBack([]byte("ERROR " + errstr))
		respchan <- listToFrames(response)
		return
	}

	command := string(parts.Front().Value.([]byte))
	switch command {
	case "PUBLISH":
		parts.Remove(parts.Front())
		if parts.Len() != 2 {
			errstr := "Wrong number of frames for PUBLISH."
			// TODO: Migrate to logging system
			fmt.Println(errstr)
			response := copyList(resptemplate)
			response.PushBack([]byte("ERROR " + errstr))
			respchan <- listToFrames(response)
		} else {
			estream := parts.Remove(parts.Front())
			data := parts.Remove(parts.Front())
			newevent := UnstoredEvent{
				Stream: estream.([]byte),
				Data: data.([]byte),
			}
			newId, err := estore.Add(newevent)
			if err != nil {
				// TODO: Migrate to logging system
				sErr := err.Error()
				fmt.Println(sErr)

				response := copyList(resptemplate)
				response.PushBack([]byte("ERROR " + sErr))
				respchan <- listToFrames(response)
			} else {
				// the event was added
				response := copyList(resptemplate)
				response.PushBack([]byte("PUBLISHED"))
				response.PushBack([]byte(newId))
				respchan <- listToFrames(response)
			}
		}
	case "QUERY":
		parts.Remove(parts.Front())
		if parts.Len() != 3 {
			errstr := "Wrong number of frames for QUERY."
			// TODO: Migrate to logging system
			fmt.Println(errstr)
			response := copyList(resptemplate)
			response.PushBack([]byte("ERROR " + errstr))
			respchan <- listToFrames(response)
		} else {
			estreamprefix := parts.Remove(parts.Front())
			fromid := parts.Remove(parts.Front())
			toid := parts.Remove(parts.Front())

			events := make(chan StoredEvent)
			req := QueryRequest{
				Stream: estreamprefix.([]byte),
				FromId: fromid.([]byte),
				ToId: toid.([]byte),
			}
			go estore.Query(req, events)
			for eventdata := range(events) {
				response := copyList(resptemplate)
				response.PushBack(eventdata.Stream)
				response.PushBack(eventdata.Id)
				response.PushBack(eventdata.Data)

				respchan <- listToFrames(response)
			}
			response := copyList(resptemplate)
			response.PushBack([]byte("END"))
			respchan <- listToFrames(response)
		}
	default:
		// TODO: Move these error strings out as constants of
		//       this package.

		// TODO: Move the chunk of code below into a separate
		// function and reuse for similar piece of code above.
		errstr := "Unknown request type."
		// TODO: Migrate to logging system
		fmt.Println(errstr)
		response := copyList(resptemplate)
		response.PushBack([]byte("ERROR " + errstr))
		respchan <- listToFrames(response)
	}
}

// Convert a doubly linked list of message frames to a slice of message
// fram
func listToFrames(l *list.List) [][]byte {
	frames := make([][]byte, l.Len())
	i := 0
	for e := l.Front(); e != nil; e = e.Next() {
		frames[i] = e.Value.([]byte)
	}
	return frames
}

// Helper function for copying a doubly linked list.
func copyList(l *list.List) *list.List {
	replica := list.New()
	replica.PushBackList(l)
	return replica
}

<|endoftext|>


<commit_before>package server

// transfer2go agent server implementation
// Author: Valentin Kuznetsov <vkuznet@gmail.com>

import (
	"crypto/tls"
	"database/sql"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	log "github.com/sirupsen/logrus"
	"github.com/vkuznet/transfer2go/core"
	"github.com/vkuznet/transfer2go/utils"

	// web profiler, see https://golang.org/pkg/net/http/pprof
	_ "net/http/pprof"
)

// Config type holds server configuration
type Config struct {
	Name           string `json:"name"`           // agent name, aka site name
	Url            string `json:"url"`            // agent url
	Catalog        string `json:"catalog"`        // catalog file name, e.g. catalog.json
	CentralCatalog string `json:"centralCatalog"` // central catalog file name, e.g. cc.json
	Protocol       string `json:"protocol"`       // backend protocol, e.g. srmv2
	Backend        string `json:"backend"`        // backend, e.g. srm
	Tool           string `json:"tool"`           // backend tool, e.g. srmcp
	ToolOpts       string `json:"toolopts"`       // options for backend tool
	Mfile          string `json:"mfile"`          // metrics file name
	Cfile          string `json:"csvfile"`        // historical data file
	Minterval      int64  `json:"minterval"`      // metrics interval
	Staticdir      string `json:"staticdir"`      // static dir defines location of static files, e.g. sql,js templates
	Workers        int    `json:"workers"`        // number of workers
	QueueSize      int    `json:"queuesize"`      // total size of the queue
	Port           int    `json:"port"`           // port number given server runs on, default 8989
	Base           string `json:"base"`           // URL base path for agent server, it will be extracted from Url
	Register       string `json:"register"`       // remote agent URL to register
	ServerKey      string `json:"serverkey"`      // server key file
	ServerCrt      string `json:"servercrt"`      // server crt file
	Type           string `json:"type"`           // Configure server type push/pull
	BufferSize     int    `json:"buffersize"`     // Size of buffered channels
	MonitorTime    int64  `json:"monitorTime"`    // Large time interval after which we need to reset monitoring calculation
	TrainInterval  string `json:"trinterval"`     // Time after which we need to retrain main agent
	RouterModel    bool   `json:"router"`         // Variable to enable the router model
}

// String returns string representation of Config data type
func (c *Config) String() string {
	return fmt.Sprintf("<Config: name=%s url=%s port=%d base=%s catalog=%s protocol=%s backend=%s tool=%s opts=%s mfile=%s minterval=%d staticdir=%s workders=%d queuesize=%d register=%s type=%s router=%v>", c.Name, c.Url, c.Port, c.Base, c.Catalog, c.Protocol, c.Backend, c.Tool, c.ToolOpts, c.Mfile, c.Minterval, c.Staticdir, c.Workers, c.QueueSize, c.Register, c.Type, c.RouterModel)
}

// AgentInfo data type
type AgentInfo struct {
	Agent string
	Alias string
}

// AgentProtocol data type
type AgentProtocol struct {
	Protocol string `json:"protocol"` // protocol name, e.g. srmv2
	Backend  string `json:"backend"`  // backend storage end-point, e.g. srm://cms-srm.cern.ch:8443/srm/managerv2?SFN=
	Tool     string `json:"tool"`     // actual executable, e.g. /usr/local/bin/srmcp
	ToolOpts string `json:"toolopts"` // options for backend tool
}

// globals used in server/handlers
var _myself, _alias, _protocol, _backend, _tool, _toolOpts string
var _agents map[string]string
var _config Config

// init
func init() {
	_agents = make(map[string]string)
}

// register a new (alias, agent) pair in agent (register)
func register(register, alias, agent string) error {
	log.WithFields(log.Fields{
		"Agent":    agent,
		"Alias":    alias,
		"Register": register,
	}).Println("Register agent as alias on register")
	// register myself with another agent
	params := AgentInfo{Agent: _myself, Alias: _alias}
	data, err := json.Marshal(params)
	if err != nil {
		log.WithFields(log.Fields{
			"Params": params,
		}).Error("Unable to marshal params", params)
	}
	url := fmt.Sprintf("%s/register", register)
	resp := utils.FetchResponse(url, data) // POST request
	// check return status code
	if resp.StatusCode != 200 {
		return fmt.Errorf("Response %s, error=%s", resp.Status, string(resp.Data))
	}
	return resp.Error
}

// helper function to register agent with all distributed agents
func registerAtAgents(aName string) {
	// register itself
	if _, ok := _agents[_alias]; ok {
		log.WithFields(log.Fields{
			"Alias":  _alias,
			"Agents": _agents,
		}).Fatal("Unable to register, alias, since this name already exists")
	}
	_agents[_alias] = _myself

	// now ask remote server for its list of agents and update internal map
	if aName != "" && len(aName) > 0 {
		err := register(aName, _alias, _myself) // submit remote registration of given agent name
		if err != nil {
			log.WithFields(log.Fields{
				"Alias": _alias,
				"Self":  _myself,
				"Name":  aName,
				"Error": err,
			}).Fatal("Unable to register")
		}
		aurl := fmt.Sprintf("%s/agents", aName)
		resp := utils.FetchResponse(aurl, []byte{})
		var remoteAgents map[string]string
		e := json.Unmarshal(resp.Data, &remoteAgents)
		if e == nil {
			for key, val := range remoteAgents {
				if _, ok := _agents[key]; !ok {
					_agents[key] = val // register remote agent/alias pair internally
				}
			}
		}
	}

	// complete registration with other agents
	for alias, agent := range _agents {
		if agent == aName || alias == _alias {
			continue
		}
		register(agent, _alias, _myself) // submit remote registration of given agent name
	}

}

// Server implementation
func Server(config Config) {
	_config = config
	_myself = config.Url
	_alias = config.Name
	_protocol = config.Protocol
	_backend = config.Backend
	_tool = config.Tool
	_toolOpts = config.ToolOpts
	utils.STATICDIR = config.Staticdir
	arr := strings.Split(_myself, "/")
	base := ""
	if len(arr) > 3 {
		base = fmt.Sprintf("/%s", strings.Join(arr[3:], "/"))
	}
	port := "8989" // default port, the port here is a string type since we'll use it later in http.ListenAndServe
	if config.Port != 0 {
		port = fmt.Sprintf("%d", config.Port)
	}
	config.Base = base
	log.WithFields(log.Fields{
		"Config": config.String(),
		"Auth":   utils.Auth,
		"Model":  config.Type,
	}).Println("Agent")

	// register self agent URI in remote agent and vice versa
	registerAtAgents(config.Register)

	// define catalog
	c, e := ioutil.ReadFile(config.Catalog)
	if e != nil {
		log.WithFields(log.Fields{
			"Error": e,
		}).Fatal("Unable to read catalog file")
	}
	err := json.Unmarshal([]byte(c), &core.TFC)
	if err != nil {
		log.WithFields(log.Fields{
			"Error": err,
		}).Fatal("Unable to parse catalog JSON file")
	}
	// open up Catalog DB
	dbtype := core.TFC.Type
	dburi := core.TFC.Uri // TODO: may be I need to change this based on DB Login/Password, check MySQL
	dbowner := core.TFC.Owner
	db, dberr := sql.Open(dbtype, dburi)
	defer db.Close()
	if dberr != nil {
		log.WithFields(log.Fields{
			"DB Error": dberr,
		}).Fatal("sql.Open")
	}
	dberr = db.Ping()
	if dberr != nil {
		log.WithFields(log.Fields{
			"DB Error": dberr,
		}).Fatal("db.Ping")
	}

	core.DB = db
	core.DBTYPE = dbtype
	core.DBSQL = core.LoadSQL(dbtype, dbowner)
	log.WithFields(log.Fields{
		"Catalog": core.TFC,
	}).Println("")

	// Define CentralCatalog
	core.CC = core.CentralCatalog{Path: config.CentralCatalog}

	// define handlers
	http.HandleFunc(fmt.Sprintf("%s/", base), AuthHandler)
	http.Handle("/html/", http.StripPrefix("/html/", http.FileServer(http.Dir("html"))))

	// initialize transfer model
	core.TransferType = config.Type

	// Check if RouterModel is enabled, then initialize router
	if config.RouterModel == true {
		log.WithFields(log.Fields{
			"TrainInterval": _config.TrainInterval,
		}).Println("Enabling router model")
		cronJob := core.NewRouter(config.TrainInterval, &_agents, config.Cfile)
		cronJob.Start()
		defer cronJob.Stop() // Stop the cron job with the server crash
	}

	// initialize job queues
	core.InitQueue(config.QueueSize, config.QueueSize, config.Mfile, config.Minterval, config.MonitorTime, config.RouterModel)

	if config.BufferSize == 0 {
		config.BufferSize = 5
	}
	// initialize task dispatcher
	dispatcher := core.NewDispatcher(config.Workers, config.BufferSize)
	dispatcher.StorageRunner()

	// initialize transfer workers
	transporter := core.NewDispatcher(config.Workers, config.BufferSize)
	transporter.TransferRunner()

	// initialize stager
	core.AgentStager = core.NewStager(config.Backend, core.TFC)

	log.WithFields(log.Fields{
		"Workers":       config.Workers,
		"QueueSize":     config.QueueSize,
		"Transfer Type": config.Type,
	}).Println("Start dispatcher")

	if utils.Auth {
		//start HTTPS server which require user certificates
		server := &http.Server{
			Addr: ":" + port,
			TLSConfig: &tls.Config{
				ClientAuth: tls.RequestClientCert,
			},
		}
		err = server.ListenAndServeTLS(config.ServerCrt, config.ServerKey)
	} else {
		err = http.ListenAndServe(":"+port, nil) // Start server without user certificates
	}

	if err != nil {
		log.WithFields(log.Fields{
			"Error": err,
		}).Fatal("ListenAndServe: ")
	}
}
<commit_msg>Change log to logs<commit_after>package server

// transfer2go agent server implementation
// Author: Valentin Kuznetsov <vkuznet@gmail.com>

import (
	"crypto/tls"
	"database/sql"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	logs "github.com/sirupsen/logrus"
	"github.com/vkuznet/transfer2go/core"
	"github.com/vkuznet/transfer2go/utils"

	// web profiler, see https://golang.org/pkg/net/http/pprof
	_ "net/http/pprof"
)

// Config type holds server configuration
type Config struct {
	Name           string `json:"name"`           // agent name, aka site name
	Url            string `json:"url"`            // agent url
	Catalog        string `json:"catalog"`        // catalog file name, e.g. catalog.json
	CentralCatalog string `json:"centralCatalog"` // central catalog file name, e.g. cc.json
	Protocol       string `json:"protocol"`       // backend protocol, e.g. srmv2
	Backend        string `json:"backend"`        // backend, e.g. srm
	Tool           string `json:"tool"`           // backend tool, e.g. srmcp
	ToolOpts       string `json:"toolopts"`       // options for backend tool
	Mfile          string `json:"mfile"`          // metrics file name
	Cfile          string `json:"csvfile"`        // historical data file
	Minterval      int64  `json:"minterval"`      // metrics interval
	Staticdir      string `json:"staticdir"`      // static dir defines location of static files, e.g. sql,js templates
	Workers        int    `json:"workers"`        // number of workers
	QueueSize      int    `json:"queuesize"`      // total size of the queue
	Port           int    `json:"port"`           // port number given server runs on, default 8989
	Base           string `json:"base"`           // URL base path for agent server, it will be extracted from Url
	Register       string `json:"register"`       // remote agent URL to register
	ServerKey      string `json:"serverkey"`      // server key file
	ServerCrt      string `json:"servercrt"`      // server crt file
	Type           string `json:"type"`           // Configure server type push/pull
	BufferSize     int    `json:"buffersize"`     // Size of buffered channels
	MonitorTime    int64  `json:"monitorTime"`    // Large time interval after which we need to reset monitoring calculation
	TrainInterval  string `json:"trinterval"`     // Time after which we need to retrain main agent
	RouterModel    bool   `json:"router"`         // Variable to enable the router model
}

// String returns string representation of Config data type
func (c *Config) String() string {
	return fmt.Sprintf("<Config: name=%s url=%s port=%d base=%s catalog=%s protocol=%s backend=%s tool=%s opts=%s mfile=%s minterval=%d staticdir=%s workders=%d queuesize=%d register=%s type=%s router=%v>", c.Name, c.Url, c.Port, c.Base, c.Catalog, c.Protocol, c.Backend, c.Tool, c.ToolOpts, c.Mfile, c.Minterval, c.Staticdir, c.Workers, c.QueueSize, c.Register, c.Type, c.RouterModel)
}

// AgentInfo data type
type AgentInfo struct {
	Agent string
	Alias string
}

// AgentProtocol data type
type AgentProtocol struct {
	Protocol string `json:"protocol"` // protocol name, e.g. srmv2
	Backend  string `json:"backend"`  // backend storage end-point, e.g. srm://cms-srm.cern.ch:8443/srm/managerv2?SFN=
	Tool     string `json:"tool"`     // actual executable, e.g. /usr/local/bin/srmcp
	ToolOpts string `json:"toolopts"` // options for backend tool
}

// globals used in server/handlers
var _myself, _alias, _protocol, _backend, _tool, _toolOpts string
var _agents map[string]string
var _config Config

// init
func init() {
	_agents = make(map[string]string)
}

// register a new (alias, agent) pair in agent (register)
func register(register, alias, agent string) error {
	logs.WithFields(logs.Fields{
		"Agent":    agent,
		"Alias":    alias,
		"Register": register,
	}).Println("Register agent as alias on register")
	// register myself with another agent
	params := AgentInfo{Agent: _myself, Alias: _alias}
	data, err := json.Marshal(params)
	if err != nil {
		logs.WithFields(logs.Fields{
			"Params": params,
		}).Error("Unable to marshal params", params)
	}
	url := fmt.Sprintf("%s/register", register)
	resp := utils.FetchResponse(url, data) // POST request
	// check return status code
	if resp.StatusCode != 200 {
		return fmt.Errorf("Response %s, error=%s", resp.Status, string(resp.Data))
	}
	return resp.Error
}

// helper function to register agent with all distributed agents
func registerAtAgents(aName string) {
	// register itself
	if _, ok := _agents[_alias]; ok {
		logs.WithFields(logs.Fields{
			"Alias":  _alias,
			"Agents": _agents,
		}).Fatal("Unable to register, alias, since this name already exists")
	}
	_agents[_alias] = _myself

	// now ask remote server for its list of agents and update internal map
	if aName != "" && len(aName) > 0 {
		err := register(aName, _alias, _myself) // submit remote registration of given agent name
		if err != nil {
			logs.WithFields(logs.Fields{
				"Alias": _alias,
				"Self":  _myself,
				"Name":  aName,
				"Error": err,
			}).Fatal("Unable to register")
		}
		aurl := fmt.Sprintf("%s/agents", aName)
		resp := utils.FetchResponse(aurl, []byte{})
		var remoteAgents map[string]string
		e := json.Unmarshal(resp.Data, &remoteAgents)
		if e == nil {
			for key, val := range remoteAgents {
				if _, ok := _agents[key]; !ok {
					_agents[key] = val // register remote agent/alias pair internally
				}
			}
		}
	}

	// complete registration with other agents
	for alias, agent := range _agents {
		if agent == aName || alias == _alias {
			continue
		}
		register(agent, _alias, _myself) // submit remote registration of given agent name
	}

}

// Server implementation
func Server(config Config) {
	_config = config
	_myself = config.Url
	_alias = config.Name
	_protocol = config.Protocol
	_backend = config.Backend
	_tool = config.Tool
	_toolOpts = config.ToolOpts
	utils.STATICDIR = config.Staticdir
	arr := strings.Split(_myself, "/")
	base := ""
	if len(arr) > 3 {
		base = fmt.Sprintf("/%s", strings.Join(arr[3:], "/"))
	}
	port := "8989" // default port, the port here is a string type since we'll use it later in http.ListenAndServe
	if config.Port != 0 {
		port = fmt.Sprintf("%d", config.Port)
	}
	config.Base = base
	logs.WithFields(logs.Fields{
		"Config": config.String(),
		"Auth":   utils.Auth,
		"Model":  config.Type,
	}).Println("Agent")

	// register self agent URI in remote agent and vice versa
	registerAtAgents(config.Register)

	// define catalog
	c, e := ioutil.ReadFile(config.Catalog)
	if e != nil {
		logs.WithFields(logs.Fields{
			"Error": e,
		}).Fatal("Unable to read catalog file")
	}
	err := json.Unmarshal([]byte(c), &core.TFC)
	if err != nil {
		logs.WithFields(logs.Fields{
			"Error": err,
		}).Fatal("Unable to parse catalog JSON file")
	}
	// open up Catalog DB
	dbtype := core.TFC.Type
	dburi := core.TFC.Uri // TODO: may be I need to change this based on DB Login/Password, check MySQL
	dbowner := core.TFC.Owner
	db, dberr := sql.Open(dbtype, dburi)
	defer db.Close()
	if dberr != nil {
		logs.WithFields(logs.Fields{
			"DB Error": dberr,
		}).Fatal("sql.Open")
	}
	dberr = db.Ping()
	if dberr != nil {
		logs.WithFields(logs.Fields{
			"DB Error": dberr,
		}).Fatal("db.Ping")
	}

	core.DB = db
	core.DBTYPE = dbtype
	core.DBSQL = core.LoadSQL(dbtype, dbowner)
	logs.WithFields(logs.Fields{
		"Catalog": core.TFC,
	}).Println("")

	// Define CentralCatalog
	core.CC = core.CentralCatalog{Path: config.CentralCatalog}

	// define handlers
	http.HandleFunc(fmt.Sprintf("%s/", base), AuthHandler)
	http.Handle("/html/", http.StripPrefix("/html/", http.FileServer(http.Dir("html"))))

	// initialize transfer model
	core.TransferType = config.Type

	// Check if RouterModel is enabled, then initialize router
	if config.RouterModel == true {
		logs.WithFields(logs.Fields{
			"TrainInterval": _config.TrainInterval,
		}).Println("Enabling router model")
		cronJob := core.NewRouter(config.TrainInterval, &_agents, config.Cfile)
		cronJob.Start()
		defer cronJob.Stop() // Stop the cron job with the server crash
	}

	// initialize job queues
	core.InitQueue(config.QueueSize, config.QueueSize, config.Mfile, config.Minterval, config.MonitorTime, config.RouterModel)

	if config.BufferSize == 0 {
		config.BufferSize = 5
	}
	// initialize task dispatcher
	dispatcher := core.NewDispatcher(config.Workers, config.BufferSize)
	dispatcher.StorageRunner()

	// initialize transfer workers
	transporter := core.NewDispatcher(config.Workers, config.BufferSize)
	transporter.TransferRunner()

	// initialize stager
	core.AgentStager = core.NewStager(config.Backend, core.TFC)

	logs.WithFields(logs.Fields{
		"Workers":       config.Workers,
		"QueueSize":     config.QueueSize,
		"Transfer Type": config.Type,
	}).Println("Start dispatcher")

	if utils.Auth {
		//start HTTPS server which require user certificates
		server := &http.Server{
			Addr: ":" + port,
			TLSConfig: &tls.Config{
				ClientAuth: tls.RequestClientCert,
			},
		}
		err = server.ListenAndServeTLS(config.ServerCrt, config.ServerKey)
	} else {
		err = http.ListenAndServe(":"+port, nil) // Start server without user certificates
	}

	if err != nil {
		logs.WithFields(logs.Fields{
			"Error": err,
		}).Fatal("ListenAndServe: ")
	}
}
<|endoftext|>


<commit_before>package server

import (
  "fmt"
  "io/ioutil"
  "net/http"
  "os"
  "strings"

  "github.com/joshuaprunier/trite/common"
)

// Light weight HTTP server implimentation - it accepts a port number and two directory paths, one for db object create definitions and another for an xtrabackup processed with the --export flag
func RunServer(tablePath string, backupPath string, port string) {
  // Make sure directory passed in has trailing slash
  if strings.HasSuffix(backupPath, "/") == false {
    backupPath = backupPath + "/"
  }

  // Ensure the backup has been prepared for transporting with --export
  check := dirWalk(backupPath, false)
  if check == false {
    fmt.Println()
    fmt.Println()
    fmt.Println("It appears that --export has not be run on your backups!")
    fmt.Println()
    fmt.Println()
    os.Exit(1)
  }

  fmt.Println()
  fmt.Println("Starting server listening on port", port)
  http.Handle("/tables/", http.StripPrefix("/tables/", http.FileServer(http.Dir(tablePath))))
  http.Handle("/backups/", http.StripPrefix("/backups/", http.FileServer(http.Dir(backupPath))))
  err := http.ListenAndServe(":"+port, nil)
  if err != nil {
    if err.Error() == "listen tcp :"+port+": bind: address already in use" {
      fmt.Println()
      fmt.Println()
      fmt.Println("ERROR: Port", port, "is already in use!")
      fmt.Println()
      fmt.Println()
      os.Exit(1)
    } else {
      common.CheckErr(err)
    }
  }
}

// Walk the backup directory and confirm there are .exp files which is proof --export was run
func dirWalk(dir string, flag bool) bool {
  files, ferr := ioutil.ReadDir(dir)
  common.CheckErr(ferr)
  for _, file := range files {
    // Check if file is a .exp, that means --export has been performed on the backup
    _, ext := common.ParseFileName(file.Name())

    // Handle sub dirs recursive function
    if file.IsDir() {
      flag := dirWalk(dir+file.Name()+"/", flag)
      if flag == true {
        return flag
      }
    } else {
      if ext == "exp" {
        flag = true
        break
      }
    }
  }
  return flag
}

<commit_msg>Added profiling for HTTP server<commit_after>package server

import (
  "fmt"
  "io/ioutil"
  "net/http"
  _ "net/http/pprof"
  "os"
  "strings"

  "github.com/joshuaprunier/trite/common"
)

// Light weight HTTP server implimentation - it accepts a port number and two directory paths, one for db object create definitions and another for an xtrabackup processed with the --export flag
func RunServer(tablePath string, backupPath string, port string) {
  // Make sure directory passed in has trailing slash
  if strings.HasSuffix(backupPath, "/") == false {
    backupPath = backupPath + "/"
  }

  // Ensure the backup has been prepared for transporting with --export
  check := dirWalk(backupPath, false)
  if check == false {
    fmt.Println()
    fmt.Println()
    fmt.Println("It appears that --export has not be run on your backups!")
    fmt.Println()
    fmt.Println()
    os.Exit(1)
  }

  fmt.Println()
  fmt.Println("Starting server listening on port", port)
  http.Handle("/tables/", http.StripPrefix("/tables/", http.FileServer(http.Dir(tablePath))))
  http.Handle("/backups/", http.StripPrefix("/backups/", http.FileServer(http.Dir(backupPath))))
  err := http.ListenAndServe(":"+port, nil)
  if err != nil {
    if err.Error() == "listen tcp :"+port+": bind: address already in use" {
      fmt.Println()
      fmt.Println()
      fmt.Println("ERROR: Port", port, "is already in use!")
      fmt.Println()
      fmt.Println()
      os.Exit(1)
    } else {
      common.CheckErr(err)
    }
  }
}

// Walk the backup directory and confirm there are .exp files which is proof --export was run
func dirWalk(dir string, flag bool) bool {
  files, ferr := ioutil.ReadDir(dir)
  common.CheckErr(ferr)
  for _, file := range files {
    // Check if file is a .exp, that means --export has been performed on the backup
    _, ext := common.ParseFileName(file.Name())

    // Handle sub dirs recursive function
    if file.IsDir() {
      flag := dirWalk(dir+file.Name()+"/", flag)
      if flag == true {
        return flag
      }
    } else {
      if ext == "exp" {
        flag = true
        break
      }
    }
  }
  return flag
}

<|endoftext|>


<commit_before>package udnssdk

import (
	"fmt"
	"log"
	"time"
)

// DirectionalPoolsService manages 'account level' 'geo' and 'ip' groups for directional-pools
type DirectionalPoolsService struct {
	client *Client
}

// DirectionalPool wraps an account-level directional-groups response from a index request
type DirectionalPool struct {
	DirectionalPoolID         string `json:"taskId"`
	DirectionalPoolStatusCode string `json:"taskStatusCode"`
	Message                   string `json:"message"`
	ResultURI                 string `json:"resultUri"`
}

// AccountLevelGeoDirectionalGroupDTO wraps an account-level, geo directonal-group response
type AccountLevelGeoDirectionalGroupDTO struct {
	Name        string   `json:"name"`
	Description string   `json:"description"`
	Codes       []string `json:"codes"`
}

// IPAddrDTO wraps an IP address range or CIDR block
type IPAddrDTO struct {
	Start   string `json:"start,omitempty"`
	End     string `json:"end,omitempty"`
	CIDR    string `json:"cidr,omitempty"`
	Address string `json:"address,omitempty"`
}

// AccountLevelIPDirectionalGroupDTO wraps an account-level, IP directional-group response
type AccountLevelIPDirectionalGroupDTO struct {
	Name        string      `json:"name"`
	Description string      `json:"description"`
	IPs         []IPAddrDTO `json:"ips"`
}

// DirectionalPoolListDTO wraps a list of account-level directional-groups response from a index request
type DirectionalPoolListDTO struct {
	DirectionalPools []DirectionalPool `json:"tasks"`
	Queryinfo        QueryInfo         `json:"queryInfo"`
	Resultinfo       ResultInfo        `json:"resultInfo"`
}

// AccountLevelGeoDirectionalGroupListDTO wraps a list of account-level, geo directional-groups response from a index request
type AccountLevelGeoDirectionalGroupListDTO struct {
	AccountName string                               `json:"zoneName"`
	GeoGroups   []AccountLevelGeoDirectionalGroupDTO `json:"geoGroups"`
	Queryinfo   QueryInfo                            `json:"queryInfo"`
	Resultinfo  ResultInfo                           `json:"resultInfo"`
}

// AccountLevelIPDirectionalGroupListDTO wraps an account-level, IP directional-group response
type AccountLevelIPDirectionalGroupListDTO struct {
	AccountName string                              `json:"zoneName"`
	IPGroups    []AccountLevelIPDirectionalGroupDTO `json:"ipGroups"`
	Queryinfo   QueryInfo                           `json:"queryInfo"`
	Resultinfo  ResultInfo                          `json:"resultInfo"`
}

// DirectionalPoolPath generates the URI for directional pools by account, type & slug ID
func DirectionalPoolPath(acct, typ, slugID string) string {
	if slugID == "" {
		return fmt.Sprintf("accounts/%s/dirgroups/%s", acct, typ)
	}
	return fmt.Sprintf("accounts/%s/dirgroups/%s/%s", acct, typ, slugID)
}

// DirectionalPoolQueryPath generates the URI for directional pools by account, type, query & offset
func DirectionalPoolQueryPath(account, typ, query string, offset int) string {
	uri := DirectionalPoolPath(account, typ, "")

	if query != "" {
		uri = fmt.Sprintf("%s?sort=NAME&query=%s&offset=%d", uri, query, offset)
	} else {
		uri = fmt.Sprintf("%s?offset=%d", uri, offset)
	}

	return uri
}

// DirectionalIPPoolQueryPath generates the URI for a directional IP pool by account, query & offset
func DirectionalIPPoolQueryPath(account, query string, offset int) string {
	return DirectionalPoolQueryPath(account, "ip", query, offset)
}

// DirectionalGeoPoolQueryPath generates the URI for a directional geo pool by account, query & offset
func DirectionalGeoPoolQueryPath(account, query string, offset int) string {
	return DirectionalPoolQueryPath(account, "geo", query, offset)
}

// GetDirectionalGeoPool requests a geo directional-pool by name & account
func (s *DirectionalPoolsService) GetDirectionalGeoPool(name, acct string) (AccountLevelGeoDirectionalGroupDTO, *Response, error) {
	uri := DirectionalPoolPath(acct, "geo", name)
	var t AccountLevelGeoDirectionalGroupDTO
	res, err := s.client.get(uri, &t)
	return t, res, err
}

// GetDirectionalIPPool requests a IP directional-pool by name & account
func (s *DirectionalPoolsService) GetDirectionalIPPool(name, acct string) (AccountLevelIPDirectionalGroupDTO, *Response, error) {
	uri := DirectionalPoolPath(acct, "ip", name)
	var t AccountLevelIPDirectionalGroupDTO
	res, err := s.client.get(uri, &t)
	return t, res, err
}

// CreateDirectionalGeoPool requests creation of a geo direcctional-pool by name & account, given a directional-pool
func (s *DirectionalPoolsService) CreateDirectionalGeoPool(name, acct string, dp AccountLevelGeoDirectionalGroupDTO) (*Response, error) {
	uri := DirectionalPoolPath(acct, "geo", name)
	return s.client.post(uri, dp, nil)
}

// CreateDirectionalIPPool requests creation of an IP directional-pool by name & account, given a directional-pool
func (s *DirectionalPoolsService) CreateDirectionalIPPool(name, acct string, dp AccountLevelIPDirectionalGroupDTO) (*Response, error) {
	uri := DirectionalPoolPath(acct, "ip", name)
	return s.client.post(uri, dp, nil)
}

// UpdateDirectionalGeoPool requests update of a geo directional-pool by name & account, given a directional-pool
func (s *DirectionalPoolsService) UpdateDirectionalGeoPool(name, acct string, dp AccountLevelGeoDirectionalGroupDTO) (*Response, error) {
	uri := DirectionalPoolPath(acct, "geo", name)
	return s.client.put(uri, dp, nil)
}

// UpdateDirectionalIPPool requests update of an IP directional-pool by name & account, given a directional-pool
func (s *DirectionalPoolsService) UpdateDirectionalIPPool(name, acct string, dp AccountLevelIPDirectionalGroupDTO) (*Response, error) {
	uri := DirectionalPoolPath(acct, "ip", name)
	return s.client.put(uri, dp, nil)
}

// ListAllDirectionalGeoPools requests all geo directional-pools, by query and account, providing pagination and error handling
func (s *DirectionalPoolsService) ListAllDirectionalGeoPools(query, account string) ([]AccountLevelGeoDirectionalGroupDTO, error) {
	// TODO: Sane Configuration for timeouts / retries
	maxerrs := 5
	waittime := 5 * time.Second

	// init accumulators
	dtos := []AccountLevelGeoDirectionalGroupDTO{}
	errcnt := 0
	offset := 0

	for {
		reqDtos, ri, res, err := s.ListDirectionalGeoPools(query, account, offset)
		if err != nil {
			if res.StatusCode >= 500 {
				errcnt = errcnt + 1
				if errcnt < maxerrs {
					time.Sleep(waittime)
					continue
				}
			}
			return dtos, err
		}

		log.Printf("[DEBUG] ResultInfo: %+v\n", ri)
		for _, d := range reqDtos {
			dtos = append(dtos, d)
		}
		if ri.ReturnedCount+ri.Offset >= ri.TotalCount {
			return dtos, nil
		}
		offset = ri.ReturnedCount + ri.Offset
		continue
	}
}

// ListDirectionalGeoPools requests list of geo directional-pools, by query & account, and an offset, returning the directional-group, the list-metadata, the actual response, or an error
func (s *DirectionalPoolsService) ListDirectionalGeoPools(query, account string, offset int) ([]AccountLevelGeoDirectionalGroupDTO, ResultInfo, *Response, error) {
	var tld AccountLevelGeoDirectionalGroupListDTO

	uri := DirectionalGeoPoolQueryPath(account, query, offset)
	res, err := s.client.get(uri, &tld)

	pis := []AccountLevelGeoDirectionalGroupDTO{}
	for _, pi := range tld.GeoGroups {
		pis = append(pis, pi)
	}
	return pis, tld.Resultinfo, res, err
}

// ListAllDirectionalIPPools requests all IP directional-pools, using pagination and error handling
func (s *DirectionalPoolsService) ListAllDirectionalIPPools(query, account string) ([]AccountLevelIPDirectionalGroupDTO, error) {
	// TODO: Sane Configuration for timeouts / retries
	maxerrs := 5
	waittime := 5 * time.Second

	// init accumulators
	gs := []AccountLevelIPDirectionalGroupDTO{}
	errcnt := 0
	offset := 0

	for {
		reqIPGroups, ri, res, err := s.ListDirectionalIPPools(query, account, offset)
		if err != nil {
			if res.StatusCode >= 500 {
				errcnt = errcnt + 1
				if errcnt < maxerrs {
					time.Sleep(waittime)
					continue
				}
			}
			return gs, err
		}

		log.Printf("ResultInfo: %+v\n", ri)
		for _, g := range reqIPGroups {
			gs = append(gs, g)
		}
		if ri.ReturnedCount+ri.Offset >= ri.TotalCount {
			return gs, nil
		}
		offset = ri.ReturnedCount + ri.Offset
		continue
	}
}

// ListDirectionalIPPools requests all IP directional-pools, by query & account, and an offset, returning the list of IP groups, list metadata & the actual response, or an error
func (s *DirectionalPoolsService) ListDirectionalIPPools(query, account string, offset int) ([]AccountLevelIPDirectionalGroupDTO, ResultInfo, *Response, error) {
	var tld AccountLevelIPDirectionalGroupListDTO

	uri := DirectionalIPPoolQueryPath(account, query, offset)
	res, err := s.client.get(uri, &tld)

	pis := []AccountLevelIPDirectionalGroupDTO{}
	for _, pi := range tld.IPGroups {
		pis = append(pis, pi)
	}

	return pis, tld.Resultinfo, res, err
}

// DeleteDirectionalGeoPool deletes a geo directional-pool
func (s *DirectionalPoolsService) DeleteDirectionalGeoPool(dp, acct string) (*Response, error) {
	path := DirectionalPoolPath(acct, "geo", dp)
	return s.client.delete(path, nil)
}

// DeleteDirectionalIPPool deletes an IP directional-pool
func (s *DirectionalPoolsService) DeleteDirectionalIPPool(dp, acct string) (*Response, error) {
	path := DirectionalPoolPath(acct, "ip", dp)
	return s.client.delete(path, nil)
}
<commit_msg>directional_pool.go: extract DirectionalPoolKey and related structs<commit_after>package udnssdk

import (
	"fmt"
	"log"
	"time"
)

// DirectionalPoolsService manages 'account level' 'geo' and 'ip' groups for directional-pools
type DirectionalPoolsService struct {
	client *Client
}

// DirectionalPool wraps an account-level directional-groups response from a index request
type DirectionalPool struct {
	DirectionalPoolID         string `json:"taskId"`
	DirectionalPoolStatusCode string `json:"taskStatusCode"`
	Message                   string `json:"message"`
	ResultURI                 string `json:"resultUri"`
}

// AccountLevelGeoDirectionalGroupDTO wraps an account-level, geo directonal-group response
type AccountLevelGeoDirectionalGroupDTO struct {
	Name        string   `json:"name"`
	Description string   `json:"description"`
	Codes       []string `json:"codes"`
}

// IPAddrDTO wraps an IP address range or CIDR block
type IPAddrDTO struct {
	Start   string `json:"start,omitempty"`
	End     string `json:"end,omitempty"`
	CIDR    string `json:"cidr,omitempty"`
	Address string `json:"address,omitempty"`
}

// AccountLevelIPDirectionalGroupDTO wraps an account-level, IP directional-group response
type AccountLevelIPDirectionalGroupDTO struct {
	Name        string      `json:"name"`
	Description string      `json:"description"`
	IPs         []IPAddrDTO `json:"ips"`
}

// DirectionalPoolListDTO wraps a list of account-level directional-groups response from a index request
type DirectionalPoolListDTO struct {
	DirectionalPools []DirectionalPool `json:"tasks"`
	Queryinfo        QueryInfo         `json:"queryInfo"`
	Resultinfo       ResultInfo        `json:"resultInfo"`
}

// AccountLevelGeoDirectionalGroupListDTO wraps a list of account-level, geo directional-groups response from a index request
type AccountLevelGeoDirectionalGroupListDTO struct {
	AccountName string                               `json:"zoneName"`
	GeoGroups   []AccountLevelGeoDirectionalGroupDTO `json:"geoGroups"`
	Queryinfo   QueryInfo                            `json:"queryInfo"`
	Resultinfo  ResultInfo                           `json:"resultInfo"`
}

// AccountLevelIPDirectionalGroupListDTO wraps an account-level, IP directional-group response
type AccountLevelIPDirectionalGroupListDTO struct {
	AccountName string                              `json:"zoneName"`
	IPGroups    []AccountLevelIPDirectionalGroupDTO `json:"ipGroups"`
	Queryinfo   QueryInfo                           `json:"queryInfo"`
	Resultinfo  ResultInfo                          `json:"resultInfo"`
}

// DirectionalPoolPath generates the URI for directional pools by account, type & slug ID
func DirectionalPoolPath(acct, typ, slugID string) string {
	p := DirectionalPoolKey{
		Account: AccountKey(acct),
		Type:    typ,
		ID:      slugID,
	}
	return p.URI()
}

// DirectionalPoolQueryPath generates the URI for directional pools by account, type, query & offset
func DirectionalPoolQueryPath(account, typ, query string, offset int) string {
	p := DirectionalPoolKey{
		Account: AccountKey(account),
		Type:    typ,
	}
	return p.QueryURI(query, offset)
}

// DirectionalIPPoolQueryPath generates the URI for a directional IP pool by account, query & offset
func DirectionalIPPoolQueryPath(account, query string, offset int) string {
	p := IPDirectionalPoolKey{Account: AccountKey(account)}
	return p.QueryURI(query, offset)
}

// DirectionalGeoPoolQueryPath generates the URI for a directional geo pool by account, query & offset
func DirectionalGeoPoolQueryPath(account, query string, offset int) string {
	p := GeoDirectionalPoolKey{Account: AccountKey(account)}
	return p.QueryURI(query, offset)
}

// GetDirectionalGeoPool requests a geo directional-pool by name & account
func (s *DirectionalPoolsService) GetDirectionalGeoPool(name, acct string) (AccountLevelGeoDirectionalGroupDTO, *Response, error) {
	p := GeoDirectionalPoolKey{
		Account: AccountKey(acct),
		ID:      name,
	}
	return s.Geos().Find(p)
}

// GetDirectionalIPPool requests a IP directional-pool by name & account
func (s *DirectionalPoolsService) GetDirectionalIPPool(name, acct string) (AccountLevelIPDirectionalGroupDTO, *Response, error) {
	p := IPDirectionalPoolKey{
		Account: AccountKey(acct),
		ID:      name,
	}
	return s.IPs().Find(p)
}

// CreateDirectionalGeoPool requests creation of a geo direcctional-pool by name & account, given a directional-pool
func (s *DirectionalPoolsService) CreateDirectionalGeoPool(name, acct string, dp AccountLevelGeoDirectionalGroupDTO) (*Response, error) {
	p := GeoDirectionalPoolKey{
		Account: AccountKey(acct),
		ID:      name,
	}
	return s.Geos().Create(p, dp)
}

// CreateDirectionalIPPool requests creation of an IP directional-pool by name & account, given a directional-pool
func (s *DirectionalPoolsService) CreateDirectionalIPPool(name, acct string, dp AccountLevelIPDirectionalGroupDTO) (*Response, error) {
	p := IPDirectionalPoolKey{
		Account: AccountKey(acct),
		ID:      name,
	}
	return s.IPs().Create(p, dp)
}

// UpdateDirectionalGeoPool requests update of a geo directional-pool by name & account, given a directional-pool
func (s *DirectionalPoolsService) UpdateDirectionalGeoPool(name, acct string, dp AccountLevelGeoDirectionalGroupDTO) (*Response, error) {
	p := GeoDirectionalPoolKey{
		Account: AccountKey(acct),
		ID:      name,
	}
	return s.Geos().Update(p, dp)
}

// UpdateDirectionalIPPool requests update of an IP directional-pool by name & account, given a directional-pool
func (s *DirectionalPoolsService) UpdateDirectionalIPPool(name, acct string, dp AccountLevelIPDirectionalGroupDTO) (*Response, error) {
	p := IPDirectionalPoolKey{
		Account: AccountKey(acct),
		ID:      name,
	}
	return s.IPs().Update(p, dp)
}

// ListAllDirectionalGeoPools requests all geo directional-pools, by query and account, providing pagination and error handling
func (s *DirectionalPoolsService) ListAllDirectionalGeoPools(query, account string) ([]AccountLevelGeoDirectionalGroupDTO, error) {
	p := GeoDirectionalPoolKey{Account: AccountKey(account)}
	return s.Geos().Select(p, query)
}

// ListDirectionalGeoPools requests list of geo directional-pools, by query & account, and an offset, returning the directional-group, the list-metadata, the actual response, or an error
func (s *DirectionalPoolsService) ListDirectionalGeoPools(query, account string, offset int) ([]AccountLevelGeoDirectionalGroupDTO, ResultInfo, *Response, error) {
	p := GeoDirectionalPoolKey{Account: AccountKey(account)}
	return s.Geos().SelectWithOffset(p, query, offset)
}

// ListAllDirectionalIPPools requests all IP directional-pools, using pagination and error handling
func (s *DirectionalPoolsService) ListAllDirectionalIPPools(query, account string) ([]AccountLevelIPDirectionalGroupDTO, error) {
	p := IPDirectionalPoolKey{Account: AccountKey(account)}
	return s.IPs().Select(p, query)
}

// ListDirectionalIPPools requests all IP directional-pools, by query & account, and an offset, returning the list of IP groups, list metadata & the actual response, or an error
func (s *DirectionalPoolsService) ListDirectionalIPPools(query, account string, offset int) ([]AccountLevelIPDirectionalGroupDTO, ResultInfo, *Response, error) {
	p := IPDirectionalPoolKey{Account: AccountKey(account)}
	return s.IPs().SelectWithOffset(p, query, offset)
}

// DeleteDirectionalGeoPool deletes a geo directional-pool
func (s *DirectionalPoolsService) DeleteDirectionalGeoPool(dp, acct string) (*Response, error) {
	p := GeoDirectionalPoolKey{
		Account: AccountKey(acct),
		ID:      dp,
	}
	return s.Geos().Delete(p)
}

// DeleteDirectionalIPPool deletes an IP directional-pool
func (s *DirectionalPoolsService) DeleteDirectionalIPPool(dp, acct string) (*Response, error) {
	p := IPDirectionalPoolKey{
		Account: AccountKey(acct),
		ID:      dp,
	}
	return s.IPs().Delete(p)
}

// ======= //

// Geos allows access to the Geo DirectionalPools API
func (s *DirectionalPoolsService) Geos() *GeoDirectionalPoolsService {
	return &GeoDirectionalPoolsService{client: s.client}
}

// IPs allows access to the IP DirectionalPools API
func (s *DirectionalPoolsService) IPs() *IPDirectionalPoolsService {
	return &IPDirectionalPoolsService{client: s.client}
}

// DirectionalPoolKey collects the identifiers of a DirectionalPool
type DirectionalPoolKey struct {
	Account AccountKey
	Type    string
	ID      string
}

// URI generates the URI for directional pools by account, type & slug ID
func (p DirectionalPoolKey) URI() string {
	if p.ID == "" {
		return fmt.Sprintf("%s/dirgroups/%s", p.Account.URI(), p.Type)
	}
	return fmt.Sprintf("%s/dirgroups/%s/%s", p.Account.URI(), p.Type, p.ID)
}

// QueryURI generates the URI for directional pools by account, type, query & offset
func (p DirectionalPoolKey) QueryURI(query string, offset int) string {
	uri := p.URI()

	if query != "" {
		uri = fmt.Sprintf("%s?sort=NAME&query=%s&offset=%d", uri, query, offset)
	} else {
		uri = fmt.Sprintf("%s?offset=%d", uri, offset)
	}

	return uri
}

// GeoDirectionalPoolKey collects the identifiers of an DirectionalPool with type Geo
type GeoDirectionalPoolKey struct {
	Account AccountKey
	ID      string
}

// DirectionalPoolKey generates the DirectionalPoolKey for the GeoDirectionalPoolKey
func (p GeoDirectionalPoolKey) DirectionalPoolKey() DirectionalPoolKey {
	return DirectionalPoolKey{
		Account: p.Account,
		Type:    "geo",
		ID:      p.ID,
	}
}

// URI generates the URI for a GeoDirectionalPool
func (p GeoDirectionalPoolKey) URI() string {
	return p.DirectionalPoolKey().URI()
}

// QueryURI generates the GeoDirectionalPool URI with query
func (p GeoDirectionalPoolKey) QueryURI(query string, offset int) string {
	return p.DirectionalPoolKey().QueryURI(query, offset)
}

// GeoDirectionalPoolsService manages 'geo' groups for directional-pools
type GeoDirectionalPoolsService struct {
	client *Client
}

// Select requests all geo directional-pools, by query and account, providing pagination and error handling
func (s *GeoDirectionalPoolsService) Select(p GeoDirectionalPoolKey, query string) ([]AccountLevelGeoDirectionalGroupDTO, error) {
	// TODO: Sane Configuration for timeouts / retries
	maxerrs := 5
	waittime := 5 * time.Second

	// init accumulators
	dtos := []AccountLevelGeoDirectionalGroupDTO{}
	errcnt := 0
	offset := 0

	for {
		reqDtos, ri, res, err := s.SelectWithOffset(p, query, offset)
		if err != nil {
			if res.StatusCode >= 500 {
				errcnt = errcnt + 1
				if errcnt < maxerrs {
					time.Sleep(waittime)
					continue
				}
			}
			return dtos, err
		}

		log.Printf("[DEBUG] ResultInfo: %+v\n", ri)
		for _, d := range reqDtos {
			dtos = append(dtos, d)
		}
		if ri.ReturnedCount+ri.Offset >= ri.TotalCount {
			return dtos, nil
		}
		offset = ri.ReturnedCount + ri.Offset
		continue
	}
}

// SelectWithOffset requests list of geo directional-pools, by query & account, and an offset, returning the directional-group, the list-metadata, the actual response, or an error
func (s *GeoDirectionalPoolsService) SelectWithOffset(p GeoDirectionalPoolKey, query string, offset int) ([]AccountLevelGeoDirectionalGroupDTO, ResultInfo, *Response, error) {
	var tld AccountLevelGeoDirectionalGroupListDTO

	res, err := s.client.get(p.QueryURI(query, offset), &tld)

	pis := []AccountLevelGeoDirectionalGroupDTO{}
	for _, pi := range tld.GeoGroups {
		pis = append(pis, pi)
	}
	return pis, tld.Resultinfo, res, err
}

// Find requests a geo directional-pool by name & account
func (s *GeoDirectionalPoolsService) Find(p GeoDirectionalPoolKey) (AccountLevelGeoDirectionalGroupDTO, *Response, error) {
	var t AccountLevelGeoDirectionalGroupDTO
	res, err := s.client.get(p.URI(), &t)
	return t, res, err
}

// Create requests creation of a DirectionalPool by DirectionalPoolKey given a directional-pool
func (s *GeoDirectionalPoolsService) Create(p GeoDirectionalPoolKey, val interface{}) (*Response, error) {
	return s.client.post(p.URI(), val, nil)
}

// Update requests update of a DirectionalPool by DirectionalPoolKey given a directional-pool
func (s *GeoDirectionalPoolsService) Update(p GeoDirectionalPoolKey, val interface{}) (*Response, error) {
	return s.client.put(p.URI(), val, nil)
}

// Delete requests deletion of a DirectionalPool
func (s *GeoDirectionalPoolsService) Delete(p GeoDirectionalPoolKey) (*Response, error) {
	return s.client.delete(p.URI(), nil)
}

// IPDirectionalPoolKey collects the identifiers of an DirectionalPool with type IP
type IPDirectionalPoolKey struct {
	Account AccountKey
	ID      string
}

// DirectionalPoolKey generates the DirectionalPoolKey for the IPDirectionalPoolKey
func (p IPDirectionalPoolKey) DirectionalPoolKey() DirectionalPoolKey {
	return DirectionalPoolKey{
		Account: p.Account,
		Type:    "ip",
		ID:      p.ID,
	}
}

// URI generates the IPDirectionalPool query URI
func (p IPDirectionalPoolKey) URI() string {
	return p.DirectionalPoolKey().URI()
}

// QueryURI generates the IPDirectionalPool URI with query
func (p IPDirectionalPoolKey) QueryURI(query string, offset int) string {
	return p.DirectionalPoolKey().QueryURI(query, offset)
}

// IPDirectionalPoolsService manages 'geo' groups for directional-pools
type IPDirectionalPoolsService struct {
	client *Client
}

// Select requests all IP directional-pools, using pagination and error handling
func (s *IPDirectionalPoolsService) Select(p IPDirectionalPoolKey, query string) ([]AccountLevelIPDirectionalGroupDTO, error) {
	// TODO: Sane Configuration for timeouts / retries
	maxerrs := 5
	waittime := 5 * time.Second

	// init accumulators
	gs := []AccountLevelIPDirectionalGroupDTO{}
	errcnt := 0
	offset := 0

	for {
		reqIPGroups, ri, res, err := s.SelectWithOffset(p, query, offset)
		if err != nil {
			if res.StatusCode >= 500 {
				errcnt = errcnt + 1
				if errcnt < maxerrs {
					time.Sleep(waittime)
					continue
				}
			}
			return gs, err
		}

		log.Printf("ResultInfo: %+v\n", ri)
		for _, g := range reqIPGroups {
			gs = append(gs, g)
		}
		if ri.ReturnedCount+ri.Offset >= ri.TotalCount {
			return gs, nil
		}
		offset = ri.ReturnedCount + ri.Offset
		continue
	}
}

// SelectWithOffset requests all IP directional-pools, by query & account, and an offset, returning the list of IP groups, list metadata & the actual response, or an error
func (s *IPDirectionalPoolsService) SelectWithOffset(p IPDirectionalPoolKey, query string, offset int) ([]AccountLevelIPDirectionalGroupDTO, ResultInfo, *Response, error) {
	var tld AccountLevelIPDirectionalGroupListDTO

	res, err := s.client.get(p.QueryURI(query, offset), &tld)

	pis := []AccountLevelIPDirectionalGroupDTO{}
	for _, pi := range tld.IPGroups {
		pis = append(pis, pi)
	}

	return pis, tld.Resultinfo, res, err
}

// Find requests a directional-pool by name & account
func (s *IPDirectionalPoolsService) Find(p IPDirectionalPoolKey) (AccountLevelIPDirectionalGroupDTO, *Response, error) {
	var t AccountLevelIPDirectionalGroupDTO
	res, err := s.client.get(p.URI(), &t)
	return t, res, err
}

// Create requests creation of a DirectionalPool by DirectionalPoolKey given a directional-pool
func (s *IPDirectionalPoolsService) Create(p IPDirectionalPoolKey, val interface{}) (*Response, error) {
	return s.client.post(p.URI(), val, nil)
}

// Update requests update of a DirectionalPool by DirectionalPoolKey given a directional-pool
func (s *IPDirectionalPoolsService) Update(p IPDirectionalPoolKey, val interface{}) (*Response, error) {
	return s.client.put(p.URI(), val, nil)
}

// Delete deletes an  directional-pool
func (s *IPDirectionalPoolsService) Delete(p IPDirectionalPoolKey) (*Response, error) {
	return s.client.delete(p.URI(), nil)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"path"
	"reflect"
	"runtime"
	"runtime/debug"
	"runtime/pprof"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/cblomart/vsphere-graphite/backend"
	"github.com/cblomart/vsphere-graphite/config"
	"github.com/cblomart/vsphere-graphite/vsphere"

	"github.com/takama/daemon"

	"code.cloudfoundry.org/bytefmt"
)

const (
	// name of the service
	name        = "vsphere-graphite"
	description = "send vsphere stats to graphite"
)

var dependencies = []string{}

var stdlog, errlog *log.Logger

// Service has embedded daemon
type Service struct {
	daemon.Daemon
}

func queryVCenter(vcenter vsphere.VCenter, conf config.Configuration, channel *chan backend.Point) {
	vcenter.Query(conf.Interval, conf.Domain, channel)
}

// Manage by daemon commands or run the daemon
func (service *Service) Manage() (string, error) {

	usage := "Usage: vsphere-graphite install | remove | start | stop | status"

	// if received any kind of command, do it
	if len(os.Args) > 1 {
		command := os.Args[1]
		switch command {
		case "install":
			return service.Install()
		case "remove":
			return service.Remove()
		case "start":
			return service.Start()
		case "stop":
			return service.Stop()
		case "status":
			return service.Status()
		default:
			return usage, nil
		}
	}

	stdlog.Println("Starting daemon:", path.Base(os.Args[0]))

	// read the configuration
	file, err := os.Open("/etc/" + path.Base(os.Args[0]) + ".json")
	if err != nil {
		return "Could not open configuration file", err
	}
	jsondec := json.NewDecoder(file)
	conf := config.Configuration{}
	err = jsondec.Decode(&conf)
	if err != nil {
		return "Could not decode configuration file", err
	}

	if conf.FlushSize == 0 {
		conf.FlushSize = 1000
	}

	if conf.CPUProfiling {
		f, err := ioutil.TempFile("/tmp", "vsphere-graphite-cpu.profile") // nolint: vetshadow
		if err != nil {
			log.Fatal("could not create CPU profile: ", err)
		}
		stdlog.Println("Will write cpu profiling to: ", f.Name())
		if err := pprof.StartCPUProfile(f); err != nil {
			log.Fatal("could not start CPU profile: ", err)
		}
		defer pprof.StopCPUProfile()
	}

	//force backend values to environement varialbles if present
	s := reflect.ValueOf(&conf.Backend).Elem()
	numfields := s.NumField()
	for i := 0; i < numfields; i++ {
		f := s.Field(i)
		if f.CanSet() {
			//exported field
			envname := strings.ToUpper(s.Type().Name() + "_" + s.Type().Field(i).Name)
			envval := os.Getenv(envname)
			if len(envval) > 0 {
				//environment variable set with name
				switch ftype := f.Type().Name(); ftype {
				case "string":
					f.SetString(envval)
				case "int":
					val, err := strconv.ParseInt(envval, 10, 64) // nolint: vetshadow
					if err == nil {
						f.SetInt(val)
					}
				}
			}
		}
	}

	for _, vcenter := range conf.VCenters {
		vcenter.Init(conf.Metrics, stdlog, errlog)
	}

	err = conf.Backend.Init(stdlog, errlog)
	if err != nil {
		return "Could not initialize backend", err
	}
	defer conf.Backend.Disconnect()

	// Set up channel on which to send signal notifications.
	// We must use a buffered channel or risk missing the signal
	// if we're not ready to receive when the signal is sent.
	interrupt := make(chan os.Signal, 1)
	signal.Notify(interrupt, os.Interrupt, os.Kill, syscall.SIGTERM) // nolint: megacheck

	// Set up a channel to receive the metrics
	metrics := make(chan backend.Point, conf.FlushSize)

	// Set up a ticker to collect metrics at givent interval
	ticker := time.NewTicker(time.Second * time.Duration(conf.Interval))
	defer ticker.Stop()

	// Start retriveing and sending metrics
	stdlog.Println("Retrieving metrics")
	for _, vcenter := range conf.VCenters {
		go queryVCenter(*vcenter, conf, &metrics)
	}

	// Memory statisctics
	var memstats runtime.MemStats
	// timer to execute memory collection
	memtimer := time.NewTimer(time.Second * time.Duration(10))
	// Memory profiling
	var mf *os.File
	if conf.MEMProfiling {
		mf, err = ioutil.TempFile("/tmp", "vsphere-graphite-mem.profile")
		if err != nil {
			log.Fatal("could not create MEM profile: ", err)
		}
		defer mf.Close() // nolint: errcheck
	}
	// buffer for points to send
	pointbuffer := make([]*backend.Point, conf.FlushSize)
	bufferindex := 0

	for {
		select {
		case value := <-metrics:
			// reset timer as a point has been revieved
			if !memtimer.Stop() {
				select {
				case <-memtimer.C:
				default:
				}
			}
			memtimer.Reset(time.Second * time.Duration(5))
			pointbuffer[bufferindex] = &value
			bufferindex++
			if bufferindex == len(pointbuffer) {
				conf.Backend.SendMetrics(pointbuffer)
				stdlog.Printf("Sent %d logs to backend", bufferindex)
				ClearBuffer(pointbuffer)
				bufferindex = 0
			}
		case <-ticker.C:
			stdlog.Println("Retrieving metrics")
			for _, vcenter := range conf.VCenters {
				go queryVCenter(*vcenter, conf, &metrics)
			}
		case <-memtimer.C:
			// sent remaining values
			conf.Backend.SendMetrics(pointbuffer)
			stdlog.Printf("Sent %d logs to backend", bufferindex)
			bufferindex = 0
			ClearBuffer(pointbuffer)
			runtime.GC()
			debug.FreeOSMemory()
			runtime.ReadMemStats(&memstats)
			stdlog.Printf("Memory usage : sys=%s alloc=%s\n", bytefmt.ByteSize(memstats.Sys), bytefmt.ByteSize(memstats.Alloc))
			if conf.MEMProfiling {
				stdlog.Println("Writing mem profiling to: ", mf.Name())
				debug.WriteHeapDump(mf.Fd())
			}
		case killSignal := <-interrupt:
			stdlog.Println("Got signal:", killSignal)
			if bufferindex > 0 {
				conf.Backend.SendMetrics(pointbuffer[:bufferindex])
				stdlog.Printf("Sent %d logs to backend", bufferindex)
			}
			if killSignal == os.Interrupt {
				return "Daemon was interrupted by system signal", nil
			}
			return "Daemon was killed", nil
		}
	}
}

// ClearBuffer : set all values in pointer array to nil
func ClearBuffer(buffer []*backend.Point) {
	for i := 0; i < len(buffer); i++ {
		buffer[i] = nil
	}
}

func init() {
	stdlog = log.New(os.Stdout, "", log.Ldate|log.Ltime)
	errlog = log.New(os.Stderr, "", log.Ldate|log.Ltime)
}

func main() {
	srv, err := daemon.New(name, description, dependencies...)
	if err != nil {
		errlog.Println("Error: ", err)
		os.Exit(1)
	}
	service := &Service{srv}
	status, err := service.Manage()
	if err != nil {
		errlog.Println(status, "Error: ", err)
		os.Exit(1)
	}
	fmt.Println(status)
}
<commit_msg>correct usage of pointer in init.<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"path"
	"reflect"
	"runtime"
	"runtime/debug"
	"runtime/pprof"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/cblomart/vsphere-graphite/backend"
	"github.com/cblomart/vsphere-graphite/config"
	"github.com/cblomart/vsphere-graphite/vsphere"

	"github.com/takama/daemon"

	"code.cloudfoundry.org/bytefmt"
)

const (
	// name of the service
	name        = "vsphere-graphite"
	description = "send vsphere stats to graphite"
)

var dependencies = []string{}

var stdlog, errlog *log.Logger

// Service has embedded daemon
type Service struct {
	daemon.Daemon
}

func queryVCenter(vcenter vsphere.VCenter, conf config.Configuration, channel *chan backend.Point) {
	vcenter.Query(conf.Interval, conf.Domain, channel)
}

// Manage by daemon commands or run the daemon
func (service *Service) Manage() (string, error) {

	usage := "Usage: vsphere-graphite install | remove | start | stop | status"

	// if received any kind of command, do it
	if len(os.Args) > 1 {
		command := os.Args[1]
		switch command {
		case "install":
			return service.Install()
		case "remove":
			return service.Remove()
		case "start":
			return service.Start()
		case "stop":
			return service.Stop()
		case "status":
			return service.Status()
		default:
			return usage, nil
		}
	}

	stdlog.Println("Starting daemon:", path.Base(os.Args[0]))

	// read the configuration
	file, err := os.Open("/etc/" + path.Base(os.Args[0]) + ".json")
	if err != nil {
		return "Could not open configuration file", err
	}
	jsondec := json.NewDecoder(file)
	conf := config.Configuration{}
	err = jsondec.Decode(&conf)
	if err != nil {
		return "Could not decode configuration file", err
	}

	if conf.FlushSize == 0 {
		conf.FlushSize = 1000
	}

	if conf.CPUProfiling {
		f, err := ioutil.TempFile("/tmp", "vsphere-graphite-cpu.profile") // nolint: vetshadow
		if err != nil {
			log.Fatal("could not create CPU profile: ", err)
		}
		stdlog.Println("Will write cpu profiling to: ", f.Name())
		if err := pprof.StartCPUProfile(f); err != nil {
			log.Fatal("could not start CPU profile: ", err)
		}
		defer pprof.StopCPUProfile()
	}

	//force backend values to environement varialbles if present
	s := reflect.ValueOf(conf.Backend).Elem()
	numfields := s.NumField()
	for i := 0; i < numfields; i++ {
		f := s.Field(i)
		if f.CanSet() {
			//exported field
			envname := strings.ToUpper(s.Type().Name() + "_" + s.Type().Field(i).Name)
			envval := os.Getenv(envname)
			if len(envval) > 0 {
				//environment variable set with name
				switch ftype := f.Type().Name(); ftype {
				case "string":
					f.SetString(envval)
				case "int":
					val, err := strconv.ParseInt(envval, 10, 64) // nolint: vetshadow
					if err == nil {
						f.SetInt(val)
					}
				}
			}
		}
	}

	for _, vcenter := range conf.VCenters {
		vcenter.Init(conf.Metrics, stdlog, errlog)
	}

	err = conf.Backend.Init(stdlog, errlog)
	if err != nil {
		return "Could not initialize backend", err
	}
	defer conf.Backend.Disconnect()

	// Set up channel on which to send signal notifications.
	// We must use a buffered channel or risk missing the signal
	// if we're not ready to receive when the signal is sent.
	interrupt := make(chan os.Signal, 1)
	signal.Notify(interrupt, os.Interrupt, os.Kill, syscall.SIGTERM) // nolint: megacheck

	// Set up a channel to receive the metrics
	metrics := make(chan backend.Point, conf.FlushSize)

	// Set up a ticker to collect metrics at givent interval
	ticker := time.NewTicker(time.Second * time.Duration(conf.Interval))
	defer ticker.Stop()

	// Start retriveing and sending metrics
	stdlog.Println("Retrieving metrics")
	for _, vcenter := range conf.VCenters {
		go queryVCenter(*vcenter, conf, &metrics)
	}

	// Memory statisctics
	var memstats runtime.MemStats
	// timer to execute memory collection
	memtimer := time.NewTimer(time.Second * time.Duration(10))
	// Memory profiling
	var mf *os.File
	if conf.MEMProfiling {
		mf, err = ioutil.TempFile("/tmp", "vsphere-graphite-mem.profile")
		if err != nil {
			log.Fatal("could not create MEM profile: ", err)
		}
		defer mf.Close() // nolint: errcheck
	}
	// buffer for points to send
	pointbuffer := make([]*backend.Point, conf.FlushSize)
	bufferindex := 0

	for {
		select {
		case value := <-metrics:
			// reset timer as a point has been revieved
			if !memtimer.Stop() {
				select {
				case <-memtimer.C:
				default:
				}
			}
			memtimer.Reset(time.Second * time.Duration(5))
			pointbuffer[bufferindex] = &value
			bufferindex++
			if bufferindex == len(pointbuffer) {
				conf.Backend.SendMetrics(pointbuffer)
				stdlog.Printf("Sent %d logs to backend", bufferindex)
				ClearBuffer(pointbuffer)
				bufferindex = 0
			}
		case <-ticker.C:
			stdlog.Println("Retrieving metrics")
			for _, vcenter := range conf.VCenters {
				go queryVCenter(*vcenter, conf, &metrics)
			}
		case <-memtimer.C:
			// sent remaining values
			conf.Backend.SendMetrics(pointbuffer)
			stdlog.Printf("Sent %d logs to backend", bufferindex)
			bufferindex = 0
			ClearBuffer(pointbuffer)
			runtime.GC()
			debug.FreeOSMemory()
			runtime.ReadMemStats(&memstats)
			stdlog.Printf("Memory usage : sys=%s alloc=%s\n", bytefmt.ByteSize(memstats.Sys), bytefmt.ByteSize(memstats.Alloc))
			if conf.MEMProfiling {
				stdlog.Println("Writing mem profiling to: ", mf.Name())
				debug.WriteHeapDump(mf.Fd())
			}
		case killSignal := <-interrupt:
			stdlog.Println("Got signal:", killSignal)
			if bufferindex > 0 {
				conf.Backend.SendMetrics(pointbuffer[:bufferindex])
				stdlog.Printf("Sent %d logs to backend", bufferindex)
			}
			if killSignal == os.Interrupt {
				return "Daemon was interrupted by system signal", nil
			}
			return "Daemon was killed", nil
		}
	}
}

// ClearBuffer : set all values in pointer array to nil
func ClearBuffer(buffer []*backend.Point) {
	for i := 0; i < len(buffer); i++ {
		buffer[i] = nil
	}
}

func init() {
	stdlog = log.New(os.Stdout, "", log.Ldate|log.Ltime)
	errlog = log.New(os.Stderr, "", log.Ldate|log.Ltime)
}

func main() {
	srv, err := daemon.New(name, description, dependencies...)
	if err != nil {
		errlog.Println("Error: ", err)
		os.Exit(1)
	}
	service := &Service{srv}
	status, err := service.Manage()
	if err != nil {
		errlog.Println(status, "Error: ", err)
		os.Exit(1)
	}
	fmt.Println(status)
}
<|endoftext|>