<commit_before>package main

import (
	"log"
	"os"
	"net"
	"io/ioutil"
	"time"
	"regexp"
	"strings"
	"strconv"
	"bufio"
)

type Server struct {
	conf Config
	mon *Monitoring
	lg log.Logger
	ch chan string
	chS chan string
}

// Sum metrics with prefix
func (s Server) sumMetricsWithPrefix() {
	for ;; time.Sleep(time.Duration(s.conf.SumInterval)*time.Second) {
		var working_list[] Metric
		chanSize := len(s.chS)
		for i := 0; i < chanSize; i++ {
			found := false
			metric := strings.Replace(<-s.chS, s.conf.SumPrefix, "", -1)
			split := regexp.MustCompile("\\s").Split(metric, 3)

			value, err := strconv.ParseFloat(split[1], 64)
			if err != nil {s.lg.Println("Can not parse value of a metric") ; continue}
			timestamp, err := strconv.ParseInt(split[2], 10, 64)
			if err != nil {s.lg.Println("Can not parse timestamp of a metric") ; continue}

			for i,_ := range working_list {
				if working_list[i].name == split[0] {
					working_list[i].amount++
					working_list[i].value += value
					working_list[i].timestamp += timestamp
					found = true
					break
				}
			}
			if !found {
				working_list = append(working_list, Metric{split[0], 1, value, timestamp})
			}
		}
		for _,val := range working_list {
			s.ch <- val.name + " " +
				strconv.FormatFloat(val.value, 'f', 2, 32) + " " + strconv.FormatInt(val.timestamp/val.amount, 10)
		}
	}
}

// Function checks and removed bad data and sorts it by SUM prefix
func (s Server)cleanAndUseIncomingData(metrics []string) {
	for _,metric := range metrics {
		if validateMetric(metric, s.conf.AllowedMetrics) {
			if strings.HasPrefix(metric, s.conf.SumPrefix) {
				if len(s.chS) < s.conf.MaxMetrics*s.conf.SumInterval{
					s.chS <- metric
				} else {
					s.mon.dropped++
				}
			} else {
				if len(s.ch) < s.conf.MaxMetrics*s.conf.ClientSendInterval {
					s.ch <- metric
				} else {
					s.mon.dropped++
				}
			}
		} else {
			s.mon.dropped++
			s.lg.Println("Removing bad metric \"" + metric + "\" from the list")
		}
	}
	metrics = nil
}

// Handles incoming requests.
func (s Server)handleRequest(conn net.Conn) {
	connbuf := bufio.NewReader(conn)
	var results_list []string
	amount := 0
	for ;; amount++ {
		metric, err := connbuf.ReadString('\n')
		if err!= nil {
			break
		}
		if amount < s.conf.MaxMetrics {
			results_list = append(results_list, strings.Replace(metric, "\n", "", -1))
		} else {
			s.mon.dropped++
		}
	}
	conn.Close()
	s.mon.got.net += amount
	s.cleanAndUseIncomingData(results_list)

	results_list = nil
}

// Reading metrics from files in folder. This is a second way how to send metrics, except direct connection
func (s Server)handleDirMetrics() {
	for ;; time.Sleep(time.Duration(s.conf.ClientSendInterval)*time.Second) {
		files, err := ioutil.ReadDir(s.conf.MetricDir)
		if err != nil {
			panic(err.Error())
		}
		for _, f := range files {
			results_list := readMetricsFromFile(s.conf.MetricDir+"/"+f.Name())
			s.mon.got.dir += len(results_list)
			s.cleanAndUseIncomingData(results_list)
		}

	}
}

func (s Server)runServer() {
	// Listen for incoming connections.
	l, err := net.Listen("tcp", s.conf.LocalBind)
	if err != nil {
		s.lg.Println("Failed to run server:", err.Error())
		os.Exit(1)
	} else {
		s.lg.Println("Server is running")
	}
	// Close the listener when the application closes.
	defer l.Close()

	// Run goroutine for reading metrics from metricDir
	go s.handleDirMetrics()
	// Run goroutine for sum metrics with prefix
	go s.sumMetricsWithPrefix()

	for {
		// Listen for an incoming connection.
		conn, err := l.Accept()
		if err != nil {
			s.lg.Println("Error accepting: ", err.Error())
			os.Exit(1)
		}
		// Handle connections in a new goroutine.
		go s.handleRequest(conn)
	}
}<commit_msg>Removed notification about "bad metric" if it is just an empty line<commit_after>package main

import (
	"log"
	"os"
	"net"
	"io/ioutil"
	"time"
	"regexp"
	"strings"
	"strconv"
	"bufio"
)

type Server struct {
	conf Config
	mon *Monitoring
	lg log.Logger
	ch chan string
	chS chan string
}

// Sum metrics with prefix
func (s Server) sumMetricsWithPrefix() {
	for ;; time.Sleep(time.Duration(s.conf.SumInterval)*time.Second) {
		var working_list[] Metric
		chanSize := len(s.chS)
		for i := 0; i < chanSize; i++ {
			found := false
			metric := strings.Replace(<-s.chS, s.conf.SumPrefix, "", -1)
			split := regexp.MustCompile("\\s").Split(metric, 3)

			value, err := strconv.ParseFloat(split[1], 64)
			if err != nil {s.lg.Println("Can not parse value of a metric") ; continue}
			timestamp, err := strconv.ParseInt(split[2], 10, 64)
			if err != nil {s.lg.Println("Can not parse timestamp of a metric") ; continue}

			for i,_ := range working_list {
				if working_list[i].name == split[0] {
					working_list[i].amount++
					working_list[i].value += value
					working_list[i].timestamp += timestamp
					found = true
					break
				}
			}
			if !found {
				working_list = append(working_list, Metric{split[0], 1, value, timestamp})
			}
		}
		for _,val := range working_list {
			s.ch <- val.name + " " +
				strconv.FormatFloat(val.value, 'f', 2, 32) + " " + strconv.FormatInt(val.timestamp/val.amount, 10)
		}
	}
}

// Function checks and removed bad data and sorts it by SUM prefix
func (s Server)cleanAndUseIncomingData(metrics []string) {
	for _,metric := range metrics {
		if validateMetric(metric, s.conf.AllowedMetrics) {
			if strings.HasPrefix(metric, s.conf.SumPrefix) {
				if len(s.chS) < s.conf.MaxMetrics*s.conf.SumInterval{
					s.chS <- metric
				} else {
					s.mon.dropped++
				}
			} else {
				if len(s.ch) < s.conf.MaxMetrics*s.conf.ClientSendInterval {
					s.ch <- metric
				} else {
					s.mon.dropped++
				}
			}
		} else {
			if metric != "" {
				s.mon.dropped++
				s.lg.Println("Removing bad metric \"" + metric + "\" from the list")
			}
		}
	}
	metrics = nil
}

// Handles incoming requests.
func (s Server)handleRequest(conn net.Conn) {
	connbuf := bufio.NewReader(conn)
	var results_list []string
	amount := 0
	for ;; amount++ {
		metric, err := connbuf.ReadString('\n')
		if err!= nil {
			break
		}
		if amount < s.conf.MaxMetrics {
			results_list = append(results_list, strings.Replace(metric, "\n", "", -1))
		} else {
			s.mon.dropped++
		}
	}
	conn.Close()
	s.mon.got.net += amount
	s.cleanAndUseIncomingData(results_list)

	results_list = nil
}

// Reading metrics from files in folder. This is a second way how to send metrics, except direct connection
func (s Server)handleDirMetrics() {
	for ;; time.Sleep(time.Duration(s.conf.ClientSendInterval)*time.Second) {
		files, err := ioutil.ReadDir(s.conf.MetricDir)
		if err != nil {
			panic(err.Error())
		}
		for _, f := range files {
			results_list := readMetricsFromFile(s.conf.MetricDir+"/"+f.Name())
			s.mon.got.dir += len(results_list)
			s.cleanAndUseIncomingData(results_list)
		}

	}
}

func (s Server)runServer() {
	// Listen for incoming connections.
	l, err := net.Listen("tcp", s.conf.LocalBind)
	if err != nil {
		s.lg.Println("Failed to run server:", err.Error())
		os.Exit(1)
	} else {
		s.lg.Println("Server is running")
	}
	// Close the listener when the application closes.
	defer l.Close()

	// Run goroutine for reading metrics from metricDir
	go s.handleDirMetrics()
	// Run goroutine for sum metrics with prefix
	go s.sumMetricsWithPrefix()

	for {
		// Listen for an incoming connection.
		conn, err := l.Accept()
		if err != nil {
			s.lg.Println("Error accepting: ", err.Error())
			os.Exit(1)
		}
		// Handle connections in a new goroutine.
		go s.handleRequest(conn)
	}
}<|endoftext|>


<commit_before>package redis

import (
    "bufio"
    "encoding/json"
    "os"
    "sync"
)

var (
    lastPublishCount = 0
    DumpFileName     = "../redisServer.dump.json"
    fileWriteMu      sync.Mutex
)

// Save the DB in background. The OK code is immediately returned. Redis forks, the parent
// continues to serve the clients, the child saves the DB on disk then exits. A client my
// be able to check if the operation succeeded using the LASTSAVE command.
// Please refer to the persistence documentation for detailed information.
//
// Return value
// Simple string reply
func BgSave(complete chan bool) string {
    if publishCount > lastPublishCount {
        lastPublishCount = publishCount

        go func() {
            fileWriteMu.Lock()
            defer fileWriteMu.Unlock()

            fo, _ := os.Create(DumpFileName)
            defer fo.Close()
            w := bufio.NewWriter(fo)
            enc := json.NewEncoder(w)

            // TODO: Lock around each key (checking existence) instead of around the whole
            // hash so that other operations may sneak in.

            hashesMu.Lock()
            enc.Encode(&allHashes)
            hashesMu.Unlock()

            listsMu.Lock()
            enc.Encode(&allLists)
            listsMu.Unlock()

            setsMu.Lock()
            enc.Encode(&allSets)
            setsMu.Unlock()

            stringsMu.Lock()
            enc.Encode(&allStrings)
            stringsMu.Unlock()

            w.Flush()

            if complete != nil {
                complete <- true
            }
        }()
    }

    return "OK"
}

func init() {
    go func() {
        fileWriteMu.Lock()
        defer fileWriteMu.Unlock()

        fo, _ := os.Open(DumpFileName)
        defer fo.Close()

        r := bufio.NewReader(fo)
        dec := json.NewDecoder(r)

        hashesMu.Lock()
        dec.Decode(&allHashes)
        hashesMu.Unlock()

        listsMu.Lock()
        dec.Decode(&allLists)
        listsMu.Unlock()

        setsMu.Lock()
        dec.Decode(&allSets)
        setsMu.Unlock()

        stringsMu.Lock()
        dec.Decode(&allStrings)
        stringsMu.Unlock()
    }()
}
<commit_msg>Indent json output for better readability<commit_after>package redis

import (
    "bufio"
    "encoding/json"
    "os"
    "sync"
)

var (
    lastPublishCount = 0
    DumpFileName     = "../redisServer.dump.json"
    fileWriteMu      sync.Mutex
)

// Save the DB in background. The OK code is immediately returned. Redis forks, the parent
// continues to serve the clients, the child saves the DB on disk then exits. A client my
// be able to check if the operation succeeded using the LASTSAVE command.
// Please refer to the persistence documentation for detailed information.
//
// Return value
// Simple string reply
func BgSave(complete chan bool) string {
    if publishCount > lastPublishCount {
        lastPublishCount = publishCount

        go func() {
            fileWriteMu.Lock()
            defer fileWriteMu.Unlock()

            fo, _ := os.Create(DumpFileName)
            defer fo.Close()
            w := bufio.NewWriter(fo)

            // TODO: Lock around each key (checking existence) instead of around the whole
            // hash so that other operations may sneak in.

            hashesMu.Lock()
            b1, err := json.MarshalIndent(&allHashes, "", "    ")
            if err != nil {
                println(err.Error())
                return
            }
            w.Write(b1)
            hashesMu.Unlock()

            listsMu.Lock()
            b2, err := json.MarshalIndent(&allLists, "", "    ")
            if err != nil {
                println(err.Error())
                return
            }
            w.Write(b2)
            listsMu.Unlock()

            setsMu.Lock()
            b3, err := json.MarshalIndent(&allSets, "", "    ")
            if err != nil {
                println(err.Error())
                return
            }
            w.Write(b3)
            setsMu.Unlock()

            stringsMu.Lock()
            b4, err := json.MarshalIndent(&allStrings, "", "    ")
            if err != nil {
                println(err.Error())
                return
            }
            w.Write(b4)
            stringsMu.Unlock()

            w.Flush()

            if complete != nil {
                complete <- true
            }
        }()
    }

    return "OK"
}

func init() {
    func() {
        fileWriteMu.Lock()
        defer fileWriteMu.Unlock()

        fo, _ := os.Open(DumpFileName)
        defer fo.Close()

        r := bufio.NewReader(fo)
        dec := json.NewDecoder(r)

        hashesMu.Lock()
        dec.Decode(&allHashes)
        hashesMu.Unlock()

        listsMu.Lock()
        dec.Decode(&allLists)
        listsMu.Unlock()

        setsMu.Lock()
        dec.Decode(&allSets)
        setsMu.Unlock()

        stringsMu.Lock()
        dec.Decode(&allStrings)
        stringsMu.Unlock()
    }()
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"code.google.com/p/goprotobuf/proto"
	"encoding/gob"
	"github.com/BenLubar/bindiff"
	"github.com/Rnoadm/wdvn/res"
	"net"
	"sync/atomic"
	"time"
)

func Listen(l net.Listener, world *World) {
	defer l.Close()

	broadcast, register, unregister := make(chan *res.Packet), make(chan chan<- *res.Packet), make(chan chan<- *res.Packet)
	go Multicast(broadcast, register, unregister)

	ch := make(chan *res.Packet)
	register <- ch
	state := make(chan State)
	go Manager(ch, state, broadcast, world)

	var connected [res.Man_count]uint64

	for {
		conn, err := l.Accept()
		if err != nil {
			panic(err)
		}

		ch := make(chan *res.Packet)
		register <- ch

		go Serve(conn, ch, broadcast, state, &connected, func() {
			go func() {
				for _ = range ch {
					// do nothing
				}
			}()
			unregister <- ch
		})
	}
}

func Multicast(broadcast <-chan *res.Packet, register, unregister <-chan chan<- *res.Packet) {
	var writers []chan<- *res.Packet

	for {
		select {
		case ch := <-register:
			writers = append(writers, ch)

		case ch := <-unregister:
			for i, ch2 := range writers {
				if ch == ch2 {
					writers = append(writers[:i], writers[i+1:]...)
					close(ch)
					break
				}
			}

		case packet := <-broadcast:
			for _, ch := range writers {
				ch <- packet
			}
		}
	}
}

func Serve(conn net.Conn, in <-chan *res.Packet, out chan<- *res.Packet, statech <-chan State, connected *[res.Man_count]uint64, disconnect func()) {
	defer disconnect()
	defer conn.Close()

	read, write := make(chan *res.Packet), make(chan *res.Packet)
	go Read(conn, read)
	go Write(conn, write)

	var man res.Man
	for man = 0; man < res.Man_count; man++ {
		if atomic.CompareAndSwapUint64(&((*connected)[man]), 0, 1) {
			break
		}
	}
	if man == res.Man_count {
		man = res.Man_Normal
		atomic.AddUint64(&((*connected)[man]), 1)
	}
	defer func() {
		atomic.AddUint64(&((*connected)[man]), ^uint64(0))
	}()

	write <- &res.Packet{
		Type: res.Type_SelectMan.Enum(),
		Man:  man.Enum(),
	}

	state := <-statech
	{
		var buf bytes.Buffer

		err := gob.NewEncoder(&buf).Encode(&state)
		if err != nil {
			panic(err)
		}

		write <- &res.Packet{
			Type: res.Type_FullState.Enum(),
			Data: buf.Bytes(),
		}
	}

	ping := time.NewTicker(time.Second)
	defer ping.Stop()
	lastPing := time.Now()

	for {
		select {
		case p := <-in:
			write <- p

		case p := <-read:
			switch p.GetType() {
			case res.Type_Ping:
				lastPing = time.Now()

			case res.Type_SelectMan:
				if atomic.CompareAndSwapUint64(&((*connected)[p.GetMan()]), 0, 1) {
					atomic.AddUint64(&((*connected)[man]), ^uint64(0))
					man = p.GetMan()
					go Send(write, p)
				}

			case res.Type_Input:
				p.Man = man.Enum()
				go Send(out, p)

			case res.Type_FullState:
				state = <-statech

				var buf bytes.Buffer

				err := gob.NewEncoder(&buf).Encode(&state)
				if err != nil {
					panic(err)
				}

				go Send(write, &res.Packet{
					Type: res.Type_FullState.Enum(),
					Data: buf.Bytes(),
				})
			}

		case <-ping.C:
			go Send(out, &res.Packet{
				Type: res.Type_Ping.Enum(),
			})

			if time.Since(lastPing) > time.Second*5 {
				return
			}
		}
	}
}

func Manager(in <-chan *res.Packet, out chan<- State, broadcast chan<- *res.Packet, world *World) {
	var state State
	var input [res.Man_count]res.Packet

	state.Lives = DefaultLives
	state.World = world
	for i := range state.Mans {
		state.Mans[i].Health = DefaultHealth
	}

	tick := time.Tick(time.Second / TicksPerSecond)

	var prev []byte

	for {
		select {
		case p := <-in:
			switch p.GetType() {
			case res.Type_Input:
				proto.Merge(&input[p.GetMan()], p)
			}

		case out <- state:

		case <-tick:
			t := state.Tick

			state.Update(&input)

			var buf bytes.Buffer
			err := gob.NewEncoder(&buf).Encode(&state)
			if err != nil {
				panic(err)
			}
			diff := bindiff.Diff(prev, buf.Bytes(), 5)
			prev = buf.Bytes()

			go Send(broadcast, &res.Packet{
				Type: res.Type_StateDiff.Enum(),
				Tick: proto.Uint64(t),
				Data: diff,
			})
		}
	}
}
<commit_msg>add some logging to the server<commit_after>package main

import (
	"bytes"
	"code.google.com/p/goprotobuf/proto"
	"encoding/gob"
	"github.com/BenLubar/bindiff"
	"github.com/Rnoadm/wdvn/res"
	"log"
	"net"
	"sync/atomic"
	"time"
)

func Listen(l net.Listener, world *World) {
	defer l.Close()

	broadcast, register, unregister := make(chan *res.Packet), make(chan chan<- *res.Packet), make(chan chan<- *res.Packet)
	go Multicast(broadcast, register, unregister)

	input := make(chan *res.Packet)
	state := make(chan State)
	go Manager(input, state, broadcast, world)

	var connected [res.Man_count]uint64

	for {
		conn, err := l.Accept()
		if err != nil {
			panic(err)
		}

		ch := make(chan *res.Packet)
		register <- ch

		go Serve(conn, ch, broadcast, state, input, &connected, func() {
			go func() {
				for _ = range ch {
					// do nothing
				}
			}()
			unregister <- ch
		})
	}
}

func Multicast(broadcast <-chan *res.Packet, register, unregister <-chan chan<- *res.Packet) {
	writers := make(map[chan<- *res.Packet]struct{})

	for {
		select {
		case ch := <-register:
			writers[ch] = struct{}{}

		case ch := <-unregister:
			delete(writers, ch)
			close(ch)

		case packet := <-broadcast:
			for ch := range writers {
				ch <- packet
			}
		}
	}
}

func Serve(conn net.Conn, in <-chan *res.Packet, out chan<- *res.Packet, statech <-chan State, input chan<- *res.Packet, connected *[res.Man_count]uint64, disconnect func()) {
	defer disconnect()
	defer conn.Close()

	read, write := make(chan *res.Packet), make(chan *res.Packet)
	go Read(conn, read)
	go Write(conn, write)

	var man res.Man
	// check for an empty slot
	for man = 0; man < res.Man_count; man++ {
		if atomic.CompareAndSwapUint64(&((*connected)[man]), 0, 1) {
			break
		}
	}
	// multiple people control normal man as a last resort
	if man == res.Man_count {
		man = res.Man_Normal
		atomic.AddUint64(&((*connected)[man]), 1)
	}
	// leave the character when we disconnect
	defer func() {
		atomic.AddUint64(&((*connected)[man]), ^uint64(0))
	}()

	log.Println(conn.RemoteAddr(), "connected for", man)

	// tell the client which man they are
	write <- &res.Packet{
		Type: res.Type_SelectMan.Enum(),
		Man:  man.Enum(),
	}

	// send full state to the client
	{
		state := <-statech
		var buf bytes.Buffer

		err := gob.NewEncoder(&buf).Encode(&state)
		if err != nil {
			panic(err)
		}

		write <- &res.Packet{
			Type: res.Type_FullState.Enum(),
			Data: buf.Bytes(),
		}
	}

	ping := time.NewTicker(time.Second)
	defer ping.Stop()
	lastPing := time.Now()

	for {
		select {
		case p := <-in:
			write <- p

		case p, ok := <-read:
			if !ok {
				log.Println(conn.RemoteAddr(), man, "disconnected: read error")
				return
			}

			switch p.GetType() {
			case res.Type_Ping:
				lastPing = time.Now()

			case res.Type_SelectMan:
				if atomic.CompareAndSwapUint64(&((*connected)[p.GetMan()]), 0, 1) {
					log.Println(conn.RemoteAddr(), "switched from", man, "to", p.GetMan())

					atomic.AddUint64(&((*connected)[man]), ^uint64(0))
					man = p.GetMan()
					go Send(write, p)
				}

			case res.Type_Input:
				p.Man = man.Enum()
				go Send(input, p)

			case res.Type_FullState:
				log.Println(conn.RemoteAddr(), "requested full state update")

				state := <-statech

				var buf bytes.Buffer

				err := gob.NewEncoder(&buf).Encode(&state)
				if err != nil {
					panic(err)
				}

				go Send(write, &res.Packet{
					Type: res.Type_FullState.Enum(),
					Data: buf.Bytes(),
				})
			}

		case <-ping.C:
			go Send(out, &res.Packet{
				Type: res.Type_Ping.Enum(),
			})

			if time.Since(lastPing) > time.Second*5 {
				log.Println(conn.RemoteAddr(), man, "disconnected: ping timeout")
				return
			}
		}
	}
}

func Manager(in <-chan *res.Packet, out chan<- State, broadcast chan<- *res.Packet, world *World) {
	var state State
	var input [res.Man_count]res.Packet

	state.Lives = DefaultLives
	state.World = world
	for i := range state.Mans {
		state.Mans[i].Health = DefaultHealth
	}

	tick := time.Tick(time.Second / TicksPerSecond)

	var prev []byte

	for {
		select {
		case p := <-in:
			switch p.GetType() {
			case res.Type_Input:
				proto.Merge(&input[p.GetMan()], p)
			}

		case out <- state:

		case <-tick:
			t := state.Tick

			state.Update(&input)

			var buf bytes.Buffer
			err := gob.NewEncoder(&buf).Encode(&state)
			if err != nil {
				panic(err)
			}
			diff := bindiff.Diff(prev, buf.Bytes(), 5)
			prev = buf.Bytes()

			go Send(broadcast, &res.Packet{
				Type: res.Type_StateDiff.Enum(),
				Tick: proto.Uint64(t),
				Data: diff,
			})
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
    "github.com/codegangsta/martini"
    "github.com/codegangsta/martini-contrib/render"
)

func main() {
    m := martini.Classic()
    m.Use(render.Renderer("templates"))
    m.Use(martini.Static("public"))

    m.Get("/", func(r render.Render) {
        r.HTML(200, "index", nil)
    })
    m.Run()
}
<commit_msg>Add: markdown parser<commit_after>package main

import (
    "bytes"
    "github.com/codegangsta/martini"
    "github.com/codegangsta/martini-contrib/render"
    "github.com/russross/blackfriday"
    "html/template"
    "io/ioutil"
    "os"
    "path/filepath"
)

func main() {
    m := martini.Classic()
    m.Use(render.Renderer("templates"))
    if martini.Env == martini.Dev {
        m.Use(martini.Static("public"))
    }

    content := parseMarkdown("data")

    m.Get("/", func(r render.Render) {
        r.HTML(200, "index", template.HTML(content))
    })

    m.Run()
}

func parseMarkdown(dir string) string {
    var content bytes.Buffer

    filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
        if _, err := filepath.Rel(dir, path); err != nil {
            return err
        }

        if !info.IsDir() {
            buf, err := ioutil.ReadFile(path)
            if err != nil {
                panic(err)
            }

            content.WriteString(string(buf))
        }

        return nil
    })

    markdown := blackfriday.MarkdownBasic(content.Bytes())

    return string(markdown)
}
<|endoftext|>


<commit_before>package main

import (
	"html/template"
	"io"
	"io/ioutil"
	"net/http"
	"regexp"
	"time"

	"database/sql"

	_ "github.com/mattn/go-sqlite3"

	"github.com/labstack/echo"
	"github.com/russross/blackfriday"
)

type Template struct{ templates *template.Template }

func (t *Template) Render(w io.Writer, name string, data interface{}, c echo.Context) error {
	return t.templates.ExecuteTemplate(w, name, data)
}

func main() {
	e := echo.New()
	db, err := sql.Open("sqlite3", "./database.sqlite")
	if err != nil {
		e.Logger.Error(err)
	}
	defer db.Close()

	e.Renderer = &Template{templates: template.Must(template.ParseGlob("assets/templates/*.html"))}

	e.File("/favicon.ico", "assets/public/favicon.ico")
	e.File("/robots.txt", "assets/public/robots.txt")
	e.File("/style.css", "assets/public/style.css")
	e.File("/index.html", "assets/public/index.html")
	e.File("/", "assets/public/index.html")

	e.GET("/TOS.md", func(c echo.Context) error {
		n, code := md2html(c, "TOS")
		return c.Render(code, "Page", n)
	})
	e.GET("/:id", func(c echo.Context) error {
		n, code := note(c, db)
		return c.Render(code, "Note", n)
	})

	e.Logger.Fatal(e.Start(":3000"))
}

type Note struct {
	ID, Title, Text   string
	Published, Edited time.Time
	Views             int
	Content           template.HTML
}

func (n Note) withTitle() Note {
	fstLine := rexpNewLine.Split(n.Text, -1)[0]
	maxLength := 25
	if len(fstLine) < 25 {
		maxLength = len(fstLine)
	}
	n.Title = rexpNonAlphaNum.ReplaceAllString(fstLine[:maxLength], "")
	return n
}

var (
	rexpNewLine     = regexp.MustCompile("[\n\r]")
	rexpNonAlphaNum = regexp.MustCompile("[`~!@#$%^&*_|+=?;:'\",.<>{}\\/]")
)

func note(c echo.Context, db *sql.DB) (Note, int) {
	stmt, err := db.Prepare("select id, text, strftime('%s', published) as published," +
		" strftime('%s',edited) as edited, password, views from notes where id = ?")
	if err != nil {
		c.Logger().Error(err)
		return note503, http.StatusServiceUnavailable
	}
	defer stmt.Close()
	row := stmt.QueryRow(c.Param("id"))
	var id, text, password, published, edited string
	var views int
	if err := row.Scan(&id, &text, &published, &edited, &password, &views); err != nil {
		c.Logger().Error(err)
		return note404, http.StatusNotFound
	}
	return Note{
		ID:      id,
		Text:    text,
		Content: mdTmplHTML([]byte(text)),
	}.withTitle(), http.StatusOK
}

func md2html(c echo.Context, name string) (Note, int) {
	path := "assets/markdown/" + name + ".md"
	mdContent, err := ioutil.ReadFile(path)
	if err != nil {
		c.Logger().Errorf("couldn't open markdown page %q: %v", path, err)
		return note503, http.StatusServiceUnavailable
	}
	return Note{Title: name, Content: mdTmplHTML(mdContent)}, http.StatusOK
}

func mdTmplHTML(content []byte) template.HTML { return template.HTML(string(blackfriday.Run(content))) }

// error notes
var note404 = Note{Title: "Not found", Content: mdTmplHTML([]byte("# 404 NOT FOUND"))}
var note503 = Note{Title: "Service unavailable", Content: mdTmplHTML([]byte("# 503 SERVICE UNAVAILABLE"))}
<commit_msg>export endpoint added<commit_after>package main

import (
	"html/template"
	"io"
	"io/ioutil"
	"net/http"
	"regexp"
	"time"

	"database/sql"

	_ "github.com/mattn/go-sqlite3"

	"github.com/labstack/echo"
	"github.com/russross/blackfriday"
)

type Template struct{ templates *template.Template }

func (t *Template) Render(w io.Writer, name string, data interface{}, c echo.Context) error {
	return t.templates.ExecuteTemplate(w, name, data)
}

func main() {
	e := echo.New()
	db, err := sql.Open("sqlite3", "./database.sqlite")
	if err != nil {
		e.Logger.Error(err)
	}
	defer db.Close()

	e.Renderer = &Template{templates: template.Must(template.ParseGlob("assets/templates/*.html"))}

	e.File("/favicon.ico", "assets/public/favicon.ico")
	e.File("/robots.txt", "assets/public/robots.txt")
	e.File("/style.css", "assets/public/style.css")
	e.File("/index.html", "assets/public/index.html")
	e.File("/", "assets/public/index.html")

	e.GET("/TOS.md", func(c echo.Context) error {
		n, code := md2html(c, "TOS")
		return c.Render(code, "Page", n)
	})
	e.GET("/:id", func(c echo.Context) error {
		n, code := note(c, db)
		return c.Render(code, "Note", n)
	})
	e.GET("/:id/export", func(c echo.Context) error {
		n, code := note(c, db)
		return c.String(code, n.Text)
	})

	e.Logger.Fatal(e.Start(":3000"))
}

type Note struct {
	ID, Title, Text   string
	Published, Edited time.Time
	Views             int
	Content           template.HTML
}

func (n Note) withTitle() Note {
	fstLine := rexpNewLine.Split(n.Text, -1)[0]
	maxLength := 25
	if len(fstLine) < 25 {
		maxLength = len(fstLine)
	}
	n.Title = rexpNonAlphaNum.ReplaceAllString(fstLine[:maxLength], "")
	return n
}

var (
	rexpNewLine     = regexp.MustCompile("[\n\r]")
	rexpNonAlphaNum = regexp.MustCompile("[`~!@#$%^&*_|+=?;:'\",.<>{}\\/]")
)

func note(c echo.Context, db *sql.DB) (Note, int) {
	stmt, err := db.Prepare("select id, text, strftime('%s', published) as published," +
		" strftime('%s',edited) as edited, password, views from notes where id = ?")
	if err != nil {
		c.Logger().Error(err)
		return note503, http.StatusServiceUnavailable
	}
	defer stmt.Close()
	row := stmt.QueryRow(c.Param("id"))
	var id, text, password, published, edited string
	var views int
	if err := row.Scan(&id, &text, &published, &edited, &password, &views); err != nil {
		c.Logger().Error(err)
		return note404, http.StatusNotFound
	}
	return Note{
		ID:      id,
		Text:    text,
		Content: mdTmplHTML([]byte(text)),
	}.withTitle(), http.StatusOK
}

func md2html(c echo.Context, name string) (Note, int) {
	path := "assets/markdown/" + name + ".md"
	mdContent, err := ioutil.ReadFile(path)
	if err != nil {
		c.Logger().Errorf("couldn't open markdown page %q: %v", path, err)
		return note503, http.StatusServiceUnavailable
	}
	return Note{Title: name, Content: mdTmplHTML(mdContent)}, http.StatusOK
}

func mdTmplHTML(content []byte) template.HTML { return template.HTML(string(blackfriday.Run(content))) }

// error notes
var note404 = Note{Title: "Not found", Content: mdTmplHTML([]byte("# 404 NOT FOUND"))}
var note503 = Note{Title: "Service unavailable", Content: mdTmplHTML([]byte("# 503 SERVICE UNAVAILABLE"))}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"time"

	"github.com/cloudfoundry-incubator/cf-lager"
	"github.com/cloudfoundry-incubator/galera-healthcheck/healthcheck"
	"github.com/imdario/mergo"
	"github.com/pivotal-cf-experimental/service-config"
	"github.com/pivotal-golang/lager"

	_ "github.com/go-sql-driver/mysql"
)

var healthchecker *healthcheck.Healthchecker

func handler(w http.ResponseWriter, r *http.Request, logger lager.Logger) {
	result, msg := healthchecker.Check()
	if result {
		w.WriteHeader(http.StatusOK)
	} else {
		w.WriteHeader(http.StatusServiceUnavailable)
	}

	body := fmt.Sprintf("Galera Cluster Node Status: %s", msg)
	fmt.Fprint(w, body)

	logger.Debug(fmt.Sprintf("Healhcheck Response Body: %s", body))
}

func main() {

	serviceConfig := service_config.New()

	flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
	pidFile := flags.String("pidFile", "", "Path to create a pid file when the healthcheck server has started")
	serviceConfig.AddFlags(flags)
	cf_lager.AddFlags(flags)
	flags.Parse(os.Args[1:])
	logger, _ := cf_lager.New("Galera Healthcheck")

	logger.Info("Starting galera healthcheck...")

	var config healthcheck.Config
	err := serviceConfig.Read(&config)
	if err != nil && err != service_config.NoConfigError {
		logger.Fatal("Failed to read config", err)
	}

	var defaults = healthcheck.Config{
		Host: "0.0.0.0",
		Port: 8080,
		DB: healthcheck.DBConfig{
			Host:     "0.0.0.0",
			Port:     3306,
			User:     "root",
			Password: "",
		},
		AvailableWhenDonor:    true,
		AvailableWhenReadOnly: false,
	}

	err = mergo.Merge(&config, defaults)
	if err != nil {
		logger.Fatal("Failed to merge user config with defaults", err)
	}

	db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%d)/", config.DB.User, config.DB.Password, config.DB.Host, config.DB.Port))
	if err != nil {
		// sql.Open may not actually check that the DB is reachable
		err = db.Ping()
	}
	if err != nil {
		logger.Fatal("Failed to open DB connection", err, lager.Data{
			"dbHost": config.DB.Host,
			"dbPort": config.DB.Port,
			"dbUser": config.DB.User,
		})
	}

	logger.Info("Opened DB connection", lager.Data{
		"dbHost": config.DB.Host,
		"dbPort": config.DB.Port,
		"dbUser": config.DB.User,
	})

	healthchecker = healthcheck.New(db, config, logger)

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		handler(w, r, logger)
	})

	address := fmt.Sprintf("%s:%d", config.Host, config.Port)
	url := fmt.Sprintf("http://%s/", address)
	logger.Info("Serving healthcheck endpoint", lager.Data{
		"url": url,
	})

	go func() {
		client := &http.Client{
			Timeout: 10 * time.Second,
		}

		logger.Info("Attempting to GET endpoint...", lager.Data{
			"url": url,
		})

		var resp *http.Response
		retryAttemptsRemaining := 3
		for ; retryAttemptsRemaining > 0; retryAttemptsRemaining-- {
			resp, err = client.Get(url)
			if err != nil {
				logger.Info("GET endpoint failed, retrying...", lager.Data{
					"url": url,
					"err": err,
				})
				time.Sleep(time.Second * 10)
			} else {
				break
			}
		}
		if retryAttemptsRemaining == 0 {
			logger.Fatal(
				"Initialization failed: Coudn't GET endpoint",
				err,
				lager.Data{
					"url":     url,
					"retries": retryAttemptsRemaining,
				})
		}
		logger.Info("GET endpoint succeeded, now accepting connections", lager.Data{
			"url":        url,
			"statusCode": resp.StatusCode,
		})

		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			logger.Fatal("Initialization failed: reading response body", err, lager.Data{
				"url":         url,
				"status-code": resp.StatusCode,
			})
		}
		logger.Info(fmt.Sprintf("Initial Response: %s", body))

		if *pidFile != "" {
			// existence of pid file means the server is running
			pid := os.Getpid()
			err = ioutil.WriteFile(*pidFile, []byte(strconv.Itoa(os.Getpid())), 0644)
			if err != nil {
				logger.Fatal("Failed to write pid file", err, lager.Data{
					"pid":     pid,
					"pidFile": *pidFile,
				})
			}
		}

		// Used by tests to deterministically know that the healthcheck is accepting incoming connections
		logger.Info("Healthcheck Started")
	}()

	server := &http.Server{Addr: address}
	server.ListenAndServe()
}
<commit_msg>Use service config library to add default options.<commit_after>package main

import (
	"database/sql"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"time"

	"github.com/cloudfoundry-incubator/cf-lager"
	"github.com/cloudfoundry-incubator/galera-healthcheck/healthcheck"
	"github.com/pivotal-cf-experimental/service-config"
	"github.com/pivotal-golang/lager"

	_ "github.com/go-sql-driver/mysql"
)

var healthchecker *healthcheck.Healthchecker

func handler(w http.ResponseWriter, r *http.Request, logger lager.Logger) {
	result, msg := healthchecker.Check()
	if result {
		w.WriteHeader(http.StatusOK)
	} else {
		w.WriteHeader(http.StatusServiceUnavailable)
	}

	body := fmt.Sprintf("Galera Cluster Node Status: %s", msg)
	fmt.Fprint(w, body)

	logger.Debug(fmt.Sprintf("Healhcheck Response Body: %s", body))
}

func main() {

	serviceConfig := service_config.New()

	flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
	pidFile := flags.String("pidFile", "", "Path to create a pid file when the healthcheck server has started")
	serviceConfig.AddFlags(flags)
	var defaults = healthcheck.Config{
		Host: "0.0.0.0",
		Port: 8080,
		DB: healthcheck.DBConfig{
			Host:     "0.0.0.0",
			Port:     3306,
			User:     "root",
			Password: "",
		},
		AvailableWhenDonor:    true,
		AvailableWhenReadOnly: false,
	}
	serviceConfig.AddDefaults(defaults)
	cf_lager.AddFlags(flags)

	flags.Parse(os.Args[1:])
	logger, _ := cf_lager.New("Galera Healthcheck")

	logger.Info("Starting galera healthcheck...")

	var config healthcheck.Config
	err := serviceConfig.Read(&config)
	if err != nil && err != service_config.NoConfigError {
		logger.Fatal("Failed to read config", err)
	}

	db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%d)/", config.DB.User, config.DB.Password, config.DB.Host, config.DB.Port))
	if err != nil {
		// sql.Open may not actually check that the DB is reachable
		err = db.Ping()
	}
	if err != nil {
		logger.Fatal("Failed to open DB connection", err, lager.Data{
			"dbHost": config.DB.Host,
			"dbPort": config.DB.Port,
			"dbUser": config.DB.User,
		})
	}

	logger.Info("Opened DB connection", lager.Data{
		"dbHost": config.DB.Host,
		"dbPort": config.DB.Port,
		"dbUser": config.DB.User,
	})

	healthchecker = healthcheck.New(db, config, logger)

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		handler(w, r, logger)
	})

	address := fmt.Sprintf("%s:%d", config.Host, config.Port)
	url := fmt.Sprintf("http://%s/", address)
	logger.Info("Serving healthcheck endpoint", lager.Data{
		"url": url,
	})

	go func() {
		client := &http.Client{
			Timeout: 10 * time.Second,
		}

		logger.Info("Attempting to GET endpoint...", lager.Data{
			"url": url,
		})

		var resp *http.Response
		retryAttemptsRemaining := 3
		for ; retryAttemptsRemaining > 0; retryAttemptsRemaining-- {
			resp, err = client.Get(url)
			if err != nil {
				logger.Info("GET endpoint failed, retrying...", lager.Data{
					"url": url,
					"err": err,
				})
				time.Sleep(time.Second * 10)
			} else {
				break
			}
		}
		if retryAttemptsRemaining == 0 {
			logger.Fatal(
				"Initialization failed: Coudn't GET endpoint",
				err,
				lager.Data{
					"url":     url,
					"retries": retryAttemptsRemaining,
				})
		}
		logger.Info("GET endpoint succeeded, now accepting connections", lager.Data{
			"url":        url,
			"statusCode": resp.StatusCode,
		})

		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			logger.Fatal("Initialization failed: reading response body", err, lager.Data{
				"url":         url,
				"status-code": resp.StatusCode,
			})
		}
		logger.Info(fmt.Sprintf("Initial Response: %s", body))

		if *pidFile != "" {
			// existence of pid file means the server is running
			pid := os.Getpid()
			err = ioutil.WriteFile(*pidFile, []byte(strconv.Itoa(os.Getpid())), 0644)
			if err != nil {
				logger.Fatal("Failed to write pid file", err, lager.Data{
					"pid":     pid,
					"pidFile": *pidFile,
				})
			}
		}

		// Used by tests to deterministically know that the healthcheck is accepting incoming connections
		logger.Info("Healthcheck Started")
	}()

	server := &http.Server{Addr: address}
	server.ListenAndServe()
}
<|endoftext|>


<commit_before>package editor

import (
	"github.com/elpinal/coco3/editor/register"
	"github.com/elpinal/coco3/screen"
)

type visual struct {
	nvCommon

	start int
}

func newVisual(s streamSet, e *editor) *visual {
	return &visual{
		nvCommon: nvCommon{
			streamSet: s,
			editor:    e,
		},

		start: e.pos,
	}
}

func (v *visual) Mode() mode {
	return modeVisual
}

func (v *visual) Runes() []rune {
	return v.buf
}

func (v *visual) Position() int {
	return v.pos
}

func (v *visual) Message() []rune {
	return []rune("-- VISUAL --")
}

func (v *visual) Highlight() *screen.Hi {
	return &screen.Hi{
		Left:  min(v.start, v.pos),
		Right: constrain(max(v.start, v.pos)+1, 0, len(v.buf)),
	}
}

func (v *visual) Run() (end continuity, next modeChanger, err error) {
	r, _, err := v.in.ReadRune()
	if err != nil {
		return end, next, err
	}
	for ('1' <= r && r <= '9') || (v.count != 0 && r == '0') {
		v.count = v.count*10 + int(r-'0')
		r1, _, err := v.streamSet.in.ReadRune()
		if err != nil {
			return end, next, err
		}
		r = r1
	}
	if v.count == 0 {
		v.count = 1
	}
	cmd, ok := visualCommands[r]
	if !ok {
		return
	}
	if m := cmd(v); m != nil {
		next = m
	}
	if v.pos == len(v.buf) {
		v.move(v.pos - 1)
	}
	v.count = 0
	return
}

type visualCommand = func(*visual) modeChanger

var visualCommands = map[rune]visualCommand{
	CharEscape: (*visual).escape,
	CharCtrlC:  (*visual).escape,
	'~':        (*visual).switchCase,
	'$':        (*visual).endline,
	'0':        (*visual).beginline,
	'A':        (*visual).appendAfter,
	'B':        (*visual).wordBack,
	'C':        (*visual).changeLine,
	'D':        (*visual).deleteLine,
	'E':        (*visual).wordEndNonBlank,
	'F':        (*visual).searchCharacterBackward,
	'I':        (*visual).insertBefore,
	'U':        (*visual).toUpper,
	'W':        (*visual).wordNonBlank,
	'b':        (*visual).wordBack,
	'c':        (*visual).change,
	'd':        (*visual).delete,
	'e':        (*visual).wordEnd,
	'f':        (*visual).searchCharacter,
	'i':        (*visual).object,
	'h':        (*visual).left,
	'l':        (*visual).right,
	'r':        (*visual).replace,
	's':        (*visual).siege,
	'o':        (*visual).swap,
	'u':        (*visual).toLower,
	'v':        (*visual).escape,
	'w':        (*visual).word,
	'y':        (*visual).yank,
}

func (v *visual) escape() modeChanger {
	return norm()
}

func (v *visual) delete() modeChanger {
	hi := v.Highlight()
	v.nvCommon.delete(hi.Left, hi.Right)
	return norm()
}

func (v *visual) change() modeChanger {
	_ = v.delete()
	return ins(v.pos == len(v.buf))
}

func (v *visual) swap() (_ modeChanger) {
	v.start, v.pos = v.pos, v.start
	return
}

func (v *visual) yank() modeChanger {
	hi := v.Highlight()
	v.nvCommon.yank(register.Unnamed, hi.Left, hi.Right)
	v.move(hi.Left)
	return norm()
}

func (v *visual) replace() modeChanger {
	r, _, _ := v.in.ReadRune()
	hi := v.Highlight()
	rs := make([]rune, hi.Right-hi.Left)
	for i := range rs {
		rs[i] = r
	}
	v.nvCommon.replace(rs, hi.Left)
	return norm()
}

func (v *visual) toUpper() modeChanger {
	hi := v.Highlight()
	v.nvCommon.toUpper(hi.Left, hi.Right)
	return norm()
}

func (v *visual) toLower() modeChanger {
	hi := v.Highlight()
	v.nvCommon.toLower(hi.Left, hi.Right)
	return norm()
}

func (v *visual) wordEnd() (_ modeChanger) {
	for i := 0; i < v.count; i++ {
		v.nvCommon.wordEnd()
	}
	return
}

func (v *visual) wordEndNonBlank() (_ modeChanger) {
	for i := 0; i < v.count; i++ {
		v.nvCommon.wordEndNonBlank()
	}
	return
}

func (v *visual) insertBefore() modeChanger {
	v.move(v.Highlight().Left)
	return ins(v.pos == len(v.buf))
}

func (v *visual) appendAfter() modeChanger {
	v.move(v.Highlight().Right)
	return ins(v.pos == len(v.buf))
}

func (v *visual) switchCase() modeChanger {
	hi := v.Highlight()
	v.editor.swapCase(hi.Left, hi.Right)
	v.move(hi.Left)
	return norm()
}

func (v *visual) deleteLine() modeChanger {
	v.editor.delete(0, len(v.buf))
	return norm()
}

func (v *visual) siege() modeChanger {
	hi := v.Highlight()
	r, _, _ := v.in.ReadRune()
	v.editor.siege(hi.Left, hi.Right, r)
	v.move(hi.Left)
	return norm()
}

func (v *visual) changeLine() modeChanger {
	v.editor.delete(0, len(v.buf))
	return ins(v.pos == len(v.buf))
}

func (v *visual) object() (_ modeChanger) {
	v.move(v.pos + 1)
	from, to := v.currentWord(false)
	if from < 0 {
		return
	}
	if v.start <= v.pos {
		v.move(to - 1)
		return
	}
	v.move(from)
	return
}
<commit_msg>Make visual.object work well when v.start > v.pos<commit_after>package editor

import (
	"github.com/elpinal/coco3/editor/register"
	"github.com/elpinal/coco3/screen"
)

type visual struct {
	nvCommon

	start int
}

func newVisual(s streamSet, e *editor) *visual {
	return &visual{
		nvCommon: nvCommon{
			streamSet: s,
			editor:    e,
		},

		start: e.pos,
	}
}

func (v *visual) Mode() mode {
	return modeVisual
}

func (v *visual) Runes() []rune {
	return v.buf
}

func (v *visual) Position() int {
	return v.pos
}

func (v *visual) Message() []rune {
	return []rune("-- VISUAL --")
}

func (v *visual) Highlight() *screen.Hi {
	return &screen.Hi{
		Left:  min(v.start, v.pos),
		Right: constrain(max(v.start, v.pos)+1, 0, len(v.buf)),
	}
}

func (v *visual) Run() (end continuity, next modeChanger, err error) {
	r, _, err := v.in.ReadRune()
	if err != nil {
		return end, next, err
	}
	for ('1' <= r && r <= '9') || (v.count != 0 && r == '0') {
		v.count = v.count*10 + int(r-'0')
		r1, _, err := v.streamSet.in.ReadRune()
		if err != nil {
			return end, next, err
		}
		r = r1
	}
	if v.count == 0 {
		v.count = 1
	}
	cmd, ok := visualCommands[r]
	if !ok {
		return
	}
	if m := cmd(v); m != nil {
		next = m
	}
	if v.pos == len(v.buf) {
		v.move(v.pos - 1)
	}
	v.count = 0
	return
}

type visualCommand = func(*visual) modeChanger

var visualCommands = map[rune]visualCommand{
	CharEscape: (*visual).escape,
	CharCtrlC:  (*visual).escape,
	'~':        (*visual).switchCase,
	'$':        (*visual).endline,
	'0':        (*visual).beginline,
	'A':        (*visual).appendAfter,
	'B':        (*visual).wordBack,
	'C':        (*visual).changeLine,
	'D':        (*visual).deleteLine,
	'E':        (*visual).wordEndNonBlank,
	'F':        (*visual).searchCharacterBackward,
	'I':        (*visual).insertBefore,
	'U':        (*visual).toUpper,
	'W':        (*visual).wordNonBlank,
	'b':        (*visual).wordBack,
	'c':        (*visual).change,
	'd':        (*visual).delete,
	'e':        (*visual).wordEnd,
	'f':        (*visual).searchCharacter,
	'i':        (*visual).object,
	'h':        (*visual).left,
	'l':        (*visual).right,
	'r':        (*visual).replace,
	's':        (*visual).siege,
	'o':        (*visual).swap,
	'u':        (*visual).toLower,
	'v':        (*visual).escape,
	'w':        (*visual).word,
	'y':        (*visual).yank,
}

func (v *visual) escape() modeChanger {
	return norm()
}

func (v *visual) delete() modeChanger {
	hi := v.Highlight()
	v.nvCommon.delete(hi.Left, hi.Right)
	return norm()
}

func (v *visual) change() modeChanger {
	_ = v.delete()
	return ins(v.pos == len(v.buf))
}

func (v *visual) swap() (_ modeChanger) {
	v.start, v.pos = v.pos, v.start
	return
}

func (v *visual) yank() modeChanger {
	hi := v.Highlight()
	v.nvCommon.yank(register.Unnamed, hi.Left, hi.Right)
	v.move(hi.Left)
	return norm()
}

func (v *visual) replace() modeChanger {
	r, _, _ := v.in.ReadRune()
	hi := v.Highlight()
	rs := make([]rune, hi.Right-hi.Left)
	for i := range rs {
		rs[i] = r
	}
	v.nvCommon.replace(rs, hi.Left)
	return norm()
}

func (v *visual) toUpper() modeChanger {
	hi := v.Highlight()
	v.nvCommon.toUpper(hi.Left, hi.Right)
	return norm()
}

func (v *visual) toLower() modeChanger {
	hi := v.Highlight()
	v.nvCommon.toLower(hi.Left, hi.Right)
	return norm()
}

func (v *visual) wordEnd() (_ modeChanger) {
	for i := 0; i < v.count; i++ {
		v.nvCommon.wordEnd()
	}
	return
}

func (v *visual) wordEndNonBlank() (_ modeChanger) {
	for i := 0; i < v.count; i++ {
		v.nvCommon.wordEndNonBlank()
	}
	return
}

func (v *visual) insertBefore() modeChanger {
	v.move(v.Highlight().Left)
	return ins(v.pos == len(v.buf))
}

func (v *visual) appendAfter() modeChanger {
	v.move(v.Highlight().Right)
	return ins(v.pos == len(v.buf))
}

func (v *visual) switchCase() modeChanger {
	hi := v.Highlight()
	v.editor.swapCase(hi.Left, hi.Right)
	v.move(hi.Left)
	return norm()
}

func (v *visual) deleteLine() modeChanger {
	v.editor.delete(0, len(v.buf))
	return norm()
}

func (v *visual) siege() modeChanger {
	hi := v.Highlight()
	r, _, _ := v.in.ReadRune()
	v.editor.siege(hi.Left, hi.Right, r)
	v.move(hi.Left)
	return norm()
}

func (v *visual) changeLine() modeChanger {
	v.editor.delete(0, len(v.buf))
	return ins(v.pos == len(v.buf))
}

func (v *visual) object() (_ modeChanger) {
	if v.start <= v.pos {
		v.move(v.pos + 1)
	} else {
		v.move(v.pos - 1)
	}
	from, to := v.currentWord(false)
	if from < 0 {
		return
	}
	if v.start <= v.pos {
		v.move(to - 1)
		return
	}
	v.move(from)
	return
}
<|endoftext|>


<commit_before>package webx

import (
	"net/http"
	"strings"

	"bitbucket.org/admpub/webx/lib/tplex"
	"github.com/labstack/echo"
	mw "github.com/labstack/echo/middleware"
)

func NewServer(name string) (s *Server) {
	s = &Server{
		Name:               name,
		Apps:               make(map[string]*App),
		apps:               make(map[string]*App),
		DefaultMiddlewares: []echo.Middleware{mw.Logger(), mw.Recover()},
		TemplateDir:        "template",
	}
	servs.Set(name, s)
	return
}

type Server struct {
	Name               string
	Apps               map[string]*App //域名关联
	apps               map[string]*App //名称关联
	DefaultMiddlewares []echo.Middleware
	TemplateEngine     *tplex.TemplateEx
	TemplateDir        string
}

func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	app, ok := s.Apps[r.Host]
	if !ok {
		app, ok = s.Apps["*"]
	}

	if ok && app.Handler != nil {
		app.Handler.ServeHTTP(w, r)
	} else {
		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
	}
}

func (s *Server) NewApp(domain string, middlewares ...echo.Middleware) *App {
	e := echo.New()
	e.Use(middlewares...)
	e.Use(s.DefaultMiddlewares...)
	r := strings.Split(domain, "@") //blog@www.blog.com
	name := ""
	if len(r) > 1 {
		name = r[0]
		domain = r[1]
	} else {
		name = domain
	}
	if s.TemplateEngine != nil {
		e.SetRenderer(s.TemplateEngine)
	}
	a := NewApp(name, e)
	s.Apps[domain] = a
	s.apps[name] = a
	return a
}

func (s *Server) InitTmpl(tmplDir ...string) *Server {
	if s.TemplateEngine != nil {
		s.TemplateEngine.Close()
	}
	if len(tmplDir) > 0 {
		s.TemplateEngine = tplex.New(tmplDir[0])
	} else {
		s.TemplateEngine = tplex.New(s.TemplateDir)
	}
	s.TemplateEngine.InitMgr(true, true)
	return s
}

func (s *Server) Run(addr ...string) {
	http.ListenAndServe(strings.Join(addr, ":"), s)
}

func (s *Server) App(args ...string) (a *App) {
	if len(args) > 0 {
		if ap, ok := s.apps[args[0]]; ok {
			a = ap
			return
		}
	}
	return
}
<commit_msg>update<commit_after>package webx

import (
	"net/http"
	"strings"

	"bitbucket.org/admpub/webx/lib/tplex"
	"github.com/labstack/echo"
	mw "github.com/labstack/echo/middleware"
)

func NewServer(name string) (s *Server) {
	s = &Server{
		Name:               name,
		Apps:               make(map[string]*App),
		apps:               make(map[string]*App),
		DefaultMiddlewares: []echo.Middleware{mw.Logger(), mw.Recover()},
		TemplateDir:        "template",
	}
	servs.Set(name, s)
	return
}

type Server struct {
	Name               string
	Apps               map[string]*App //域名关联
	apps               map[string]*App //名称关联
	DefaultMiddlewares []echo.Middleware
	TemplateEngine     *tplex.TemplateEx
	TemplateDir        string
}

func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	app, ok := s.Apps[r.Host]
	if !ok {
		app, ok = s.Apps["*"]
	}

	if ok && app.Handler != nil {
		app.Handler.ServeHTTP(w, r)
	} else {
		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
	}
}

func (s *Server) NewApp(domain string, middlewares ...echo.Middleware) *App {
	r := strings.Split(domain, "@") //blog@www.blog.com
	name := ""
	if len(r) > 1 {
		name = r[0]
		domain = r[1]
	} else {
		name = domain
	}
	if name == "*" {
		if a, ok := s.apps[name]; ok {
			return a
		}
	}
	e := echo.New()
	e.Use(middlewares...)
	e.Use(s.DefaultMiddlewares...)
	if s.TemplateEngine != nil {
		e.SetRenderer(s.TemplateEngine)
	}
	a := NewApp(name, e)
	s.Apps[domain] = a
	s.apps[name] = a
	return a
}

func (s *Server) InitTmpl(tmplDir ...string) *Server {
	if s.TemplateEngine != nil {
		s.TemplateEngine.Close()
	}
	if len(tmplDir) > 0 {
		s.TemplateEngine = tplex.New(tmplDir[0])
	} else {
		s.TemplateEngine = tplex.New(s.TemplateDir)
	}
	s.TemplateEngine.InitMgr(true, true)
	return s
}

func (s *Server) Run(addr ...string) {
	http.ListenAndServe(strings.Join(addr, ":"), s)
}

func (s *Server) App(args ...string) (a *App) {
	var name string
	if len(args) > 0 {
		name = args[0]
		if ap, ok := s.apps[name]; ok {
			a = ap
			return
		}
	}
	if name == "" {
		name = "*"
	}
	return s.NewApp(name)
}
<|endoftext|>


<commit_before>package fastrpc

import (
	"bufio"
	"crypto/tls"
	"fmt"
	"github.com/valyala/fasthttp"
	"io"
	"log"
	"net"
	"os"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

// HandlerCtx is an interface implementing context passed to Server.Handler
type HandlerCtx interface {
	// ConcurrencyLimitError must set the response
	// to 'concurrency limit exceeded' error.
	ConcurrencyLimitError(concurrency int)

	// Init must prepare ctx for reading the next request.
	Init(conn net.Conn, logger fasthttp.Logger)

	// ReadRequest must read request from br.
	ReadRequest(br *bufio.Reader) error

	// WriteResponse must write response to bw.
	WriteResponse(bw *bufio.Writer) error
}

// Server accepts rpc requests from Client.
type Server struct {
	// SniffHeader is the header read from each client connection.
	//
	// The server sends the same header to each client.
	SniffHeader string

	// ProtocolVersion is the version of HandlerCtx.ReadRequest
	// and HandlerCtx.WriteResponse.
	//
	// The ProtocolVersion must be changed each time HandlerCtx.ReadRequest
	// or HandlerCtx.WriteResponse changes the underlying format.
	ProtocolVersion byte

	// NewHandlerCtx must return new HandlerCtx
	NewHandlerCtx func() HandlerCtx

	// Handler must process incoming requests.
	//
	// The handler must return either ctx passed to the call
	// or new non-nil ctx.
	//
	// The handler may return ctx passed to the call only if the ctx
	// is no longer used after returning from the handler.
	// Otherwise new ctx must be returned.
	Handler func(ctx HandlerCtx) HandlerCtx

	// CompressType is the compression type used for responses.
	//
	// CompressFlate is used by default.
	CompressType CompressType

	// Concurrency is the maximum number of concurrent goroutines
	// with Server.Handler the server may run.
	//
	// DefaultConcurrency is used by default.
	Concurrency int

	// TLSConfig is TLS (aka SSL) config used for accepting encrypted
	// client connections.
	//
	// Encrypted connections may be used for transferring sensitive
	// information over untrusted networks.
	//
	// By default server accepts only unencrypted connections.
	TLSConfig *tls.Config

	// MaxBatchDelay is the maximum duration before ready responses
	// are sent to the client.
	//
	// Responses' batching may reduce network bandwidth usage and CPU usage.
	//
	// By default responses are sent immediately to the client.
	MaxBatchDelay time.Duration

	// Maximum duration for reading the full request (including body).
	//
	// This also limits the maximum lifetime for idle connections.
	//
	// By default request read timeout is unlimited.
	ReadTimeout time.Duration

	// Maximum duration for writing the full response (including body).
	//
	// By default response write timeout is unlimited.
	WriteTimeout time.Duration

	// ReadBufferSize is the size for read buffer.
	//
	// DefaultReadBufferSize is used by default.
	ReadBufferSize int

	// WriteBufferSize is the size for write buffer.
	//
	// DefaultWriteBufferSize is used by default.
	WriteBufferSize int

	// Logger, which is used by the Server.
	//
	// Standard logger from log package is used by default.
	Logger fasthttp.Logger

	// PipelineRequests enables requests' pipelining.
	//
	// Requests from a single client are processed serially
	// if is set to true.
	//
	// Enabling requests' pipelining may be useful in the following cases:
	//
	//   - if requests from a single client must be processed serially;
	//   - if the Server.Handler doesn't block and maximum throughput
	//     must be achieved for requests' processing.
	//
	// By default requests from a single client are processed concurrently.
	PipelineRequests bool

	workItemPool sync.Pool

	concurrencyCount uint32
}

func (s *Server) concurrency() int {
	concurrency := s.Concurrency
	if concurrency <= 0 {
		concurrency = DefaultConcurrency
	}
	return concurrency
}

// Serve serves rpc requests accepted from the given listener.
func (s *Server) Serve(ln net.Listener) error {
	if s.Handler == nil {
		panic("BUG: Server.Handler must be set")
	}
	for {
		conn, err := ln.Accept()
		if err != nil {
			if conn != nil {
				panic("BUG: net.Listener returned non-nil conn and non-nil error")
			}
			if netErr, ok := err.(net.Error); ok && netErr.Temporary() {
				s.logger().Printf("fastrpc.Server: temporary error when accepting new connections: %s", netErr)
				time.Sleep(time.Second)
				continue
			}
			if err != io.EOF && !strings.Contains(err.Error(), "use of closed network connection") {
				s.logger().Printf("fastrpc.Server: permanent error when accepting new connections: %s", err)
				return err
			}
			return nil
		}
		if conn == nil {
			panic("BUG: net.Listener returned (nil, nil)")
		}

		go func() {
			laddr := conn.LocalAddr().String()
			raddr := conn.RemoteAddr().String()
			if err := s.serveConn(conn); err != nil {
				s.logger().Printf("fastrpc.Server: error on connection %q<->%q: %s", laddr, raddr, err)
			}
		}()
	}
}

func (s *Server) serveConn(conn net.Conn) error {
	cfg := &handshakeConfig{
		sniffHeader:       []byte(s.SniffHeader),
		protocolVersion:   s.ProtocolVersion,
		conn:              conn,
		readBufferSize:    s.ReadBufferSize,
		writeBufferSize:   s.WriteBufferSize,
		writeCompressType: s.CompressType,
		tlsConfig:         s.TLSConfig,
		isServer:          true,
	}
	br, bw, err := newBufioConn(cfg)
	if err != nil {
		conn.Close()
		return err
	}

	stopCh := make(chan struct{})

	pendingResponses := make(chan *serverWorkItem, s.concurrency())
	readerDone := make(chan error, 1)
	go func() {
		readerDone <- s.connReader(br, conn, pendingResponses, stopCh)
	}()

	writerDone := make(chan error, 1)
	go func() {
		writerDone <- s.connWriter(bw, conn, pendingResponses, stopCh)
	}()

	select {
	case err = <-readerDone:
		conn.Close()
		close(stopCh)
		<-writerDone
	case err = <-writerDone:
		conn.Close()
		close(stopCh)
		<-readerDone
	}
	return err
}

func (s *Server) connReader(br *bufio.Reader, conn net.Conn, pendingResponses chan<- *serverWorkItem, stopCh <-chan struct{}) error {
	logger := s.logger()
	concurrency := s.concurrency()
	readTimeout := s.ReadTimeout
	var lastReadDeadline time.Time
	for {
		wi := s.acquireWorkItem()

		if readTimeout > 0 {
			// Optimization: update read deadline only if more than 25%
			// of the last read deadline exceeded.
			// See https://github.com/golang/go/issues/15133 for details.
			t := coarseTimeNow()
			if t.Sub(lastReadDeadline) > (readTimeout >> 2) {
				if err := conn.SetReadDeadline(t.Add(readTimeout)); err != nil {
					return fmt.Errorf("cannot update read deadline: %s", err)
				}
				lastReadDeadline = t
			}
		}

		if n, err := io.ReadFull(br, wi.reqID[:]); err != nil {
			if n == 0 {
				// Ignore error if no bytes are read, since
				// the client may just close the connection.
				return nil
			}
			return fmt.Errorf("cannot read request ID: %s", err)
		}

		wi.ctx.Init(conn, logger)
		if err := wi.ctx.ReadRequest(br); err != nil {
			return fmt.Errorf("cannot read request: %s", err)
		}

		n := int(atomic.AddUint32(&s.concurrencyCount, 1))
		if n > concurrency {
			atomic.AddUint32(&s.concurrencyCount, ^uint32(0))
			wi.ctx.ConcurrencyLimitError(concurrency)
			if !pushPendingResponse(pendingResponses, wi, stopCh) {
				return nil
			}
			continue
		}

		if s.PipelineRequests {
			s.handleRequest(wi, pendingResponses, stopCh)
			atomic.AddUint32(&s.concurrencyCount, ^uint32(0))
		} else {
			go func(wi *serverWorkItem) {
				s.handleRequest(wi, pendingResponses, stopCh)
				atomic.AddUint32(&s.concurrencyCount, ^uint32(0))
			}(wi)
		}
	}
}

func (s *Server) handleRequest(wi *serverWorkItem, pendingResponses chan<- *serverWorkItem, stopCh <-chan struct{}) {
	ctxNew := s.Handler(wi.ctx)
	if ctxNew != wi.ctx {
		if ctxNew == nil {
			panic("BUG: Server.Handler mustn't return nil")
		}
		// The current ctx may be still in use by the handler.
		// So create new wi for passing to pendingResponses.
		reqID := wi.reqID
		wi = s.acquireWorkItem()
		wi.reqID = reqID
		wi.ctx = ctxNew
	}
	pushPendingResponse(pendingResponses, wi, stopCh)
}

func pushPendingResponse(pendingResponses chan<- *serverWorkItem, wi *serverWorkItem, stopCh <-chan struct{}) bool {
	select {
	case pendingResponses <- wi:
	default:
		select {
		case pendingResponses <- wi:
		case <-stopCh:
			return false
		}
	}
	return true
}

func (s *Server) connWriter(bw *bufio.Writer, conn net.Conn, pendingResponses <-chan *serverWorkItem, stopCh <-chan struct{}) error {
	var wi *serverWorkItem

	var (
		flushTimer    = getFlushTimer()
		flushCh       <-chan time.Time
		flushAlwaysCh = make(chan time.Time)
	)
	defer putFlushTimer(flushTimer)

	close(flushAlwaysCh)
	maxBatchDelay := s.MaxBatchDelay
	if maxBatchDelay < 0 {
		maxBatchDelay = 0
	}

	writeTimeout := s.WriteTimeout
	var lastWriteDeadline time.Time
	for {
		select {
		case wi = <-pendingResponses:
		default:
			select {
			case wi = <-pendingResponses:
			case <-stopCh:
				return nil
			case <-flushCh:
				if err := bw.Flush(); err != nil {
					return fmt.Errorf("cannot flush response data to client: %s", err)
				}
				flushCh = nil
				continue
			}
		}

		if writeTimeout > 0 {
			// Optimization: update write deadline only if more than 25%
			// of the last write deadline exceeded.
			// See https://github.com/golang/go/issues/15133 for details.
			t := coarseTimeNow()
			if t.Sub(lastWriteDeadline) > (writeTimeout >> 2) {
				if err := conn.SetReadDeadline(t.Add(writeTimeout)); err != nil {
					return fmt.Errorf("cannot update write deadline: %s", err)
				}
				lastWriteDeadline = t
			}
		}

		if _, err := bw.Write(wi.reqID[:]); err != nil {
			return fmt.Errorf("cannot write response ID: %s", err)
		}
		if err := wi.ctx.WriteResponse(bw); err != nil {
			return fmt.Errorf("cannot write response: %s", err)
		}

		s.releaseWorkItem(wi)

		// re-arm flush channel
		if len(pendingResponses) == 0 {
			if maxBatchDelay > 0 {
				resetFlushTimer(flushTimer, maxBatchDelay)
				flushCh = flushTimer.C
			} else {
				flushCh = flushAlwaysCh
			}
		}
	}
}

type serverWorkItem struct {
	ctx   HandlerCtx
	reqID [4]byte
}

func (s *Server) acquireWorkItem() *serverWorkItem {
	v := s.workItemPool.Get()
	if v == nil {
		return &serverWorkItem{
			ctx: s.NewHandlerCtx(),
		}
	}
	return v.(*serverWorkItem)
}

func (s *Server) releaseWorkItem(wi *serverWorkItem) {
	s.workItemPool.Put(wi)
}

var defaultLogger = log.New(os.Stderr, "", log.LstdFlags)

func (s *Server) logger() fasthttp.Logger {
	if s.Logger != nil {
		return s.Logger
	}
	return defaultLogger
}
<commit_msg>Server: do not send response for SendNowait request<commit_after>package fastrpc

import (
	"bufio"
	"crypto/tls"
	"fmt"
	"github.com/valyala/fasthttp"
	"io"
	"log"
	"net"
	"os"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

// HandlerCtx is an interface implementing context passed to Server.Handler
type HandlerCtx interface {
	// ConcurrencyLimitError must set the response
	// to 'concurrency limit exceeded' error.
	ConcurrencyLimitError(concurrency int)

	// Init must prepare ctx for reading the next request.
	Init(conn net.Conn, logger fasthttp.Logger)

	// ReadRequest must read request from br.
	ReadRequest(br *bufio.Reader) error

	// WriteResponse must write response to bw.
	WriteResponse(bw *bufio.Writer) error
}

// Server accepts rpc requests from Client.
type Server struct {
	// SniffHeader is the header read from each client connection.
	//
	// The server sends the same header to each client.
	SniffHeader string

	// ProtocolVersion is the version of HandlerCtx.ReadRequest
	// and HandlerCtx.WriteResponse.
	//
	// The ProtocolVersion must be changed each time HandlerCtx.ReadRequest
	// or HandlerCtx.WriteResponse changes the underlying format.
	ProtocolVersion byte

	// NewHandlerCtx must return new HandlerCtx
	NewHandlerCtx func() HandlerCtx

	// Handler must process incoming requests.
	//
	// The handler must return either ctx passed to the call
	// or new non-nil ctx.
	//
	// The handler may return ctx passed to the call only if the ctx
	// is no longer used after returning from the handler.
	// Otherwise new ctx must be returned.
	Handler func(ctx HandlerCtx) HandlerCtx

	// CompressType is the compression type used for responses.
	//
	// CompressFlate is used by default.
	CompressType CompressType

	// Concurrency is the maximum number of concurrent goroutines
	// with Server.Handler the server may run.
	//
	// DefaultConcurrency is used by default.
	Concurrency int

	// TLSConfig is TLS (aka SSL) config used for accepting encrypted
	// client connections.
	//
	// Encrypted connections may be used for transferring sensitive
	// information over untrusted networks.
	//
	// By default server accepts only unencrypted connections.
	TLSConfig *tls.Config

	// MaxBatchDelay is the maximum duration before ready responses
	// are sent to the client.
	//
	// Responses' batching may reduce network bandwidth usage and CPU usage.
	//
	// By default responses are sent immediately to the client.
	MaxBatchDelay time.Duration

	// Maximum duration for reading the full request (including body).
	//
	// This also limits the maximum lifetime for idle connections.
	//
	// By default request read timeout is unlimited.
	ReadTimeout time.Duration

	// Maximum duration for writing the full response (including body).
	//
	// By default response write timeout is unlimited.
	WriteTimeout time.Duration

	// ReadBufferSize is the size for read buffer.
	//
	// DefaultReadBufferSize is used by default.
	ReadBufferSize int

	// WriteBufferSize is the size for write buffer.
	//
	// DefaultWriteBufferSize is used by default.
	WriteBufferSize int

	// Logger, which is used by the Server.
	//
	// Standard logger from log package is used by default.
	Logger fasthttp.Logger

	// PipelineRequests enables requests' pipelining.
	//
	// Requests from a single client are processed serially
	// if is set to true.
	//
	// Enabling requests' pipelining may be useful in the following cases:
	//
	//   - if requests from a single client must be processed serially;
	//   - if the Server.Handler doesn't block and maximum throughput
	//     must be achieved for requests' processing.
	//
	// By default requests from a single client are processed concurrently.
	PipelineRequests bool

	workItemPool sync.Pool

	concurrencyCount uint32
}

func (s *Server) concurrency() int {
	concurrency := s.Concurrency
	if concurrency <= 0 {
		concurrency = DefaultConcurrency
	}
	return concurrency
}

// Serve serves rpc requests accepted from the given listener.
func (s *Server) Serve(ln net.Listener) error {
	if s.Handler == nil {
		panic("BUG: Server.Handler must be set")
	}
	for {
		conn, err := ln.Accept()
		if err != nil {
			if conn != nil {
				panic("BUG: net.Listener returned non-nil conn and non-nil error")
			}
			if netErr, ok := err.(net.Error); ok && netErr.Temporary() {
				s.logger().Printf("fastrpc.Server: temporary error when accepting new connections: %s", netErr)
				time.Sleep(time.Second)
				continue
			}
			if err != io.EOF && !strings.Contains(err.Error(), "use of closed network connection") {
				s.logger().Printf("fastrpc.Server: permanent error when accepting new connections: %s", err)
				return err
			}
			return nil
		}
		if conn == nil {
			panic("BUG: net.Listener returned (nil, nil)")
		}

		go func() {
			laddr := conn.LocalAddr().String()
			raddr := conn.RemoteAddr().String()
			if err := s.serveConn(conn); err != nil {
				s.logger().Printf("fastrpc.Server: error on connection %q<->%q: %s", laddr, raddr, err)
			}
		}()
	}
}

func (s *Server) serveConn(conn net.Conn) error {
	cfg := &handshakeConfig{
		sniffHeader:       []byte(s.SniffHeader),
		protocolVersion:   s.ProtocolVersion,
		conn:              conn,
		readBufferSize:    s.ReadBufferSize,
		writeBufferSize:   s.WriteBufferSize,
		writeCompressType: s.CompressType,
		tlsConfig:         s.TLSConfig,
		isServer:          true,
	}
	br, bw, err := newBufioConn(cfg)
	if err != nil {
		conn.Close()
		return err
	}

	stopCh := make(chan struct{})

	pendingResponses := make(chan *serverWorkItem, s.concurrency())
	readerDone := make(chan error, 1)
	go func() {
		readerDone <- s.connReader(br, conn, pendingResponses, stopCh)
	}()

	writerDone := make(chan error, 1)
	go func() {
		writerDone <- s.connWriter(bw, conn, pendingResponses, stopCh)
	}()

	select {
	case err = <-readerDone:
		conn.Close()
		close(stopCh)
		<-writerDone
	case err = <-writerDone:
		conn.Close()
		close(stopCh)
		<-readerDone
	}
	return err
}

func (s *Server) connReader(br *bufio.Reader, conn net.Conn, pendingResponses chan<- *serverWorkItem, stopCh <-chan struct{}) error {
	logger := s.logger()
	concurrency := s.concurrency()
	readTimeout := s.ReadTimeout
	var lastReadDeadline time.Time
	for {
		wi := s.acquireWorkItem()

		if readTimeout > 0 {
			// Optimization: update read deadline only if more than 25%
			// of the last read deadline exceeded.
			// See https://github.com/golang/go/issues/15133 for details.
			t := coarseTimeNow()
			if t.Sub(lastReadDeadline) > (readTimeout >> 2) {
				if err := conn.SetReadDeadline(t.Add(readTimeout)); err != nil {
					return fmt.Errorf("cannot update read deadline: %s", err)
				}
				lastReadDeadline = t
			}
		}

		if n, err := io.ReadFull(br, wi.reqID[:]); err != nil {
			if n == 0 {
				// Ignore error if no bytes are read, since
				// the client may just close the connection.
				return nil
			}
			return fmt.Errorf("cannot read request ID: %s", err)
		}

		wi.ctx.Init(conn, logger)
		if err := wi.ctx.ReadRequest(br); err != nil {
			return fmt.Errorf("cannot read request: %s", err)
		}

		n := int(atomic.AddUint32(&s.concurrencyCount, 1))
		if n > concurrency {
			atomic.AddUint32(&s.concurrencyCount, ^uint32(0))
			wi.ctx.ConcurrencyLimitError(concurrency)
			if !pushPendingResponse(pendingResponses, wi, stopCh) {
				return nil
			}
			continue
		}

		if s.PipelineRequests {
			s.handleRequest(wi, pendingResponses, stopCh)
			atomic.AddUint32(&s.concurrencyCount, ^uint32(0))
		} else {
			go func(wi *serverWorkItem) {
				s.handleRequest(wi, pendingResponses, stopCh)
				atomic.AddUint32(&s.concurrencyCount, ^uint32(0))
			}(wi)
		}
	}
}

func (s *Server) handleRequest(wi *serverWorkItem, pendingResponses chan<- *serverWorkItem, stopCh <-chan struct{}) {
	reqID := wi.reqID
	ctxNew := s.Handler(wi.ctx)
	if isZeroReqID(reqID) {
		// Do not send response for SendNowait request.
		if ctxNew == wi.ctx {
			s.releaseWorkItem(wi)
		}
		return
	}

	if ctxNew != wi.ctx {
		if ctxNew == nil {
			panic("BUG: Server.Handler mustn't return nil")
		}
		// The current ctx may be still in use by the handler.
		// So create new wi for passing to pendingResponses.
		wi = s.acquireWorkItem()
		wi.reqID = reqID
		wi.ctx = ctxNew
	}
	pushPendingResponse(pendingResponses, wi, stopCh)
}

func pushPendingResponse(pendingResponses chan<- *serverWorkItem, wi *serverWorkItem, stopCh <-chan struct{}) bool {
	select {
	case pendingResponses <- wi:
	default:
		select {
		case pendingResponses <- wi:
		case <-stopCh:
			return false
		}
	}
	return true
}

func (s *Server) connWriter(bw *bufio.Writer, conn net.Conn, pendingResponses <-chan *serverWorkItem, stopCh <-chan struct{}) error {
	var wi *serverWorkItem

	var (
		flushTimer    = getFlushTimer()
		flushCh       <-chan time.Time
		flushAlwaysCh = make(chan time.Time)
	)
	defer putFlushTimer(flushTimer)

	close(flushAlwaysCh)
	maxBatchDelay := s.MaxBatchDelay
	if maxBatchDelay < 0 {
		maxBatchDelay = 0
	}

	writeTimeout := s.WriteTimeout
	var lastWriteDeadline time.Time
	for {
		select {
		case wi = <-pendingResponses:
		default:
			select {
			case wi = <-pendingResponses:
			case <-stopCh:
				return nil
			case <-flushCh:
				if err := bw.Flush(); err != nil {
					return fmt.Errorf("cannot flush response data to client: %s", err)
				}
				flushCh = nil
				continue
			}
		}

		if writeTimeout > 0 {
			// Optimization: update write deadline only if more than 25%
			// of the last write deadline exceeded.
			// See https://github.com/golang/go/issues/15133 for details.
			t := coarseTimeNow()
			if t.Sub(lastWriteDeadline) > (writeTimeout >> 2) {
				if err := conn.SetReadDeadline(t.Add(writeTimeout)); err != nil {
					return fmt.Errorf("cannot update write deadline: %s", err)
				}
				lastWriteDeadline = t
			}
		}

		if _, err := bw.Write(wi.reqID[:]); err != nil {
			return fmt.Errorf("cannot write response ID: %s", err)
		}
		if err := wi.ctx.WriteResponse(bw); err != nil {
			return fmt.Errorf("cannot write response: %s", err)
		}

		s.releaseWorkItem(wi)

		// re-arm flush channel
		if len(pendingResponses) == 0 {
			if maxBatchDelay > 0 {
				resetFlushTimer(flushTimer, maxBatchDelay)
				flushCh = flushTimer.C
			} else {
				flushCh = flushAlwaysCh
			}
		}
	}
}

type serverWorkItem struct {
	ctx   HandlerCtx
	reqID [4]byte
}

func (s *Server) acquireWorkItem() *serverWorkItem {
	v := s.workItemPool.Get()
	if v == nil {
		return &serverWorkItem{
			ctx: s.NewHandlerCtx(),
		}
	}
	return v.(*serverWorkItem)
}

func (s *Server) releaseWorkItem(wi *serverWorkItem) {
	s.workItemPool.Put(wi)
}

var defaultLogger = log.New(os.Stderr, "", log.LstdFlags)

func (s *Server) logger() fasthttp.Logger {
	if s.Logger != nil {
		return s.Logger
	}
	return defaultLogger
}

func isZeroReqID(reqID [4]byte) bool {
	return reqID[0] == 0 && reqID[1] == 0 && reqID[2] == 0 && reqID[3] == 0
}
<|endoftext|>


<commit_before>// Copyright 2015 Igor Dolzhikov. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.

package spawn

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httputil"
	"os"
	"regexp"
	"sort"
	"time"

	"github.com/takama/router"
)

const (

	// VERSION - current version of the service
	VERSION = "0.1.10"

	// DATE - revision date of the service
	DATE = "2015-04-03T17:40:17Z"

	// MaxSignals - maximum count of update signals
	MaxSignals = 1000

	// MaxJobs - maximum count of update jobs for every bundle
	MaxJobs = 100000

	// DefaultTimeout is a timeout for the worker's response
	DefaultTimeout time.Duration = 10

	// HTTP methods, which should be queued
	protocolHTTP = "http"
	methodGET    = "GET"
	methodPOST   = "POST"
	methodPUT    = "PUT"
	methodDELETE = "DELETE"

	// Job signals
	responseSignal = iota
	nodeJobSignal
)

// simplest logger, which initialized during starts of the application
var (
	stdlog = log.New(os.Stdout, "", log.Ldate|log.Ltime)
	errlog = log.New(os.Stderr, "", log.Ldate|log.Ltime|log.Lshortfile)
)

// Server Record
type Server struct {

	// Server name/description
	Name string

	// Node Bundle contains the Node records
	Nodes *NodeBundle
	// contains filtered or unexported fields

	// Embeded router
	*router.Router

	// Response signal channel
	response chan struct{}

	// responseTimeout is a timeout for worker's response
	responseTimeout time.Duration

	// job signal channel
	job chan int

	// quit signal channel
	quit chan struct{}

	// round robin mode
	roundRobin bool

	// nodes will queried according to priority
	byPriority bool

	// nodes health check
	check HealthCheck

	// Queue Bundle contains the queue records
	queues *queueBundle

	// Node connection transport
	transport http.RoundTripper
}

// HealthCheck contains parameters which used for checking node
type HealthCheck struct {

	// health check time of the node in seconds
	Seconds time.Duration `json:"seconds"`

	// url which will be checked
	URL string `json:"url"`

	// regexp pattern for extended check analyze
	Pattern string `json:"regexp"`
}

// NewServer creates a new server which contains the nodes/queues
func NewServer(name string) (*Server, error) {

	// Init the router
	r := router.New()
	r.PanicHandler = panicHandler
	r.NotFound = notFound
	r.Logger = logger
	r.CustomHandler = baseHandler

	// Init the Server
	server := &Server{
		Name:            name,
		Router:          r,
		responseTimeout: DefaultTimeout,
		response:        make(chan struct{}),
		job:             make(chan int, MaxSignals),
		quit:            make(chan struct{}),
		transport:       http.DefaultTransport,
	}

	// Create and init nodes bundle
	server.Nodes = &NodeBundle{
		Server:  server,
		records: make(map[string]map[uint64]Node),
	}

	// Create and init queues bundle
	server.queues = &queueBundle{records: make(map[string]*queue)}

	return server, nil
}

// Run the server, init the handlers, init the specified modes.
// If transport http.RoundTripper is not defined used default transport.
// http.RoundTripper contains callback function which handle
// all incoming requests and get responses/errors
func (server *Server) Run(
	hostPort, apiHostPort string,
	transport http.RoundTripper,
	nodes []Node,
	roundRobin, byPriority bool,
	check HealthCheck,
) (status string, err error) {

	// if used round-robin mode
	if roundRobin {
		stdlog.Println(server.Name, "will used 'round-robin' mode")
		server.roundRobin = roundRobin
	}

	// if used by-priority mode
	if byPriority {
		stdlog.Println("Nodes will queried according to priority")
		server.byPriority = byPriority
	}

	// Init the Nodes update channel
	server.Nodes.update = make(chan nodeJob, MaxJobs)

	// Starts the worker which manage server's jobs
	go server.manage()

	// Init the Nodes settings
	if !server.Nodes.SetAll(nodes) {
		status = server.Name + " is not loaded"
		err = errors.New("The nodes settings in config have incorrect values")
		return
	}

	// Init a health check settings
	server.check = check

	// The info handler returns a system status of the application
	server.GET("/info", infoHandler)

	// Lists methods, which display how to use API
	server.GET("/list", displayAllMethods)
	server.GET("/list/nodes", displayAllNodeMethods)
	server.GET("/list/nodes/get", displayGetNodeMethods)
	server.GET("/list/nodes/set", displaySetNodeMethods)
	server.GET("/list/nodes/delete", displayDeleteNodeMethods)

	// Init API methods for the Nodes
	server.GET("/nodes/:host/:port", server.Nodes.getRecord)
	server.GET("/nodes/:host", server.Nodes.getAllRecordsByHost)
	server.GET("/nodes", server.Nodes.getAllRecords)
	server.PUT("/nodes/:host/:port", server.Nodes.putRecord)
	server.PUT("/nodes", server.Nodes.putAllRecords)
	server.DELETE("/nodes/:host/:port", server.Nodes.deleteRecord)
	server.DELETE("/nodes/:host", server.Nodes.deleteAllRecordsByHost)
	server.DELETE("/nodes", server.Nodes.deleteAllRecords)

	go server.Listen(apiHostPort)
	go func() {
		p := &proxy{transport: server}
		if transport != nil {
			p.transport = transport
		}
		if err := http.ListenAndServe(hostPort, p); err != nil {
			errlog.Fatal(err)
		}
	}()

	status = server.Name + " loaded successfully"

	return
}

// Shutdown closes the server graceful
func (server *Server) Shutdown() (status string, err error) {

	// Set timer to wait one minute
	timeout := time.NewTimer(time.Second * 30)

	// a unwanted response sweeps if exist
	for {
		select {
		case <-server.response:
			continue
		default:
		}
		break
	}

	// sends a 'quit' signal
	server.quit <- struct{}{}

	status = server.Name + " connections closed"
	select {

	// Exit by timeout if jobs did not done
	case <-timeout.C:
		err = errors.New("timeout")
		return
	// Exit after all jobs done
	case <-server.response:
		return
	}
}

// Manage routine which manage all jobs
func (server *Server) manage() {
	defer func() {
		if recovery := recover(); recovery != nil {
			errlog.Println("Recovered in Manage routine", recovery)
			// Recover routine
			go server.manage()
		} else {
			stdlog.Println("Manage routine stoped")
			server.response <- struct{}{}
		}
	}()
	for {
		select {
		case job := <-server.job:
			server.doJob(job)
			continue
		default:
		}
		select {
		case job := <-server.job:
			server.doJob(job)
			continue
		case <-server.quit:
			return
		}
	}
}

// Do updates depended by the signal
func (server *Server) doJob(signal int) {
	defer func() {
		if recovery := recover(); recovery != nil {
			errlog.Println("Recovered in do job routine", recovery)
		}
	}()
	switch signal {
	case responseSignal:
		server.response <- struct{}{}
	case nodeJobSignal:
		server.Nodes.updateRecords()
		server.Nodes.InitRing()
	}
}

// RoundTrip manages all requests/responses
func (server *Server) RoundTrip(request *http.Request) (*http.Response, error) {

	// Add "X-Forwarded-For" to repost remote host IP
	if request.Header.Get("X-Forwarded-For") == "" {
		request.Header.Add("X-Forwarded-For", request.RemoteAddr)
	}

	// Use HTTP scheme
	request.URL.Scheme = protocolHTTP

	// If requests should not be queued, get result immediately
	if request.Method != methodPOST &&
		request.Method != methodPUT &&
		request.Method != methodDELETE {

		return server.processReceive(request)
	}

	return server.processUpdate(request)
}

// call 'GET' and others requests to the node using defined mode
func (server *Server) processReceive(request *http.Request) (*http.Response, error) {
	if server.roundRobin {

		// Use round robin to get data from the host
		for count := 0; count < server.Nodes.ring.Len(); count++ {
			if node, ok := server.Nodes.CurrentFromRing(); ok &&
				node.Active && !node.Maintenance {

				// The host is active and is not in maintenance
				request.URL.Host = fmt.Sprintf("%s:%d", node.Host, node.Port)

				// Prepare next host
				server.Nodes.TwistRing()

				if server.checkNode(request.URL.Host) {
					response, err := server.transport.RoundTrip(request)
					if err == nil {
						// If response is sucess, return
						return response, nil
					}
					errlog.Println(err)
				}
			} else {

				// Use next host if not active or maintenance mode
				server.Nodes.TwistRing()
			}
		}
	} else {

		// If is not round robin mode, use first registered host
		if nodes, total := server.Nodes.GetAll(); total > 0 {
			if server.byPriority {
				sort.Sort(byPriority(nodes))
			}
			for _, node := range nodes {
				if node.Active && !node.Maintenance {

					// The host is active and is not in maintenance
					request.URL.Host = fmt.Sprintf("%s:%d", node.Host, node.Port)
					if server.checkNode(request.URL.Host) {
						response, err := http.DefaultTransport.RoundTrip(request)
						if err == nil {
							// If response is sucess, return
							return response, nil
						}
						errlog.Println(err)
					}
				}
			}
		}
	}

	return nil, errors.New("Warning: no one of the nodes is active")
}

// call 'PUT', 'POST', 'DELETE' request to the node
func (server *Server) processUpdate(request *http.Request) (*http.Response, error) {
	// grab update request
	proxyRequestData, err := httputil.DumpRequest(request, true)
	if err != nil {

		// if unsuccessful, return error
		return nil, err
	}
	var host string
	var response *http.Response
	if nodes, total := server.Nodes.GetAll(); total > 0 {
		answer := make(chan *http.Response, total)
		done := make(chan struct{}, total)
		for _, node := range nodes {
			if node.Active {

				host = fmt.Sprintf("%s:%d", node.Host, node.Port)

				// create new queue job
				job := &queueJob{
					done:   done,
					query:  make(chan []byte, 1),
					answer: answer,
				}
				job.query <- proxyRequestData

				queue, _ := server.queues.check(host)
				queue.jobs <- job
				queue.task <- doJobTask
			}
		}
		timeout := time.NewTimer(time.Second * server.responseTimeout)
		for {
			select {
			case response = <-answer:
				return response, nil
			case <-timeout.C:
				return response, errors.New("timeout")
			}
		}
	}
	return response, errors.New("The nodes are not defined")
}

// worker receive a data from the queue and send it to the node
func (server *Server) worker(q *queue) {
	defer func() {
		if recovery := recover(); recovery != nil {
			errlog.Println("Recovered in worker routine", recovery)
			// the worker recovers again
			go server.worker(q)
		} else {
			q.response <- struct{}{}
			stdlog.Println("Worker closed for", q.id)
		}
	}()
	stdlog.Println("Worker started for", q.id)
	for {
		select {
		case task := <-q.task:
			switch task {
			case doJobTask:
				server.doUpdate(q)
			}
			continue
		default:
		}
		select {
		case task := <-q.task:
			switch task {
			case doJobTask:
				server.doUpdate(q)
			}
			continue
		case <-q.quit:
			return
		case <-q.ask:
			q.response <- struct{}{}
		}
	}
}

func (server *Server) doUpdate(q *queue) {
	// check the node
	for {
		if server.checkNode(q.id) {
			break
		}
		stdlog.Println("Node", q.id, "does not ready for updates")
		stdlog.Println("try again in", server.check.Seconds, "seconds")
		timeout := time.NewTimer(time.Second * server.check.Seconds)
		select {
		//  Repeat by timeout
		case <-timeout.C:
			continue
		case <-q.quit:
			q.task <- doJobTask
			return
		case <-q.ask:
			q.response <- struct{}{}
		}
	}
	// if the node is alive, post data
	job := <-q.jobs
	data := <-job.query
	if response, err := server.dispatchRequest(q.id, data); err != nil {

		// Job does not done
		errlog.Println(err)

	} else {

		// job done
		if len(job.done) == 0 {
			// send first response and done signal
			job.done <- struct{}{}
			job.answer <- response
		} else {
			// just close connection
			response.Body.Close()
		}
	}
}

// check the node
func (server *Server) checkNode(host string) bool {
	response, err := http.Get(protocolHTTP + "://" + host + server.check.URL)
	if err != nil {
		return false
	}

	defer response.Body.Close()
	// if pattern does not exist, should be true
	if server.check.Pattern == "" {
		return true
	}

	data, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return false
	}
	// check of regexp pattern
	valid := regexp.MustCompile(server.check.Pattern)
	return valid.MatchString(string(data))
}

// Reproduce request to specified node and capture response
func (server *Server) dispatchRequest(host string, data []byte) (*http.Response, error) {
	reader := bufio.NewReader(bytes.NewBuffer(data))
	request, err := http.ReadRequest(reader)
	if err != nil {
		return nil, err
	}
	request.Body = ioutil.NopCloser(reader)
	request.URL.Scheme = protocolHTTP
	request.URL.Host = host

	response, err := server.transport.RoundTrip(request)
	if err != nil {
		return nil, err
	}
	return response, nil
}
<commit_msg>Bumped version number to 0.1.11<commit_after>// Copyright 2015 Igor Dolzhikov. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.

package spawn

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httputil"
	"os"
	"regexp"
	"sort"
	"time"

	"github.com/takama/router"
)

const (

	// VERSION - current version of the service
	VERSION = "0.1.11"

	// DATE - revision date of the service
	DATE = "2015-04-06T00:40:17Z"

	// MaxSignals - maximum count of update signals
	MaxSignals = 1000

	// MaxJobs - maximum count of update jobs for every bundle
	MaxJobs = 100000

	// DefaultTimeout is a timeout for the worker's response
	DefaultTimeout time.Duration = 10

	// HTTP methods, which should be queued
	protocolHTTP = "http"
	methodGET    = "GET"
	methodPOST   = "POST"
	methodPUT    = "PUT"
	methodDELETE = "DELETE"

	// Job signals
	responseSignal = iota
	nodeJobSignal
)

// simplest logger, which initialized during starts of the application
var (
	stdlog = log.New(os.Stdout, "", log.Ldate|log.Ltime)
	errlog = log.New(os.Stderr, "", log.Ldate|log.Ltime|log.Lshortfile)
)

// Server Record
type Server struct {

	// Server name/description
	Name string

	// Node Bundle contains the Node records
	Nodes *NodeBundle
	// contains filtered or unexported fields

	// Embeded router
	*router.Router

	// Response signal channel
	response chan struct{}

	// responseTimeout is a timeout for worker's response
	responseTimeout time.Duration

	// job signal channel
	job chan int

	// quit signal channel
	quit chan struct{}

	// round robin mode
	roundRobin bool

	// nodes will queried according to priority
	byPriority bool

	// nodes health check
	check HealthCheck

	// Queue Bundle contains the queue records
	queues *queueBundle

	// Node connection transport
	transport http.RoundTripper
}

// HealthCheck contains parameters which used for checking node
type HealthCheck struct {

	// health check time of the node in seconds
	Seconds time.Duration `json:"seconds"`

	// url which will be checked
	URL string `json:"url"`

	// regexp pattern for extended check analyze
	Pattern string `json:"regexp"`
}

// NewServer creates a new server which contains the nodes/queues
func NewServer(name string) (*Server, error) {

	// Init the router
	r := router.New()
	r.PanicHandler = panicHandler
	r.NotFound = notFound
	r.Logger = logger
	r.CustomHandler = baseHandler

	// Init the Server
	server := &Server{
		Name:            name,
		Router:          r,
		responseTimeout: DefaultTimeout,
		response:        make(chan struct{}),
		job:             make(chan int, MaxSignals),
		quit:            make(chan struct{}),
		transport:       http.DefaultTransport,
	}

	// Create and init nodes bundle
	server.Nodes = &NodeBundle{
		Server:  server,
		records: make(map[string]map[uint64]Node),
	}

	// Create and init queues bundle
	server.queues = &queueBundle{records: make(map[string]*queue)}

	return server, nil
}

// Run the server, init the handlers, init the specified modes.
// If transport http.RoundTripper is not defined used default transport.
// http.RoundTripper contains callback function which handle
// all incoming requests and get responses/errors
func (server *Server) Run(
	hostPort, apiHostPort string,
	transport http.RoundTripper,
	nodes []Node,
	roundRobin, byPriority bool,
	check HealthCheck,
) (status string, err error) {

	// if used round-robin mode
	if roundRobin {
		stdlog.Println(server.Name, "will used 'round-robin' mode")
		server.roundRobin = roundRobin
	}

	// if used by-priority mode
	if byPriority {
		stdlog.Println("Nodes will queried according to priority")
		server.byPriority = byPriority
	}

	// Init the Nodes update channel
	server.Nodes.update = make(chan nodeJob, MaxJobs)

	// Starts the worker which manage server's jobs
	go server.manage()

	// Init the Nodes settings
	if !server.Nodes.SetAll(nodes) {
		status = server.Name + " is not loaded"
		err = errors.New("The nodes settings in config have incorrect values")
		return
	}

	// Init a health check settings
	server.check = check

	// The info handler returns a system status of the application
	server.GET("/info", infoHandler)

	// Lists methods, which display how to use API
	server.GET("/list", displayAllMethods)
	server.GET("/list/nodes", displayAllNodeMethods)
	server.GET("/list/nodes/get", displayGetNodeMethods)
	server.GET("/list/nodes/set", displaySetNodeMethods)
	server.GET("/list/nodes/delete", displayDeleteNodeMethods)

	// Init API methods for the Nodes
	server.GET("/nodes/:host/:port", server.Nodes.getRecord)
	server.GET("/nodes/:host", server.Nodes.getAllRecordsByHost)
	server.GET("/nodes", server.Nodes.getAllRecords)
	server.PUT("/nodes/:host/:port", server.Nodes.putRecord)
	server.PUT("/nodes", server.Nodes.putAllRecords)
	server.DELETE("/nodes/:host/:port", server.Nodes.deleteRecord)
	server.DELETE("/nodes/:host", server.Nodes.deleteAllRecordsByHost)
	server.DELETE("/nodes", server.Nodes.deleteAllRecords)

	go server.Listen(apiHostPort)
	go func() {
		p := &proxy{transport: server}
		if transport != nil {
			p.transport = transport
		}
		if err := http.ListenAndServe(hostPort, p); err != nil {
			errlog.Fatal(err)
		}
	}()

	status = server.Name + " loaded successfully"

	return
}

// Shutdown closes the server graceful
func (server *Server) Shutdown() (status string, err error) {

	// Set timer to wait one minute
	timeout := time.NewTimer(time.Second * 30)

	// a unwanted response sweeps if exist
	for {
		select {
		case <-server.response:
			continue
		default:
		}
		break
	}

	// sends a 'quit' signal
	server.quit <- struct{}{}

	status = server.Name + " connections closed"
	select {

	// Exit by timeout if jobs did not done
	case <-timeout.C:
		err = errors.New("timeout")
		return
	// Exit after all jobs done
	case <-server.response:
		return
	}
}

// Manage routine which manage all jobs
func (server *Server) manage() {
	defer func() {
		if recovery := recover(); recovery != nil {
			errlog.Println("Recovered in Manage routine", recovery)
			// Recover routine
			go server.manage()
		} else {
			stdlog.Println("Manage routine stoped")
			server.response <- struct{}{}
		}
	}()
	for {
		select {
		case job := <-server.job:
			server.doJob(job)
			continue
		default:
		}
		select {
		case job := <-server.job:
			server.doJob(job)
			continue
		case <-server.quit:
			return
		}
	}
}

// Do updates depended by the signal
func (server *Server) doJob(signal int) {
	defer func() {
		if recovery := recover(); recovery != nil {
			errlog.Println("Recovered in do job routine", recovery)
		}
	}()
	switch signal {
	case responseSignal:
		server.response <- struct{}{}
	case nodeJobSignal:
		server.Nodes.updateRecords()
		server.Nodes.InitRing()
	}
}

// RoundTrip manages all requests/responses
func (server *Server) RoundTrip(request *http.Request) (*http.Response, error) {

	// Add "X-Forwarded-For" to repost remote host IP
	if request.Header.Get("X-Forwarded-For") == "" {
		request.Header.Add("X-Forwarded-For", request.RemoteAddr)
	}

	// Use HTTP scheme
	request.URL.Scheme = protocolHTTP

	// If requests should not be queued, get result immediately
	if request.Method != methodPOST &&
		request.Method != methodPUT &&
		request.Method != methodDELETE {

		return server.processReceive(request)
	}

	return server.processUpdate(request)
}

// call 'GET' and others requests to the node using defined mode
func (server *Server) processReceive(request *http.Request) (*http.Response, error) {
	if server.roundRobin {

		// Use round robin to get data from the host
		for count := 0; count < server.Nodes.ring.Len(); count++ {
			if node, ok := server.Nodes.CurrentFromRing(); ok &&
				node.Active && !node.Maintenance {

				// The host is active and is not in maintenance
				request.URL.Host = fmt.Sprintf("%s:%d", node.Host, node.Port)

				// Prepare next host
				server.Nodes.TwistRing()

				if server.checkNode(request.URL.Host) {
					response, err := server.transport.RoundTrip(request)
					if err == nil {
						// If response is sucess, return
						return response, nil
					}
					errlog.Println(err)
				}
			} else {

				// Use next host if not active or maintenance mode
				server.Nodes.TwistRing()
			}
		}
	} else {

		// If is not round robin mode, use first registered host
		if nodes, total := server.Nodes.GetAll(); total > 0 {
			if server.byPriority {
				sort.Sort(byPriority(nodes))
			}
			for _, node := range nodes {
				if node.Active && !node.Maintenance {

					// The host is active and is not in maintenance
					request.URL.Host = fmt.Sprintf("%s:%d", node.Host, node.Port)
					if server.checkNode(request.URL.Host) {
						response, err := http.DefaultTransport.RoundTrip(request)
						if err == nil {
							// If response is sucess, return
							return response, nil
						}
						errlog.Println(err)
					}
				}
			}
		}
	}

	return nil, errors.New("Warning: no one of the nodes is active")
}

// call 'PUT', 'POST', 'DELETE' request to the node
func (server *Server) processUpdate(request *http.Request) (*http.Response, error) {
	// grab update request
	proxyRequestData, err := httputil.DumpRequest(request, true)
	if err != nil {

		// if unsuccessful, return error
		return nil, err
	}
	var host string
	var response *http.Response
	if nodes, total := server.Nodes.GetAll(); total > 0 {
		answer := make(chan *http.Response, total)
		done := make(chan struct{}, total)
		for _, node := range nodes {
			if node.Active {

				host = fmt.Sprintf("%s:%d", node.Host, node.Port)

				// create new queue job
				job := &queueJob{
					done:   done,
					query:  make(chan []byte, 1),
					answer: answer,
				}
				job.query <- proxyRequestData

				queue, _ := server.queues.check(host)
				queue.jobs <- job
				queue.task <- doJobTask
			}
		}
		timeout := time.NewTimer(time.Second * server.responseTimeout)
		for {
			select {
			case response = <-answer:
				return response, nil
			case <-timeout.C:
				return response, errors.New("timeout")
			}
		}
	}
	return response, errors.New("The nodes are not defined")
}

// worker receive a data from the queue and send it to the node
func (server *Server) worker(q *queue) {
	defer func() {
		if recovery := recover(); recovery != nil {
			errlog.Println("Recovered in worker routine", recovery)
			// the worker recovers again
			go server.worker(q)
		} else {
			q.response <- struct{}{}
			stdlog.Println("Worker closed for", q.id)
		}
	}()
	stdlog.Println("Worker started for", q.id)
	for {
		select {
		case task := <-q.task:
			switch task {
			case doJobTask:
				server.doUpdate(q)
			}
			continue
		default:
		}
		select {
		case task := <-q.task:
			switch task {
			case doJobTask:
				server.doUpdate(q)
			}
			continue
		case <-q.quit:
			return
		case <-q.ask:
			q.response <- struct{}{}
		}
	}
}

func (server *Server) doUpdate(q *queue) {
	// check the node
	for {
		if server.checkNode(q.id) {
			break
		}
		stdlog.Println("Node", q.id, "does not ready for updates")
		stdlog.Println("try again in", server.check.Seconds, "seconds")
		timeout := time.NewTimer(time.Second * server.check.Seconds)
		select {
		//  Repeat by timeout
		case <-timeout.C:
			continue
		case <-q.quit:
			q.task <- doJobTask
			return
		case <-q.ask:
			q.response <- struct{}{}
		}
	}
	// if the node is alive, post data
	job := <-q.jobs
	data := <-job.query
	if response, err := server.dispatchRequest(q.id, data); err != nil {

		// Job does not done
		errlog.Println(err)

	} else {

		// job done
		if len(job.done) == 0 {
			// send first response and done signal
			job.done <- struct{}{}
			job.answer <- response
		} else {
			// just close connection
			response.Body.Close()
		}
	}
}

// check the node
func (server *Server) checkNode(host string) bool {
	response, err := http.Get(protocolHTTP + "://" + host + server.check.URL)
	if err != nil {
		return false
	}

	defer response.Body.Close()
	// if pattern does not exist, should be true
	if server.check.Pattern == "" {
		return true
	}

	data, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return false
	}
	// check of regexp pattern
	valid := regexp.MustCompile(server.check.Pattern)
	return valid.MatchString(string(data))
}

// Reproduce request to specified node and capture response
func (server *Server) dispatchRequest(host string, data []byte) (*http.Response, error) {
	reader := bufio.NewReader(bytes.NewBuffer(data))
	request, err := http.ReadRequest(reader)
	if err != nil {
		return nil, err
	}
	request.Body = ioutil.NopCloser(reader)
	request.URL.Scheme = protocolHTTP
	request.URL.Host = host

	response, err := server.transport.RoundTrip(request)
	if err != nil {
		return nil, err
	}
	return response, nil
}
<|endoftext|>