<commit_before>// Copyright 2016 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"errors"
	"log"
	"net/http"
	"os"
	"os/signal"
	"regexp"
	"runtime"
	"runtime/pprof"
	"strings"
	"syscall"
	"time"

	"github.com/codegangsta/cli"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/tsuru/planb/backend"
	"github.com/tsuru/planb/reverseproxy"
	"github.com/tsuru/planb/router"
	"github.com/tsuru/planb/tls"
)

func handleSignals(server interface {
	Stop()
}) {
	sigChan := make(chan os.Signal, 3)
	go func() {
		for sig := range sigChan {
			if sig == os.Interrupt || sig == os.Kill {
				server.Stop()
			}
			if sig == syscall.SIGUSR1 {
				pprof.Lookup("goroutine").WriteTo(os.Stdout, 2)
			}
			if sig == syscall.SIGUSR2 {
				go startProfiling()
			}
		}
	}()
	signal.Notify(sigChan, os.Interrupt, os.Kill, syscall.SIGUSR1, syscall.SIGUSR2)
}

func startProfiling() {
	cpufile, _ := os.OpenFile("./planb_cpu.pprof", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
	memfile, _ := os.OpenFile("./planb_mem.pprof", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
	lockfile, _ := os.OpenFile("./planb_lock.pprof", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
	log.Println("enabling profile...")
	runtime.GC()
	pprof.WriteHeapProfile(memfile)
	memfile.Close()
	runtime.SetBlockProfileRate(1)
	time.Sleep(30 * time.Second)
	pprof.Lookup("block").WriteTo(lockfile, 0)
	runtime.SetBlockProfileRate(0)
	lockfile.Close()
	pprof.StartCPUProfile(cpufile)
	time.Sleep(30 * time.Second)
	pprof.StopCPUProfile()
	cpufile.Close()
	log.Println("profiling done")
}

func runServer(c *cli.Context) {
	var rp reverseproxy.ReverseProxy
	switch c.String("engine") {
	case "native":
		rp = &reverseproxy.NativeReverseProxy{}
	case "fasthttp":
		rp = &reverseproxy.FastReverseProxy{}
	case "sni":
		rp = &reverseproxy.SNIReverseProxy{}
	default:
		log.Fatal(errors.New("invalid engine"))
	}
	log.Printf("Using %T engine\n", rp)
	readOpts := backend.RedisOptions{
		Network:       c.String("read-redis-network"),
		Host:          c.String("read-redis-host"),
		Port:          c.Int("read-redis-port"),
		SentinelAddrs: c.String("read-redis-sentinel-addrs"),
		SentinelName:  c.String("read-redis-sentinel-name"),
		Password:      c.String("read-redis-password"),
		DB:            c.Int("read-redis-db"),
	}
	writeOpts := backend.RedisOptions{
		Network:       c.String("write-redis-network"),
		Host:          c.String("write-redis-host"),
		Port:          c.Int("write-redis-port"),
		SentinelAddrs: c.String("write-redis-sentinel-addrs"),
		SentinelName:  c.String("write-redis-sentinel-name"),
		Password:      c.String("write-redis-password"),
		DB:            c.Int("write-redis-db"),
	}
	routesBE, err := backend.NewRedisBackend(readOpts, writeOpts)
	if err != nil {
		log.Fatal(err)
	}
	if c.Bool("active-healthcheck") {
		err = routesBE.StartMonitor()
		if err != nil {
			log.Fatal(err)
		}
	}
	r := router.Router{
		Backend:        routesBE,
		LogPath:        c.String("access-log"),
		DeadBackendTTL: c.Int("dead-backend-time"),
		CacheEnabled:   c.Bool("backend-cache"),
	}
	err = r.Init()
	if err != nil {
		log.Fatal(err)
	}
	err = rp.Initialize(reverseproxy.ReverseProxyConfig{
		Router:            &r,
		RequestIDHeader:   http.CanonicalHeaderKey(c.String("request-id-header")),
		FlushInterval:     time.Duration(c.Int("flush-interval")) * time.Millisecond,
		DialTimeout:       time.Duration(c.Int("dial-timeout")) * time.Second,
		RequestTimeout:    time.Duration(c.Int("request-timeout")) * time.Second,
		ReadTimeout:       c.Duration("client-read-timeout"),
		ReadHeaderTimeout: c.Duration("client-read-header-timeout"),
		WriteTimeout:      c.Duration("client-write-timeout"),
		IdleTimeout:       c.Duration("client-idle-timeout"),
	})
	if err != nil {
		log.Fatal(err)
	}

	listener := &router.RouterListener{
		ReverseProxy: rp,
		Listen:       c.String("listen"),
		TLSListen:    c.String("tls-listen"),
		CertLoader:   getCertificateLoader(c, readOpts),
	}

	if addr := c.String("metrics-address"); addr != "" {
		handler := http.NewServeMux()
		handler.Handle("/metrics", promhttp.Handler())
		go func() {
			log.Fatal(http.ListenAndServe(addr, handler))
		}()
	}

	handleSignals(listener)
	listener.Serve()

	r.Stop()
	routesBE.StopMonitor()
}

func getCertificateLoader(c *cli.Context, readOpts backend.RedisOptions) tls.CertificateLoader {
	if c.String("tls-listen") == "" {
		return nil
	}

	from := c.String("load-certificates-from")
	switch from {
	case "redis":
		client, err := readOpts.Client()
		if err != nil {
			log.Fatal(err)
		}

		return tls.NewRedisCertificateLoader(client)
	default:
		return tls.NewFSCertificateLoader(from)
	}
}

func fixUsage(s string) string {
	linebreakRegexp := regexp.MustCompile(`\n{1}[\t ]*`)
	s = linebreakRegexp.ReplaceAllString(s, " ")
	parts := strings.Split(s, " ")
	currLen := 0
	lastPart := 0
	var lines []string
	for i := range parts {
		if currLen+len(parts[i])+1 > 65 {
			lines = append(lines, strings.Join(parts[lastPart:i], " "))
			currLen = 0
			lastPart = i
		}
		currLen += len(parts[i]) + 1
	}
	lines = append(lines, strings.Join(parts[lastPart:], " "))
	return strings.Join(lines, "\n\t")
}

func main() {
	oldStringer := cli.FlagStringer
	cli.FlagStringer = func(flag cli.Flag) string {
		usage := oldStringer(flag)
		usageIdx := strings.LastIndex(usage, "\t")
		if usageIdx != -1 {
			usage = usage[:usageIdx] + "\t" + fixUsage(usage[usageIdx+1:])
		}
		return usage
	}
	app := cli.NewApp()
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "listen, l",
			Value: "0.0.0.0:8989",
			Usage: "Address to listen",
		},
		cli.StringFlag{
			Name:  "tls-listen",
			Usage: "Address to listen with tls",
		},
		cli.StringFlag{
			Name:  "metrics-address",
			Usage: "Address to expose prometheus /metrics",
		},
		cli.StringFlag{
			Name:  "load-certificates-from",
			Value: "redis",
			Usage: `Path where certificate will found.
If value equals 'redis' certificate will be loaded from redis service.`,
		},
		cli.StringFlag{
			Name:  "read-redis-network",
			Value: "tcp",
		},
		cli.StringFlag{
			Name:  "read-redis-host",
			Value: "127.0.0.1",
		},
		cli.IntFlag{
			Name:  "read-redis-port",
			Value: 6379,
		},
		cli.StringFlag{
			Name:  "read-redis-sentinel-addrs",
			Usage: "Comma separated list of redis addresses",
		},
		cli.StringFlag{
			Name: "read-redis-sentinel-name",
		},
		cli.StringFlag{
			Name: "read-redis-password",
		},
		cli.IntFlag{
			Name: "read-redis-db",
		},
		cli.StringFlag{
			Name:  "write-redis-network",
			Value: "tcp",
		},
		cli.StringFlag{
			Name:  "write-redis-host",
			Value: "127.0.0.1",
		},
		cli.IntFlag{
			Name:  "write-redis-port",
			Value: 6379,
		},
		cli.StringFlag{
			Name:  "write-redis-sentinel-addrs",
			Usage: "Comma separated list of redis addresses",
		},
		cli.StringFlag{
			Name: "write-redis-sentinel-name",
		},
		cli.StringFlag{
			Name: "write-redis-password",
		},
		cli.IntFlag{
			Name: "write-redis-db",
		},
		cli.StringFlag{
			Name:  "access-log",
			Value: "./access.log",
			Usage: `File path where access log will be written.
If value equals 'syslog' log will be sent to local syslog.
The value 'none' can be used to disable access logs.`,
		},
		cli.IntFlag{
			Name:  "request-timeout",
			Value: 30,
			Usage: "Total backend request timeout in seconds",
		},
		cli.IntFlag{
			Name:  "dial-timeout",
			Value: 10,
			Usage: "Dial backend request timeout in seconds",
		},
		cli.DurationFlag{
			Name:  "client-read-timeout",
			Value: 0,
			Usage: "Maximum duration for reading the entire request, including the body",
		},
		cli.DurationFlag{
			Name:  "client-read-header-timeout",
			Value: 0,
			Usage: "Amount of time allowed to read request headers",
		},
		cli.DurationFlag{
			Name:  "client-write-timeout",
			Value: 0,
			Usage: "Maximum duration before timing out writes of the response",
		},
		cli.DurationFlag{
			Name:  "client-idle-timeout",
			Value: 0,
			Usage: "Maximum amount of time to wait for the next request when keep-alives are enabled",
		},
		cli.IntFlag{
			Name:  "dead-backend-time",
			Value: 30,
			Usage: "Time in seconds a backend will remain disabled after a network failure",
		},
		cli.IntFlag{
			Name:  "flush-interval",
			Value: 10,
			Usage: "Time in milliseconds to flush the proxied request",
		},
		cli.StringFlag{
			Name:  "request-id-header",
			Usage: "Header to enable message tracking",
		},
		cli.BoolFlag{
			Name: "active-healthcheck",
		},
		cli.StringFlag{
			Name:  "engine",
			Value: "native",
			Usage: "Reverse proxy engine, options are 'native', 'sni' and 'fasthttp'. Using 'sni' and 'fasthttp' is highly experimental and not recommended for production environments.",
		},
		cli.BoolFlag{
			Name:  "backend-cache",
			Usage: "Enable caching backend results for 2 seconds. This may cause temporary inconsistencies.",
		},
	}
	app.Version = "0.1.14"
	app.Name = "planb"
	app.Usage = "http and websockets reverse proxy"
	app.Action = runServer
	app.Author = "tsuru team"
	app.Email = "https://github.com/tsuru/planb"
	app.Run(os.Args)
}
<commit_msg>bump version to 0.1.15<commit_after>// Copyright 2016 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"errors"
	"log"
	"net/http"
	"os"
	"os/signal"
	"regexp"
	"runtime"
	"runtime/pprof"
	"strings"
	"syscall"
	"time"

	"github.com/codegangsta/cli"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/tsuru/planb/backend"
	"github.com/tsuru/planb/reverseproxy"
	"github.com/tsuru/planb/router"
	"github.com/tsuru/planb/tls"
)

func handleSignals(server interface {
	Stop()
}) {
	sigChan := make(chan os.Signal, 3)
	go func() {
		for sig := range sigChan {
			if sig == os.Interrupt || sig == os.Kill {
				server.Stop()
			}
			if sig == syscall.SIGUSR1 {
				pprof.Lookup("goroutine").WriteTo(os.Stdout, 2)
			}
			if sig == syscall.SIGUSR2 {
				go startProfiling()
			}
		}
	}()
	signal.Notify(sigChan, os.Interrupt, os.Kill, syscall.SIGUSR1, syscall.SIGUSR2)
}

func startProfiling() {
	cpufile, _ := os.OpenFile("./planb_cpu.pprof", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
	memfile, _ := os.OpenFile("./planb_mem.pprof", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
	lockfile, _ := os.OpenFile("./planb_lock.pprof", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
	log.Println("enabling profile...")
	runtime.GC()
	pprof.WriteHeapProfile(memfile)
	memfile.Close()
	runtime.SetBlockProfileRate(1)
	time.Sleep(30 * time.Second)
	pprof.Lookup("block").WriteTo(lockfile, 0)
	runtime.SetBlockProfileRate(0)
	lockfile.Close()
	pprof.StartCPUProfile(cpufile)
	time.Sleep(30 * time.Second)
	pprof.StopCPUProfile()
	cpufile.Close()
	log.Println("profiling done")
}

func runServer(c *cli.Context) {
	var rp reverseproxy.ReverseProxy
	switch c.String("engine") {
	case "native":
		rp = &reverseproxy.NativeReverseProxy{}
	case "fasthttp":
		rp = &reverseproxy.FastReverseProxy{}
	case "sni":
		rp = &reverseproxy.SNIReverseProxy{}
	default:
		log.Fatal(errors.New("invalid engine"))
	}
	log.Printf("Using %T engine\n", rp)
	readOpts := backend.RedisOptions{
		Network:       c.String("read-redis-network"),
		Host:          c.String("read-redis-host"),
		Port:          c.Int("read-redis-port"),
		SentinelAddrs: c.String("read-redis-sentinel-addrs"),
		SentinelName:  c.String("read-redis-sentinel-name"),
		Password:      c.String("read-redis-password"),
		DB:            c.Int("read-redis-db"),
	}
	writeOpts := backend.RedisOptions{
		Network:       c.String("write-redis-network"),
		Host:          c.String("write-redis-host"),
		Port:          c.Int("write-redis-port"),
		SentinelAddrs: c.String("write-redis-sentinel-addrs"),
		SentinelName:  c.String("write-redis-sentinel-name"),
		Password:      c.String("write-redis-password"),
		DB:            c.Int("write-redis-db"),
	}
	routesBE, err := backend.NewRedisBackend(readOpts, writeOpts)
	if err != nil {
		log.Fatal(err)
	}
	if c.Bool("active-healthcheck") {
		err = routesBE.StartMonitor()
		if err != nil {
			log.Fatal(err)
		}
	}
	r := router.Router{
		Backend:        routesBE,
		LogPath:        c.String("access-log"),
		DeadBackendTTL: c.Int("dead-backend-time"),
		CacheEnabled:   c.Bool("backend-cache"),
	}
	err = r.Init()
	if err != nil {
		log.Fatal(err)
	}
	err = rp.Initialize(reverseproxy.ReverseProxyConfig{
		Router:            &r,
		RequestIDHeader:   http.CanonicalHeaderKey(c.String("request-id-header")),
		FlushInterval:     time.Duration(c.Int("flush-interval")) * time.Millisecond,
		DialTimeout:       time.Duration(c.Int("dial-timeout")) * time.Second,
		RequestTimeout:    time.Duration(c.Int("request-timeout")) * time.Second,
		ReadTimeout:       c.Duration("client-read-timeout"),
		ReadHeaderTimeout: c.Duration("client-read-header-timeout"),
		WriteTimeout:      c.Duration("client-write-timeout"),
		IdleTimeout:       c.Duration("client-idle-timeout"),
	})
	if err != nil {
		log.Fatal(err)
	}

	listener := &router.RouterListener{
		ReverseProxy: rp,
		Listen:       c.String("listen"),
		TLSListen:    c.String("tls-listen"),
		CertLoader:   getCertificateLoader(c, readOpts),
	}

	if addr := c.String("metrics-address"); addr != "" {
		handler := http.NewServeMux()
		handler.Handle("/metrics", promhttp.Handler())
		go func() {
			log.Fatal(http.ListenAndServe(addr, handler))
		}()
	}

	handleSignals(listener)
	listener.Serve()

	r.Stop()
	routesBE.StopMonitor()
}

func getCertificateLoader(c *cli.Context, readOpts backend.RedisOptions) tls.CertificateLoader {
	if c.String("tls-listen") == "" {
		return nil
	}

	from := c.String("load-certificates-from")
	switch from {
	case "redis":
		client, err := readOpts.Client()
		if err != nil {
			log.Fatal(err)
		}

		return tls.NewRedisCertificateLoader(client)
	default:
		return tls.NewFSCertificateLoader(from)
	}
}

func fixUsage(s string) string {
	linebreakRegexp := regexp.MustCompile(`\n{1}[\t ]*`)
	s = linebreakRegexp.ReplaceAllString(s, " ")
	parts := strings.Split(s, " ")
	currLen := 0
	lastPart := 0
	var lines []string
	for i := range parts {
		if currLen+len(parts[i])+1 > 65 {
			lines = append(lines, strings.Join(parts[lastPart:i], " "))
			currLen = 0
			lastPart = i
		}
		currLen += len(parts[i]) + 1
	}
	lines = append(lines, strings.Join(parts[lastPart:], " "))
	return strings.Join(lines, "\n\t")
}

func main() {
	oldStringer := cli.FlagStringer
	cli.FlagStringer = func(flag cli.Flag) string {
		usage := oldStringer(flag)
		usageIdx := strings.LastIndex(usage, "\t")
		if usageIdx != -1 {
			usage = usage[:usageIdx] + "\t" + fixUsage(usage[usageIdx+1:])
		}
		return usage
	}
	app := cli.NewApp()
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "listen, l",
			Value: "0.0.0.0:8989",
			Usage: "Address to listen",
		},
		cli.StringFlag{
			Name:  "tls-listen",
			Usage: "Address to listen with tls",
		},
		cli.StringFlag{
			Name:  "metrics-address",
			Usage: "Address to expose prometheus /metrics",
		},
		cli.StringFlag{
			Name:  "load-certificates-from",
			Value: "redis",
			Usage: `Path where certificate will found.
If value equals 'redis' certificate will be loaded from redis service.`,
		},
		cli.StringFlag{
			Name:  "read-redis-network",
			Value: "tcp",
		},
		cli.StringFlag{
			Name:  "read-redis-host",
			Value: "127.0.0.1",
		},
		cli.IntFlag{
			Name:  "read-redis-port",
			Value: 6379,
		},
		cli.StringFlag{
			Name:  "read-redis-sentinel-addrs",
			Usage: "Comma separated list of redis addresses",
		},
		cli.StringFlag{
			Name: "read-redis-sentinel-name",
		},
		cli.StringFlag{
			Name: "read-redis-password",
		},
		cli.IntFlag{
			Name: "read-redis-db",
		},
		cli.StringFlag{
			Name:  "write-redis-network",
			Value: "tcp",
		},
		cli.StringFlag{
			Name:  "write-redis-host",
			Value: "127.0.0.1",
		},
		cli.IntFlag{
			Name:  "write-redis-port",
			Value: 6379,
		},
		cli.StringFlag{
			Name:  "write-redis-sentinel-addrs",
			Usage: "Comma separated list of redis addresses",
		},
		cli.StringFlag{
			Name: "write-redis-sentinel-name",
		},
		cli.StringFlag{
			Name: "write-redis-password",
		},
		cli.IntFlag{
			Name: "write-redis-db",
		},
		cli.StringFlag{
			Name:  "access-log",
			Value: "./access.log",
			Usage: `File path where access log will be written.
If value equals 'syslog' log will be sent to local syslog.
The value 'none' can be used to disable access logs.`,
		},
		cli.IntFlag{
			Name:  "request-timeout",
			Value: 30,
			Usage: "Total backend request timeout in seconds",
		},
		cli.IntFlag{
			Name:  "dial-timeout",
			Value: 10,
			Usage: "Dial backend request timeout in seconds",
		},
		cli.DurationFlag{
			Name:  "client-read-timeout",
			Value: 0,
			Usage: "Maximum duration for reading the entire request, including the body",
		},
		cli.DurationFlag{
			Name:  "client-read-header-timeout",
			Value: 0,
			Usage: "Amount of time allowed to read request headers",
		},
		cli.DurationFlag{
			Name:  "client-write-timeout",
			Value: 0,
			Usage: "Maximum duration before timing out writes of the response",
		},
		cli.DurationFlag{
			Name:  "client-idle-timeout",
			Value: 0,
			Usage: "Maximum amount of time to wait for the next request when keep-alives are enabled",
		},
		cli.IntFlag{
			Name:  "dead-backend-time",
			Value: 30,
			Usage: "Time in seconds a backend will remain disabled after a network failure",
		},
		cli.IntFlag{
			Name:  "flush-interval",
			Value: 10,
			Usage: "Time in milliseconds to flush the proxied request",
		},
		cli.StringFlag{
			Name:  "request-id-header",
			Usage: "Header to enable message tracking",
		},
		cli.BoolFlag{
			Name: "active-healthcheck",
		},
		cli.StringFlag{
			Name:  "engine",
			Value: "native",
			Usage: "Reverse proxy engine, options are 'native', 'sni' and 'fasthttp'. Using 'sni' and 'fasthttp' is highly experimental and not recommended for production environments.",
		},
		cli.BoolFlag{
			Name:  "backend-cache",
			Usage: "Enable caching backend results for 2 seconds. This may cause temporary inconsistencies.",
		},
	}
	app.Version = "0.1.15"
	app.Name = "planb"
	app.Usage = "http and websockets reverse proxy"
	app.Action = runServer
	app.Author = "tsuru team"
	app.Email = "https://github.com/tsuru/planb"
	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"github.com/gorilla/websocket"
	"log"
	"net/http"
	"os"
)

func main() {

	if len(os.Args) != 2 {
		fmt.Fprintln(os.Stderr, "usage: nozzle <port>")
		os.Exit(1)
	}

	port := fmt.Sprintf(":%s", os.Args[1])
	log.Printf("Starting WebSocket server on port %v\n", port)

	subscribers := make(chan chan []byte)
	go readStdin(subscribers)

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		handler(w, r, subscribers)
	})
	if err := http.ListenAndServe(port, nil); err != nil {
		log.Fatalln(err)
	}
}

func readStdin(subscribers chan chan []byte) {
	allSubscribers := make([]chan []byte, 0)

	stdin := make(chan []byte)
	go func(stdin chan []byte) {
		reader := bufio.NewReader(os.Stdin)
		for {
			line, _, err := reader.ReadLine()
			if err != nil {
				fmt.Println(err)
				return
			}
			stdin <- line
		}
	}(stdin)

	for {
		select {
		case subscription := <-subscribers:
			allSubscribers = append(allSubscribers, subscription)
		case line := <-stdin:
			activeSubscribers := make([]chan []byte, 0)
			for i := range allSubscribers {
				_, ok := <-allSubscribers[i]
				if !ok {
					continue
				} else {
					allSubscribers[i] <- line
					activeSubscribers = append(activeSubscribers, allSubscribers[i])
				}
			}
			allSubscribers = activeSubscribers
		}
	}
}

func handler(w http.ResponseWriter, r *http.Request, subscribers chan chan []byte) {
	conn, err := websocket.Upgrade(w, r, w.Header(), 4096, 4096)
	if err != nil {
		log.Println(err)
		return
	}

	log.Printf("Incoming connection: %s\n", r.RemoteAddr)
	subscription := make(chan []byte)
	subscribers <- subscription
	subscription <- []byte("acc")
	defer close(subscription)
	defer log.Printf("Connection closed: %s\n", r.RemoteAddr)

	log.Printf("Stream started: STDIN -> %v\n", r.RemoteAddr)

	for {
		line := <-subscription
		if err = conn.WriteMessage(websocket.TextMessage, line); err != nil {
			log.Println(err)
			return
		}
		subscription <- []byte("acc")
	}
}
<commit_msg>proper logging<commit_after>package main

import (
	"bufio"
	"fmt"
	"github.com/gorilla/websocket"
	"log"
	"net/http"
	"os"
)

func main() {

	if len(os.Args) != 2 {
		fmt.Fprintln(os.Stderr, "usage: nozzle <port>")
		os.Exit(1)
	}

	port := fmt.Sprintf(":%s", os.Args[1])
	log.Printf("Starting WebSocket server on port %v\n", port)

	subscribers := make(chan chan []byte)
	go readStdin(subscribers)

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		handler(w, r, subscribers)
	})
	if err := http.ListenAndServe(port, nil); err != nil {
		log.Fatalln(err)
	}
}

func readStdin(subscribers chan chan []byte) {
	allSubscribers := make([]chan []byte, 0)

	stdin := make(chan []byte)
	go func(stdin chan []byte) {
		reader := bufio.NewReader(os.Stdin)
		for {
			line, _, err := reader.ReadLine()
			if err != nil {
				log.Fatalln(err)
			}
			stdin <- line
		}
	}(stdin)

	for {
		select {
		case subscription := <-subscribers:
			allSubscribers = append(allSubscribers, subscription)
		case line := <-stdin:
			activeSubscribers := make([]chan []byte, 0)
			for i := range allSubscribers {
				_, ok := <-allSubscribers[i]
				if !ok {
					continue
				} else {
					allSubscribers[i] <- line
					activeSubscribers = append(activeSubscribers, allSubscribers[i])
				}
			}
			allSubscribers = activeSubscribers
		}
	}
}

func handler(w http.ResponseWriter, r *http.Request, subscribers chan chan []byte) {
	conn, err := websocket.Upgrade(w, r, w.Header(), 4096, 4096)
	if err != nil {
		log.Println(err)
		return
	}

	log.Printf("Incoming connection: %s\n", r.RemoteAddr)
	subscription := make(chan []byte)
	subscribers <- subscription
	subscription <- []byte("acc")
	defer close(subscription)
	defer log.Printf("Connection closed: %s\n", r.RemoteAddr)

	log.Printf("Stream started: STDIN -> %v\n", r.RemoteAddr)

	for {
		line := <-subscription
		if err = conn.WriteMessage(websocket.TextMessage, line); err != nil {
			log.Println(err)
			return
		}
		subscription <- []byte("acc")
	}
}
<|endoftext|>


<commit_before>//Package memory provides a lightweight in memory store for onecache
//Do take a look at other stores
package memory

import (
	"sync"
	"time"

	"github.com/adelowo/onecache"
)

//Represents an inmemory store
type InMemoryStore struct {
	lock sync.RWMutex
	data map[string][]byte
}

//Returns a new instance of the Inmemory store
func NewInMemoryStore() *InMemoryStore {
	return &InMemoryStore{data: make(map[string][]byte)}
}

func (i *InMemoryStore) Set(key string, data interface{}, expires time.Duration) error {
	i.lock.RLock()

	defer i.lock.RUnlock()

	item := &onecache.Item{ExpiresAt: time.Now().Add(expires), Data: data}

	b, err := item.Bytes()

	if err != nil {
		return err
	}

	i.data[key] = b

	return nil
}

func (i *InMemoryStore) Get(key string) (interface{}, error) {
	i.lock.RLock()
	defer i.lock.RUnlock()

	bytes, ok := i.data[key]

	if !ok {
		return nil, onecache.ErrCacheMiss
	}

	item, err := onecache.BytesToItem(bytes)

	if item.IsExpired() {
		i.Delete(key)
		return nil, onecache.ErrCacheMiss
	}

	if err != nil {
		return nil, err
	}

	return item.Data, nil
}

func (i *InMemoryStore) Delete(key string) error {
	i.lock.RLock()
	defer i.lock.RUnlock()

	_, ok := i.data[key]

	if !ok {
		return onecache.ErrCacheMiss
	}

	delete(i.data, key)

	return nil
}

func (i *InMemoryStore) Flush() error {
	i.lock.RLock()
	defer i.lock.RUnlock()

	i.data = make(map[string][]byte)

	return nil
}
<commit_msg>Increment cache data for the memory store<commit_after>//Package memory provides a lightweight in memory store for onecache
//Do take a look at other stores
package memory

import (
	"sync"
	"time"

	"github.com/adelowo/onecache"
)

//Represents an inmemory store
type InMemoryStore struct {
	lock sync.RWMutex
	data map[string][]byte
}

//Returns a new instance of the Inmemory store
func NewInMemoryStore() *InMemoryStore {
	return &InMemoryStore{data: make(map[string][]byte)}
}

func (i *InMemoryStore) Set(key string, data interface{}, expires time.Duration) error {
	i.lock.RLock()

	defer i.lock.RUnlock()

	item := &onecache.Item{ExpiresAt: time.Now().Add(expires), Data: data}

	b, err := item.Bytes()

	if err != nil {
		return err
	}

	i.data[key] = b

	return nil
}

func (i *InMemoryStore) Get(key string) (interface{}, error) {
	i.lock.RLock()
	defer i.lock.RUnlock()

	bytes, ok := i.data[key]

	if !ok {
		return nil, onecache.ErrCacheMiss
	}

	item, err := onecache.BytesToItem(bytes)

	if item.IsExpired() {
		i.Delete(key)
		return nil, onecache.ErrCacheMiss
	}

	if err != nil {
		return nil, err
	}

	return item.Data, nil
}

func (i *InMemoryStore) Delete(key string) error {
	i.lock.RLock()
	defer i.lock.RUnlock()

	_, ok := i.data[key]

	if !ok {
		return onecache.ErrCacheMiss
	}

	delete(i.data, key)

	return nil
}

func (i *InMemoryStore) Flush() error {
	i.lock.RLock()
	defer i.lock.RUnlock()

	i.data = make(map[string][]byte)

	return nil
}

func (i *InMemoryStore) Increment(key string, steps int) error {

	i.lock.RLock()
	defer i.lock.RUnlock()

	if !i.has(key) {
		return onecache.ErrCacheMiss
	}

	bytes := i.data[key]

	item, err := onecache.BytesToItem(bytes)

	if err != nil {
		return err
	}

	item.Data, err = onecache.Increment(item.Data, steps)

	if err != nil {
		return err
	}

	b, err := item.Bytes()

	if err != nil {
		return err
	}

	i.data[key] = b

	return nil

}

func (i *InMemoryStore) has(key string) bool {

	_, ok := i.data[key]

	return ok
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"github.com/alinpopa/barvin/handlers/slack"
	"github.com/op/go-logging"
	"os"
	"sync"
)

func main() {
	userID := flag.String("userid", "", "The privileged slack userid.")
	token := flag.String("token", "", "Slack token to connect.")
	flag.Parse()

	var format = logging.MustStringFormatter(
		`%{color}%{time:15:04:05.000} %{shortfunc} >>> %{level} %{id:03x} %{message}%{color:reset}`,
	)
	loggingBackend := logging.NewLogBackend(os.Stderr, "", 0)
	backend2Formatter := logging.NewBackendFormatter(loggingBackend, format)
	loggingBackendLeveled := logging.AddModuleLevel(loggingBackend)
	loggingBackendLeveled.SetLevel(logging.DEBUG, "")
	logging.SetBackend(backend2Formatter)

	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		restartChannel := make(chan string)
		go func() {
			restartChannel <- "Initial start"
		}()
		for {
			msg := <-restartChannel
			go slack.SlackHandler(msg, restartChannel, *userID, *token)
		}
	}()
	wg.Wait()
}
<commit_msg>Add date as part of the log messages.<commit_after>package main

import (
	"flag"
	"github.com/alinpopa/barvin/handlers/slack"
	"github.com/op/go-logging"
	"os"
	"sync"
)

func main() {
	userID := flag.String("userid", "", "The privileged slack userid.")
	token := flag.String("token", "", "Slack token to connect.")
	flag.Parse()

	var format = logging.MustStringFormatter(
		`%{color}%{time:2006-01-02T15:04:05.000} %{shortfunc} >>> %{level} %{id:03x} %{message}%{color:reset}`,
	)
	loggingBackend := logging.NewLogBackend(os.Stderr, "", 0)
	backend2Formatter := logging.NewBackendFormatter(loggingBackend, format)
	loggingBackendLeveled := logging.AddModuleLevel(loggingBackend)
	loggingBackendLeveled.SetLevel(logging.DEBUG, "")
	logging.SetBackend(backend2Formatter)

	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		restartChannel := make(chan string)
		go func() {
			restartChannel <- "Initial start"
		}()
		for {
			msg := <-restartChannel
			go slack.SlackHandler(msg, restartChannel, *userID, *token)
		}
	}()
	wg.Wait()
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	"database/sql"
	_ "github.com/go-sql-driver/mysql"
	"github.com/jmoiron/sqlx"
)

const VERSIONS_PREFIX = ".sys.v#."

var GLOBAL_FLAGS *cliFlags

type cliFlags struct {
	dbUsername string
	dbPassword string
	dbHost     string
	dbPort     int
	dbName     string
	noTouchDb  bool
	eosMGMURL  string
	debug      bool
	userPrefix string
}

func parseFlags() *cliFlags {
	flags := &cliFlags{}
	flag.StringVar(&flags.dbUsername, "username", "", "The username to connect to the db")
	flag.StringVar(&flags.dbPassword, "password", "", "The password to connect to the db")
	flag.StringVar(&flags.dbHost, "host", "", "The host of the db")
	flag.IntVar(&flags.dbPort, "port", 0, "The port of the db")
	flag.StringVar(&flags.dbName, "dbname", "", "The name of the database")
	flag.BoolVar(&flags.noTouchDb, "notouchdb", false, "With dry run enbaled the changes are not commited to the db")
	flag.StringVar(&flags.eosMGMURL, "eosmgmurl", "root://eospps-slave.cern.ch", "The EOS MGM URL")
	flag.StringVar(&flags.userPrefix, "userprefix", "/eos/scratch/user/", "The path under users reside")
	flag.BoolVar(&flags.debug, "debug", false, "Print debug information")
	flag.Parse()
	GLOBAL_FLAGS = flags
	return flags
}

type shareInfo struct {
	ID          int64          `db:"id"`
	ShareType   int            `db:"share_type"`
	ShareWith   sql.NullString `db:"share_with"`
	UIDOwner    string         `db:"uid_owner"`
	Parent      sql.NullInt64  `db:"parent"`
	ItemType    sql.NullString `db:"item_type"`
	ItemSource  sql.NullString `db:"item_source"`
	ItemTarget  sql.NullString `db:"item_target"`
	FileSource  sql.NullInt64  `db:"file_source"`
	FileTarget  sql.NullString `db:"file_target"`
	Permissions string         `db:"permissions"`
	STime       int            `db:"stime"`
	Accepted    int            `db:"accepted"`
	Expiration  time.Time      `db:"expiration"`
	Token       sql.NullString `db:"token"`
	MailSend    int            `db:"mail_send"`
}

type sqlDriver struct {
	db *sqlx.DB
}

func newSQLDriver(flags *cliFlags) (*sqlDriver, error) {
	dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", flags.dbUsername, flags.dbPassword, flags.dbHost, flags.dbPort, flags.dbName)
	d, err := sqlx.Connect("mysql", dsn)
	if err != nil {
		return nil, err
	}
	return &sqlDriver{db: d}, nil
}
func (d *sqlDriver) getAllShares() ([]shareInfo, error) {
	var entries []shareInfo
	err := d.db.Select(&entries, "SELECT id,share_type,item_source,item_target,file_source,file_target from oc_share where share_type=3 and item_type='file' ORDER BY id;")
	if err != nil {
		return nil, err
	}
	return entries, nil
}

type Metadata struct {
	Inode int64
	Path  string
	UID   string
	GID   string
	Size  int64
}

func (d *sqlDriver) executeCMD(cmd *exec.Cmd) (string, string, error) {
	outBuf := &bytes.Buffer{}
	errBuf := &bytes.Buffer{}
	cmd.Stdout = outBuf
	cmd.Stderr = errBuf
	err := cmd.Run()
	if GLOBAL_FLAGS.debug {
		fmt.Printf("DEBUG CMD: %+v ERR:%s", cmd, err)
	}
	return outBuf.String(), errBuf.String(), err
}

// getMetadataFromEos returns the metadata fo the file/folder given the inode
func (d *sqlDriver) getMetadataFromEOS(ID int64) (*Metadata, error) {
	cmd := exec.Command("/usr/bin/eos", "-r", "0", "0", "file", "info", fmt.Sprintf("inode:%d", ID), "-m")
	stdout, _, err := d.executeCMD(cmd)
	if err != nil {
		return nil, err
	}
	return d.parseFileInfo(stdout)
}

// getMetadataFromEosPath returns the metadata fo the file/folder given the eos path
func (d *sqlDriver) getMetadataFromEOSPath(path string) (*Metadata, error) {
	cmd := exec.Command("/usr/bin/eos", "-r", "0", "0", "file", "info", path, "-m")
	stdout, _, err := d.executeCMD(cmd)
	if err != nil {
		return nil, err
	}
	return d.parseFileInfo(stdout)
}

// getVersionsFolderMetadata returns the metadata associated to the versions folder of the file with the inode passed
// if the versions folder does not exists it will try to create it.
func (d *sqlDriver) getVersionsFolderMetadata(fileMeta *Metadata) (*Metadata, error) {
	// obtain the versions folder path from the file path
	dirName := path.Dir(fileMeta.Path)
	baseName := path.Base(fileMeta.Path)
	versionsPath := path.Join(dirName, VERSIONS_PREFIX+baseName)
	versionsMeta, err := d.getMetadataFromEOSPath(versionsPath)
	if err != nil {
		// if versions folder does not exists (eos err code == 2) trigger creation of this folder
		if exiterr, ok := err.(*exec.ExitError); ok {
			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
				if status.ExitStatus() == 2 {
					err = d.createVersionsFolder(fileMeta)
					if err != nil {
						return nil, err
					}

					// if we are talking to the slave the creation of the versions folder may not has been replicated, so we retry ever
					maxRetries := 5
					err := fmt.Errorf("Version not created yet")
					var versionsMeta *Metadata

					for maxRetries > 0 && err != nil {
						_versionsMeta, _err := d.getMetadataFromEOSPath(versionsPath)
						err = _err
						versionsMeta = _versionsMeta
						maxRetries--

					}
					if err != nil {
						return nil, err
					}
					return versionsMeta, nil
				}
			}
			return nil, err
		}
		return nil, err
	}
	return versionsMeta, nil
}

func (d *sqlDriver) parseFileInfo(raw string) (*Metadata, error) {
	kv := make(map[string]string)
	partsBySpace := strings.Split(raw, " ") // we have [keylength.file=14 file=/eos/pps/proc/ container=3 ...}
	for _, p := range partsBySpace {
		partsByEqual := strings.Split(p, "=") // we have kv pairs like [ keylength.file 14]
		if len(partsByEqual) == 2 {
			kv[partsByEqual[0]] = partsByEqual[1]
		}
	}
	// fix eos path because the kv pair file=path could contains whitespace and the whitespace is the pair separator. Not very smart :(
	fileLength := kv["keylength.file"]
	fileLengthInt64, err := strconv.ParseInt(fileLength, 10, 64)
	if err != nil {
		return nil, err
	}
	startIndex := int64(14) + int64(len(fileLength)) + 7
	kv["file"] = raw[startIndex : startIndex+fileLengthInt64]

	inodeInt64, err := strconv.ParseInt(kv["ino"], 10, 64)
	if err != nil {
		return nil, err
	}
	sizeInt64, err := strconv.ParseInt(kv["size"], 10, 64)
	m := &Metadata{Inode: inodeInt64, Path: kv["file"], UID: kv["uid"], GID: kv["gid"], Size: sizeInt64}
	return m, nil
}
func (d *sqlDriver) createVersionsFolder(fileMeta *Metadata) error {
	cmd := exec.Command("/usr/bin/eos", "-r", fileMeta.UID, fileMeta.GID, "cp", fileMeta.Path, fileMeta.Path)
	_, stderr, err := d.executeCMD(cmd)
	if err != nil {
		fmt.Fprintln(os.Stderr, stderr)
		return err
	}
	return nil
}
func (d *sqlDriver) updateShareTable(shareInfo *shareInfo, versionsMeta *Metadata) error {
	fmt.Printf("RECORD: %d UPDATE oc_share SET item_source=%s,item_target=%s,file_source=%d,file_target=%s WHERE id=%d\n", shareInfo.ID, fmt.Sprintf("%d", versionsMeta.Inode), "/"+fmt.Sprintf("%d", versionsMeta.Inode), versionsMeta.Inode, "/"+path.Base(versionsMeta.Path), shareInfo.ID)
	if GLOBAL_FLAGS.noTouchDb {
		return nil
	}
	query := "UPDATE oc_share SET item_source=?,item_target=?,file_source=?,file_target=? WHERE id=?"
	stmt, err := d.db.Prepare(query)
	if err != nil {
		return err
	}
	defer stmt.Close()
	result, err := stmt.Exec(fmt.Sprintf("%d", versionsMeta.Inode), "/"+fmt.Sprintf("%d", versionsMeta.Inode), versionsMeta.Inode, "/"+path.Base(versionsMeta.Path), shareInfo.ID)
	if err != nil {
		return err
	}
	numRowsAffected, err := result.RowsAffected()
	if err != nil {
		return err
	}
	if numRowsAffected == 0 || numRowsAffected > 1 {
		return fmt.Errorf("Cannot updated share because share id %d does not exists anymore", shareInfo.ID)
	}
	return nil
}
func main() {
	flags := parseFlags()
	os.Setenv("EOS_MGM_URL", flags.eosMGMURL)
	d, err := newSQLDriver(flags)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	shares, err := d.getAllShares()
	if err != nil {
		fmt.Fprintln(os.Stderr, "Cannot get all shares because ", err)
		os.Exit(1)
	}
	if len(shares) == 0 {
		fmt.Fprintln(os.Stderr, "oc_share table does not contain public share files")
		os.Exit(1)
	}

	const maxConcurrency = 20 // for example
	var throttle = make(chan int, maxConcurrency)

	var wg sync.WaitGroup
	for _, s := range shares {
		throttle <- 1 // whatever number
		wg.Add(1)
		go func(d *sqlDriver, s shareInfo, wg *sync.WaitGroup, throttle chan int) {
			defer wg.Done()
			defer func() {
				<-throttle
			}()
			meta, err := d.getMetadataFromEOS(s.FileSource.Int64)
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
				return
			} else {
				fmt.Printf("RECORD: %d info:file id:%d share_type:%d item_source:%s item_target:%s file_source:%d file_target:%s eospath:%s uid:%s gid:%s\n", s.ID, s.ID, s.ShareType, s.ItemSource.String, s.ItemTarget.String, s.FileSource.Int64, s.FileTarget.String, strconv.Quote(meta.Path), meta.UID, meta.GID)
				parts := strings.Split(path.Clean(meta.Path), "/")
				parentDir := parts[len(parts)-2]
				if strings.HasPrefix(path.Base(meta.Path), VERSIONS_PREFIX) {
					fmt.Printf("RECORD: %d ALREADY POINTS TO THE VERSION FOLDER\n", s.ID)
					return
				}
				if !strings.HasPrefix(meta.Path, GLOBAL_FLAGS.userPrefix) {
					fmt.Printf("RECORD: %d FILE NOT UNDER HOME DIRECTORY\n", s.ID)
					return
				}
				if strings.HasPrefix(parentDir, VERSIONS_PREFIX) {
					fmt.Printf("RECORD: %d POINTS TO A VERSION\n", s.ID)
					versionFolder := path.Dir(meta.Path)
					versionsMeta, err := d.getMetadataFromEOSPath(versionFolder)
					if err != nil {
						fmt.Fprintln(os.Stderr, err)
						return
					}
					fmt.Printf("RECORD: %d info:versionfolder id:%d path:%s\n", s.ID, versionsMeta.Inode, versionsMeta.Path)
					err = d.updateShareTable(&s, versionsMeta)
					if err != nil {
						fmt.Fprintln(os.Stderr, err)
						return
					}
					return
				}
				versionsMeta, err := d.getVersionsFolderMetadata(meta)
				if err != nil {
					fmt.Fprintln(os.Stderr, err)
					return
				} else {
					fmt.Printf("RECORD: %d info:versionfolder id:%d path:%s\n", s.ID, versionsMeta.Inode, versionsMeta.Path)
					err = d.updateShareTable(&s, versionsMeta)
					if err != nil {
						fmt.Fprintln(os.Stderr, err)
						return
					}
				}
			}
		}(d, s, &wg, throttle)
	}
	wg.Wait()
	fmt.Printf("Sucess. Dry run: %t\n", GLOBAL_FLAGS.noTouchDb)
	os.Exit(0)
}
<commit_msg>Added possibility to run the migration just for one user<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	"database/sql"
	_ "github.com/go-sql-driver/mysql"
	"github.com/jmoiron/sqlx"
)

const VERSIONS_PREFIX = ".sys.v#."

var GLOBAL_FLAGS *cliFlags

type cliFlags struct {
	dbUsername string
	dbPassword string
	dbHost     string
	dbPort     int
	dbName     string
	noTouchDb  bool
	eosMGMURL  string
	debug      bool
	userPrefix string
	user       string
}

func parseFlags() *cliFlags {
	flags := &cliFlags{}
	flag.StringVar(&flags.dbUsername, "username", "", "The username to connect to the db")
	flag.StringVar(&flags.dbPassword, "password", "", "The password to connect to the db")
	flag.StringVar(&flags.dbHost, "host", "", "The host of the db")
	flag.IntVar(&flags.dbPort, "port", 0, "The port of the db")
	flag.StringVar(&flags.dbName, "dbname", "", "The name of the database")
	flag.BoolVar(&flags.noTouchDb, "notouchdb", false, "With dry run enbaled the changes are not commited to the db")
	flag.StringVar(&flags.eosMGMURL, "eosmgmurl", "root://eospps-slave.cern.ch", "The EOS MGM URL")
	flag.StringVar(&flags.userPrefix, "userprefix", "/eos/scratch/user/", "The path under users reside")
	flag.StringVar(&flags.user, "user", "ourense", "Run the migration just for this user")
	flag.BoolVar(&flags.debug, "debug", false, "Print debug information")
	flag.Parse()
	GLOBAL_FLAGS = flags
	return flags
}

type shareInfo struct {
	ID          int64          `db:"id"`
	ShareType   int            `db:"share_type"`
	ShareWith   sql.NullString `db:"share_with"`
	UIDOwner    string         `db:"uid_owner"`
	Parent      sql.NullInt64  `db:"parent"`
	ItemType    sql.NullString `db:"item_type"`
	ItemSource  sql.NullString `db:"item_source"`
	ItemTarget  sql.NullString `db:"item_target"`
	FileSource  sql.NullInt64  `db:"file_source"`
	FileTarget  sql.NullString `db:"file_target"`
	Permissions string         `db:"permissions"`
	STime       int            `db:"stime"`
	Accepted    int            `db:"accepted"`
	Expiration  time.Time      `db:"expiration"`
	Token       sql.NullString `db:"token"`
	MailSend    int            `db:"mail_send"`
}

type sqlDriver struct {
	db *sqlx.DB
}

func newSQLDriver(flags *cliFlags) (*sqlDriver, error) {
	dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", flags.dbUsername, flags.dbPassword, flags.dbHost, flags.dbPort, flags.dbName)
	d, err := sqlx.Connect("mysql", dsn)
	if err != nil {
		return nil, err
	}
	return &sqlDriver{db: d}, nil
}
func (d *sqlDriver) getAllShares() ([]shareInfo, error) {
	var entries []shareInfo
	selectStmt := "SELECT id,share_type,item_source,item_target,file_source,file_target from oc_share where share_type=3 and item_type='file' ORDER BY id;"
	if GLOBAL_FLAGS.user != "" {
		selectStmt = fmt.Sprintf("SELECT id,share_type,item_source,item_target,file_source,file_target from oc_share where share_type=3 and item_type='file' and uid_owner='%s' ORDER BY id;", GLOBAL_FLAGS.user)
	}
	err := d.db.Select(&entries, selectStmt)
	if err != nil {
		return nil, err
	}
	return entries, nil
}

type Metadata struct {
	Inode int64
	Path  string
	UID   string
	GID   string
	Size  int64
}

func (d *sqlDriver) executeCMD(cmd *exec.Cmd) (string, string, error) {
	outBuf := &bytes.Buffer{}
	errBuf := &bytes.Buffer{}
	cmd.Stdout = outBuf
	cmd.Stderr = errBuf
	err := cmd.Run()
	if GLOBAL_FLAGS.debug {
		fmt.Printf("DEBUG CMD: %+v ERR:%s", cmd, err)
	}
	return outBuf.String(), errBuf.String(), err
}

// getMetadataFromEos returns the metadata fo the file/folder given the inode
func (d *sqlDriver) getMetadataFromEOS(ID int64) (*Metadata, error) {
	cmd := exec.Command("/usr/bin/eos", "-r", "0", "0", "file", "info", fmt.Sprintf("inode:%d", ID), "-m")
	stdout, _, err := d.executeCMD(cmd)
	if err != nil {
		return nil, err
	}
	return d.parseFileInfo(stdout)
}

// getMetadataFromEosPath returns the metadata fo the file/folder given the eos path
func (d *sqlDriver) getMetadataFromEOSPath(path string) (*Metadata, error) {
	cmd := exec.Command("/usr/bin/eos", "-r", "0", "0", "file", "info", path, "-m")
	stdout, _, err := d.executeCMD(cmd)
	if err != nil {
		return nil, err
	}
	return d.parseFileInfo(stdout)
}

// getVersionsFolderMetadata returns the metadata associated to the versions folder of the file with the inode passed
// if the versions folder does not exists it will try to create it.
func (d *sqlDriver) getVersionsFolderMetadata(fileMeta *Metadata) (*Metadata, error) {
	// obtain the versions folder path from the file path
	dirName := path.Dir(fileMeta.Path)
	baseName := path.Base(fileMeta.Path)
	versionsPath := path.Join(dirName, VERSIONS_PREFIX+baseName)
	versionsMeta, err := d.getMetadataFromEOSPath(versionsPath)
	if err != nil {
		// if versions folder does not exists (eos err code == 2) trigger creation of this folder
		if exiterr, ok := err.(*exec.ExitError); ok {
			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
				if status.ExitStatus() == 2 {
					err = d.createVersionsFolder(fileMeta)
					if err != nil {
						return nil, err
					}

					// if we are talking to the slave the creation of the versions folder may not has been replicated, so we retry ever
					maxRetries := 5
					err := fmt.Errorf("Version not created yet")
					var versionsMeta *Metadata

					for maxRetries > 0 && err != nil {
						_versionsMeta, _err := d.getMetadataFromEOSPath(versionsPath)
						err = _err
						versionsMeta = _versionsMeta
						maxRetries--

					}
					if err != nil {
						return nil, err
					}
					return versionsMeta, nil
				}
			}
			return nil, err
		}
		return nil, err
	}
	return versionsMeta, nil
}

func (d *sqlDriver) parseFileInfo(raw string) (*Metadata, error) {
	kv := make(map[string]string)
	partsBySpace := strings.Split(raw, " ") // we have [keylength.file=14 file=/eos/pps/proc/ container=3 ...}
	for _, p := range partsBySpace {
		partsByEqual := strings.Split(p, "=") // we have kv pairs like [ keylength.file 14]
		if len(partsByEqual) == 2 {
			kv[partsByEqual[0]] = partsByEqual[1]
		}
	}
	// fix eos path because the kv pair file=path could contains whitespace and the whitespace is the pair separator. Not very smart :(
	fileLength := kv["keylength.file"]
	fileLengthInt64, err := strconv.ParseInt(fileLength, 10, 64)
	if err != nil {
		return nil, err
	}
	startIndex := int64(14) + int64(len(fileLength)) + 7
	kv["file"] = raw[startIndex : startIndex+fileLengthInt64]

	inodeInt64, err := strconv.ParseInt(kv["ino"], 10, 64)
	if err != nil {
		return nil, err
	}
	sizeInt64, err := strconv.ParseInt(kv["size"], 10, 64)
	m := &Metadata{Inode: inodeInt64, Path: kv["file"], UID: kv["uid"], GID: kv["gid"], Size: sizeInt64}
	return m, nil
}
func (d *sqlDriver) createVersionsFolder(fileMeta *Metadata) error {
	cmd := exec.Command("/usr/bin/eos", "-r", fileMeta.UID, fileMeta.GID, "cp", fileMeta.Path, fileMeta.Path)
	_, stderr, err := d.executeCMD(cmd)
	if err != nil {
		fmt.Fprintln(os.Stderr, stderr)
		return err
	}
	return nil
}
func (d *sqlDriver) updateShareTable(shareInfo *shareInfo, versionsMeta *Metadata) error {
	fmt.Printf("RECORD: %d UPDATE oc_share SET item_source=%s,item_target=%s,file_source=%d,file_target=%s WHERE id=%d\n", shareInfo.ID, fmt.Sprintf("%d", versionsMeta.Inode), "/"+fmt.Sprintf("%d", versionsMeta.Inode), versionsMeta.Inode, "/"+path.Base(versionsMeta.Path), shareInfo.ID)
	if GLOBAL_FLAGS.noTouchDb {
		return nil
	}
	query := "UPDATE oc_share SET item_source=?,item_target=?,file_source=?,file_target=? WHERE id=?"
	stmt, err := d.db.Prepare(query)
	if err != nil {
		return err
	}
	defer stmt.Close()
	result, err := stmt.Exec(fmt.Sprintf("%d", versionsMeta.Inode), "/"+fmt.Sprintf("%d", versionsMeta.Inode), versionsMeta.Inode, "/"+path.Base(versionsMeta.Path), shareInfo.ID)
	if err != nil {
		return err
	}
	numRowsAffected, err := result.RowsAffected()
	if err != nil {
		return err
	}
	if numRowsAffected == 0 || numRowsAffected > 1 {
		return fmt.Errorf("Cannot updated share because share id %d does not exists anymore", shareInfo.ID)
	}
	return nil
}
func main() {
	flags := parseFlags()
	os.Setenv("EOS_MGM_URL", flags.eosMGMURL)
	d, err := newSQLDriver(flags)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	shares, err := d.getAllShares()
	if err != nil {
		fmt.Fprintln(os.Stderr, "Cannot get all shares because ", err)
		os.Exit(1)
	}
	if len(shares) == 0 {
		fmt.Fprintln(os.Stderr, "oc_share table does not contain public share files")
		os.Exit(1)
	}

	const maxConcurrency = 20 // for example
	var throttle = make(chan int, maxConcurrency)

	var wg sync.WaitGroup
	for _, s := range shares {
		throttle <- 1 // whatever number
		wg.Add(1)
		go func(d *sqlDriver, s shareInfo, wg *sync.WaitGroup, throttle chan int) {
			defer wg.Done()
			defer func() {
				<-throttle
			}()
			meta, err := d.getMetadataFromEOS(s.FileSource.Int64)
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
				return
			} else {
				fmt.Printf("RECORD: %d info:file id:%d share_type:%d item_source:%s item_target:%s file_source:%d file_target:%s eospath:%s uid:%s gid:%s\n", s.ID, s.ID, s.ShareType, s.ItemSource.String, s.ItemTarget.String, s.FileSource.Int64, s.FileTarget.String, strconv.Quote(meta.Path), meta.UID, meta.GID)
				parts := strings.Split(path.Clean(meta.Path), "/")
				parentDir := parts[len(parts)-2]
				if strings.HasPrefix(path.Base(meta.Path), VERSIONS_PREFIX) {
					fmt.Printf("RECORD: %d ALREADY POINTS TO THE VERSION FOLDER\n", s.ID)
					return
				}
				if !strings.HasPrefix(meta.Path, GLOBAL_FLAGS.userPrefix) {
					fmt.Printf("RECORD: %d FILE NOT UNDER HOME DIRECTORY\n", s.ID)
					return
				}
				if strings.HasPrefix(parentDir, VERSIONS_PREFIX) {
					fmt.Printf("RECORD: %d POINTS TO A VERSION\n", s.ID)
					versionFolder := path.Dir(meta.Path)
					versionsMeta, err := d.getMetadataFromEOSPath(versionFolder)
					if err != nil {
						fmt.Fprintln(os.Stderr, err)
						return
					}
					fmt.Printf("RECORD: %d info:versionfolder id:%d path:%s\n", s.ID, versionsMeta.Inode, versionsMeta.Path)
					err = d.updateShareTable(&s, versionsMeta)
					if err != nil {
						fmt.Fprintln(os.Stderr, err)
						return
					}
					return
				}
				versionsMeta, err := d.getVersionsFolderMetadata(meta)
				if err != nil {
					fmt.Fprintln(os.Stderr, err)
					return
				} else {
					fmt.Printf("RECORD: %d info:versionfolder id:%d path:%s\n", s.ID, versionsMeta.Inode, versionsMeta.Path)
					err = d.updateShareTable(&s, versionsMeta)
					if err != nil {
						fmt.Fprintln(os.Stderr, err)
						return
					}
				}
			}
		}(d, s, &wg, throttle)
	}
	wg.Wait()
	fmt.Printf("Sucess. Dry run: %t\n", GLOBAL_FLAGS.noTouchDb)
	os.Exit(0)
}
<|endoftext|>


<commit_before>package main

import "github.com/go-martini/martini"

func main() {
	// Install HAP devices
	InstallX10Devices()

	// Setup REST API
	m := martini.Classic()
	m.Get("/", func() string {
		return "Hello world!"
	})
	m.Run()
}
<commit_msg>set port<commit_after>package main

import "github.com/go-martini/martini"

func main() {
	// Install HAP devices
	InstallX10Devices()

	// Setup REST API
	m := martini.Classic()
	m.Get("/", func() string {
		return "Hello world!"
	})
	m.RunOnAddr(":5591")
}
<|endoftext|>


<commit_before>package main

// This software is an expense tracker i made to read the transactions exported from
// my bank account.

import (
	"fmt"
	"go-cli-bank/categories"
	"go-cli-bank/infrastructure"
	"go-cli-bank/reports"
	"os"

	docopt "github.com/docopt/docopt-go"
	// flag "github.com/ogier/pflag"
)

// CommandCreateCategory handles category creation command
func CommandCreateCategory(name string) error {

	dbHandler := infrastructure.DatabaseHandler{}
	cr := categories.CategoryRepository{DBHandler: &dbHandler}
	i := categories.Interactor{Repository: &cr}
	_, err := i.NewCategory(name)
	if err != nil {
		return err
	}

	return nil
}

// CommandShowReport handles report commands
func CommandShowReport(inputFilePath string) error {
	file, err := infrastructure.OpenFile(inputFilePath)
	if err != nil {
		return err
	}
	defer file.Close()

	err = reports.MonthlyReport(file)
	if err != nil {
		return err
	}

	return nil
}

func errorf(format string, args ...interface{}) {
	fmt.Fprintln(os.Stderr, format, args)
	os.Exit(2)
}

var intro = "Your command line money manager.\n"

var usage = `Usage:
	go-cli-bank -h | --help
	go-cli-bank report --input <file>
	go-cli-bank category new <name>`

var options = `
Options:
	--input    The path to the transactions list.
	-h --help     Show this screen.`

func main() {

	arguments, _ := docopt.Parse(intro+usage+options, nil, true, "Go CLI Bank 0.0.1", false)

	if arguments["category"].(bool) && arguments["new"].(bool) {
		if err := CommandCreateCategory(arguments["<name>"].(string)); err != nil {
			errorf("Failed to create category: %v\n", err)
		}
	}

	if arguments["report"].(bool) {
		if err := CommandShowReport(arguments["<file>"].(string)); err != nil {
			errorf("Failed to show report: %v\n", err)
		}
	}
}
<commit_msg>Changes error message. Still temporary<commit_after>package main

// This software is an expense tracker i made to read the transactions exported from
// my bank account.

import (
	"fmt"
	"go-cli-bank/categories"
	"go-cli-bank/infrastructure"
	"go-cli-bank/reports"
	"os"

	docopt "github.com/docopt/docopt-go"
	// flag "github.com/ogier/pflag"
)

// CommandCreateCategory handles category creation command
func CommandCreateCategory(name string) error {

	dbHandler := infrastructure.DatabaseHandler{}
	cr := categories.CategoryRepository{DBHandler: &dbHandler}
	i := categories.Interactor{Repository: &cr}
	_, err := i.NewCategory(name)
	if err != nil {
		return err
	}

	return nil
}

// CommandShowReport handles report commands
func CommandShowReport(inputFilePath string) error {
	file, err := infrastructure.OpenFile(inputFilePath)
	if err != nil {
		return err
	}
	defer file.Close()

	err = reports.MonthlyReport(file)
	if err != nil {
		return err
	}

	return nil
}

func errorf(format string, args ...interface{}) {
	fmt.Fprintln(os.Stderr, format, args)
	os.Exit(2)
}

var intro = "Your command line money manager.\n"

var usage = `Usage:
	go-cli-bank -h | --help
	go-cli-bank report --input <file>
	go-cli-bank category new <name>`

var options = `
Options:
	--input    The path to the transactions list.
	-h --help     Show this screen.`

func main() {

	arguments, _ := docopt.Parse(intro+usage+options, nil, true, "Go CLI Bank 0.0.1", false)

	if arguments["category"].(bool) && arguments["new"].(bool) {
		if err := CommandCreateCategory(arguments["<name>"].(string)); err != nil {
			errorf("Error:", err)
		}
	}

	if arguments["report"].(bool) {
		if err := CommandShowReport(arguments["<file>"].(string)); err != nil {
			errorf("Error:", err)
		}
	}
}
<|endoftext|>


<commit_before>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"

	"github.com/line/line-bot-sdk-go/linebot"
)

var ImgSrv string = "https://img-cache-server.herokuapp.com/"
var bot *linebot.Client

//PetDB :
var PetDB *Pets

func main() {
	var err error
	PetDB = NewPets()
	bot, err = linebot.New(os.Getenv("ChannelSecret"), os.Getenv("ChannelAccessToken"))
	log.Println("Bot:", bot, " err:", err)
	http.HandleFunc("/callback", callbackHandler)
	port := os.Getenv("PORT")
	addr := fmt.Sprintf(":%s", port)
	http.ListenAndServe(addr, nil)
}

func getSecureImageAddress(oriAdd string) string {
	eURL := url.QueryEscape(oriAdd)
	imgGetUrl := fmt.Sprintf("%surl?%s", ImgSrv, eURL)
	log.Println("eURL:", eURL, " url:", imgGetUrl, " ImgApi:", ImgSrv)

	response, err := http.Get(imgGetUrl)
	if err != nil {
		log.Println("Error while downloading:", err)
		return ""
	}
	defer response.Body.Close()

	if response.StatusCode != 200 {
		return ""
	}

	totalBody, err := ioutil.ReadAll(response.Body)
	if err != nil {
		log.Println("Error while parsing:", err)
		return ""
	}
	log.Println("Got data:", string(totalBody))
	return fmt.Sprintf("%simgs?%s.jpg", ImgSrv, string(totalBody))

}

func callbackHandler(w http.ResponseWriter, r *http.Request) {
	events, err := bot.ParseRequest(r)
	if err != nil {
		if err == linebot.ErrInvalidSignature {
			w.WriteHeader(400)
		} else {
			w.WriteHeader(500)
		}
		return
	}
	for _, event := range events {
		if event.Type == linebot.EventTypeMessage {
			switch message := event.Message.(type) {
			case *linebot.TextMessage:
				var pet *Pet
				log.Println(message.Text)
				inText := strings.ToLower(message.Text)
				if strings.Contains(inText, "狗") || strings.Contains(inText, "dog") {
					pet = PetDB.GetNextDog()
				} else if strings.Contains(inText, "貓") || strings.Contains(inText, "cat") {
					pet = PetDB.GetNextCat()
				}

				if pet == nil {
					pet = PetDB.GetNextPet()
				}

				out := fmt.Sprintf("您好，目前的動物名為%s, 所在地為:%s, 電話為:%s ", pet.Name, pet.Resettlement, pet.Phone)
				log.Println("Current msg:", out)
				if _, err = bot.ReplyMessage(event.ReplyToken, linebot.NewTextMessage(out)).Do(); err != nil {
					log.Print(err)
				}
			}
		}
	}
}
<commit_msg>Add beacon support<commit_after>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"

	"github.com/line/line-bot-sdk-go/linebot"
)

var ImgSrv string = "https://img-cache-server.herokuapp.com/"
var bot *linebot.Client

//PetDB :
var PetDB *Pets

func main() {
	var err error
	PetDB = NewPets()
	bot, err = linebot.New(os.Getenv("ChannelSecret"), os.Getenv("ChannelAccessToken"))
	log.Println("Bot:", bot, " err:", err)
	http.HandleFunc("/callback", callbackHandler)
	port := os.Getenv("PORT")
	addr := fmt.Sprintf(":%s", port)
	http.ListenAndServe(addr, nil)
}

func getSecureImageAddress(oriAdd string) string {
	eURL := url.QueryEscape(oriAdd)
	imgGetUrl := fmt.Sprintf("%surl?%s", ImgSrv, eURL)
	log.Println("eURL:", eURL, " url:", imgGetUrl, " ImgApi:", ImgSrv)

	response, err := http.Get(imgGetUrl)
	if err != nil {
		log.Println("Error while downloading:", err)
		return ""
	}
	defer response.Body.Close()

	if response.StatusCode != 200 {
		return ""
	}

	totalBody, err := ioutil.ReadAll(response.Body)
	if err != nil {
		log.Println("Error while parsing:", err)
		return ""
	}
	log.Println("Got data:", string(totalBody))
	return fmt.Sprintf("%simgs?%s.jpg", ImgSrv, string(totalBody))

}

func callbackHandler(w http.ResponseWriter, r *http.Request) {
	events, err := bot.ParseRequest(r)
	if err != nil {
		if err == linebot.ErrInvalidSignature {
			w.WriteHeader(400)
		} else {
			w.WriteHeader(500)
		}
		return
	}
	for _, event := range events {
		switch event.Type {
		case linebot.EventTypeMessage:
			switch message := event.Message.(type) {
			case *linebot.TextMessage:
				var pet *Pet
				log.Println(message.Text)
				inText := strings.ToLower(message.Text)
				if strings.Contains(inText, "狗") || strings.Contains(inText, "dog") {
					pet = PetDB.GetNextDog()
				} else if strings.Contains(inText, "貓") || strings.Contains(inText, "cat") {
					pet = PetDB.GetNextCat()
				}

				if pet == nil {
					pet = PetDB.GetNextPet()
				}

				out := fmt.Sprintf("您好，目前的動物名為%s, 所在地為:%s, 電話為:%s ", pet.Name, pet.Resettlement, pet.Phone)
				log.Println("Current msg:", out)
				if _, err = bot.ReplyMessage(event.ReplyToken, linebot.NewTextMessage(out)).Do(); err != nil {
					log.Print(err)
				}
			}
		case linebot.EventTypeBeacon:
			log.Println(" Beacon event....")
			if b := event.Beacon; b != nil {
				ret := fmt.Sprintln("Msg:", string(b.DeviceMessage), " hwid:", b.Hwid, " type:", b.Type)
				if _, err = bot.ReplyMessage(event.ReplyToken, linebot.NewTextMessage(ret)).Do(); err != nil {
					log.Print(err)
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package rrredis

import (
	"fmt"
	"gopkg.in/redis.v5"
	"sync"
	"time"
)

type RedisOptions struct {
	dialTimeout  time.Duration
	db           int
	password     string
	connPoolSize int
}

var (
	cp              = &clientPool{pool: make(map[string]*redis.Client)}
	defaultPoolSize = 500
)

type clientPool struct {
	pool map[string]*redis.Client
	mu   sync.RWMutex
}

func (s *clientPool) add(addr string, rc *redis.Client) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.pool[addr] = rc
}

func (s *clientPool) get(addr string) *redis.Client {
	s.mu.RLock()
	defer s.mu.RUnlock()
	if v, ok := s.pool[addr]; ok {
		return v
	}
	return nil
}

// connect once when program start if you wanna customize redis options
func Connect(addr string, opt *RedisOptions) error {
	if addr == "" || opt == nil {
		return fmt.Errorf("Redis addr empty or options nil")
	}

	client := redis.NewClient(&redis.Options{
		Addr:        addr,
		DB:          opt.db,
		DialTimeout: opt.dialTimeout,
		PoolSize:    opt.connPoolSize,
		Password:    opt.password,
	})
	if client == nil {
		return fmt.Errorf(fmt.Sprintf("Connect to redis [%s] fail", addr))
	}
	if _, err := client.Ping().Result(); err != nil {
		return err
	}
	cp.add(addr, client)
	return nil
}

// get redis client before you want to do some operations
func GetRedisClient(addr string) (error, *RedisClient) {
	if c := cp.get(addr); c != nil {
		return nil, &RedisClient{c: c}
	}
	err := Connect(addr, &RedisOptions{
		db:           0,
		dialTimeout:  1 * time.Second,
		connPoolSize: defaultPoolSize,
		password:     "",
	})
	if err != nil {
		return fmt.Errorf("Couldn't find redis client for [%s], failed when try to get a new one, %s", addr, err), nil
	}
	return GetRedisClient(addr)
}

type RedisClient struct {
	c *redis.Client
}

// list of redis operations
// hash set
func (c *RedisClient) HMSet(key string, fields map[string]string) error {
	status := c.c.HMSet(key, fields)
	return status.Err()
}

func (c *RedisClient) HMGet(key string, fields ...string) (interface{}, error) {
	status := c.c.HMGet(key, fields...)
	return status.Result()
}

func (c *RedisClient) HMDelete(key string, fields ...string) error {
	status := c.c.HDel(key, fields...)
	return status.Err()
}

func (c *RedisClient) HMExists(key, field string) error {
	status := c.c.HExists(key, field)
	return status.Err()
}

func (c *RedisClient) Exists(key string) bool {
	return c.c.Exists(key).Val()
}

func (c *RedisClient) Expire(key string, t time.Duration) (bool, error) {
	status := c.c.Expire(key, t)
	return status.Result()
}

func (c *RedisClient) Rename(okey, nkey string) error {
	status := c.c.Rename(okey, nkey)
	return status.Err()
}

func (c *RedisClient) Keys(pattern string) ([]string, error) {
	status := c.c.Keys(pattern)
	return status.Result()
}

func (c *RedisClient) KeyExists(key string) (bool, error) {
	status := c.c.Exists(key)
	return status.Result()
}

func (c *RedisClient) Del(key string) error {
	status := c.c.Del(key)
	return status.Err()
}

func (c *RedisClient) Set(key string, value string, expir time.Duration) error {
	status := c.c.Set(key, value, expir)
	return status.Err()
}

func (c *RedisClient) Get(key string) ([]byte, error) {
	status := c.c.Get(key)
	b, _ := status.Bytes()
	return b, status.Err()
}

// redis list operations
func (c *RedisClient) LPop(key string) (string, error) {
	status := c.c.LPop(key)
	return status.Result()
}

func (c *RedisClient) RPush(key string, values ...interface{}) (int64, error) {
	status := c.c.RPush(key, values...)
	return status.Result()
}

func (c *RedisClient) LLen(key string) (int64, error) {
	status := c.c.LLen(key)
	return status.Result()
}

// redis sorted set operations
func (c *RedisClient) ZInterStore(dest string, aggregate string, keys ...string) error {
	weights := make([]float64, len(keys))
	for i, n := 0, len(keys); i < n; i++ {
		weights[i] = 1
	}
	t := redis.ZStore{
		weights,
		aggregate,
	}
	status := c.c.ZInterStore(dest, t, keys...)
	return status.Err()
}

func (c *RedisClient) ZCard(key string) (int64, error) {
	status := c.c.ZCard(key)
	return status.Result()
}

func (c *RedisClient) ZRange(key string, start, stop int64) ([]string, error) {
	status := c.c.ZRange(key, start, stop)
	return status.Result()
}

func (c *RedisClient) ZRevRange(key string, start, stop int64) ([]string, error) {
	status := c.c.ZRevRange(key, start, stop)
	return status.Result()
}

func (c *RedisClient) ZAddBatch(key string, score_lst []float64, data_lst []interface{}) int64 {
	mem_lst := make([]redis.Z, 0)
	for i := 0; i < len(score_lst); i++ {
		tmp := redis.Z{
			Score:  score_lst[i],
			Member: data_lst[i],
		}

		mem_lst = append(mem_lst, tmp)
	}
	status := c.c.ZAdd(key, mem_lst...)
	return status.Val()
}

func (c *RedisClient) ZRangeByScore(key, min, max string, offset, count int64) ([]string, error) {
	opt := &redis.ZRangeBy{
		Min:    min,
		Max:    max,
		Offset: offset,
		Count:  count,
	}
	status := c.c.ZRangeByScore(key, *opt)
	return status.Result()
}

func (c *RedisClient) ZRangeByScoreWithScores(key, min, max string, offset, count int64) ([]redis.Z, error) {
	opt := &redis.ZRangeBy{
		Min:    min,
		Max:    max,
		Offset: offset,
		Count:  count,
	}
	status := c.c.ZRangeByScoreWithScores(key, *opt)
	return status.Result()
}
<commit_msg>export redis.Nil<commit_after>package rrredis

import (
	"fmt"
	"gopkg.in/redis.v5"
	"sync"
	"time"
)

var (
	Nil = redis.Nil
)

type RedisOptions struct {
	dialTimeout  time.Duration
	db           int
	password     string
	connPoolSize int
}

var (
	cp              = &clientPool{pool: make(map[string]*redis.Client)}
	defaultPoolSize = 500
)

type clientPool struct {
	pool map[string]*redis.Client
	mu   sync.RWMutex
}

func (s *clientPool) add(addr string, rc *redis.Client) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.pool[addr] = rc
}

func (s *clientPool) get(addr string) *redis.Client {
	s.mu.RLock()
	defer s.mu.RUnlock()
	if v, ok := s.pool[addr]; ok {
		return v
	}
	return nil
}

// if you wanna customize redis options, connect once when program start
func Connect(addr string, opt *RedisOptions) error {
	if addr == "" || opt == nil {
		return fmt.Errorf("Redis addr empty or options nil")
	}

	client := redis.NewClient(&redis.Options{
		Addr:        addr,
		DB:          opt.db,
		DialTimeout: opt.dialTimeout,
		PoolSize:    opt.connPoolSize,
		Password:    opt.password,
	})
	if client == nil {
		return fmt.Errorf(fmt.Sprintf("Connect to redis [%s] fail", addr))
	}
	if _, err := client.Ping().Result(); err != nil {
		return err
	}
	cp.add(addr, client)
	return nil
}

// get redis client before you want to do some operations
func GetRedisClient(addr string) (error, *RedisClient) {
	if c := cp.get(addr); c != nil {
		return nil, &RedisClient{c: c}
	}
	err := Connect(addr, &RedisOptions{
		db:           0,
		dialTimeout:  1 * time.Second,
		connPoolSize: defaultPoolSize,
		password:     "",
	})
	if err != nil {
		return fmt.Errorf("Couldn't find redis client for [%s], failed when try to get a new one, %s", addr, err), nil
	}
	return GetRedisClient(addr)
}

type RedisClient struct {
	c *redis.Client
}

// list of redis operations
// hash set
func (c *RedisClient) HMSet(key string, fields map[string]string) error {
	status := c.c.HMSet(key, fields)
	return status.Err()
}

func (c *RedisClient) HMGet(key string, fields ...string) (interface{}, error) {
	status := c.c.HMGet(key, fields...)
	return status.Result()
}

func (c *RedisClient) HMDelete(key string, fields ...string) error {
	status := c.c.HDel(key, fields...)
	return status.Err()
}

func (c *RedisClient) HMExists(key, field string) (bool, error) {
	status := c.c.HExists(key, field)
	return status.Result()
}

func (c *RedisClient) Exists(key string) bool {
	return c.c.Exists(key).Val()
}

func (c *RedisClient) Expire(key string, t time.Duration) (bool, error) {
	status := c.c.Expire(key, t)
	return status.Result()
}

func (c *RedisClient) Rename(okey, nkey string) error {
	status := c.c.Rename(okey, nkey)
	return status.Err()
}

func (c *RedisClient) Keys(pattern string) ([]string, error) {
	status := c.c.Keys(pattern)
	return status.Result()
}

func (c *RedisClient) KeyExists(key string) (bool, error) {
	status := c.c.Exists(key)
	return status.Result()
}

func (c *RedisClient) Del(key string) error {
	status := c.c.Del(key)
	return status.Err()
}

func (c *RedisClient) Set(key string, value string, expir time.Duration) error {
	status := c.c.Set(key, value, expir)
	return status.Err()
}

func (c *RedisClient) Get(key string) ([]byte, error) {
	status := c.c.Get(key)
	b, _ := status.Bytes()
	return b, status.Err()
}

// redis list operations
func (c *RedisClient) LPop(key string) (string, error) {
	status := c.c.LPop(key)
	return status.Result()
}

func (c *RedisClient) RPush(key string, values ...interface{}) (int64, error) {
	status := c.c.RPush(key, values...)
	return status.Result()
}

func (c *RedisClient) LLen(key string) (int64, error) {
	status := c.c.LLen(key)
	return status.Result()
}

// redis sorted set operations
func (c *RedisClient) ZInterStore(dest string, aggregate string, keys ...string) error {
	weights := make([]float64, len(keys))
	for i, n := 0, len(keys); i < n; i++ {
		weights[i] = 1
	}
	t := redis.ZStore{
		weights,
		aggregate,
	}
	status := c.c.ZInterStore(dest, t, keys...)
	return status.Err()
}

func (c *RedisClient) ZCard(key string) (int64, error) {
	status := c.c.ZCard(key)
	return status.Result()
}

func (c *RedisClient) ZRange(key string, start, stop int64) ([]string, error) {
	status := c.c.ZRange(key, start, stop)
	return status.Result()
}

func (c *RedisClient) ZRevRange(key string, start, stop int64) ([]string, error) {
	status := c.c.ZRevRange(key, start, stop)
	return status.Result()
}

func (c *RedisClient) ZAddBatch(key string, score_lst []float64, data_lst []interface{}) int64 {
	mem_lst := make([]redis.Z, 0)
	for i := 0; i < len(score_lst); i++ {
		tmp := redis.Z{
			Score:  score_lst[i],
			Member: data_lst[i],
		}

		mem_lst = append(mem_lst, tmp)
	}
	status := c.c.ZAdd(key, mem_lst...)
	return status.Val()
}

func (c *RedisClient) ZRangeByScore(key, min, max string, offset, count int64) ([]string, error) {
	opt := &redis.ZRangeBy{
		Min:    min,
		Max:    max,
		Offset: offset,
		Count:  count,
	}
	status := c.c.ZRangeByScore(key, *opt)
	return status.Result()
}

func (c *RedisClient) ZRangeByScoreWithScores(key, min, max string, offset, count int64) ([]redis.Z, error) {
	opt := &redis.ZRangeBy{
		Min:    min,
		Max:    max,
		Offset: offset,
		Count:  count,
	}
	status := c.c.ZRangeByScoreWithScores(key, *opt)
	return status.Result()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"

	"github.com/bigroom/vision/tunnel"
	"github.com/gorilla/websocket"
)

var (
	messages chan tunnel.MessageArgs
	clients  map[string][]*conn
)

func main() {
	var (
		host = "localhost"
		port = "8080"
	)

	messages = make(chan tunnel.MessageArgs)
	clients = make(map[string][]*conn)

	go tunnel.NewRPCServer(messages, host, port)
	go messageLoop()

	http.HandleFunc("/", dispatchHandler)

	log.Println(http.ListenAndServe("localhost:6060", nil))
}

func messageLoop() {
	for {
		log.Println("Waiting on message...")
		m := <-messages
		fmt.Println("Got message")

		for _, u := range clients[m.Key()] {
			fmt.Println("Writing message")
			err := u.c.WriteMessage(websocket.TextMessage, []byte(m.String()))
			if err != nil {
				fmt.Println("error (sending message):", err)
			}
		}
	}
}

var upgrader = websocket.Upgrader{
	CheckOrigin: func(r *http.Request) bool { return true },
}

func dispatchHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Add("Access-Control-Allow-Methods", "PUT")
	w.Header().Add("Access-Control-Allow-Headers", "Content-Type")

	c, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Println("couldn't upgrade:", err)
		return
	}

	defer c.Close()

	user := &conn{
		c: c,
	}

	for {
		_, message, err := user.c.ReadMessage()
		if err != nil {
			log.Println("error reading:", err)
			return
		}

		msg := string(message)
		if msg[:3] == "SET" {
			log.Println("Adding user to channel")
			channel := msg[3:]
			clients[channel] = append(clients[channel], user)
		}

	}
}

type conn struct {
	c        *websocket.Conn
	channels []string
}
<commit_msg>Fix logging<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"

	"github.com/bigroom/vision/tunnel"
	"github.com/gorilla/websocket"
)

var (
	messages chan tunnel.MessageArgs
	clients  map[string][]*conn
)

func main() {
	var (
		host = "localhost"
		port = "8080"
	)

	messages = make(chan tunnel.MessageArgs)
	clients = make(map[string][]*conn)

	go tunnel.NewRPCServer(messages, host, port)
	go messageLoop()

	http.HandleFunc("/", dispatchHandler)

	log.Println(http.ListenAndServe("localhost:6060", nil))
}

func messageLoop() {
	for {
		log.Println("Waiting on message...")
		m := <-messages
		log.Println("Got message")
		log.Printf("Sending to channel with key: %v", m.Key())

		for _, u := range clients[m.Key()] {
			fmt.Println("Writing message")
			err := u.c.WriteMessage(websocket.TextMessage, []byte(m.String()))
			if err != nil {
				fmt.Println("error (sending message):", err)
			}
		}
	}
}

var upgrader = websocket.Upgrader{
	CheckOrigin: func(r *http.Request) bool { return true },
}

func dispatchHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Add("Access-Control-Allow-Methods", "PUT")
	w.Header().Add("Access-Control-Allow-Headers", "Content-Type")

	c, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Println("couldn't upgrade:", err)
		return
	}

	defer c.Close()

	user := &conn{
		c: c,
	}

	for {
		_, message, err := user.c.ReadMessage()
		if err != nil {
			log.Println("error reading:", err)
			return
		}

		msg := string(message)
		if msg[:3] == "SET" {
			channel := msg[3:]
			log.Printf("Adding user to channel %v", channel)
			clients[channel] = append(clients[channel], user)
		}

	}
}

type conn struct {
	c        *websocket.Conn
	channels []string
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/jrperritt/gophercloud/rackspace"
	rsV1Servers "github.com/jrperritt/gophercloud/rackspace/compute/v1/servers"
	"github.com/rackspace/gophercloud"
	osV2Servers "github.com/rackspace/gophercloud/openstack/compute/v2/servers"
	"github.com/rackspace/gophercloud/pagination"
	rsV2Servers "github.com/rackspace/gophercloud/rackspace/compute/v2/servers"
	"github.com/rackspace/gophercloud/rackspace/identity/v2/tokens"
)

const (
	metadataKey     = "rax:reboot_window"
	metadataTimeFmt = "2006-01-02T15:04:05Z"
)

func main() {
	outputToCSV := flag.Bool("csv", false, "Output a CSV file")
	flag.Parse()

	if flag.NArg() != 2 {
		fmt.Println("You must supply a username and API key after all other args.")
		os.Exit(1)
	}

	username, apiKey := flag.Arg(0), flag.Arg(1)

	opts := gophercloud.AuthOptions{
		Username: username,
		APIKey:   apiKey,
	}

	provider, err := rackspace.AuthenticatedClient(opts)
	if err != nil {
		fmt.Printf("Unable to authenticate: %v\n", err)
		os.Exit(1)
	}

	regions, fg := Regions(provider, opts)

	fmt.Printf("Regions with a compute endpoint: %s\n", strings.Join(regions, ", "))
	if fg {
		fmt.Println("You have a first-gen endpoint, too.")
	}

	var numUnaffected int
	var entries []entry

	// Iterate through regions with an NG compute endpoint. Collect data about each server.
	for _, region := range regions {
		compute, err := rackspace.NewComputeV2(provider, gophercloud.EndpointOpts{
			Region: region,
		})
		if err != nil {
			fmt.Printf("Unable to locate a v2 compute endpoint in region %s: %v\n", region, err)
			continue
		}

		err = rsV2Servers.List(compute, nil).EachPage(func(page pagination.Page) (bool, error) {
			s, err := osV2Servers.ExtractServers(page)
			if err != nil {
				return false, err
			}

			for _, server := range s {
				entry, err := ConstructEntry(server, "Next Gen", region)
				if err != nil {
					if strings.Contains(err.Error(), "not present") {
						numUnaffected++
						continue
					}
					fmt.Printf("%s\n", err)
					continue
				} else {
					entries = append(entries, *entry)
				}
			}

			return true, nil
		})
	}

	// Iterate through regions with an FG compute endpoint. Collect data about each server.
	compute, err := rackspace.NewComputeV1(provider, gophercloud.EndpointOpts{
		Availability: gophercloud.AvailabilityPublic,
	})
	if err != nil {
		fmt.Printf("Unable to locate a v1 compute endpoint. Skipping...\n")
	} else {
		err = rsV1Servers.List(compute, rsV1Servers.ListOpts{}).EachPage(func(page pagination.Page) (bool, error) {
			s, err := osV2Servers.ExtractServers(page)
			if err != nil {
				return false, err
			}

			for _, server := range s {
				entry, err := ConstructEntry(server, "First Gen", "DFW")
				if err != nil {
					if strings.Contains(err.Error(), "not present") {
						numUnaffected++
						continue
					}
					fmt.Printf("%s\n", err)
					continue
				} else {
					entries = append(entries, *entry)
				}
			}

			return true, nil
		})
		if err != nil {
			fmt.Printf("Error listing servers: %+v\n", err)
		}
	}

	if *outputToCSV {
		outputCSV(entries)
	} else {
		if numUnaffected > 0 {
			fmt.Printf("%d of your servers will not require a reboot. The following %d servers will need to be rebooted", numUnaffected, len(entries))
		}
		outputTabular(entries)
	}
}

// Regions acquires the service catalog and returns a slice of every region that contains a next-gen
// server endpoint, and a boolean indicating whether or not this customer has access to FG servers.
func Regions(provider *gophercloud.ProviderClient, opts gophercloud.AuthOptions) ([]string, bool) {
	service := rackspace.NewIdentityV2(provider)

	result := tokens.Create(service, tokens.WrapOptions(opts))
	catalog, err := result.ExtractServiceCatalog()
	if err != nil {
		fmt.Printf("Unable to retrieve the service catalog: %v\n", err)
		os.Exit(1)
	}

	var regions []string
	var fg bool
	for _, entry := range catalog.Entries {
		if entry.Type == "compute" {
			for _, endpoint := range entry.Endpoints {
				if endpoint.Region == "" {
					fg = true
				} else {
					regions = append(regions, endpoint.Region)
				}
			}
		}
	}
	return regions, fg
}

// ConstructEntry extracts the metadata key and builds an entry for a server.
func ConstructEntry(server osV2Servers.Server, genType, region string) (*entry, error) {
	window, ok := server.Metadata[metadataKey]
	if !ok {
		return nil, fmt.Errorf("Metadatum %s was not present in the result for server %s", metadataKey, server.ID)
	}

	windowString, ok := window.(string)
	if !ok {
		return nil, fmt.Errorf("Metadatum %s for server %s was not a string: %#v", metadataKey, server.ID, window)
	}

	// Expected format: 2014-01-28T00:00:00Z;2014-01-28T03:00:00Z

	parts := strings.Split(windowString, ";")
	if len(parts) != 2 {
		return nil, fmt.Errorf("Unexpected metadatum format for server %s: %s", server.ID, windowString)
	}

	start, err := time.Parse(metadataTimeFmt, parts[0])
	if err != nil {
		return nil, fmt.Errorf("Unable to parse window start time for server %s: %s", server.ID, parts[0])
	}

	end, err := time.Parse(metadataTimeFmt, parts[1])
	if err != nil {
		return nil, fmt.Errorf("Unable to parse window end time for server %s: %s", server.ID, parts[1])
	}

	e := &entry{
		Server:      server,
		Region:      region,
		GenType:     genType,
		WindowStart: start,
		WindowEnd:   end,
	}
	return e, nil
}
<commit_msg>don't output servers if entries is empty<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/jrperritt/gophercloud/rackspace"
	rsV1Servers "github.com/jrperritt/gophercloud/rackspace/compute/v1/servers"
	"github.com/rackspace/gophercloud"
	osV2Servers "github.com/rackspace/gophercloud/openstack/compute/v2/servers"
	"github.com/rackspace/gophercloud/pagination"
	rsV2Servers "github.com/rackspace/gophercloud/rackspace/compute/v2/servers"
	"github.com/rackspace/gophercloud/rackspace/identity/v2/tokens"
)

const (
	metadataKey     = "rax:reboot_window"
	metadataTimeFmt = "2006-01-02T15:04:05Z"
)

func main() {
	outputToCSV := flag.Bool("csv", false, "Output a CSV file")
	flag.Parse()

	if flag.NArg() != 2 {
		fmt.Println("You must supply a username and API key after all other args.")
		os.Exit(1)
	}

	username, apiKey := flag.Arg(0), flag.Arg(1)

	opts := gophercloud.AuthOptions{
		Username: username,
		APIKey:   apiKey,
	}

	provider, err := rackspace.AuthenticatedClient(opts)
	if err != nil {
		fmt.Printf("Unable to authenticate: %v\n", err)
		os.Exit(1)
	}

	regions, fg := Regions(provider, opts)

	fmt.Printf("Regions with a compute endpoint: %s\n", strings.Join(regions, ", "))
	if fg {
		fmt.Println("You have a first-gen endpoint, too.")
	}

	var numUnaffected int
	var entries []entry

	// Iterate through regions with an NG compute endpoint. Collect data about each server.
	for _, region := range regions {
		compute, err := rackspace.NewComputeV2(provider, gophercloud.EndpointOpts{
			Region: region,
		})
		if err != nil {
			fmt.Printf("Unable to locate a v2 compute endpoint in region %s: %v\n", region, err)
			continue
		}

		err = rsV2Servers.List(compute, nil).EachPage(func(page pagination.Page) (bool, error) {
			s, err := osV2Servers.ExtractServers(page)
			if err != nil {
				return false, err
			}

			for _, server := range s {
				entry, err := ConstructEntry(server, "Next Gen", region)
				if err != nil {
					if strings.Contains(err.Error(), "not present") {
						numUnaffected++
						continue
					}
					fmt.Printf("%s\n", err)
					continue
				} else {
					entries = append(entries, *entry)
				}
			}

			return true, nil
		})
	}

	// Iterate through regions with an FG compute endpoint. Collect data about each server.
	compute, err := rackspace.NewComputeV1(provider, gophercloud.EndpointOpts{
		Availability: gophercloud.AvailabilityPublic,
	})
	if err != nil {
		fmt.Printf("Unable to locate a v1 compute endpoint. Skipping...\n")
	} else {
		err = rsV1Servers.List(compute, rsV1Servers.ListOpts{}).EachPage(func(page pagination.Page) (bool, error) {
			s, err := osV2Servers.ExtractServers(page)
			if err != nil {
				return false, err
			}

			for _, server := range s {
				entry, err := ConstructEntry(server, "First Gen", "DFW")
				if err != nil {
					if strings.Contains(err.Error(), "not present") {
						numUnaffected++
						continue
					}
					fmt.Printf("%s\n", err)
					continue
				} else {
					entries = append(entries, *entry)
				}
			}

			return true, nil
		})
		if err != nil {
			fmt.Printf("Error listing servers: %+v\n", err)
		}
	}

	if len(entries) > 0 {
		if *outputToCSV {
			outputCSV(entries)
		} else {
			if numUnaffected > 0 {
				fmt.Printf("%d of your servers will not require a reboot. The following %d servers will need to be rebooted", numUnaffected, len(entries))
			}
			outputTabular(entries)
		}
	}
}

// Regions acquires the service catalog and returns a slice of every region that contains a next-gen
// server endpoint, and a boolean indicating whether or not this customer has access to FG servers.
func Regions(provider *gophercloud.ProviderClient, opts gophercloud.AuthOptions) ([]string, bool) {
	service := rackspace.NewIdentityV2(provider)

	result := tokens.Create(service, tokens.WrapOptions(opts))
	catalog, err := result.ExtractServiceCatalog()
	if err != nil {
		fmt.Printf("Unable to retrieve the service catalog: %v\n", err)
		os.Exit(1)
	}

	var regions []string
	var fg bool
	for _, entry := range catalog.Entries {
		if entry.Type == "compute" {
			for _, endpoint := range entry.Endpoints {
				if endpoint.Region == "" {
					fg = true
				} else {
					regions = append(regions, endpoint.Region)
				}
			}
		}
	}
	return regions, fg
}

// ConstructEntry extracts the metadata key and builds an entry for a server.
func ConstructEntry(server osV2Servers.Server, genType, region string) (*entry, error) {
	window, ok := server.Metadata[metadataKey]
	if !ok {
		return nil, fmt.Errorf("Metadatum %s was not present in the result for server %s", metadataKey, server.ID)
	}

	windowString, ok := window.(string)
	if !ok {
		return nil, fmt.Errorf("Metadatum %s for server %s was not a string: %#v", metadataKey, server.ID, window)
	}

	// Expected format: 2014-01-28T00:00:00Z;2014-01-28T03:00:00Z

	parts := strings.Split(windowString, ";")
	if len(parts) != 2 {
		return nil, fmt.Errorf("Unexpected metadatum format for server %s: %s", server.ID, windowString)
	}

	start, err := time.Parse(metadataTimeFmt, parts[0])
	if err != nil {
		return nil, fmt.Errorf("Unable to parse window start time for server %s: %s", server.ID, parts[0])
	}

	end, err := time.Parse(metadataTimeFmt, parts[1])
	if err != nil {
		return nil, fmt.Errorf("Unable to parse window end time for server %s: %s", server.ID, parts[1])
	}

	e := &entry{
		Server:      server,
		Region:      region,
		GenType:     genType,
		WindowStart: start,
		WindowEnd:   end,
	}
	return e, nil
}
<|endoftext|>


<commit_before>package consensus

import (
	"bufio"
	"bytes"
	"fmt"
	"math/rand"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/pkg/errors"
	"github.com/tendermint/abci/example/dummy"
	bc "github.com/tendermint/tendermint/blockchain"
	cfg "github.com/tendermint/tendermint/config"
	"github.com/tendermint/tendermint/proxy"
	sm "github.com/tendermint/tendermint/state"
	"github.com/tendermint/tendermint/types"
	auto "github.com/tendermint/tmlibs/autofile"
	"github.com/tendermint/tmlibs/db"
	"github.com/tendermint/tmlibs/log"
)

// WALWithNBlocks generates a consensus WAL. It does this by spining up a
// stripped down version of node (proxy app, event bus, consensus state) with a
// persistent dummy application and special consensus wal instance
// (byteBufferWAL) and waits until numBlocks are created. Then it returns a WAL
// content.
func WALWithNBlocks(numBlocks int) (data []byte, err error) {
	config := getConfig()

	app := dummy.NewPersistentDummyApplication(filepath.Join(config.DBDir(), "wal_generator"))

	logger := log.TestingLogger().With("wal_generator", "wal_generator")

	/////////////////////////////////////////////////////////////////////////////
	// COPY PASTE FROM node.go WITH A FEW MODIFICATIONS
	// NOTE: we can't import node package because of circular dependency
	privValidatorFile := config.PrivValidatorFile()
	privValidator := types.LoadOrGenPrivValidatorFS(privValidatorFile)
	genDoc, err := types.GenesisDocFromFile(config.GenesisFile())
	if err != nil {
		return nil, errors.Wrap(err, "failed to read genesis file")
	}
	stateDB := db.NewMemDB()
	blockStoreDB := db.NewMemDB()
	state, err := sm.MakeGenesisState(stateDB, genDoc)
	state.SetLogger(logger.With("module", "state"))
	if err != nil {
		return nil, errors.Wrap(err, "failed to make genesis state")
	}
	blockStore := bc.NewBlockStore(blockStoreDB)
	handshaker := NewHandshaker(state, blockStore)
	proxyApp := proxy.NewAppConns(proxy.NewLocalClientCreator(app), handshaker)
	proxyApp.SetLogger(logger.With("module", "proxy"))
	if err := proxyApp.Start(); err != nil {
		return nil, errors.Wrap(err, "failed to start proxy app connections")
	}
	defer proxyApp.Stop()
	eventBus := types.NewEventBus()
	eventBus.SetLogger(logger.With("module", "events"))
	if err := eventBus.Start(); err != nil {
		return nil, errors.Wrap(err, "failed to start event bus")
	}
	mempool := types.MockMempool{}
	consensusState := NewConsensusState(config.Consensus, state.Copy(), proxyApp.Consensus(), blockStore, mempool)
	consensusState.SetLogger(logger)
	consensusState.SetEventBus(eventBus)
	if privValidator != nil {
		consensusState.SetPrivValidator(privValidator)
	}
	// END OF COPY PASTE
	/////////////////////////////////////////////////////////////////////////////

	// set consensus wal to buffered WAL, which will write all incoming msgs to buffer
	var b bytes.Buffer
	wr := bufio.NewWriter(&b)
	numBlocksWritten := make(chan struct{})
	wal := newByteBufferWAL(NewWALEncoder(wr), int64(numBlocks), numBlocksWritten)
	// see wal.go#103
	wal.Save(EndHeightMessage{0})
	consensusState.wal = wal

	if err := consensusState.Start(); err != nil {
		return nil, errors.Wrap(err, "failed to start consensus state")
	}
	defer consensusState.Stop()

	select {
	case <-numBlocksWritten:
		wr.Flush()
		return b.Bytes(), nil
	case <-time.After(1 * time.Minute):
		wr.Flush()
		return b.Bytes(), fmt.Errorf("waited too long for tendermint to produce %d blocks (grep logs for `wal_generator`)", numBlocks)
	}
}

// f**ing long, but unique for each test
func makePathname() string {
	// get path
	p, err := os.Getwd()
	if err != nil {
		panic(err)
	}
	// fmt.Println(p)
	sep := string(filepath.Separator)
	return strings.Replace(p, sep, "_", -1)
}

func randPort() int {
	// returns between base and base + spread
	base, spread := 20000, 20000
	return base + rand.Intn(spread)
}

func makeAddrs() (string, string, string) {
	start := randPort()
	return fmt.Sprintf("tcp://0.0.0.0:%d", start),
		fmt.Sprintf("tcp://0.0.0.0:%d", start+1),
		fmt.Sprintf("tcp://0.0.0.0:%d", start+2)
}

// getConfig returns a config for test cases
func getConfig() *cfg.Config {
	pathname := makePathname()
	c := cfg.ResetTestRoot(pathname)

	// and we use random ports to run in parallel
	tm, rpc, grpc := makeAddrs()
	c.P2P.ListenAddress = tm
	c.RPC.ListenAddress = rpc
	c.RPC.GRPCListenAddress = grpc
	return c
}

// byteBufferWAL is a WAL which writes all msgs to a byte buffer. Writing stops
// when the heightToStop is reached. Client will be notified via
// signalWhenStopsTo channel.
type byteBufferWAL struct {
	enc               *WALEncoder
	stopped           bool
	heightToStop      int64
	signalWhenStopsTo chan<- struct{}
}

// needed for determinism
var fixedTime, _ = time.Parse(time.RFC3339, "2017-01-02T15:04:05Z")

func newByteBufferWAL(enc *WALEncoder, nBlocks int64, signalStop chan<- struct{}) *byteBufferWAL {
	return &byteBufferWAL{
		enc:               enc,
		heightToStop:      nBlocks,
		signalWhenStopsTo: signalStop,
	}
}

// Save writes message to the internal buffer except when heightToStop is
// reached, in which case it will signal the caller via signalWhenStopsTo and
// skip writing.
func (w *byteBufferWAL) Save(m WALMessage) {
	if w.stopped {
		return
	}

	if endMsg, ok := m.(EndHeightMessage); ok {
		if endMsg.Height == w.heightToStop {
			w.signalWhenStopsTo <- struct{}{}
			w.stopped = true
			return
		}
	}

	err := w.enc.Encode(&TimedWALMessage{fixedTime, m})
	if err != nil {
		panic(fmt.Sprintf("failed to encode the msg %v", m))
	}
}

func (w *byteBufferWAL) Group() *auto.Group {
	panic("not implemented")
}
func (w *byteBufferWAL) SearchForEndHeight(height int64, options *WALSearchOptions) (gr *auto.GroupReader, found bool, err error) {
	return nil, false, nil
}

func (w *byteBufferWAL) Start() error { return nil }
func (w *byteBufferWAL) Stop() error  { return nil }
func (w *byteBufferWAL) Wait()        {}
<commit_msg>stop eventBus<commit_after>package consensus

import (
	"bufio"
	"bytes"
	"fmt"
	"math/rand"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/pkg/errors"
	"github.com/tendermint/abci/example/dummy"
	bc "github.com/tendermint/tendermint/blockchain"
	cfg "github.com/tendermint/tendermint/config"
	"github.com/tendermint/tendermint/proxy"
	sm "github.com/tendermint/tendermint/state"
	"github.com/tendermint/tendermint/types"
	auto "github.com/tendermint/tmlibs/autofile"
	"github.com/tendermint/tmlibs/db"
	"github.com/tendermint/tmlibs/log"
)

// WALWithNBlocks generates a consensus WAL. It does this by spining up a
// stripped down version of node (proxy app, event bus, consensus state) with a
// persistent dummy application and special consensus wal instance
// (byteBufferWAL) and waits until numBlocks are created. Then it returns a WAL
// content.
func WALWithNBlocks(numBlocks int) (data []byte, err error) {
	config := getConfig()

	app := dummy.NewPersistentDummyApplication(filepath.Join(config.DBDir(), "wal_generator"))

	logger := log.TestingLogger().With("wal_generator", "wal_generator")
	logger.Info("generating WAL (last height msg excluded)", "numBlocks", numBlocks)

	/////////////////////////////////////////////////////////////////////////////
	// COPY PASTE FROM node.go WITH A FEW MODIFICATIONS
	// NOTE: we can't import node package because of circular dependency
	privValidatorFile := config.PrivValidatorFile()
	privValidator := types.LoadOrGenPrivValidatorFS(privValidatorFile)
	genDoc, err := types.GenesisDocFromFile(config.GenesisFile())
	if err != nil {
		return nil, errors.Wrap(err, "failed to read genesis file")
	}
	stateDB := db.NewMemDB()
	blockStoreDB := db.NewMemDB()
	state, err := sm.MakeGenesisState(stateDB, genDoc)
	state.SetLogger(logger.With("module", "state"))
	if err != nil {
		return nil, errors.Wrap(err, "failed to make genesis state")
	}
	blockStore := bc.NewBlockStore(blockStoreDB)
	handshaker := NewHandshaker(state, blockStore)
	proxyApp := proxy.NewAppConns(proxy.NewLocalClientCreator(app), handshaker)
	proxyApp.SetLogger(logger.With("module", "proxy"))
	if err := proxyApp.Start(); err != nil {
		return nil, errors.Wrap(err, "failed to start proxy app connections")
	}
	defer proxyApp.Stop()
	eventBus := types.NewEventBus()
	eventBus.SetLogger(logger.With("module", "events"))
	if err := eventBus.Start(); err != nil {
		return nil, errors.Wrap(err, "failed to start event bus")
	}
	defer eventBus.Stop()
	mempool := types.MockMempool{}
	consensusState := NewConsensusState(config.Consensus, state.Copy(), proxyApp.Consensus(), blockStore, mempool)
	consensusState.SetLogger(logger)
	consensusState.SetEventBus(eventBus)
	if privValidator != nil {
		consensusState.SetPrivValidator(privValidator)
	}
	// END OF COPY PASTE
	/////////////////////////////////////////////////////////////////////////////

	// set consensus wal to buffered WAL, which will write all incoming msgs to buffer
	var b bytes.Buffer
	wr := bufio.NewWriter(&b)
	numBlocksWritten := make(chan struct{})
	wal := newByteBufferWAL(NewWALEncoder(wr), int64(numBlocks), numBlocksWritten)
	// see wal.go#103
	wal.Save(EndHeightMessage{0})
	consensusState.wal = wal

	if err := consensusState.Start(); err != nil {
		return nil, errors.Wrap(err, "failed to start consensus state")
	}
	defer consensusState.Stop()

	select {
	case <-numBlocksWritten:
		wr.Flush()
		return b.Bytes(), nil
	case <-time.After(1 * time.Minute):
		wr.Flush()
		return b.Bytes(), fmt.Errorf("waited too long for tendermint to produce %d blocks (grep logs for `wal_generator`)", numBlocks)
	}
}

// f**ing long, but unique for each test
func makePathname() string {
	// get path
	p, err := os.Getwd()
	if err != nil {
		panic(err)
	}
	// fmt.Println(p)
	sep := string(filepath.Separator)
	return strings.Replace(p, sep, "_", -1)
}

func randPort() int {
	// returns between base and base + spread
	base, spread := 20000, 20000
	return base + rand.Intn(spread)
}

func makeAddrs() (string, string, string) {
	start := randPort()
	return fmt.Sprintf("tcp://0.0.0.0:%d", start),
		fmt.Sprintf("tcp://0.0.0.0:%d", start+1),
		fmt.Sprintf("tcp://0.0.0.0:%d", start+2)
}

// getConfig returns a config for test cases
func getConfig() *cfg.Config {
	pathname := makePathname()
	c := cfg.ResetTestRoot(pathname)

	// and we use random ports to run in parallel
	tm, rpc, grpc := makeAddrs()
	c.P2P.ListenAddress = tm
	c.RPC.ListenAddress = rpc
	c.RPC.GRPCListenAddress = grpc
	return c
}

// byteBufferWAL is a WAL which writes all msgs to a byte buffer. Writing stops
// when the heightToStop is reached. Client will be notified via
// signalWhenStopsTo channel.
type byteBufferWAL struct {
	enc               *WALEncoder
	stopped           bool
	heightToStop      int64
	signalWhenStopsTo chan<- struct{}
}

// needed for determinism
var fixedTime, _ = time.Parse(time.RFC3339, "2017-01-02T15:04:05Z")

func newByteBufferWAL(enc *WALEncoder, nBlocks int64, signalStop chan<- struct{}) *byteBufferWAL {
	return &byteBufferWAL{
		enc:               enc,
		heightToStop:      nBlocks,
		signalWhenStopsTo: signalStop,
	}
}

// Save writes message to the internal buffer except when heightToStop is
// reached, in which case it will signal the caller via signalWhenStopsTo and
// skip writing.
func (w *byteBufferWAL) Save(m WALMessage) {
	if w.stopped {
		return
	}

	if endMsg, ok := m.(EndHeightMessage); ok {
		if endMsg.Height == w.heightToStop {
			w.signalWhenStopsTo <- struct{}{}
			w.stopped = true
			return
		}
	}

	err := w.enc.Encode(&TimedWALMessage{fixedTime, m})
	if err != nil {
		panic(fmt.Sprintf("failed to encode the msg %v", m))
	}
}

func (w *byteBufferWAL) Group() *auto.Group {
	panic("not implemented")
}
func (w *byteBufferWAL) SearchForEndHeight(height int64, options *WALSearchOptions) (gr *auto.GroupReader, found bool, err error) {
	return nil, false, nil
}

func (w *byteBufferWAL) Start() error { return nil }
func (w *byteBufferWAL) Stop() error  { return nil }
func (w *byteBufferWAL) Wait()        {}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strconv"

	"github.com/najeira/ltsv"

	"golang.org/x/crypto/ssh/terminal"
)

func main() {
	if terminal.IsTerminal(1) {
		args := os.Args

		reader := ltsv.NewReader(os.Stdin)
		records, _ := reader.ReadAll()

		var result float64
		if args[1] == "sum" {
			result = sum(args[2], records)
		}

		fmt.Println(result)
	}
}

func sum(key string, ltsv []map[string]string) float64 {
	result := 0.0
	for _, record := range ltsv {
		for k, v := range record {
			if k == key {
				f64, _ := strconv.ParseFloat(v, 64)
				result += f64
			}
		}
	}

	return result
}
<commit_msg>calculate average value<commit_after>package main

import (
	"fmt"
	"os"
	"strconv"

	"github.com/najeira/ltsv"

	"golang.org/x/crypto/ssh/terminal"
)

func main() {
	if terminal.IsTerminal(1) {
		args := os.Args

		reader := ltsv.NewReader(os.Stdin)
		records, _ := reader.ReadAll()

		var result float64

		switch args[1] {
		case "sum":
			result = sum(args[2], records)

		case "ave":
			result = ave(args[2], records)
		}

		fmt.Println(result)
	}
}

func sum(key string, ltsv []map[string]string) float64 {
	result := 0.0
	for _, record := range ltsv {
		for k, v := range record {
			if k == key {
				f64, _ := strconv.ParseFloat(v, 64)
				result += f64
			}
		}
	}

	return result
}

func ave(key string, ltsv []map[string]string) float64 {
	sum := sum(key, ltsv)
	return sum / float64(len(ltsv))
}
<|endoftext|>