<commit_before>package tunnel

import (
	"bytes"
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/gorilla/websocket"
	"github.com/ondevice/ondevice/logg"
	"github.com/ondevice/ondevice/util"
)

// Tunnel -- an ondevice.io tunnel ()
type Tunnel struct {
	Connection

	// Side -- either DeviceSide or ClientSide
	Side      string
	connected chan util.APIError
	wdog      *util.Watchdog // the client will use this to periodically send 'meta:ping' messages, the device will respond (and kick the Watchdog in the process)
	lastPing  time.Time

	readEOF, writeEOF bool

	// metrics:
	bytesRead, bytesWritten int64
	startTs                 time.Time

	// listeners
	DataListeners    []func(data []byte)
	EOFListeners     []func()
	TimeoutListeners []func()
}

const (
	// ClientSide -- This Tunnel instance represents the client side of the tunnel (see Tunnel.Side)
	ClientSide = "client"
	// DeviceSide -- This Tunnel instance represents the device side of the tunnel (see Tunnel.Side)
	DeviceSide = "device"
)

// GetErrorCodeName -- returns a string representing the given 'HTTP-ish' tunnel error code
func GetErrorCodeName(code int) string {
	switch code {
	case 400:
		return "Bad Request"
	case 403:
		return "Access Denied"
	case 404:
		return "Not Found"
	case 503:
		return "Service Unavailable"
	}

	return ""
}

func (t *Tunnel) _initTunnel(side string) {
	t.connected = make(chan util.APIError)
	t.Side = side
	t.startTs = time.Now()
	t.CloseListeners = append([]func(){t._onClose}, t.CloseListeners...)
}

// SendEOF -- send an EOF to the remote end of the tunnel (i.e. close the write channel)
func (t *Tunnel) SendEOF() {
	if t.writeEOF == true {
		logg.Debug("Attempted to close already closed write channel")
		return
	}

	logg.Info("sending EOF")
	t.writeEOF = true
	t.SendBinary([]byte("meta:EOF"))
	t._checkClose()
}

func (t *Tunnel) Write(data []byte) {
	msg := append([]byte("data:"), data...)

	t.bytesWritten += int64(len(data))
	t.SendBinary(msg)
}

func (t *Tunnel) onMessage(_type int, msg []byte) {
	parts := bytes.SplitN(msg, []byte(":"), 2)

	if _type != websocket.BinaryMessage {
		logg.Error("Got non-binary message over the tunnel")
		return
	}
	if len(parts) < 2 {
		logg.Error("Missing colon in tunnel message")
		return
	}
	msgType := string(parts[0])
	msg = parts[1]

	if msgType == "meta" {
		parts = bytes.SplitN(msg, []byte(":"), 2)
		metaType := string(parts[0])

		if metaType == "ping" {
			//logg.Debug("got tunnel ping")
			pong := []byte("meta:pong")
			t.lastPing = time.Now()
			t.wdog.Kick()

			if len(parts) > 1 {
				pong = append(pong, byte(':'))
				pong = append(pong, msg[5:]...)
			}
			t.SendBinary(pong)
		} else if metaType == "pong" {
			logg.Debug("got tunnel pong: ", string(msg))
			t.lastPing = time.Now()
		} else if metaType == "connected" {
			logg.Debug("connected")
			t.connected <- nil
		} else if metaType == "EOF" {
			t._onEOF()
			t._checkClose()
		} else {
			t._error(util.NewAPIError(util.OtherError, "Unsupported meta message: ", metaType))
		}
	} else if msgType == "data" {
		t.bytesRead += int64(len(msg))

		if len(t.DataListeners) == 0 {
			panic("Tunnel: Missing OnData handler")
		}

		// call listeners
		for _, cb := range t.DataListeners {
			cb(msg)
		}
	} else if msgType == "error" {
		parts := strings.SplitN(string(msg), ":", 2)
		var code int
		var errMsg string
		if len(parts) == 1 {
			errMsg = parts[0]
		} else {
			code, _ = strconv.Atoi(parts[0])
			errMsg = parts[1]
		}

		err := util.NewAPIError(code, errMsg)
		if t.connected != nil {
			t.connected <- err
		}
		t._error(err)
	} else {
		logg.Warning("Unsupported tunnel message type: ", msgType)
	}
}

// _checkClose -- after sending/receiving EOF this method checks if the tunnel
// should be closed
func (t *Tunnel) _checkClose() {
	if t.readEOF && t.writeEOF {
		logg.Debug("EOF on both channels, closing tunnel - side: ", t.Side)
		if t.Side == DeviceSide {
			// it's the client's job to actually close the tunnel - but if it doesn't
			// do that in time, we'll do it ourselves
			time.AfterFunc(10*time.Second, t.Close)
		} else if t.Side == ClientSide {
			t.Close()
		} else {
			logg.Warning("Unsupported tunnel side: ", t.Side)
		}
	}
}

func (t *Tunnel) _error(err util.APIError) {
	if len(t.ErrorListeners) == 0 {
		logg.Error(err)
	}
	for _, cb := range t.ErrorListeners {
		cb(err)
	}
}

func (t *Tunnel) _onClose() {
	t.writeEOF = true // no need to send an EOF over a closed tunnel
	t._onEOF()        // always fire the EOF signal

	// print log message and stop timers
	duration := time.Now().Sub(t.startTs)
	msg := fmt.Sprintf("Tunnel closed, bytesRead=%d, bytesWritten=%d, duration=%s", t.bytesRead, t.bytesWritten, duration.String())
	if t.Side == ClientSide {
		logg.Debug(msg)
	} else if t.Side == DeviceSide {
		logg.Info(msg)
	}

	// TODO stop timers
}

func (t *Tunnel) _onEOF() {
	logg.Debug("Tunnel._onEOF()")
	if t.readEOF == true {
		return
	}
	t.readEOF = true

	// call listeners
	for _, cb := range t.EOFListeners {
		cb()
	}
}

func (t *Tunnel) _onTimeout() {
	for _, cb := range t.TimeoutListeners {
		cb()
	}
}
<commit_msg>tunnel.ws: decreased log verbosity<commit_after>package tunnel

import (
	"bytes"
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/gorilla/websocket"
	"github.com/ondevice/ondevice/logg"
	"github.com/ondevice/ondevice/util"
)

// Tunnel -- an ondevice.io tunnel ()
type Tunnel struct {
	Connection

	// Side -- either DeviceSide or ClientSide
	Side      string
	connected chan util.APIError
	wdog      *util.Watchdog // the client will use this to periodically send 'meta:ping' messages, the device will respond (and kick the Watchdog in the process)
	lastPing  time.Time

	readEOF, writeEOF bool

	// metrics:
	bytesRead, bytesWritten int64
	startTs                 time.Time

	// listeners
	DataListeners    []func(data []byte)
	EOFListeners     []func()
	TimeoutListeners []func()
}

const (
	// ClientSide -- This Tunnel instance represents the client side of the tunnel (see Tunnel.Side)
	ClientSide = "client"
	// DeviceSide -- This Tunnel instance represents the device side of the tunnel (see Tunnel.Side)
	DeviceSide = "device"
)

// GetErrorCodeName -- returns a string representing the given 'HTTP-ish' tunnel error code
func GetErrorCodeName(code int) string {
	switch code {
	case 400:
		return "Bad Request"
	case 403:
		return "Access Denied"
	case 404:
		return "Not Found"
	case 503:
		return "Service Unavailable"
	}

	return ""
}

func (t *Tunnel) _initTunnel(side string) {
	t.connected = make(chan util.APIError)
	t.Side = side
	t.startTs = time.Now()
	t.CloseListeners = append([]func(){t._onClose}, t.CloseListeners...)
}

// SendEOF -- send an EOF to the remote end of the tunnel (i.e. close the write channel)
func (t *Tunnel) SendEOF() {
	if t.writeEOF == true {
		logg.Debug("Attempted to close already closed write channel")
		return
	}

	logg.Debug("sending EOF")
	t.writeEOF = true
	t.SendBinary([]byte("meta:EOF"))
	t._checkClose()
}

func (t *Tunnel) Write(data []byte) {
	msg := append([]byte("data:"), data...)

	t.bytesWritten += int64(len(data))
	t.SendBinary(msg)
}

func (t *Tunnel) onMessage(_type int, msg []byte) {
	parts := bytes.SplitN(msg, []byte(":"), 2)

	if _type != websocket.BinaryMessage {
		logg.Error("Got non-binary message over the tunnel")
		return
	}
	if len(parts) < 2 {
		logg.Error("Missing colon in tunnel message")
		return
	}
	msgType := string(parts[0])
	msg = parts[1]

	if msgType == "meta" {
		parts = bytes.SplitN(msg, []byte(":"), 2)
		metaType := string(parts[0])

		if metaType == "ping" {
			//logg.Debug("got tunnel ping")
			pong := []byte("meta:pong")
			t.lastPing = time.Now()
			t.wdog.Kick()

			if len(parts) > 1 {
				pong = append(pong, byte(':'))
				pong = append(pong, msg[5:]...)
			}
			t.SendBinary(pong)
		} else if metaType == "pong" {
			logg.Debug("got tunnel pong: ", string(msg))
			t.lastPing = time.Now()
		} else if metaType == "connected" {
			logg.Debug("connected")
			t.connected <- nil
		} else if metaType == "EOF" {
			t._onEOF()
			t._checkClose()
		} else {
			t._error(util.NewAPIError(util.OtherError, "Unsupported meta message: ", metaType))
		}
	} else if msgType == "data" {
		t.bytesRead += int64(len(msg))

		if len(t.DataListeners) == 0 {
			panic("Tunnel: Missing OnData handler")
		}

		// call listeners
		for _, cb := range t.DataListeners {
			cb(msg)
		}
	} else if msgType == "error" {
		parts := strings.SplitN(string(msg), ":", 2)
		var code int
		var errMsg string
		if len(parts) == 1 {
			errMsg = parts[0]
		} else {
			code, _ = strconv.Atoi(parts[0])
			errMsg = parts[1]
		}

		err := util.NewAPIError(code, errMsg)
		if t.connected != nil {
			t.connected <- err
		}
		t._error(err)
	} else {
		logg.Warning("Unsupported tunnel message type: ", msgType)
	}
}

// _checkClose -- after sending/receiving EOF this method checks if the tunnel
// should be closed
func (t *Tunnel) _checkClose() {
	if t.readEOF && t.writeEOF {
		logg.Debug("EOF on both channels, closing tunnel - side: ", t.Side)
		if t.Side == DeviceSide {
			// it's the client's job to actually close the tunnel - but if it doesn't
			// do that in time, we'll do it ourselves
			time.AfterFunc(10*time.Second, t.Close)
		} else if t.Side == ClientSide {
			t.Close()
		} else {
			logg.Warning("Unsupported tunnel side: ", t.Side)
		}
	}
}

func (t *Tunnel) _error(err util.APIError) {
	if len(t.ErrorListeners) == 0 {
		logg.Error(err)
	}
	for _, cb := range t.ErrorListeners {
		cb(err)
	}
}

func (t *Tunnel) _onClose() {
	t.writeEOF = true // no need to send an EOF over a closed tunnel
	t._onEOF()        // always fire the EOF signal

	// print log message and stop timers
	duration := time.Now().Sub(t.startTs)
	msg := fmt.Sprintf("Tunnel closed, bytesRead=%d, bytesWritten=%d, duration=%s", t.bytesRead, t.bytesWritten, duration.String())
	if t.Side == ClientSide {
		logg.Debug(msg)
	} else if t.Side == DeviceSide {
		logg.Info(msg)
	}

	// TODO stop timers
}

func (t *Tunnel) _onEOF() {
	logg.Debug("Tunnel._onEOF()")
	if t.readEOF == true {
		return
	}
	t.readEOF = true

	// call listeners
	for _, cb := range t.EOFListeners {
		cb()
	}
}

func (t *Tunnel) _onTimeout() {
	for _, cb := range t.TimeoutListeners {
		cb()
	}
}
<|endoftext|>


<commit_before>//JDSU CellAdvisor Web-Live Program
//Copyright (C) 2015 Jihyuk Bok <tomahawk28@gmail.com>
//
//Permission is hereby granted, free of charge, to any person obtaining
//a copy of this software and associated documentation files (the "Software"),
//to deal in the Software without restriction, including without limitation
//the rights to use, copy, modify, merge, publish, distribute, sublicense,
//and/or sell copies of the Software, and to permit persons to whom the
//Software is furnished to do so, subject to the following conditions:
//
//The above copyright notice and this permission notice shall be included
//in all copies or substantial portions of the Software.
//
//THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
//EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
//OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
//IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
//DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
//TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
//OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package main

import (
	"expvar"
	"flag"
	"fmt"
	"log"
	"sync"
	"text/template"
	"time"

	"net/http"

	"github.com/tomahawk28/cell"
)

var (
	httpAddr        = flag.String("http", ":8040", "Listen Address")
	cellAdvisorAddr = flag.String("celladdr", "10.82.26.12", "CellAdvisor Address")
	pollPeriod      = flag.Duration("poll", 30*time.Second, "Poll Period")
)

var (
	screenCache = ScreenCache{time.Now(), []byte{}, sync.RWMutex{}}
	mu          = sync.Mutex{}
	tmpl        = template.Must(template.ParseFiles("template.html"))
)

var (
	sendSuccessCount    = expvar.NewInt("sendSuccessCount")
	receiveSucessCount  = expvar.NewInt("receiveSucessCount")
	sendPendingCount    = expvar.NewInt("sendPendingCount")
	receivePendingCount = expvar.NewInt("receivePendingCount")
)

type Request struct {
	command string
	args    map[string]string
	result  chan []byte
}

type ScreenCache struct {
	last  time.Time
	cache []byte
	mu    sync.RWMutex
}

func Poller(done <-chan struct{}, in <-chan *Request, cell *cell.CellAdvisor, thread_number int) {
	var err error
	for {
		select {
		case <-done:
			log.Println("Cancellation signal received")
			return
		case r := <-in:
			log.Println("Thread ", thread_number, ":", r.command)
			switch r.command {
			case "keyp":
				scpicmd := fmt.Sprintf("KEYP:%s", r.args["value"])
				cell.SendSCPI(scpicmd)
				sendResult(done, r.result, []byte{})
			case "touch":
				scpicmd := fmt.Sprintf("KEYP %s %s", r.args["x"], r.args["y"])
				cell.SendSCPI(scpicmd)
				sendResult(done, r.result, []byte{})
			case "screen":
				go func() {
					screenCache.mu.Lock()
					defer screenCache.mu.Unlock()
					if time.Now().Sub(screenCache.last).Seconds() > 1 {
						screenCache.last = time.Now()
						screenCache.cache, err = cell.GetScreen()
						if err != nil {
							log.Println(err.Error())
						}
					}
					sendResult(done, r.result, screenCache.cache)
				}()
			case "heartbeat":
				msg, err := cell.GetStatusMessage()
				if err != nil {
					log.Println(err.Error())
				}
				sendResult(done, r.result, msg)
			}
		case <-time.After(time.Second * 15):
			mu.Lock()
			msg, err := cell.GetStatusMessage()
			if err != nil {
				log.Println(err.Error())
			}
			log.Println("Hearbeat:", thread_number, string(msg))
			mu.Unlock()
		}
	}
}

func NewRequest(command string, args map[string]string) *Request {
	return &Request{command, args, make(chan []byte)}
}

func sendResult(done <-chan struct{}, pipe chan<- []byte, result []byte) {
	select {
	case pipe <- result:
		sendSuccessCount.Add(1)
	case <-time.After(time.Second * 2):
		log.Println("Sending Timeout")
		sendPendingCount.Add(1)
	case <-done:
		return
	}
}
func receiveResult(done <-chan struct{}, pipe <-chan []byte) []byte {
	select {
	case result := <-pipe:
		receiveSucessCount.Add(1)
		return result
	case <-time.After(time.Second * 5):
		log.Println("Receive Timeout")
		receivePendingCount.Add(1)
	case <-done:
	}
	return []byte{}
}

func main() {

	done := make(chan struct{})
	defer close(done)

	flag.Parse()
	cell_list := []cell.CellAdvisor{cell.NewCellAdvisor(*cellAdvisorAddr),
		cell.NewCellAdvisor(*cellAdvisorAddr),
		cell.NewCellAdvisor(*cellAdvisorAddr)}
	//cell := cell.NewCellAdvisor(*cellAdvisorAddr)

	request_channel := make(chan *Request, len(cell_list))
	for i, _ := range cell_list {
		go Poller(done, request_channel, &cell_list[i], i)
	}

	http.HandleFunc("/screen", func(w http.ResponseWriter, req *http.Request) {
		w.Header().Set("Content-Type", "image/jpeg")
		request_object := NewRequest("screen", nil)
		request_channel <- request_object

		w.Write(receiveResult(done, request_object.result))
	})
	http.HandleFunc("/touch", func(w http.ResponseWriter, req *http.Request) {
		query := req.URL.Query()
		x, y := query.Get("x"), query.Get("y")
		if x != "" && y != "" {
			request_object := NewRequest("touch", map[string]string{"x": x, "y": y})
			request_channel <- request_object
			w.Write(receiveResult(done, request_object.result))
		} else {
			fmt.Fprintf(w, "Coordination not given")
			w.WriteHeader(http.StatusBadRequest)
		}
	})
	http.HandleFunc("/keyp", func(w http.ResponseWriter, req *http.Request) {
		err := req.ParseForm()
		if err != nil {
			fmt.Fprintf(w, "Form Parse error")
			w.WriteHeader(http.StatusBadRequest)
			return
		}
		value := req.FormValue("value")

		if value != "" {
			request_object := NewRequest("keyp", map[string]string{"value": value})
			request_channel <- request_object
			w.Write(receiveResult(done, request_object.result))

		} else {
			fmt.Fprintf(w, "Keypad name not given")
			w.WriteHeader(http.StatusBadRequest)
		}
	})
	http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		err := tmpl.Execute(w, nil)
		if err != nil {
			panic(err)
		}
	})
	fs := http.FileServer(http.Dir("static"))
	http.Handle("/static/", http.StripPrefix("/static/", fs))
	log.Fatal(http.ListenAndServe(*httpAddr, nil))
}
<commit_msg>Adjust timeout period for sending result to http response<commit_after>//JDSU CellAdvisor Web-Live Program
//Copyright (C) 2015 Jihyuk Bok <tomahawk28@gmail.com>
//
//Permission is hereby granted, free of charge, to any person obtaining
//a copy of this software and associated documentation files (the "Software"),
//to deal in the Software without restriction, including without limitation
//the rights to use, copy, modify, merge, publish, distribute, sublicense,
//and/or sell copies of the Software, and to permit persons to whom the
//Software is furnished to do so, subject to the following conditions:
//
//The above copyright notice and this permission notice shall be included
//in all copies or substantial portions of the Software.
//
//THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
//EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
//OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
//IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
//DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
//TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
//OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package main

import (
	"expvar"
	"flag"
	"fmt"
	"log"
	"sync"
	"text/template"
	"time"

	"net/http"

	"github.com/tomahawk28/cell"
)

var (
	httpAddr        = flag.String("http", ":8040", "Listen Address")
	cellAdvisorAddr = flag.String("celladdr", "10.82.26.12", "CellAdvisor Address")
	pollPeriod      = flag.Duration("poll", 30*time.Second, "Poll Period")
)

var (
	screenCache = ScreenCache{time.Now(), []byte{}, sync.RWMutex{}}
	mu          = sync.Mutex{}
	tmpl        = template.Must(template.ParseFiles("template.html"))
)

var (
	sendSuccessCount    = expvar.NewInt("sendSuccessCount")
	receiveSucessCount  = expvar.NewInt("receiveSucessCount")
	sendPendingCount    = expvar.NewInt("sendPendingCount")
	receivePendingCount = expvar.NewInt("receivePendingCount")
)

type Request struct {
	command string
	args    map[string]string
	result  chan []byte
}

type ScreenCache struct {
	last  time.Time
	cache []byte
	mu    sync.RWMutex
}

func Poller(done <-chan struct{}, in <-chan *Request, cell *cell.CellAdvisor, thread_number int) {
	var err error
	for {
		select {
		case <-done:
			log.Println("Cancellation signal received")
			return
		case r := <-in:
			log.Println("Thread ", thread_number, ":", r.command)
			switch r.command {
			case "keyp":
				scpicmd := fmt.Sprintf("KEYP:%s", r.args["value"])
				cell.SendSCPI(scpicmd)
				sendResult(done, r.result, []byte{})
			case "touch":
				scpicmd := fmt.Sprintf("KEYP %s %s", r.args["x"], r.args["y"])
				cell.SendSCPI(scpicmd)
				sendResult(done, r.result, []byte{})
			case "screen":
				go func() {
					screenCache.mu.Lock()
					defer screenCache.mu.Unlock()
					if time.Now().Sub(screenCache.last).Seconds() > 1 {
						screenCache.last = time.Now()
						screenCache.cache, err = cell.GetScreen()
						if err != nil {
							log.Println(err.Error())
						}
					}
					sendResult(done, r.result, screenCache.cache)
				}()
			case "heartbeat":
				msg, err := cell.GetStatusMessage()
				if err != nil {
					log.Println(err.Error())
				}
				sendResult(done, r.result, msg)
			}
		case <-time.After(time.Second * 15):
			mu.Lock()
			msg, err := cell.GetStatusMessage()
			if err != nil {
				log.Println(err.Error())
			}
			log.Println("Hearbeat:", thread_number, string(msg))
			mu.Unlock()
		}
	}
}

func NewRequest(command string, args map[string]string) *Request {
	return &Request{command, args, make(chan []byte)}
}

func sendResult(done <-chan struct{}, pipe chan<- []byte, result []byte) {
	select {
	case pipe <- result:
		sendSuccessCount.Add(1)
	case <-time.After(time.Second * 3):
		log.Println("Sending Timeout")
		sendPendingCount.Add(1)
	case <-done:
		return
	}
}
func receiveResult(done <-chan struct{}, pipe <-chan []byte) []byte {
	select {
	case result := <-pipe:
		receiveSucessCount.Add(1)
		return result
	case <-time.After(time.Second * 5):
		log.Println("Receive Timeout")
		receivePendingCount.Add(1)
	case <-done:
	}
	return []byte{}
}

func main() {

	done := make(chan struct{})
	defer close(done)

	flag.Parse()
	cell_list := []cell.CellAdvisor{cell.NewCellAdvisor(*cellAdvisorAddr),
		cell.NewCellAdvisor(*cellAdvisorAddr),
		cell.NewCellAdvisor(*cellAdvisorAddr)}
	//cell := cell.NewCellAdvisor(*cellAdvisorAddr)

	request_channel := make(chan *Request, len(cell_list))
	for i, _ := range cell_list {
		go Poller(done, request_channel, &cell_list[i], i)
	}

	http.HandleFunc("/screen", func(w http.ResponseWriter, req *http.Request) {
		w.Header().Set("Content-Type", "image/jpeg")
		request_object := NewRequest("screen", nil)
		request_channel <- request_object

		w.Write(receiveResult(done, request_object.result))
	})
	http.HandleFunc("/touch", func(w http.ResponseWriter, req *http.Request) {
		query := req.URL.Query()
		x, y := query.Get("x"), query.Get("y")
		if x != "" && y != "" {
			request_object := NewRequest("touch", map[string]string{"x": x, "y": y})
			request_channel <- request_object
			w.Write(receiveResult(done, request_object.result))
		} else {
			fmt.Fprintf(w, "Coordination not given")
			w.WriteHeader(http.StatusBadRequest)
		}
	})
	http.HandleFunc("/keyp", func(w http.ResponseWriter, req *http.Request) {
		err := req.ParseForm()
		if err != nil {
			fmt.Fprintf(w, "Form Parse error")
			w.WriteHeader(http.StatusBadRequest)
			return
		}
		value := req.FormValue("value")

		if value != "" {
			request_object := NewRequest("keyp", map[string]string{"value": value})
			request_channel <- request_object
			w.Write(receiveResult(done, request_object.result))

		} else {
			fmt.Fprintf(w, "Keypad name not given")
			w.WriteHeader(http.StatusBadRequest)
		}
	})
	http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		err := tmpl.Execute(w, nil)
		if err != nil {
			panic(err)
		}
	})
	fs := http.FileServer(http.Dir("static"))
	http.Handle("/static/", http.StripPrefix("/static/", fs))
	log.Fatal(http.ListenAndServe(*httpAddr, nil))
}
<|endoftext|>


<commit_before>/* This simnple example demonstrates some of the color facilities of ncurses */

package main

/* Note that is not considered idiomatic Go to import curses this way */
import . "code.google.com/p/goncurses"

func main() {
	stdscr, _ := Init()
	defer End()
	StartColor()

	Raw(true)
	Echo(true)
	InitPair(1, C_BLUE, C_WHITE)
	InitPair(2, C_BLACK, C_CYAN)

	// An example of trying to set an invalid color pair
	err := InitPair(255, C_BLACK, C_CYAN)
	stdscr.Print("An intentional error: %s", err.Error())

	stdscr.Keypad(true)
	stdscr.MovePrint(12, 30, "Hello, World!!!")
	stdscr.Refresh()
	stdscr.GetChar()
	// Note that background doesn't just accept colours but will fill
	// any blank positions with the supplied character too
	stdscr.SetBackground(Character('-' | ColorPair(2)))
	stdscr.ColorOn(1)
	stdscr.MovePrint(13, 30, "Hello, World in Color!!!")
	stdscr.ColorOff(1)
	stdscr.Refresh()
	stdscr.GetChar()
}
<commit_msg>Update color example with changes to Print and other improvements<commit_after>/* This simnple example demonstrates some of the color facilities of ncurses */

package main

/* Note that is not considered idiomatic Go to import curses this way */
import (
	. "code.google.com/p/goncurses"
	"log"
)

func main() {
	stdscr, err := Init()
	if err != nil {
		log.Fatal("init:", err)
	}
	defer End()
	StartColor()

	Raw(true)
	Echo(false)
	InitPair(1, C_BLUE, C_WHITE)
	InitPair(2, C_BLACK, C_CYAN)

	stdscr.Println("Type 'q' to proceed and again to exit")

	// An example of trying to set an invalid color pair
	err = InitPair(255, C_BLACK, C_CYAN)
	stdscr.Println("An intentional error:", err)

	stdscr.Keypad(true)
	stdscr.MovePrint(12, 30, "Hello, World!!!")
	stdscr.Refresh()
	stdscr.GetChar()
	// Note that background doesn't just accept colours but will fill
	// any blank positions with the supplied character too
	stdscr.SetBackground(Character('-' | ColorPair(2)))
	stdscr.ColorOn(1)
	stdscr.MovePrint(13, 30, "Hello, World in Color!!!")
	stdscr.ColorOff(1)
	stdscr.Refresh()
	stdscr.GetChar()
}
<|endoftext|>


<commit_before>package ace

import "html/template"

// Defaults
const (
	defaultExtension  = "ace"
	defaultDelimLeft  = "{{"
	defaultDelimRight = "}}"
)

// Options represents options for the template engine.
type Options struct {
	// Extension represents an extension of files.
	Extension string
	// DelimLeft represents a left delimiter for the html template.
	DelimLeft string
	// DelimRight represents a right delimiter for the html template.
	DelimRight string
	// Cache represents a flag which means whether Ace caches the parsed
	// templates or no.
	Cache bool
	// BaseDir represents a base directory of the Ace templates.
	BaseDir string
	// Asset loads and returns the asset for the given name.
	// If this function is set, Ace load the template data from
	// this function instead of the template files.
	Asset func(name string) ([]byte, error)
	// FuncMap represents a template.FuncMap which is set to
	// the result template.
	FuncMap template.FuncMap
}

// initializeOptions initializes the options
func initializeOptions(opts *Options) *Options {
	if opts == nil {
		opts = &Options{}
	}

	if opts.Extension == "" {
		opts.Extension = defaultExtension
	}

	if opts.DelimLeft == "" {
		opts.DelimLeft = defaultDelimLeft
	}

	if opts.DelimRight == "" {
		opts.DelimRight = defaultDelimRight
	}

	return opts
}
<commit_msg>Update options.go<commit_after>package ace

import "html/template"

// Defaults
const (
	defaultExtension  = "ace"
	defaultDelimLeft  = "{{"
	defaultDelimRight = "}}"
)

// Options represents options for the template engine.
type Options struct {
	// Extension represents an extension of files.
	Extension string
	// DelimLeft represents a left delimiter for the html template.
	DelimLeft string
	// DelimRight represents a right delimiter for the html template.
	DelimRight string
	// Cache represents a flag which means whether Ace caches the parsed
	// templates or not.
	// This option should be true in production.
	Cache bool
	// BaseDir represents a base directory of the Ace templates.
	BaseDir string
	// Asset loads and returns the asset for the given name.
	// If this function is set, Ace load the template data from
	// this function instead of the template files.
	Asset func(name string) ([]byte, error)
	// FuncMap represents a template.FuncMap which is set to
	// the result template.
	FuncMap template.FuncMap
}

// initializeOptions initializes the options
func initializeOptions(opts *Options) *Options {
	if opts == nil {
		opts = &Options{}
	}

	if opts.Extension == "" {
		opts.Extension = defaultExtension
	}

	if opts.DelimLeft == "" {
		opts.DelimLeft = defaultDelimLeft
	}

	if opts.DelimRight == "" {
		opts.DelimRight = defaultDelimRight
	}

	return opts
}
<|endoftext|>


<commit_before>package types

import (
	"errors"
	"io"

	"github.com/tendermint/tendermint/account"
	"github.com/tendermint/tendermint/binary"
	. "github.com/tendermint/tendermint/common"
)

var (
	ErrTxInvalidAddress       = errors.New("Error invalid address")
	ErrTxDuplicateAddress     = errors.New("Error duplicate address")
	ErrTxInvalidAmount        = errors.New("Error invalid amount")
	ErrTxInsufficientFunds    = errors.New("Error insufficient funds")
	ErrTxInsufficientGasPrice = errors.New("Error insufficient gas price")
	ErrTxUnknownPubKey        = errors.New("Error unknown pubkey")
	ErrTxInvalidPubKey        = errors.New("Error invalid pubkey")
	ErrTxInvalidSignature     = errors.New("Error invalid signature")
	ErrTxInvalidString        = errors.New("Error invalid string")
	ErrIncorrectOwner         = errors.New("Error incorrect owner")
)

type ErrTxInvalidSequence struct {
	Got      uint64
	Expected uint64
}

func (e ErrTxInvalidSequence) Error() string {
	return Fmt("Error invalid sequence. Got %d, expected %d", e.Got, e.Expected)
}

/*
Tx (Transaction) is an atomic operation on the ledger state.

Account Txs:
 - SendTx         Send coins to address
 - CallTx         Send a msg to a contract that runs in the vm
 - NameTx	  Store some value under a name in the global namereg

Validation Txs:
 - BondTx         New validator posts a bond
 - UnbondTx       Validator leaves
 - DupeoutTx      Validator dupes out (equivocates)
*/
type Tx interface {
	WriteSignBytes(chainID string, w io.Writer, n *int64, err *error)
}

// Types of Tx implementations
const (
	// Account transactions
	TxTypeSend = byte(0x01)
	TxTypeCall = byte(0x02)
	TxTypeName = byte(0x03)

	// Validation transactions
	TxTypeBond    = byte(0x11)
	TxTypeUnbond  = byte(0x12)
	TxTypeRebond  = byte(0x13)
	TxTypeDupeout = byte(0x14)
)

// for binary.readReflect
var _ = binary.RegisterInterface(
	struct{ Tx }{},
	binary.ConcreteType{&SendTx{}, TxTypeSend},
	binary.ConcreteType{&CallTx{}, TxTypeCall},
	binary.ConcreteType{&NameTx{}, TxTypeName},
	binary.ConcreteType{&BondTx{}, TxTypeBond},
	binary.ConcreteType{&UnbondTx{}, TxTypeUnbond},
	binary.ConcreteType{&RebondTx{}, TxTypeRebond},
	binary.ConcreteType{&DupeoutTx{}, TxTypeDupeout},
)

//-----------------------------------------------------------------------------

type TxInput struct {
	Address   []byte            `json:"address"`   // Hash of the PubKey
	Amount    uint64            `json:"amount"`    // Must not exceed account balance
	Sequence  uint              `json:"sequence"`  // Must be 1 greater than the last committed TxInput
	Signature account.Signature `json:"signature"` // Depends on the PubKey type and the whole Tx
	PubKey    account.PubKey    `json:"pub_key"`   // Must not be nil, may be nil
}

func (txIn *TxInput) ValidateBasic() error {
	if len(txIn.Address) != 20 {
		return ErrTxInvalidAddress
	}
	if txIn.Amount == 0 {
		return ErrTxInvalidAmount
	}
	return nil
}

func (txIn *TxInput) WriteSignBytes(w io.Writer, n *int64, err *error) {
	binary.WriteTo([]byte(Fmt(`{"address":"%X","amount":%v,"sequence":%v}`, txIn.Address, txIn.Amount, txIn.Sequence)), w, n, err)
}

func (txIn *TxInput) String() string {
	return Fmt("TxInput{%X,%v,%v,%v,%v}", txIn.Address, txIn.Amount, txIn.Sequence, txIn.Signature, txIn.PubKey)
}

//-----------------------------------------------------------------------------

type TxOutput struct {
	Address []byte `json:"address"` // Hash of the PubKey
	Amount  uint64 `json:"amount"`  // The sum of all outputs must not exceed the inputs.
}

func (txOut *TxOutput) ValidateBasic() error {
	if len(txOut.Address) != 20 {
		return ErrTxInvalidAddress
	}
	if txOut.Amount == 0 {
		return ErrTxInvalidAmount
	}
	return nil
}

func (txOut *TxOutput) WriteSignBytes(w io.Writer, n *int64, err *error) {
	binary.WriteTo([]byte(Fmt(`{"address":"%X","amount":%v}`, txOut.Address, txOut.Amount)), w, n, err)
}

func (txOut *TxOutput) String() string {
	return Fmt("TxOutput{%X,%v}", txOut.Address, txOut.Amount)
}

//-----------------------------------------------------------------------------

type SendTx struct {
	Inputs  []*TxInput  `json:"inputs"`
	Outputs []*TxOutput `json:"outputs"`
}

func (tx *SendTx) WriteSignBytes(chainID string, w io.Writer, n *int64, err *error) {
	binary.WriteTo([]byte(Fmt(`{"chain_id":"%s"`, chainID)), w, n, err)
	binary.WriteTo([]byte(Fmt(`,"tx":[%v,{"inputs":[`, TxTypeSend)), w, n, err)
	for i, in := range tx.Inputs {
		in.WriteSignBytes(w, n, err)
		if i != len(tx.Inputs)-1 {
			binary.WriteTo([]byte(","), w, n, err)
		}
	}
	binary.WriteTo([]byte(`],"outputs":[`), w, n, err)
	for i, out := range tx.Outputs {
		out.WriteSignBytes(w, n, err)
		if i != len(tx.Outputs)-1 {
			binary.WriteTo([]byte(","), w, n, err)
		}
	}
	binary.WriteTo([]byte(`]}]}`), w, n, err)
}

func (tx *SendTx) String() string {
	return Fmt("SendTx{%v -> %v}", tx.Inputs, tx.Outputs)
}

//-----------------------------------------------------------------------------

type CallTx struct {
	Input    *TxInput `json:"input"`
	Address  []byte   `json:"address"`
	GasLimit uint64   `json:"gas_limit"`
	Fee      uint64   `json:"fee"`
	Data     []byte   `json:"data"`
}

func (tx *CallTx) WriteSignBytes(chainID string, w io.Writer, n *int64, err *error) {
	binary.WriteTo([]byte(Fmt(`{"chain_id":"%s"`, chainID)), w, n, err)
	binary.WriteTo([]byte(Fmt(`,"tx":[%v,{"address":"%X","data":"%X"`, TxTypeCall, tx.Address, tx.Data)), w, n, err)
	binary.WriteTo([]byte(Fmt(`,"fee":%v,"gas_limit":%v,"input":`, tx.Fee, tx.GasLimit)), w, n, err)
	tx.Input.WriteSignBytes(w, n, err)
	binary.WriteTo([]byte(`}]}`), w, n, err)
}

func (tx *CallTx) String() string {
	return Fmt("CallTx{%v -> %x: %x}", tx.Input, tx.Address, tx.Data)
}

//-----------------------------------------------------------------------------

type NameTx struct {
	Input *TxInput `json:"input"`
	Name  string   `json:"name"`
	Data  string   `json:"data"`
	Fee   uint64   `json:"fee"`
}

func (tx *NameTx) WriteSignBytes(chainID string, w io.Writer, n *int64, err *error) {
	binary.WriteTo([]byte(Fmt(`{"chain_id":"%s"`, chainID)), w, n, err)
	binary.WriteTo([]byte(Fmt(`,"tx":[%v,{"name":"%s","data":"%s"`, TxTypeName, tx.Name, tx.Data)), w, n, err)
	binary.WriteTo([]byte(Fmt(`,"fee":%v,"input":`, tx.Fee)), w, n, err)
	tx.Input.WriteSignBytes(w, n, err)
	binary.WriteTo([]byte(`}]}`), w, n, err)
}

func (tx *NameTx) ValidateStrings() error {
	if len(tx.Name) == 0 {
		return errors.New("Name must not be empty")
	}
	if len(tx.Name) > MaxNameLength {
		return errors.New(Fmt("Name is too long. Max %d bytes", MaxNameLength))
	}
	if len(tx.Data) > MaxDataLength {
		return errors.New(Fmt("Data is too long. Max %d bytes", MaxDataLength))
	}

	if !validateNameRegEntryName(tx.Name) {
		return errors.New(Fmt("Invalid characters found in NameTx.Name (%s). Only alphanumeric, underscores, and forward slashes allowed", tx.Name))
	}

	if !validateNameRegEntryData(tx.Data) {
		return errors.New(Fmt("Invalid characters found in NameTx.Data (%s). Only the kind of things found in a JSON file are allowed", tx.Data))
	}

	return nil
}

func (tx *NameTx) BaseEntryCost() uint64 {
	return BaseEntryCost(tx.Name, tx.Data)
}

func (tx *NameTx) String() string {
	return Fmt("NameTx{%v -> %s: %s}", tx.Input, tx.Name, tx.Data)
}

//-----------------------------------------------------------------------------

type BondTx struct {
	PubKey    account.PubKeyEd25519    `json:"pub_key"`
	Signature account.SignatureEd25519 `json:"signature"`
	Inputs    []*TxInput               `json:"inputs"`
	UnbondTo  []*TxOutput              `json:"unbond_to"`
}

func (tx *BondTx) WriteSignBytes(chainID string, w io.Writer, n *int64, err *error) {
	binary.WriteTo([]byte(Fmt(`{"chain_id":"%s"`, chainID)), w, n, err)
	binary.WriteTo([]byte(Fmt(`,"tx":[%v,{"inputs":[`, TxTypeBond)), w, n, err)
	for i, in := range tx.Inputs {
		in.WriteSignBytes(w, n, err)
		if i != len(tx.Inputs)-1 {
			binary.WriteTo([]byte(","), w, n, err)
		}
	}
	binary.WriteTo([]byte(Fmt(`],"pub_key":`)), w, n, err)
	binary.WriteTo(binary.JSONBytes(tx.PubKey), w, n, err)
	binary.WriteTo([]byte(`,"unbond_to":[`), w, n, err)
	for i, out := range tx.UnbondTo {
		out.WriteSignBytes(w, n, err)
		if i != len(tx.UnbondTo)-1 {
			binary.WriteTo([]byte(","), w, n, err)
		}
	}
	binary.WriteTo([]byte(`]}]}`), w, n, err)
}

func (tx *BondTx) String() string {
	return Fmt("BondTx{%v: %v -> %v}", tx.PubKey, tx.Inputs, tx.UnbondTo)
}

//-----------------------------------------------------------------------------

type UnbondTx struct {
	Address   []byte                   `json:"address"`
	Height    uint                     `json:"height"`
	Signature account.SignatureEd25519 `json:"signature"`
}

func (tx *UnbondTx) WriteSignBytes(chainID string, w io.Writer, n *int64, err *error) {
	binary.WriteTo([]byte(Fmt(`{"chain_id":"%s"`, chainID)), w, n, err)
	binary.WriteTo([]byte(Fmt(`,"tx":[%v,{"address":"%X","height":%v}]}`, TxTypeUnbond, tx.Address, tx.Height)), w, n, err)
}

func (tx *UnbondTx) String() string {
	return Fmt("UnbondTx{%X,%v,%v}", tx.Address, tx.Height, tx.Signature)
}

//-----------------------------------------------------------------------------

type RebondTx struct {
	Address   []byte                   `json:"address"`
	Height    uint                     `json:"height"`
	Signature account.SignatureEd25519 `json:"signature"`
}

func (tx *RebondTx) WriteSignBytes(chainID string, w io.Writer, n *int64, err *error) {
	binary.WriteTo([]byte(Fmt(`{"chain_id":"%s"`, chainID)), w, n, err)
	binary.WriteTo([]byte(Fmt(`,"tx":[%v,{"address":"%X","height":%v}]}`, TxTypeRebond, tx.Address, tx.Height)), w, n, err)
}

func (tx *RebondTx) String() string {
	return Fmt("RebondTx{%X,%v,%v}", tx.Address, tx.Height, tx.Signature)
}

//-----------------------------------------------------------------------------

type DupeoutTx struct {
	Address []byte `json:"address"`
	VoteA   Vote   `json:"vote_a"`
	VoteB   Vote   `json:"vote_b"`
}

func (tx *DupeoutTx) WriteSignBytes(chainID string, w io.Writer, n *int64, err *error) {
	panic("DupeoutTx has no sign bytes")
}

func (tx *DupeoutTx) String() string {
	return Fmt("DupeoutTx{%X,%v,%v}", tx.Address, tx.VoteA, tx.VoteB)
}

//-----------------------------------------------------------------------------

func TxId(chainID string, tx Tx) []byte {
	signBytes := account.SignBytes(chainID, tx)
	return binary.BinaryRipemd160(signBytes)
}
<commit_msg>sign-bytes field ordering & escaping of strings<commit_after>package types

import (
	"encoding/json"
	"errors"
	"io"

	"github.com/tendermint/tendermint/account"
	"github.com/tendermint/tendermint/binary"
	. "github.com/tendermint/tendermint/common"
)

var (
	ErrTxInvalidAddress       = errors.New("Error invalid address")
	ErrTxDuplicateAddress     = errors.New("Error duplicate address")
	ErrTxInvalidAmount        = errors.New("Error invalid amount")
	ErrTxInsufficientFunds    = errors.New("Error insufficient funds")
	ErrTxInsufficientGasPrice = errors.New("Error insufficient gas price")
	ErrTxUnknownPubKey        = errors.New("Error unknown pubkey")
	ErrTxInvalidPubKey        = errors.New("Error invalid pubkey")
	ErrTxInvalidSignature     = errors.New("Error invalid signature")
	ErrTxInvalidString        = errors.New("Error invalid string")
	ErrIncorrectOwner         = errors.New("Error incorrect owner")
)

type ErrTxInvalidSequence struct {
	Got      uint64
	Expected uint64
}

func (e ErrTxInvalidSequence) Error() string {
	return Fmt("Error invalid sequence. Got %d, expected %d", e.Got, e.Expected)
}

/*
Tx (Transaction) is an atomic operation on the ledger state.

Account Txs:
 - SendTx         Send coins to address
 - CallTx         Send a msg to a contract that runs in the vm
 - NameTx	  Store some value under a name in the global namereg

Validation Txs:
 - BondTx         New validator posts a bond
 - UnbondTx       Validator leaves
 - DupeoutTx      Validator dupes out (equivocates)
*/
type Tx interface {
	WriteSignBytes(chainID string, w io.Writer, n *int64, err *error)
}

// Types of Tx implementations
const (
	// Account transactions
	TxTypeSend = byte(0x01)
	TxTypeCall = byte(0x02)
	TxTypeName = byte(0x03)

	// Validation transactions
	TxTypeBond    = byte(0x11)
	TxTypeUnbond  = byte(0x12)
	TxTypeRebond  = byte(0x13)
	TxTypeDupeout = byte(0x14)
)

// for binary.readReflect
var _ = binary.RegisterInterface(
	struct{ Tx }{},
	binary.ConcreteType{&SendTx{}, TxTypeSend},
	binary.ConcreteType{&CallTx{}, TxTypeCall},
	binary.ConcreteType{&NameTx{}, TxTypeName},
	binary.ConcreteType{&BondTx{}, TxTypeBond},
	binary.ConcreteType{&UnbondTx{}, TxTypeUnbond},
	binary.ConcreteType{&RebondTx{}, TxTypeRebond},
	binary.ConcreteType{&DupeoutTx{}, TxTypeDupeout},
)

//-----------------------------------------------------------------------------

type TxInput struct {
	Address   []byte            `json:"address"`   // Hash of the PubKey
	Amount    uint64            `json:"amount"`    // Must not exceed account balance
	Sequence  uint              `json:"sequence"`  // Must be 1 greater than the last committed TxInput
	Signature account.Signature `json:"signature"` // Depends on the PubKey type and the whole Tx
	PubKey    account.PubKey    `json:"pub_key"`   // Must not be nil, may be nil
}

func (txIn *TxInput) ValidateBasic() error {
	if len(txIn.Address) != 20 {
		return ErrTxInvalidAddress
	}
	if txIn.Amount == 0 {
		return ErrTxInvalidAmount
	}
	return nil
}

func (txIn *TxInput) WriteSignBytes(w io.Writer, n *int64, err *error) {
	binary.WriteTo([]byte(Fmt(`{"address":"%X","amount":%v,"sequence":%v}`, txIn.Address, txIn.Amount, txIn.Sequence)), w, n, err)
}

func (txIn *TxInput) String() string {
	return Fmt("TxInput{%X,%v,%v,%v,%v}", txIn.Address, txIn.Amount, txIn.Sequence, txIn.Signature, txIn.PubKey)
}

//-----------------------------------------------------------------------------

type TxOutput struct {
	Address []byte `json:"address"` // Hash of the PubKey
	Amount  uint64 `json:"amount"`  // The sum of all outputs must not exceed the inputs.
}

func (txOut *TxOutput) ValidateBasic() error {
	if len(txOut.Address) != 20 {
		return ErrTxInvalidAddress
	}
	if txOut.Amount == 0 {
		return ErrTxInvalidAmount
	}
	return nil
}

func (txOut *TxOutput) WriteSignBytes(w io.Writer, n *int64, err *error) {
	binary.WriteTo([]byte(Fmt(`{"address":"%X","amount":%v}`, txOut.Address, txOut.Amount)), w, n, err)
}

func (txOut *TxOutput) String() string {
	return Fmt("TxOutput{%X,%v}", txOut.Address, txOut.Amount)
}

//-----------------------------------------------------------------------------

type SendTx struct {
	Inputs  []*TxInput  `json:"inputs"`
	Outputs []*TxOutput `json:"outputs"`
}

func (tx *SendTx) WriteSignBytes(chainID string, w io.Writer, n *int64, err *error) {
	binary.WriteTo([]byte(Fmt(`{"chain_id":%s`, jsonEscape(chainID))), w, n, err)
	binary.WriteTo([]byte(Fmt(`,"tx":[%v,{"inputs":[`, TxTypeSend)), w, n, err)
	for i, in := range tx.Inputs {
		in.WriteSignBytes(w, n, err)
		if i != len(tx.Inputs)-1 {
			binary.WriteTo([]byte(","), w, n, err)
		}
	}
	binary.WriteTo([]byte(`],"outputs":[`), w, n, err)
	for i, out := range tx.Outputs {
		out.WriteSignBytes(w, n, err)
		if i != len(tx.Outputs)-1 {
			binary.WriteTo([]byte(","), w, n, err)
		}
	}
	binary.WriteTo([]byte(`]}]}`), w, n, err)
}

func (tx *SendTx) String() string {
	return Fmt("SendTx{%v -> %v}", tx.Inputs, tx.Outputs)
}

//-----------------------------------------------------------------------------

type CallTx struct {
	Input    *TxInput `json:"input"`
	Address  []byte   `json:"address"`
	GasLimit uint64   `json:"gas_limit"`
	Fee      uint64   `json:"fee"`
	Data     []byte   `json:"data"`
}

func (tx *CallTx) WriteSignBytes(chainID string, w io.Writer, n *int64, err *error) {
	binary.WriteTo([]byte(Fmt(`{"chain_id":%s`, jsonEscape(chainID))), w, n, err)
	binary.WriteTo([]byte(Fmt(`,"tx":[%v,{"address":"%X","data":"%X"`, TxTypeCall, tx.Address, tx.Data)), w, n, err)
	binary.WriteTo([]byte(Fmt(`,"fee":%v,"gas_limit":%v,"input":`, tx.Fee, tx.GasLimit)), w, n, err)
	tx.Input.WriteSignBytes(w, n, err)
	binary.WriteTo([]byte(`}]}`), w, n, err)
}

func (tx *CallTx) String() string {
	return Fmt("CallTx{%v -> %x: %x}", tx.Input, tx.Address, tx.Data)
}

//-----------------------------------------------------------------------------

type NameTx struct {
	Input *TxInput `json:"input"`
	Name  string   `json:"name"`
	Data  string   `json:"data"`
	Fee   uint64   `json:"fee"`
}

func (tx *NameTx) WriteSignBytes(chainID string, w io.Writer, n *int64, err *error) {
	binary.WriteTo([]byte(Fmt(`{"chain_id":%s`, jsonEscape(chainID))), w, n, err)
	binary.WriteTo([]byte(Fmt(`,"tx":[%v,{"data":%s,"fee":%v`, TxTypeName, jsonEscape(tx.Data), tx.Fee)), w, n, err)
	binary.WriteTo([]byte(Fmt(`,"input":`, tx.Input)), w, n, err)
	tx.Input.WriteSignBytes(w, n, err)
	binary.WriteTo([]byte(Fmt(`,"name":%s`, jsonEscape(tx.Name))), w, n, err)
	binary.WriteTo([]byte(`}]}`), w, n, err)
}

func (tx *NameTx) ValidateStrings() error {
	if len(tx.Name) == 0 {
		return errors.New("Name must not be empty")
	}
	if len(tx.Name) > MaxNameLength {
		return errors.New(Fmt("Name is too long. Max %d bytes", MaxNameLength))
	}
	if len(tx.Data) > MaxDataLength {
		return errors.New(Fmt("Data is too long. Max %d bytes", MaxDataLength))
	}

	if !validateNameRegEntryName(tx.Name) {
		return errors.New(Fmt("Invalid characters found in NameTx.Name (%s). Only alphanumeric, underscores, and forward slashes allowed", tx.Name))
	}

	if !validateNameRegEntryData(tx.Data) {
		return errors.New(Fmt("Invalid characters found in NameTx.Data (%s). Only the kind of things found in a JSON file are allowed", tx.Data))
	}

	return nil
}

func (tx *NameTx) BaseEntryCost() uint64 {
	return BaseEntryCost(tx.Name, tx.Data)
}

func (tx *NameTx) String() string {
	return Fmt("NameTx{%v -> %s: %s}", tx.Input, tx.Name, tx.Data)
}

//-----------------------------------------------------------------------------

type BondTx struct {
	PubKey    account.PubKeyEd25519    `json:"pub_key"`
	Signature account.SignatureEd25519 `json:"signature"`
	Inputs    []*TxInput               `json:"inputs"`
	UnbondTo  []*TxOutput              `json:"unbond_to"`
}

func (tx *BondTx) WriteSignBytes(chainID string, w io.Writer, n *int64, err *error) {
	binary.WriteTo([]byte(Fmt(`{"chain_id":%s`, jsonEscape(chainID))), w, n, err)
	binary.WriteTo([]byte(Fmt(`,"tx":[%v,{"inputs":[`, TxTypeBond)), w, n, err)
	for i, in := range tx.Inputs {
		in.WriteSignBytes(w, n, err)
		if i != len(tx.Inputs)-1 {
			binary.WriteTo([]byte(","), w, n, err)
		}
	}
	binary.WriteTo([]byte(Fmt(`],"pub_key":`)), w, n, err)
	binary.WriteTo(binary.JSONBytes(tx.PubKey), w, n, err)
	binary.WriteTo([]byte(`,"unbond_to":[`), w, n, err)
	for i, out := range tx.UnbondTo {
		out.WriteSignBytes(w, n, err)
		if i != len(tx.UnbondTo)-1 {
			binary.WriteTo([]byte(","), w, n, err)
		}
	}
	binary.WriteTo([]byte(`]}]}`), w, n, err)
}

func (tx *BondTx) String() string {
	return Fmt("BondTx{%v: %v -> %v}", tx.PubKey, tx.Inputs, tx.UnbondTo)
}

//-----------------------------------------------------------------------------

type UnbondTx struct {
	Address   []byte                   `json:"address"`
	Height    uint                     `json:"height"`
	Signature account.SignatureEd25519 `json:"signature"`
}

func (tx *UnbondTx) WriteSignBytes(chainID string, w io.Writer, n *int64, err *error) {
	binary.WriteTo([]byte(Fmt(`{"chain_id":%s`, jsonEscape(chainID))), w, n, err)
	binary.WriteTo([]byte(Fmt(`,"tx":[%v,{"address":"%X","height":%v}]}`, TxTypeUnbond, tx.Address, tx.Height)), w, n, err)
}

func (tx *UnbondTx) String() string {
	return Fmt("UnbondTx{%X,%v,%v}", tx.Address, tx.Height, tx.Signature)
}

//-----------------------------------------------------------------------------

type RebondTx struct {
	Address   []byte                   `json:"address"`
	Height    uint                     `json:"height"`
	Signature account.SignatureEd25519 `json:"signature"`
}

func (tx *RebondTx) WriteSignBytes(chainID string, w io.Writer, n *int64, err *error) {
	binary.WriteTo([]byte(Fmt(`{"chain_id":%s`, jsonEscape(chainID))), w, n, err)
	binary.WriteTo([]byte(Fmt(`,"tx":[%v,{"address":"%X","height":%v}]}`, TxTypeRebond, tx.Address, tx.Height)), w, n, err)
}

func (tx *RebondTx) String() string {
	return Fmt("RebondTx{%X,%v,%v}", tx.Address, tx.Height, tx.Signature)
}

//-----------------------------------------------------------------------------

type DupeoutTx struct {
	Address []byte `json:"address"`
	VoteA   Vote   `json:"vote_a"`
	VoteB   Vote   `json:"vote_b"`
}

func (tx *DupeoutTx) WriteSignBytes(chainID string, w io.Writer, n *int64, err *error) {
	panic("DupeoutTx has no sign bytes")
}

func (tx *DupeoutTx) String() string {
	return Fmt("DupeoutTx{%X,%v,%v}", tx.Address, tx.VoteA, tx.VoteB)
}

//-----------------------------------------------------------------------------

func TxId(chainID string, tx Tx) []byte {
	signBytes := account.SignBytes(chainID, tx)
	return binary.BinaryRipemd160(signBytes)
}

//--------------------------------------------------------------------------------

// Contract: This function is deterministic and completely reversible.
func jsonEscape(str string) string {
	escapedBytes, err := json.Marshal(str)
	if err != nil {
		panic(Fmt("Error json-escaping a string", str))
	}
	return string(escapedBytes)
}
<|endoftext|>


<commit_before>package ui

import (
	"errors"
	"fmt"
	"log"
	"os"
	"regexp"
	"strconv"
	"strings"

	"github.com/acarl005/stripansi"
	"github.com/errata-ai/vale/config"
	"github.com/mattn/go-colorable"
)

type valeError struct {
	line int
	path string
	text string
	code string
}

var logger = log.New(os.Stderr, "", 0)
var header = regexp.MustCompile(`(\w+) .+ \[(.+):(\d+):(\d+)\]`)

func init() {
	// https://github.com/logrusorgru/aurora/issues/2#issuecomment-299014211
	logger.SetOutput(colorable.NewColorableStderr())
}

func parseError(err error) (valeError, error) {
	var parsed valeError

	plain := stripansi.Strip(err.Error())
	lines := strings.Split(plain, "\n\n")

	if len(lines) < 3 {
		return parsed, errors.New("missing body")
	} else if !header.MatchString(lines[0]) {
		return parsed, errors.New("missing header")
	}

	groups := header.FindStringSubmatch(lines[0])

	parsed.code = groups[1]
	parsed.path = groups[2]

	i, err := strconv.Atoi(groups[3])
	if err != nil {
		return parsed, errors.New("missing line")
	}

	parsed.line = i
	parsed.text = lines[len(lines)-2]

	return parsed, nil
}

// ShowError displays the given error in the user-specified format.
func ShowError(err error, config *config.Config) {
	parsed, failed := parseError(err)
	switch config.Output {
	case "JSON":
		var data interface{}

		if failed != nil {
			data = struct {
				Code string
				Text string
			}{
				Text: err.Error(),
				Code: "E100",
			}
		} else {
			data = struct {
				Line int
				Path string
				Text string
				Code string
			}{
				Line: parsed.line,
				Path: parsed.path,
				Text: parsed.text,
				Code: parsed.code,
			}
		}

		logger.Fatalln(getJSON(data))
	case "line":
		var data string

		if failed != nil {
			data = err.Error()
		} else {
			data = fmt.Sprintf("%s:%d:%s:%s",
				parsed.path, parsed.line, parsed.code, parsed.text)
		}

		logger.Fatalln(data)
	default:
		logger.Fatalln(err)
	}
}
<commit_msg>refactor: include error column in JSON output<commit_after>package ui

import (
	"errors"
	"fmt"
	"log"
	"os"
	"regexp"
	"strconv"
	"strings"

	"github.com/acarl005/stripansi"
	"github.com/errata-ai/vale/config"
	"github.com/mattn/go-colorable"
)

type valeError struct {
	line int
	path string
	text string
	code string
	span int
}

var logger = log.New(os.Stderr, "", 0)
var header = regexp.MustCompile(`(\w+) .+ \[(.+):(\d+):(\d+)\]`)

func init() {
	// https://github.com/logrusorgru/aurora/issues/2#issuecomment-299014211
	logger.SetOutput(colorable.NewColorableStderr())
}

func parseError(err error) (valeError, error) {
	var parsed valeError

	plain := stripansi.Strip(err.Error())
	lines := strings.Split(plain, "\n\n")

	if len(lines) < 3 {
		return parsed, errors.New("missing body")
	} else if !header.MatchString(lines[0]) {
		return parsed, errors.New("missing header")
	}

	groups := header.FindStringSubmatch(lines[0])

	parsed.code = groups[1]
	parsed.path = groups[2]

	i, err := strconv.Atoi(groups[3])
	if err != nil {
		return parsed, errors.New("missing line")
	}
	parsed.line = i

	i, err = strconv.Atoi(groups[4])
	if err != nil {
		return parsed, errors.New("missing span")
	}
	parsed.span = i

	parsed.text = lines[len(lines)-2]
	return parsed, nil
}

// ShowError displays the given error in the user-specified format.
func ShowError(err error, config *config.Config) {
	parsed, failed := parseError(err)
	switch config.Output {
	case "JSON":
		var data interface{}

		if failed != nil {
			data = struct {
				Code string
				Text string
			}{
				Text: err.Error(),
				Code: "E100",
			}
		} else {
			data = struct {
				Line int
				Path string
				Text string
				Code string
				Span int
			}{
				Line: parsed.line,
				Path: parsed.path,
				Text: parsed.text,
				Code: parsed.code,
				Span: parsed.span,
			}
		}

		logger.Fatalln(getJSON(data))
	case "line":
		var data string

		if failed != nil {
			data = err.Error()
		} else {
			data = fmt.Sprintf("%s:%d:%s:%s",
				parsed.path, parsed.line, parsed.code, parsed.text)
		}

		logger.Fatalln(data)
	default:
		logger.Fatalln(err)
	}
}
<|endoftext|>


<commit_before>// Package tokenizer implements file tokenization used by the enry file
// classifier. This package is an implementation detail of enry and should not
// be imported by other packages.
package tokenizer

import (
	"bytes"

	"gopkg.in/src-d/enry.v1/regex"
)

const byteLimit = 100000

// Tokenize returns classification tokens from content. The tokens returned
// should match what the Linguist library returns. At most the first 100KB of
// content are tokenized.
func Tokenize(content []byte) []string {
	if len(content) > byteLimit {
		content = content[:byteLimit]
	}

	// Copy the input so that changes wrought by the tokenization steps do not
	// modify the caller's copy of the input. See #196.
	content = append([]byte(nil), content...)

	tokens := make([][]byte, 0, 50)
	for _, extract := range extractTokens {
		var extractedTokens [][]byte
		content, extractedTokens = extract(content)
		tokens = append(tokens, extractedTokens...)
	}

	return toString(tokens)
}

func toString(tokens [][]byte) []string {
	stokens := make([]string, 0, len(tokens))
	for _, token := range tokens {
		stokens = append(stokens, string(token))
	}

	return stokens
}

var (
	extractTokens = []func(content []byte) (replacedContent []byte, tokens [][]byte){
		// The order to must be this
		extractAndReplaceShebang,
		extractAndReplaceSGML,
		skipCommentsAndLiterals,
		extractAndReplacePunctuation,
		extractAndReplaceRegular,
		extractAndReplaceOperator,
		extractRemainders,
	}

	// Differences between golang regexp and oniguruma:
	// 1. no (?s) in oniguruma - makes dot match \n
	// 2. no (?U) in oniguruma - ungreedy *
	// 3. (?m) implies dot matches \n in oniguruma
	// 4. oniguruma handles \w differently - impossible, but true
	//
	// Workarounds:
	// 1. (.|\n)
	// 2. replace * with *?
	// 3. replace . with [^\n]
	// 4. replace \w with [0-9A-Za-z_]
	//
	// Original golang regexps:
	//
	// reLiteralStringQuotes = regexp.MustCompile(`(?sU)(".*"|'.*')`)
	// reSingleLineComment   = regexp.MustCompile(`(?m)(//|--|#|%|")\s(.*$)`)
	// reMultilineComment    = regexp.MustCompile(`(?sU)(/\*.*\*/|<!--.*-->|\{-.*-\}|\(\*.*\*\)|""".*"""|'''.*''')`)
	// reLiteralNumber       = regexp.MustCompile(`(0x[0-9A-Fa-f]([0-9A-Fa-f]|\.)*|\d(\d|\.)*)([uU][lL]{0,2}|([eE][-+]\d*)?[fFlL]*)`)
	// reShebang             = regexp.MustCompile(`(?m)^#!(?:/\w+)*/(?:(\w+)|\w+(?:\s*\w+=\w+\s*)*\s*(\w+))(?:\s*-\w+\s*)*$`)
	// rePunctuation         = regexp.MustCompile(`;|\{|\}|\(|\)|\[|\]`)
	// reSGML                = regexp.MustCompile(`(?sU)(<\/?[^\s<>=\d"']+)(?:\s.*\/?>|>)`)
	// reSGMLComment         = regexp.MustCompile(`(?sU)(<!--.*-->)`)
	// reSGMLAttributes      = regexp.MustCompile(`\s+(\w+=)|\s+([^\s>]+)`)
	// reSGMLLoneAttribute   = regexp.MustCompile(`(\w+)`)
	// reRegularToken        = regexp.MustCompile(`[\w\.@#\/\*]+`)
	// reOperators           = regexp.MustCompile(`<<?|\+|\-|\*|\/|%|&&?|\|\|?`)
	//
	// These regexps were converted to work in the same way for both engines:
	//
	reLiteralStringQuotes = regex.MustCompile(`("(.|\n)*?"|'(.|\n)*?')`)
	reSingleLineComment   = regex.MustCompile(`(?m)(//|--|#|%|")\s([^\n]*$)`)
	reMultilineComment    = regex.MustCompile(`(/\*(.|\n)*?\*/|<!--(.|\n)*?-->|\{-(.|\n)*?-\}|\(\*(.|\n)*?\*\)|"""(.|\n)*?"""|'''(.|\n)*?''')`)
	reLiteralNumber       = regex.MustCompile(`(0x[0-9A-Fa-f]([0-9A-Fa-f]|\.)*|\d(\d|\.)*)([uU][lL]{0,2}|([eE][-+]\d*)?[fFlL]*)`)
	reShebang             = regex.MustCompile(`(?m)^#!(?:/[0-9A-Za-z_]+)*/(?:([0-9A-Za-z_]+)|[0-9A-Za-z_]+(?:\s*[0-9A-Za-z_]+=[0-9A-Za-z_]+\s*)*\s*([0-9A-Za-z_]+))(?:\s*-[0-9A-Za-z_]+\s*)*$`)
	rePunctuation         = regex.MustCompile(`;|\{|\}|\(|\)|\[|\]`)
	reSGML                = regex.MustCompile(`(<\/?[^\s<>=\d"']+)(?:\s(.|\n)*?\/?>|>)`)
	reSGMLComment         = regex.MustCompile(`(<!--(.|\n)*?-->)`)
	reSGMLAttributes      = regex.MustCompile(`\s+([0-9A-Za-z_]+=)|\s+([^\s>]+)`)
	reSGMLLoneAttribute   = regex.MustCompile(`([0-9A-Za-z_]+)`)
	reRegularToken        = regex.MustCompile(`[0-9A-Za-z_\.@#\/\*]+`)
	reOperators           = regex.MustCompile(`<<?|\+|\-|\*|\/|%|&&?|\|\|?`)

	regexToSkip = []regex.EnryRegexp{
		// The order must be this
		reLiteralStringQuotes,
		reMultilineComment,
		reSingleLineComment,
		reLiteralNumber,
	}
)

func extractAndReplaceShebang(content []byte) ([]byte, [][]byte) {
	var shebangTokens [][]byte
	matches := reShebang.FindAllSubmatch(content, -1)
	if matches != nil {
		shebangTokens = make([][]byte, 0, 2)
		for _, match := range matches {
			shebangToken := getShebangToken(match)
			shebangTokens = append(shebangTokens, shebangToken)
		}

		reShebang.ReplaceAll(content, []byte(` `))
	}

	return content, shebangTokens
}

func getShebangToken(matchedShebang [][]byte) []byte {
	const prefix = `SHEBANG#!`
	var token []byte
	for i := 1; i < len(matchedShebang); i++ {
		if len(matchedShebang[i]) > 0 {
			token = matchedShebang[i]
			break
		}
	}

	tokenShebang := append([]byte(prefix), token...)
	return tokenShebang
}

func commonExtractAndReplace(content []byte, re regex.EnryRegexp) ([]byte, [][]byte) {
	tokens := re.FindAll(content, -1)
	content = re.ReplaceAll(content, []byte(` `))
	return content, tokens
}

func extractAndReplacePunctuation(content []byte) ([]byte, [][]byte) {
	return commonExtractAndReplace(content, rePunctuation)
}

func extractAndReplaceRegular(content []byte) ([]byte, [][]byte) {
	return commonExtractAndReplace(content, reRegularToken)
}

func extractAndReplaceOperator(content []byte) ([]byte, [][]byte) {
	return commonExtractAndReplace(content, reOperators)
}

func extractAndReplaceSGML(content []byte) ([]byte, [][]byte) {
	var SGMLTokens [][]byte
	matches := reSGML.FindAllSubmatch(content, -1)
	if matches != nil {
		SGMLTokens = make([][]byte, 0, 2)
		for _, match := range matches {
			if reSGMLComment.Match(match[0]) {
				continue
			}

			token := append(match[1], '>')
			SGMLTokens = append(SGMLTokens, token)
			attributes := getSGMLAttributes(match[0])
			SGMLTokens = append(SGMLTokens, attributes...)
		}

		content = reSGML.ReplaceAll(content, []byte(` `))
	}

	return content, SGMLTokens
}

func getSGMLAttributes(SGMLTag []byte) [][]byte {
	var attributes [][]byte
	matches := reSGMLAttributes.FindAllSubmatch(SGMLTag, -1)
	if matches != nil {
		attributes = make([][]byte, 0, 5)
		for _, match := range matches {
			if len(match[1]) != 0 {
				attributes = append(attributes, match[1])
			}

			if len(match[2]) != 0 {
				loneAttributes := reSGMLLoneAttribute.FindAll(match[2], -1)
				attributes = append(attributes, loneAttributes...)
			}
		}
	}

	return attributes
}

func skipCommentsAndLiterals(content []byte) ([]byte, [][]byte) {
	for _, skip := range regexToSkip {
		content = skip.ReplaceAll(content, []byte(` `))
	}

	return content, nil
}

func extractRemainders(content []byte) ([]byte, [][]byte) {
	splitted := bytes.Fields(content)
	remainderTokens := make([][]byte, 0, len(splitted)*3)
	for _, remainder := range splitted {
		remainders := bytes.Split(remainder, nil)
		remainderTokens = append(remainderTokens, remainders...)
	}

	return content, remainderTokens
}
<commit_msg>Apply suggestions from review.<commit_after>// Package tokenizer implements file tokenization used by the enry file
// classifier. This package is an implementation detail of enry and should not
// be imported by other packages.
package tokenizer

import (
	"bytes"

	"gopkg.in/src-d/enry.v1/regex"
)

const byteLimit = 100000

// Tokenize returns language-agnostic lexical tokens from content. The tokens
// returned should match what the Linguist library returns. At most the first
// 100KB of content are tokenized.
func Tokenize(content []byte) []string {
	if len(content) > byteLimit {
		content = content[:byteLimit]
	}

	// Copy the input so that changes wrought by the tokenization steps do not
	// modify the caller's copy of the input. See #196.
	content = append([]byte(nil), content...)

	tokens := make([][]byte, 0, 50)
	for _, extract := range extractTokens {
		var extractedTokens [][]byte
		content, extractedTokens = extract(content)
		tokens = append(tokens, extractedTokens...)
	}

	return toString(tokens)
}

func toString(tokens [][]byte) []string {
	stokens := make([]string, 0, len(tokens))
	for _, token := range tokens {
		stokens = append(stokens, string(token))
	}

	return stokens
}

var (
	extractTokens = []func(content []byte) (replacedContent []byte, tokens [][]byte){
		// The order to must be this
		extractAndReplaceShebang,
		extractAndReplaceSGML,
		skipCommentsAndLiterals,
		extractAndReplacePunctuation,
		extractAndReplaceRegular,
		extractAndReplaceOperator,
		extractRemainders,
	}

	// Differences between golang regexp and oniguruma:
	// 1. no (?s) in oniguruma - makes dot match \n
	// 2. no (?U) in oniguruma - ungreedy *
	// 3. (?m) implies dot matches \n in oniguruma
	// 4. oniguruma handles \w differently - impossible, but true
	//
	// Workarounds:
	// 1. (.|\n)
	// 2. replace * with *?
	// 3. replace . with [^\n]
	// 4. replace \w with [0-9A-Za-z_]
	//
	// Original golang regexps:
	//
	// reLiteralStringQuotes = regexp.MustCompile(`(?sU)(".*"|'.*')`)
	// reSingleLineComment   = regexp.MustCompile(`(?m)(//|--|#|%|")\s(.*$)`)
	// reMultilineComment    = regexp.MustCompile(`(?sU)(/\*.*\*/|<!--.*-->|\{-.*-\}|\(\*.*\*\)|""".*"""|'''.*''')`)
	// reLiteralNumber       = regexp.MustCompile(`(0x[0-9A-Fa-f]([0-9A-Fa-f]|\.)*|\d(\d|\.)*)([uU][lL]{0,2}|([eE][-+]\d*)?[fFlL]*)`)
	// reShebang             = regexp.MustCompile(`(?m)^#!(?:/\w+)*/(?:(\w+)|\w+(?:\s*\w+=\w+\s*)*\s*(\w+))(?:\s*-\w+\s*)*$`)
	// rePunctuation         = regexp.MustCompile(`;|\{|\}|\(|\)|\[|\]`)
	// reSGML                = regexp.MustCompile(`(?sU)(<\/?[^\s<>=\d"']+)(?:\s.*\/?>|>)`)
	// reSGMLComment         = regexp.MustCompile(`(?sU)(<!--.*-->)`)
	// reSGMLAttributes      = regexp.MustCompile(`\s+(\w+=)|\s+([^\s>]+)`)
	// reSGMLLoneAttribute   = regexp.MustCompile(`(\w+)`)
	// reRegularToken        = regexp.MustCompile(`[\w\.@#\/\*]+`)
	// reOperators           = regexp.MustCompile(`<<?|\+|\-|\*|\/|%|&&?|\|\|?`)
	//
	// These regexps were converted to work in the same way for both engines:
	//
	reLiteralStringQuotes = regex.MustCompile(`("(.|\n)*?"|'(.|\n)*?')`)
	reSingleLineComment   = regex.MustCompile(`(?m)(//|--|#|%|")\s([^\n]*$)`)
	reMultilineComment    = regex.MustCompile(`(/\*(.|\n)*?\*/|<!--(.|\n)*?-->|\{-(.|\n)*?-\}|\(\*(.|\n)*?\*\)|"""(.|\n)*?"""|'''(.|\n)*?''')`)
	reLiteralNumber       = regex.MustCompile(`(0x[0-9A-Fa-f]([0-9A-Fa-f]|\.)*|\d(\d|\.)*)([uU][lL]{0,2}|([eE][-+]\d*)?[fFlL]*)`)
	reShebang             = regex.MustCompile(`(?m)^#!(?:/[0-9A-Za-z_]+)*/(?:([0-9A-Za-z_]+)|[0-9A-Za-z_]+(?:\s*[0-9A-Za-z_]+=[0-9A-Za-z_]+\s*)*\s*([0-9A-Za-z_]+))(?:\s*-[0-9A-Za-z_]+\s*)*$`)
	rePunctuation         = regex.MustCompile(`;|\{|\}|\(|\)|\[|\]`)
	reSGML                = regex.MustCompile(`(<\/?[^\s<>=\d"']+)(?:\s(.|\n)*?\/?>|>)`)
	reSGMLComment         = regex.MustCompile(`(<!--(.|\n)*?-->)`)
	reSGMLAttributes      = regex.MustCompile(`\s+([0-9A-Za-z_]+=)|\s+([^\s>]+)`)
	reSGMLLoneAttribute   = regex.MustCompile(`([0-9A-Za-z_]+)`)
	reRegularToken        = regex.MustCompile(`[0-9A-Za-z_\.@#\/\*]+`)
	reOperators           = regex.MustCompile(`<<?|\+|\-|\*|\/|%|&&?|\|\|?`)

	regexToSkip = []regex.EnryRegexp{
		// The order must be this
		reLiteralStringQuotes,
		reMultilineComment,
		reSingleLineComment,
		reLiteralNumber,
	}
)

func extractAndReplaceShebang(content []byte) ([]byte, [][]byte) {
	var shebangTokens [][]byte
	matches := reShebang.FindAllSubmatch(content, -1)
	if matches != nil {
		shebangTokens = make([][]byte, 0, 2)
		for _, match := range matches {
			shebangToken := getShebangToken(match)
			shebangTokens = append(shebangTokens, shebangToken)
		}

		reShebang.ReplaceAll(content, []byte(` `))
	}

	return content, shebangTokens
}

func getShebangToken(matchedShebang [][]byte) []byte {
	const prefix = `SHEBANG#!`
	var token []byte
	for i := 1; i < len(matchedShebang); i++ {
		if len(matchedShebang[i]) > 0 {
			token = matchedShebang[i]
			break
		}
	}

	tokenShebang := append([]byte(prefix), token...)
	return tokenShebang
}

func commonExtractAndReplace(content []byte, re regex.EnryRegexp) ([]byte, [][]byte) {
	tokens := re.FindAll(content, -1)
	content = re.ReplaceAll(content, []byte(` `))
	return content, tokens
}

func extractAndReplacePunctuation(content []byte) ([]byte, [][]byte) {
	return commonExtractAndReplace(content, rePunctuation)
}

func extractAndReplaceRegular(content []byte) ([]byte, [][]byte) {
	return commonExtractAndReplace(content, reRegularToken)
}

func extractAndReplaceOperator(content []byte) ([]byte, [][]byte) {
	return commonExtractAndReplace(content, reOperators)
}

func extractAndReplaceSGML(content []byte) ([]byte, [][]byte) {
	var SGMLTokens [][]byte
	matches := reSGML.FindAllSubmatch(content, -1)
	if matches != nil {
		SGMLTokens = make([][]byte, 0, 2)
		for _, match := range matches {
			if reSGMLComment.Match(match[0]) {
				continue
			}

			token := append(match[1], '>')
			SGMLTokens = append(SGMLTokens, token)
			attributes := getSGMLAttributes(match[0])
			SGMLTokens = append(SGMLTokens, attributes...)
		}

		content = reSGML.ReplaceAll(content, []byte(` `))
	}

	return content, SGMLTokens
}

func getSGMLAttributes(SGMLTag []byte) [][]byte {
	var attributes [][]byte
	matches := reSGMLAttributes.FindAllSubmatch(SGMLTag, -1)
	if matches != nil {
		attributes = make([][]byte, 0, 5)
		for _, match := range matches {
			if len(match[1]) != 0 {
				attributes = append(attributes, match[1])
			}

			if len(match[2]) != 0 {
				loneAttributes := reSGMLLoneAttribute.FindAll(match[2], -1)
				attributes = append(attributes, loneAttributes...)
			}
		}
	}

	return attributes
}

func skipCommentsAndLiterals(content []byte) ([]byte, [][]byte) {
	for _, skip := range regexToSkip {
		content = skip.ReplaceAll(content, []byte(` `))
	}

	return content, nil
}

func extractRemainders(content []byte) ([]byte, [][]byte) {
	splitted := bytes.Fields(content)
	remainderTokens := make([][]byte, 0, len(splitted)*3)
	for _, remainder := range splitted {
		remainders := bytes.Split(remainder, nil)
		remainderTokens = append(remainderTokens, remainders...)
	}

	return content, remainderTokens
}
<|endoftext|>


<commit_before>package main_test

import (
	"fmt"
	"net/url"
	"os/exec"
	"strings"

	"github.com/cloudfoundry-incubator/auctioneer"
	"github.com/cloudfoundry-incubator/bbs"
	bbstestrunner "github.com/cloudfoundry-incubator/bbs/cmd/bbs/testrunner"
	"github.com/cloudfoundry-incubator/consuladapter"
	"github.com/cloudfoundry-incubator/consuladapter/consulrunner"
	"github.com/cloudfoundry-incubator/locket"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	"github.com/cloudfoundry/storeadapter"
	"github.com/cloudfoundry/storeadapter/storerunner/etcdstorerunner"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/pivotal-golang/clock"
	"github.com/pivotal-golang/lager"
	"github.com/pivotal-golang/lager/lagertest"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"

	"testing"
	"time"
)

var auctioneerProcess ifrit.Process

var auctioneerPath string

var dotNetStack = "dot-net"
var dotNetRootFSURL = models.PreloadedRootFS(dotNetStack)
var linuxStack = "linux"
var linuxRootFSURL = models.PreloadedRootFS(linuxStack)
var dotNetCell, linuxCell *FakeCell

var auctioneerServerPort int
var auctioneerAddress string
var runner *ginkgomon.Runner

var etcdPort int
var etcdRunner *etcdstorerunner.ETCDClusterRunner
var etcdClient storeadapter.StoreAdapter

var consulRunner *consulrunner.ClusterRunner
var consulSession *consuladapter.Session

var auctioneerClient auctioneer.Client

var bbsArgs bbstestrunner.Args
var bbsBinPath string
var bbsURL *url.URL
var bbsRunner *ginkgomon.Runner
var bbsProcess ifrit.Process
var bbsClient bbs.Client

var locketClient locket.Client
var logger lager.Logger

func TestAuctioneer(t *testing.T) {
	// these integration tests can take a bit, especially under load;
	// 1 second is too harsh
	SetDefaultEventuallyTimeout(10 * time.Second)

	RegisterFailHandler(Fail)
	RunSpecs(t, "Auctioneer Cmd Suite")
}

var _ = SynchronizedBeforeSuite(func() []byte {
	bbsConfig, err := gexec.Build("github.com/cloudfoundry-incubator/bbs/cmd/bbs", "-race")
	Expect(err).NotTo(HaveOccurred())

	compiledAuctioneerPath, err := gexec.Build("github.com/cloudfoundry-incubator/auctioneer/cmd/auctioneer", "-race")
	Expect(err).NotTo(HaveOccurred())
	return []byte(strings.Join([]string{compiledAuctioneerPath, bbsConfig}, ","))
}, func(pathsByte []byte) {
	path := string(pathsByte)
	compiledAuctioneerPath := strings.Split(path, ",")[0]
	bbsBinPath = strings.Split(path, ",")[1]

	bbsBinPath = strings.Split(path, ",")[1]
	auctioneerPath = string(compiledAuctioneerPath)

	auctioneerServerPort = 1800 + GinkgoParallelNode()
	auctioneerAddress = fmt.Sprintf("http://127.0.0.1:%d", auctioneerServerPort)

	etcdPort = 5001 + GinkgoParallelNode()
	etcdRunner = etcdstorerunner.NewETCDClusterRunner(etcdPort, 1, nil)
	etcdClient = etcdRunner.Adapter(nil)

	consulRunner = consulrunner.NewClusterRunner(
		9001+config.GinkgoConfig.ParallelNode*consulrunner.PortOffsetLength,
		1,
		"http",
	)

	auctioneerClient = auctioneer.NewClient(auctioneerAddress)

	logger = lagertest.NewTestLogger("test")

	consulRunner.Start()
	consulRunner.WaitUntilReady()

	bbsAddress := fmt.Sprintf("127.0.0.1:%d", 13000+GinkgoParallelNode())

	bbsURL = &url.URL{
		Scheme: "http",
		Host:   bbsAddress,
	}

	bbsClient = bbs.NewClient(bbsURL.String())

	etcdUrl := fmt.Sprintf("http://127.0.0.1:%d", etcdPort)
	bbsArgs = bbstestrunner.Args{
		Address:           bbsAddress,
		AdvertiseURL:      bbsURL.String(),
		AuctioneerAddress: auctioneerAddress,
		EtcdCluster:       etcdUrl,
		ConsulCluster:     consulRunner.ConsulCluster(),
	}
})

var _ = BeforeEach(func() {
	consulRunner.Reset()
	etcdRunner.Start()

	bbsRunner = bbstestrunner.New(bbsBinPath, bbsArgs)
	bbsProcess = ginkgomon.Invoke(bbsRunner)

	consulSession = consulRunner.NewSession("a-session")

	locketClient = locket.NewClient(consulSession, clock.NewClock(), logger)

	runner = ginkgomon.New(ginkgomon.Config{
		Name: "auctioneer",
		Command: exec.Command(
			auctioneerPath,
			"-bbsAddress", bbsURL.String(),
			"-listenAddr", fmt.Sprintf("0.0.0.0:%d", auctioneerServerPort),
			"-lockRetryInterval", "1s",
			"-consulCluster", consulRunner.ConsulCluster(),
		),
		StartCheck: "auctioneer.started",
	})

	dotNetCell = SpinUpFakeCell(locketClient, "dot-net-cell", dotNetStack)
	linuxCell = SpinUpFakeCell(locketClient, "linux-cell", linuxStack)
})

var _ = AfterEach(func() {
	ginkgomon.Kill(auctioneerProcess)
	etcdRunner.Stop()
	ginkgomon.Kill(bbsProcess)
	dotNetCell.Stop()
	linuxCell.Stop()
})

var _ = SynchronizedAfterSuite(func() {
	if etcdRunner != nil {
		etcdRunner.Stop()
	}
	if consulRunner != nil {
		consulRunner.Stop()
	}
}, func() {
	gexec.CleanupBuildArtifacts()
})
<commit_msg>Pass encryption key to BBS<commit_after>package main_test

import (
	"fmt"
	"net/url"
	"os/exec"
	"strings"

	"github.com/cloudfoundry-incubator/auctioneer"
	"github.com/cloudfoundry-incubator/bbs"
	bbstestrunner "github.com/cloudfoundry-incubator/bbs/cmd/bbs/testrunner"
	"github.com/cloudfoundry-incubator/consuladapter"
	"github.com/cloudfoundry-incubator/consuladapter/consulrunner"
	"github.com/cloudfoundry-incubator/locket"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	"github.com/cloudfoundry/storeadapter"
	"github.com/cloudfoundry/storeadapter/storerunner/etcdstorerunner"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/pivotal-golang/clock"
	"github.com/pivotal-golang/lager"
	"github.com/pivotal-golang/lager/lagertest"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"

	"testing"
	"time"
)

var auctioneerProcess ifrit.Process

var auctioneerPath string

var dotNetStack = "dot-net"
var dotNetRootFSURL = models.PreloadedRootFS(dotNetStack)
var linuxStack = "linux"
var linuxRootFSURL = models.PreloadedRootFS(linuxStack)
var dotNetCell, linuxCell *FakeCell

var auctioneerServerPort int
var auctioneerAddress string
var runner *ginkgomon.Runner

var etcdPort int
var etcdRunner *etcdstorerunner.ETCDClusterRunner
var etcdClient storeadapter.StoreAdapter

var consulRunner *consulrunner.ClusterRunner
var consulSession *consuladapter.Session

var auctioneerClient auctioneer.Client

var bbsArgs bbstestrunner.Args
var bbsBinPath string
var bbsURL *url.URL
var bbsRunner *ginkgomon.Runner
var bbsProcess ifrit.Process
var bbsClient bbs.Client

var locketClient locket.Client
var logger lager.Logger

func TestAuctioneer(t *testing.T) {
	// these integration tests can take a bit, especially under load;
	// 1 second is too harsh
	SetDefaultEventuallyTimeout(10 * time.Second)

	RegisterFailHandler(Fail)
	RunSpecs(t, "Auctioneer Cmd Suite")
}

var _ = SynchronizedBeforeSuite(func() []byte {
	bbsConfig, err := gexec.Build("github.com/cloudfoundry-incubator/bbs/cmd/bbs", "-race")
	Expect(err).NotTo(HaveOccurred())

	compiledAuctioneerPath, err := gexec.Build("github.com/cloudfoundry-incubator/auctioneer/cmd/auctioneer", "-race")
	Expect(err).NotTo(HaveOccurred())
	return []byte(strings.Join([]string{compiledAuctioneerPath, bbsConfig}, ","))
}, func(pathsByte []byte) {
	path := string(pathsByte)
	compiledAuctioneerPath := strings.Split(path, ",")[0]
	bbsBinPath = strings.Split(path, ",")[1]

	bbsBinPath = strings.Split(path, ",")[1]
	auctioneerPath = string(compiledAuctioneerPath)

	auctioneerServerPort = 1800 + GinkgoParallelNode()
	auctioneerAddress = fmt.Sprintf("http://127.0.0.1:%d", auctioneerServerPort)

	etcdPort = 5001 + GinkgoParallelNode()
	etcdRunner = etcdstorerunner.NewETCDClusterRunner(etcdPort, 1, nil)
	etcdClient = etcdRunner.Adapter(nil)

	consulRunner = consulrunner.NewClusterRunner(
		9001+config.GinkgoConfig.ParallelNode*consulrunner.PortOffsetLength,
		1,
		"http",
	)

	auctioneerClient = auctioneer.NewClient(auctioneerAddress)

	logger = lagertest.NewTestLogger("test")

	consulRunner.Start()
	consulRunner.WaitUntilReady()

	bbsAddress := fmt.Sprintf("127.0.0.1:%d", 13000+GinkgoParallelNode())

	bbsURL = &url.URL{
		Scheme: "http",
		Host:   bbsAddress,
	}

	bbsClient = bbs.NewClient(bbsURL.String())

	etcdUrl := fmt.Sprintf("http://127.0.0.1:%d", etcdPort)
	bbsArgs = bbstestrunner.Args{
		Address:           bbsAddress,
		AdvertiseURL:      bbsURL.String(),
		AuctioneerAddress: auctioneerAddress,
		EtcdCluster:       etcdUrl,
		ConsulCluster:     consulRunner.ConsulCluster(),

		EncryptionKeys: []string{"label:key"},
		ActiveKeyLabel: "label",
	}
})

var _ = BeforeEach(func() {
	consulRunner.Reset()
	etcdRunner.Start()

	bbsRunner = bbstestrunner.New(bbsBinPath, bbsArgs)
	bbsProcess = ginkgomon.Invoke(bbsRunner)

	consulSession = consulRunner.NewSession("a-session")

	locketClient = locket.NewClient(consulSession, clock.NewClock(), logger)

	runner = ginkgomon.New(ginkgomon.Config{
		Name: "auctioneer",
		Command: exec.Command(
			auctioneerPath,
			"-bbsAddress", bbsURL.String(),
			"-listenAddr", fmt.Sprintf("0.0.0.0:%d", auctioneerServerPort),
			"-lockRetryInterval", "1s",
			"-consulCluster", consulRunner.ConsulCluster(),
		),
		StartCheck: "auctioneer.started",
	})

	dotNetCell = SpinUpFakeCell(locketClient, "dot-net-cell", dotNetStack)
	linuxCell = SpinUpFakeCell(locketClient, "linux-cell", linuxStack)
})

var _ = AfterEach(func() {
	ginkgomon.Kill(auctioneerProcess)
	etcdRunner.Stop()
	ginkgomon.Kill(bbsProcess)
	dotNetCell.Stop()
	linuxCell.Stop()
})

var _ = SynchronizedAfterSuite(func() {
	if etcdRunner != nil {
		etcdRunner.Stop()
	}
	if consulRunner != nil {
		consulRunner.Stop()
	}
}, func() {
	gexec.CleanupBuildArtifacts()
})
<|endoftext|>


<commit_before>package main

import (
	"os"

	log "github.com/sirupsen/logrus"
	"github.com/spf13/cobra"

	"github.com/authelia/authelia/internal/utils"
)

func buildAutheliaBinary() {
	cmd := utils.CommandWithStdout("go", "build", "-o", "../../"+OutputDir+"/authelia")
	cmd.Dir = "cmd/authelia"

	cmd.Env = append(os.Environ(),
		"GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=1")

	err := cmd.Run()
	if err != nil {
		log.Fatal(err)
	}
}

func buildFrontend() {
	cmd := utils.CommandWithStdout("yarn", "install")
	cmd.Dir = webDirectory

	err := cmd.Run()
	if err != nil {
		log.Fatal(err)
	}

	cmd = utils.CommandWithStdout("yarn", "build")
	cmd.Dir = webDirectory

	cmd.Env = append(os.Environ(), "INLINE_RUNTIME_CHUNK=false")

	err = cmd.Run()
	if err != nil {
		log.Fatal(err)
	}

	cmd = utils.CommandWithStdout("rm", "-rf", "internal/server/public_html")

	err = cmd.Run()
	if err != nil {
		log.Fatal(err)
	}

	err = os.Rename("web/build", "internal/server/public_html")
	if err != nil {
		log.Fatal(err)
	}
}

func buildSwagger() {
	swaggerVer := "3.45.0"
	cmd := utils.CommandWithStdout("bash", "-c", "wget -q https://github.com/swagger-api/swagger-ui/archive/v"+swaggerVer+".tar.gz -O ./v"+swaggerVer+".tar.gz")

	err := cmd.Run()
	if err != nil {
		log.Fatal(err)
	}

	cmd = utils.CommandWithStdout("cp", "-r", "api", "internal/server/public_html")

	err = cmd.Run()
	if err != nil {
		log.Fatal(err)
	}

	cmd = utils.CommandWithStdout("tar", "-C", "internal/server/public_html/api", "--exclude=index.html", "--strip-components=2", "-xf", "v"+swaggerVer+".tar.gz", "swagger-ui-"+swaggerVer+"/dist")

	err = cmd.Run()
	if err != nil {
		log.Fatal(err)
	}

	cmd = utils.CommandWithStdout("rm", "./v"+swaggerVer+".tar.gz")

	err = cmd.Run()
	if err != nil {
		log.Fatal(err)
	}
}

func cleanAssets() {
	if err := os.Rename("internal/server/public_html", OutputDir+"/public_html"); err != nil {
		log.Fatal(err)
	}

	cmd := utils.CommandWithStdout("mkdir", "-p", "internal/server/public_html/api")

	if err := cmd.Run(); err != nil {
		log.Fatal(err)
	}

	cmd = utils.CommandWithStdout("bash", "-c", "touch internal/server/public_html/{index.html,api/index.html,api/openapi.yml}")

	if err := cmd.Run(); err != nil {
		log.Fatal(err)
	}
}

// Build build Authelia.
func Build(cobraCmd *cobra.Command, args []string) {
	log.Info("Building Authelia...")

	Clean(cobraCmd, args)

	log.Debug("Creating `" + OutputDir + "` directory")
	err := os.MkdirAll(OutputDir, os.ModePerm)

	if err != nil {
		log.Fatal(err)
	}

	log.Debug("Building Authelia frontend...")
	buildFrontend()

	log.Debug("Building swagger-ui frontend...")
	buildSwagger()

	log.Debug("Building Authelia Go binary...")
	buildAutheliaBinary()
	cleanAssets()
}
<commit_msg>build(deps): update swagger-ui to v3.46.0 (#1891)<commit_after>package main

import (
	"os"

	log "github.com/sirupsen/logrus"
	"github.com/spf13/cobra"

	"github.com/authelia/authelia/internal/utils"
)

func buildAutheliaBinary() {
	cmd := utils.CommandWithStdout("go", "build", "-o", "../../"+OutputDir+"/authelia")
	cmd.Dir = "cmd/authelia"

	cmd.Env = append(os.Environ(),
		"GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=1")

	err := cmd.Run()
	if err != nil {
		log.Fatal(err)
	}
}

func buildFrontend() {
	cmd := utils.CommandWithStdout("yarn", "install")
	cmd.Dir = webDirectory

	err := cmd.Run()
	if err != nil {
		log.Fatal(err)
	}

	cmd = utils.CommandWithStdout("yarn", "build")
	cmd.Dir = webDirectory

	cmd.Env = append(os.Environ(), "INLINE_RUNTIME_CHUNK=false")

	err = cmd.Run()
	if err != nil {
		log.Fatal(err)
	}

	cmd = utils.CommandWithStdout("rm", "-rf", "internal/server/public_html")

	err = cmd.Run()
	if err != nil {
		log.Fatal(err)
	}

	err = os.Rename("web/build", "internal/server/public_html")
	if err != nil {
		log.Fatal(err)
	}
}

func buildSwagger() {
	swaggerVer := "3.46.0"
	cmd := utils.CommandWithStdout("bash", "-c", "wget -q https://github.com/swagger-api/swagger-ui/archive/v"+swaggerVer+".tar.gz -O ./v"+swaggerVer+".tar.gz")

	err := cmd.Run()
	if err != nil {
		log.Fatal(err)
	}

	cmd = utils.CommandWithStdout("cp", "-r", "api", "internal/server/public_html")

	err = cmd.Run()
	if err != nil {
		log.Fatal(err)
	}

	cmd = utils.CommandWithStdout("tar", "-C", "internal/server/public_html/api", "--exclude=index.html", "--strip-components=2", "-xf", "v"+swaggerVer+".tar.gz", "swagger-ui-"+swaggerVer+"/dist")

	err = cmd.Run()
	if err != nil {
		log.Fatal(err)
	}

	cmd = utils.CommandWithStdout("rm", "./v"+swaggerVer+".tar.gz")

	err = cmd.Run()
	if err != nil {
		log.Fatal(err)
	}
}

func cleanAssets() {
	if err := os.Rename("internal/server/public_html", OutputDir+"/public_html"); err != nil {
		log.Fatal(err)
	}

	cmd := utils.CommandWithStdout("mkdir", "-p", "internal/server/public_html/api")

	if err := cmd.Run(); err != nil {
		log.Fatal(err)
	}

	cmd = utils.CommandWithStdout("bash", "-c", "touch internal/server/public_html/{index.html,api/index.html,api/openapi.yml}")

	if err := cmd.Run(); err != nil {
		log.Fatal(err)
	}
}

// Build build Authelia.
func Build(cobraCmd *cobra.Command, args []string) {
	log.Info("Building Authelia...")

	Clean(cobraCmd, args)

	log.Debug("Creating `" + OutputDir + "` directory")
	err := os.MkdirAll(OutputDir, os.ModePerm)

	if err != nil {
		log.Fatal(err)
	}

	log.Debug("Building Authelia frontend...")
	buildFrontend()

	log.Debug("Building swagger-ui frontend...")
	buildSwagger()

	log.Debug("Building Authelia Go binary...")
	buildAutheliaBinary()
	cleanAssets()
}
<|endoftext|>


<commit_before>package command

import (
	"flag"
	"fmt"
	"strings"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
)

type UnderReplicated struct {
	Ui  cli.Ui
	Cmd string

	zone string
}

func (this *UnderReplicated) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("underreplicated", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", "", "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if this.zone == "" {
		forSortedZones(func(zkzone *zk.ZkZone) {
			zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
				this.displayUnderReplicatedPartitionsOfCluster(zkcluster)
			})

			printSwallowedErrors(this.Ui, zkzone)
		})

		return
	}

	// a single zone
	ensureZoneValid(this.zone)
	zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
	zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		this.displayUnderReplicatedPartitionsOfCluster(zkcluster)
	})

	printSwallowedErrors(this.Ui, zkzone)

	return
}

func (this *UnderReplicated) displayUnderReplicatedPartitionsOfCluster(zkcluster *zk.ZkCluster) {
	this.Ui.Output(zkcluster.Name())

	brokerList := zkcluster.BrokerList()
	if len(brokerList) == 0 {
		this.Ui.Output(fmt.Sprintf("%4s%s", " ", color.Red("empty brokers")))
		return
	}

	kfk, err := sarama.NewClient(brokerList, sarama.NewConfig())
	if err != nil {
		this.Ui.Output(color.Yellow("%4s%+v %s", " ", brokerList, err.Error()))

		return
	}
	defer kfk.Close()

	topics, err := kfk.Topics()
	swallow(err)
	if len(topics) == 0 {
		return
	}

	for _, topic := range topics {
		// get partitions and check if some dead
		alivePartitions, err := kfk.WritablePartitions(topic)
		if err != nil {
			this.Ui.Error(color.Red("topic[%s] cannot fetch writable partitions: %v", topic, err))
			continue
		}
		partions, err := kfk.Partitions(topic)
		if err != nil {
			this.Ui.Error(color.Red("topic[%s] cannot fetch partitions: %v", topic, err))
			continue
		}
		if len(alivePartitions) != len(partions) {
			this.Ui.Error(fmt.Sprintf("topic[%s] has %s partitions: %+v/%+v",
				topic, color.Red("dead"), alivePartitions, partions))
		}

		for _, partitionID := range alivePartitions {
			replicas, err := kfk.Replicas(topic, partitionID)
			if err != nil {
				this.Ui.Error(color.Red("topic[%s] P:%d: %v", topic, partitionID, err))
				continue
			}

			isr := zkcluster.Isr(topic, partitionID)

			underReplicated := false
			if len(isr) != len(replicas) {
				underReplicated = true
			}

			if underReplicated {
				leader, err := kfk.Leader(topic, partitionID)
				swallow(err)

				latestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetNewest)
				swallow(err)

				oldestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetOldest)
				swallow(err)

				this.Ui.Output(color.Red("\t%s Partition:%d Leader:%d Replicas:%+v Isr:%+v Offset:%d Num:%d",
					topic,
					partitionID, leader.ID(), replicas, isr,
					latestOffset, latestOffset-oldestOffset))
			}
		}
	}
}

func (*UnderReplicated) Synopsis() string {
	return "Display under-replicated partitions"
}

func (this *UnderReplicated) Help() string {
	help := fmt.Sprintf(`
Usage: %s underreplicated [options]

    Display under-replicated partitions

Options:

    -z zone
`, this.Cmd)
	return strings.TrimSpace(help)
}
<commit_msg>alignment<commit_after>package command

import (
	"flag"
	"fmt"
	"strings"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
)

type UnderReplicated struct {
	Ui  cli.Ui
	Cmd string

	zone string
}

func (this *UnderReplicated) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("underreplicated", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", "", "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if this.zone == "" {
		forSortedZones(func(zkzone *zk.ZkZone) {
			this.Ui.Output(zkzone.Name())
			zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
				this.displayUnderReplicatedPartitionsOfCluster(zkcluster)
			})

			printSwallowedErrors(this.Ui, zkzone)
		})

		return
	}

	// a single zone
	ensureZoneValid(this.zone)
	zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
	this.Ui.Output(zkzone.Name())
	zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		this.displayUnderReplicatedPartitionsOfCluster(zkcluster)
	})

	printSwallowedErrors(this.Ui, zkzone)

	return
}

func (this *UnderReplicated) displayUnderReplicatedPartitionsOfCluster(zkcluster *zk.ZkCluster) {
	this.Ui.Output(fmt.Sprintf("%s %s", strings.Repeat(" ", 4), zkcluster.Name()))

	brokerList := zkcluster.BrokerList()
	if len(brokerList) == 0 {
		this.Ui.Output(fmt.Sprintf("%4s%s", " ", color.Red("empty brokers")))
		return
	}

	kfk, err := sarama.NewClient(brokerList, sarama.NewConfig())
	if err != nil {
		this.Ui.Output(color.Yellow("%4s%+v %s", " ", brokerList, err.Error()))

		return
	}
	defer kfk.Close()

	topics, err := kfk.Topics()
	swallow(err)
	if len(topics) == 0 {
		return
	}

	for _, topic := range topics {
		// get partitions and check if some dead
		alivePartitions, err := kfk.WritablePartitions(topic)
		if err != nil {
			this.Ui.Error(color.Red("topic[%s] cannot fetch writable partitions: %v", topic, err))
			continue
		}
		partions, err := kfk.Partitions(topic)
		if err != nil {
			this.Ui.Error(color.Red("topic[%s] cannot fetch partitions: %v", topic, err))
			continue
		}
		if len(alivePartitions) != len(partions) {
			this.Ui.Error(fmt.Sprintf("topic[%s] has %s partitions: %+v/%+v",
				topic, color.Red("dead"), alivePartitions, partions))
		}

		for _, partitionID := range alivePartitions {
			replicas, err := kfk.Replicas(topic, partitionID)
			if err != nil {
				this.Ui.Error(color.Red("topic[%s] P:%d: %v", topic, partitionID, err))
				continue
			}

			isr := zkcluster.Isr(topic, partitionID)

			underReplicated := false
			if len(isr) != len(replicas) {
				underReplicated = true
			}

			if underReplicated {
				leader, err := kfk.Leader(topic, partitionID)
				swallow(err)

				latestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetNewest)
				swallow(err)

				oldestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetOldest)
				swallow(err)

				this.Ui.Output(color.Red("\t%s Partition:%d Leader:%d Replicas:%+v Isr:%+v Offset:%d Num:%d",
					topic,
					partitionID, leader.ID(), replicas, isr,
					latestOffset, latestOffset-oldestOffset))
			}
		}
	}
}

func (*UnderReplicated) Synopsis() string {
	return "Display under-replicated partitions"
}

func (this *UnderReplicated) Help() string {
	help := fmt.Sprintf(`
Usage: %s underreplicated [options]

    Display under-replicated partitions

Options:

    -z zone
`, this.Cmd)
	return strings.TrimSpace(help)
}
<|endoftext|>